diff --git a/.cursor/rules/rust-best-practices.mdc b/.cursor/rules/rust-best-practices.mdc new file mode 100644 index 0000000000..e8511c67d4 --- /dev/null +++ b/.cursor/rules/rust-best-practices.mdc @@ -0,0 +1,109 @@ +--- +description: +globs: backend/**/*.rs +alwaysApply: false +--- +# Windmill Backend - Rust Best Practices + +## Project Structure + +Windmill uses a workspace-based architecture with multiple crates: + +- **windmill-api**: API server functionality +- **windmill-worker**: Job execution +- **windmill-common**: Shared code used by all crates +- **windmill-queue**: Job & flow queuing +- **windmill-audit**: Audit logging +- Other specialized crates (git-sync, autoscaling, etc.) + +## Adding New Code + +### Module Organization + +- Place new code in the appropriate crate based on functionality +- For API endpoints, create or modify files in `windmill-api/src/` organized by domain +- For shared functionality, use `windmill-common/src/` +- Use the `_ee.rs` suffix for enterprise-only modules +- Follow existing patterns for file structure and organization + +### Error Handling + +- Use the custom `Error` enum from `windmill-common::error` +- Return `Result` or `JsonResult` for functions that can fail +- Use the `?` operator for error propagation +- Add location tracking to errors using `#[track_caller]` + +### Database Operations + +- Use `sqlx` for database operations with prepared statements +- Leverage existing database helper functions in `db.rs` modules +- Use transactions for multi-step operations +- Handle database errors properly + +### API Endpoints + +- Follow existing patterns in the `windmill-api` crate +- Use axum's routing system and extractors +- Group related routes together +- Use consistent response formats (JSON) +- Follow proper authentication and authorization patterns + +## Performance Optimizations + +When generating code, especially involving `serde`, `sqlx`, and `tokio`, prioritize performance by applying the following principles: + +### Serde Optimizations (Serialization & Deserialization) + +- **Specify Structure Explicitly:** When defining structs for Serde (`#[derive(Serialize, Deserialize)]`), use `#[serde(...` attributes extensively. This includes: + * `#[serde(rename = "...")]` or `#[serde(alias = "...")]` to map external names precisely, avoiding dynamic lookups. + * `#[serde(default)]` for optional fields with default values, reducing parsing complexity. + * `#[serde(skip_serializing_if = "...")]` to avoid writing fields that meet a certain condition (e.g., `Option::is_none()`, `Vec::is_empty()`, or a custom function), reducing output size and serialization work. + * `#[serde(skip_serializing)]` or `#[serde(skip_deserializing)]` for fields that should *not* be included. +- **Prefer Borrowing:** Where possible and safe (data lifetime allows), use `Cow<'a, str>` or `&'a str` (with `#[serde(borrow)]`) instead of `String` for string fields during deserialization. This avoids allocating new strings, enabling zero-copy reading from the input buffer. Apply this principle to byte slices (`&'a [u8]` / `Cow<'a, [u8]>`) and potentially borrowed vectors as well. +- **Avoid Intermediate `Value`:** Unless the data structure is truly dynamic or unknown at compile time, deserialize directly into a well-defined struct or enum rather than into `serde_json::Value` (or equivalent for other formats). This avoids unnecessary heap allocations and type switching. + +### SQLx Optimizations (Database Interaction) + +- **Select Only Necessary Columns:** In `SELECT` queries, list specific column names rather than using `SELECT *`. This reduces data transferred from the database and the work needed for hydration/deserialization. +- **Batch Operations:** For multiple `INSERT`, `UPDATE`, or `DELETE` statements, prefer executing them in a single query if the database and driver support it efficiently (e.g., `INSERT INTO ... VALUES (...), (...), ...`). This minimizes round trips to the database. +- **Avoid N+1 Queries:** Do not loop through results of one query and execute a separate query for each item (e.g., fetching users, then querying for each user's profile in a loop). Instead, use JOINs or a single query with an `IN` clause to fetch related data efficiently. +- **Deserialize Directly:** Use `#[derive(FromRow)]` on structs and ensure the struct fields match the selected columns in the query. This allows SQLx to hydrate objects directly, avoiding intermediate data structures. +- **Parameterize Queries:** Always use SQLx's query methods (`.bind(...)`) to pass values as parameters rather than string formatting. This prevents SQL injection and allows the database to cache query plans, improving performance on repeated executions. + +### Tokio Optimizations (Asynchronous Runtime) + +- **Avoid Blocking Operations:** **Crucially**, never perform blocking operations (synchronous file I/O, `std::thread::sleep`, CPU-bound loops, `std::sync::Mutex::lock`, blocking network calls without `tokio::net`) directly within an `async fn` or a standard `tokio::spawn` task. Blocking pauses the entire worker thread, potentially starving other tasks. Use `tokio::task::spawn_blocking` for CPU-intensive work or blocking I/O. +- **Use Tokio's Async Primitives:** Prefer `tokio::sync` (channels, mutexes, semaphores), `tokio::io`, `tokio::net`, and `tokio::time` over their `std` counterparts in asynchronous contexts. These are designed to yield control back to the scheduler. +- **Manage Concurrency:** Be mindful of how many tasks are spawned. Creating a new task for every tiny piece of work can introduce overhead. Group related asynchronous operations where appropriate. +- **Handle Shared State Efficiently:** Use `Arc` for shared ownership in concurrent tasks. When shared state needs mutation, prefer `tokio::sync::Mutex` over `std::sync::Mutex` in `async` code. Consider `tokio::sync::RwLock` if reads significantly outnumber writes. Minimize the duration for which locks are held. +- **Understand `.await`:** Place `.await` strategically to allow the runtime to switch to other ready tasks. Ensure that `.await` points to genuinely asynchronous operations. +- **Backpressure:** If dealing with data streams or queues between tasks, implement backpressure mechanisms (e.g., bounded channels like `tokio::sync::mpsc::channel`) to prevent one component from overwhelming another or critical resources like the database. + +## Enterprise Features + +- Use feature flags for enterprise functionality +- Conditionally compile with `#[cfg(feature = "enterprise")]` +- Isolate enterprise code in separate modules + +## Code Style + +- Group imports by external and internal crates +- Place struct/enum definitions before implementations +- Group similar functionality together +- Use descriptive naming consistent with the codebase +- Follow existing patterns for async code using tokio + +## Testing + +- Write unit tests for core functionality +- Use the `#[cfg(test)]` module for test code +- For database tests, use the existing test utilities + +## Common Crates Used + +- **tokio**: For async runtime +- **axum**: For web server and routing +- **sqlx**: For database operations +- **serde**: For serialization/deserialization +- **tracing**: For logging and diagnostics +- **reqwest**: For HTTP client functionality \ No newline at end of file diff --git a/.cursor/rules/svelte5-best-practices.mdc b/.cursor/rules/svelte5-best-practices.mdc new file mode 100644 index 0000000000..c2fa3baae8 --- /dev/null +++ b/.cursor/rules/svelte5-best-practices.mdc @@ -0,0 +1,229 @@ +--- +description: +globs: frontend/src/**/*.svelte +alwaysApply: false +--- +# Svelte 5 Best Practices + +This guide outlines best practices for developing with Svelte 5, incorporating the new Runes API and other modern Svelte features. They should be applied on every new files created, but not on existing svelte 4 files unless specifically asked to. + +## Reactivity with Runes + +Svelte 5 introduces Runes for more explicit and flexible reactivity. + +1. **Embrace Runes for State Management**: + * Use `$state` for reactive local component state. + ```svelte + + + + ``` + * Use `$derived` for computed values based on other reactive state. + ```svelte + + +

{count} * 2 = {doubled}

+ ``` + * Use `$effect` for side effects that need to run when reactive values change (e.g., logging, manual DOM manipulation, data fetching). Remember `$effect` does not run on the server. + ```svelte + + ``` + +2. **Props with `$props`**: + * Declare component props using `$props()`. This offers better clarity and flexibility compared to `export let`. + ```svelte + + +

Name: {name}

+

Age: {age}

+ ``` + * For bindable props, use `$bindable`. + ```svelte + + + + ``` + +## Event Handling + +* **Use direct event attributes**: Svelte 5 moves away from `on:` directives for DOM events. + * **Do**: `` + * **Don't**: `` +* **For component events, prefer callback props**: Instead of `createEventDispatcher`, pass functions as props. + ```svelte + + + +

Message from child: {message}

+ + + + + ``` + +## Snippets for Content Projection + +* **Use `{#snippet ...}` and `{@render ...}` instead of slots**: Snippets are more powerful and flexible. + ```svelte + + + + + {#snippet title()} + My Awesome Title + {/snippet} + {#snippet content()} +

Some interesting content here.

+ {/snippet} + + + + + +
+
{@render title()}
+
{@render content()}
+
+ ``` +* Default content is passed via the `children` prop (which is a snippet). + ```svelte + + +
+ {@render children?.()} +
+ ``` + +## Component Design + +1. **Create Small, Reusable Components**: Break down complex UIs into smaller, focused components. Each component should have a single responsibility. This also aids performance by limiting the scope of reactivity updates. +2. **Descriptive Naming**: Use clear and descriptive names for variables, functions, and components. +3. **Minimize Logic in Components**: Move complex business logic to utility functions or services. Keep components focused on presentation and interaction. + +## State Management (Stores) + +1. **Segment Stores**: Avoid a single global store. Create multiple stores, each responsible for a specific piece of global state (e.g., `userStore.js`, `themeStore.js`). This can help limit reactivity updates to only the parts of the UI that depend on specific state segments. +2. **Use Custom Stores for Complex Logic**: For stores with related methods, create custom stores. + ```javascript + // counterStore.js + import { writable } from 'svelte/store'; + + function createCounter() { + const { subscribe, set, update } = writable(0); + + return { + subscribe, + increment: () => update(n => n + 1), + decrement: () => update(n => n - 1), + reset: () => set(0) + }; + } + export const counter = createCounter(); + ``` +3. **Use Context API for Localized State**: For state shared within a component subtree, consider Svelte's context API (`setContext`, `getContext`) instead of global stores when the state doesn't need to be truly global. + +## Performance Optimizations (Svelte 5) + +When generating Svelte 5 code, prioritize frontend performance by applying the following principles: + +### General Svelte 5 Principles + +- **Leverage the Compiler:** Trust Svelte's compiler to generate optimized JavaScript. Avoid manual DOM manipulation (`document.querySelector`, etc.) unless absolutely necessary for integrating third-party libraries that lack Svelte adapters. +- **Keep Components Small and Focused:** Reinforcing from Component Design, smaller components lead to less complex reactivity graphs and more targeted, efficient updates. + +### Reactivity & State Management + +- **Optimize Computations with `$derived`:** Always use `$derived` for computed values that depend on other state. This ensures the computation only runs when its specific dependencies change, avoiding unnecessary work compared to recomputing derived values in `$effect` or less efficient methods. +- **Minimize `$effect` Usage:** Use `$effect` sparingly and only for true side effects that interact with the outside world or non-Svelte state. Avoid putting complex logic or state updates *within* an `$effect` unless those updates are explicitly intended as a reaction to external changes or non-Svelte state. Excessive or complex effects can impact rendering performance. +- **Structure State for Fine-Grained Updates:** Design your `$state` objects or variables such that updates affect only the necessary parts of the UI. Avoid putting too much unrelated state into a single large object that gets frequently updated, as this can potentially trigger broader updates than necessary. Consider normalizing complex, nested state. + +### List Rendering (`{#each}`) + +- **Mandate `key` Attribute:** Always use a `key` attribute (`{#each items as item (item.id)}`) that refers to a unique, stable identifier for each item in a list. This is critical for allowing Svelte to efficiently update, reorder, add, or remove list items without destroying and re-creating unnecessary DOM elements and component instances. + +### Component Loading & Bundling + +- **Implement Lazy Loading/Code Splitting:** For routes, components, or modules that are not immediately needed on page load, use dynamic imports (`import(...)`) to split the code bundle. SvelteKit handles this automatically for routes, but it can be applied manually to components using helper patterns if needed. +- **Be Mindful of Third-Party Libraries:** When incorporating external libraries, import only the necessary functions or components to minimize the final bundle size. Prefer libraries designed to be tree-shakeable. + +### Rendering & DOM + +- **Use CSS for Animations/Transitions:** Prefer CSS animations or transitions where possible for performance. Svelte's built-in `transition:` directive is also highly optimized and should be used for complex state-driven transitions, but simple cases can often use plain CSS. +- **Optimize Image Loading:** Implement best practices for images: use optimized formats (WebP, AVIF), lazy loading (`loading="lazy"`), and responsive images (``, `srcset`) to avoid loading unnecessarily large images. + +### Server-Side Rendering (SSR) & Hydration + +- **Ensure SSR Compatibility:** Write components that can be rendered on the server for faster initial page loads. Avoid relying on browser-specific APIs (like `window` or `document`) in the main ` + +
+ Hello +
+ ``` +5. **Stay Updated**: Keep Svelte and its related packages up to date to benefit from the latest features, performance improvements, and security fixes. \ No newline at end of file diff --git a/.env b/.env index f09d57bc24..ad887513cd 100644 --- a/.env +++ b/.env @@ -1,5 +1,13 @@ -DATABASE_URL=postgres://postgres:changeme@localhost:5432/windmill?sslmode=disable -DENO_PATH=/opt/homebrew/bin/deno -BUN_PATH=/opt/homebrew/bin/bun +DATABASE_URL=postgres://postgres:changeme@db/windmill?sslmode=disable -WM_IMAGE=ghcr.io/windmill-labs/windmill:main \ No newline at end of file +# For Enterprise Edition, use: +# WM_IMAGE=ghcr.io/windmill-labs/windmill-ee:main +WM_IMAGE=ghcr.io/windmill-labs/windmill:main + + +# To use another port than :80, setup the Caddyfile and the caddy section of the docker-compose to your needs: https://caddyserver.com/docs/getting-started +# To have caddy take care of automatic TLS + +# To rotate logs, set the following variables: +#LOG_MAX_SIZE=10m +#LOG_MAX_FILE=3 \ No newline at end of file diff --git a/.github/change-versions.sh b/.github/change-versions.sh index 699b87b0d7..454f373038 100755 --- a/.github/change-versions.sh +++ b/.github/change-versions.sh @@ -24,4 +24,4 @@ sed -i -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile sed -i -zE "s/name = \"windmill\"\nversion = \"[^\"]*\"\\n(.*)/name = \"windmill\"\nversion = \"$VERSION\"\\n\\1/" ${root_dirpath}/backend/Cargo.lock -cd ${root_dirpath}/frontend && npm i --package-lock-only +cd ${root_dirpath}/frontend && npm i --package-lock-only --ignore-scripts diff --git a/.github/workflows/build-publish-rh-image.yml b/.github/workflows/build-publish-rh-image.yml index 030c617777..b919de6e18 100644 --- a/.github/workflows/build-publish-rh-image.yml +++ b/.github/workflows/build-publish-rh-image.yml @@ -64,7 +64,7 @@ jobs: platforms: linux/amd64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp secrets: | rh_username=${{ secrets.RH_USERNAME }} rh_password=${{ secrets.RH_PASSWORD }} @@ -81,7 +81,7 @@ jobs: platforms: linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp secrets: | rh_username=${{ secrets.RH_USERNAME }} rh_password=${{ secrets.RH_PASSWORD }} diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml index 785ade3201..ac1476d76e 100644 --- a/.github/workflows/build_windows_worker_.yml +++ b/.github/workflows/build_windows_worker_.yml @@ -51,7 +51,7 @@ jobs: $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static" mkdir frontend/build && cd backend New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force - cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages + cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,mcp - name: Rename binary with corresponding architecture run: | Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe" diff --git a/.github/workflows/docker-image-rpi4.yml b/.github/workflows/docker-image-rpi4.yml index b275ad2154..bb270aac8a 100644 --- a/.github/workflows/docker-image-rpi4.yml +++ b/.github/workflows/docker-image-rpi4.yml @@ -67,7 +67,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=embedding,parquet,openidconnect,license,http_trigger,zip,oauth2,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core + features=embedding,parquet,openidconnect,license,http_trigger,zip,oauth2,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev ${{ steps.meta-public.outputs.tags }} diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 06f8003c96..216d7cefea 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -92,7 +92,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,agent_worker_server,all_languages,deno_core + features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,agent_worker_server,all_languages,deno_core,mcp tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} ${{ steps.meta-public.outputs.tags }} @@ -154,7 +154,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,agent_worker_server,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,static_frontend,all_languages,deno_core + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,agent_worker_server,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} ${{ steps.meta-ee-public.outputs.tags }} diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml index 5b591abc3c..057a858881 100644 --- a/.github/workflows/publish_windows_worker.yml +++ b/.github/workflows/publish_windows_worker.yml @@ -53,7 +53,7 @@ jobs: $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static" mkdir frontend/build && cd backend New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force - cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,static_frontend,all_languages + cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,static_frontend,all_languages,mcp - name: Rename binary with corresponding architecture run: | Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe" diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c5ee479f4..2814c7bb72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,142 @@ # Changelog +## [1.490.0](https://github.com/windmill-labs/windmill/compare/v1.489.0...v1.490.0) (2025-05-12) + + +### Features + +* preprocessor refactor ([#5629](https://github.com/windmill-labs/windmill/issues/5629)) ([254c3cf](https://github.com/windmill-labs/windmill/commit/254c3cf8eff32071d5290429aafd26992527fbca)) + + +### Bug Fixes + +* add back missing query args from http trigger object + correct wm_trigger shape ([#5722](https://github.com/windmill-labs/windmill/issues/5722)) ([66798df](https://github.com/windmill-labs/windmill/commit/66798df38464d732864627ae27a0e51e9518c609)) +* fix date input issue with initializer ([0cd9293](https://github.com/windmill-labs/windmill/commit/0cd92932f0e0998fc30ac02065d292ec35db5cae)) +* improve agents workers handling of WHITELIST_ENVS ([7c69959](https://github.com/windmill-labs/windmill/commit/7c699598533dade9713d976d8dd90fc657ebb503)) +* improve error display of nativets exceptions ([a3c76fb](https://github.com/windmill-labs/windmill/commit/a3c76fb10cba4d18547e66e47edec84833172b64)) +* make ansible more resilient to invalid lockfiles ([b51568c](https://github.com/windmill-labs/windmill/commit/b51568c166e29ec5ee4053fb14abda2fe6d46488)) + +## [1.489.0](https://github.com/windmill-labs/windmill/compare/v1.488.0...v1.489.0) (2025-05-08) + + +### Features + +* raise error if end early in flow ([#5653](https://github.com/windmill-labs/windmill/issues/5653)) ([242a565](https://github.com/windmill-labs/windmill/commit/242a5654285b0a3bf222c80e82f6861ffafed838)) + +## [1.488.0](https://github.com/windmill-labs/windmill/compare/v1.487.0...v1.488.0) (2025-05-07) + + +### Features + +* handle . in interpolated args ([0ac8e47](https://github.com/windmill-labs/windmill/commit/0ac8e477d6fb7c5a7699a198fce9d18a08aff68c)) + + +### Bug Fixes + +* fix azure object storage regression due to object_store regression ([df9f827](https://github.com/windmill-labs/windmill/commit/df9f827d103def27166a767044373bd0754285e2)) +* performance and stability improvement to fetch last deployed script ([75d9924](https://github.com/windmill-labs/windmill/commit/75d992449c845fd11c9a317d401c405e7d78e1ec)) + +## [1.487.0](https://github.com/windmill-labs/windmill/compare/v1.486.1...v1.487.0) (2025-05-06) + + +### Features + +* critical alert if disk near full ([#5549](https://github.com/windmill-labs/windmill/issues/5549)) ([4fd0561](https://github.com/windmill-labs/windmill/commit/4fd056123907337efb5f5669975b337973a124cc)) + + +### Bug Fixes + +* ansible in agent mode can use inventory.ini ([9bdd301](https://github.com/windmill-labs/windmill/commit/9bdd301f5296fbfb631df9ff9100e92e0984ff64)) + +## [1.486.1](https://github.com/windmill-labs/windmill/compare/v1.486.0...v1.486.1) (2025-05-04) + + +### Bug Fixes + +* improve MultiSelectWrapper behavior ([36da8ae](https://github.com/windmill-labs/windmill/commit/36da8aec080742e13f23e1dee12b3954947f53dd)) + +## [1.486.0](https://github.com/windmill-labs/windmill/compare/v1.485.3...v1.486.0) (2025-05-01) + + +### Features + +* add run now directly on schedule drawer and duplicate schedule option ([#5674](https://github.com/windmill-labs/windmill/issues/5674)) ([dfb947f](https://github.com/windmill-labs/windmill/commit/dfb947ff37c688f54a32de5aa3c5c3d142cb80f4)) +* Database Manager ([#5586](https://github.com/windmill-labs/windmill/issues/5586)) ([41c15fc](https://github.com/windmill-labs/windmill/commit/41c15fc78aaf844c559d3d6c772e04ecce436e9d)) +* Integrate MCP with hub ([#5685](https://github.com/windmill-labs/windmill/issues/5685)) ([ec701a9](https://github.com/windmill-labs/windmill/commit/ec701a9ee74c9d890b54234362392deca63a77c7)) + + +### Bug Fixes + +* Ai Chat: do not send tools if empty + respond even if tool fails ([#5692](https://github.com/windmill-labs/windmill/issues/5692)) ([9c55040](https://github.com/windmill-labs/windmill/commit/9c55040e47e76af8b7e2864b82fa30505545dcb5)) +* do not track relative deps for scripts with raw defined deps from CLI ([#5696](https://github.com/windmill-labs/windmill/issues/5696)) ([7eb9d7d](https://github.com/windmill-labs/windmill/commit/7eb9d7d46cb48ae69a3fd3ff852a57abae450a3b)) +* improve CLI file scanning performances ([0916978](https://github.com/windmill-labs/windmill/commit/09169784bd2d0ab7acf5f40dc86f36f1cae967b7)) + +## [1.485.3](https://github.com/windmill-labs/windmill/compare/v1.485.2...v1.485.3) (2025-04-29) + + +### Bug Fixes + +* improve performance of background cleanup monitoring operations ([18dced3](https://github.com/windmill-labs/windmill/commit/18dced3c748cd5305f0934b26e50d69899563723)) + +## [1.485.2](https://github.com/windmill-labs/windmill/compare/v1.485.1...v1.485.2) (2025-04-29) + + +### Bug Fixes + +* improve agent workers for deployed scripts ([60018aa](https://github.com/windmill-labs/windmill/commit/60018aadf62cecadf111e019d3600513a89810f1)) +* make `#(extra_)requirements:` work better with pins ([#5680](https://github.com/windmill-labs/windmill/issues/5680)) ([1ab4160](https://github.com/windmill-labs/windmill/commit/1ab41603f4fd1526d0c944396ef250b184aed1f4)) +* **python:** handle better relative imports with requirements or extra_requirements ([f662cf5](https://github.com/windmill-labs/windmill/commit/f662cf5d75beed8fd114ba171cbe0fa8e4b2773f)) + +## [1.485.1](https://github.com/windmill-labs/windmill/compare/v1.485.0...v1.485.1) (2025-04-28) + + +### Bug Fixes + +* improve mcp mode api ([cf77ff0](https://github.com/windmill-labs/windmill/commit/cf77ff088b8382b861113120589de58f7cf241d0)) +* MCP handle long names + invalid char in prop key + fix for not found resource type ([#5668](https://github.com/windmill-labs/windmill/issues/5668)) ([eadae95](https://github.com/windmill-labs/windmill/commit/eadae95a42d679bf8792bdefd8b9d19dbcbc4b57)) +* skip_flow_update for dependency tracking table ([#5670](https://github.com/windmill-labs/windmill/issues/5670)) ([35b69da](https://github.com/windmill-labs/windmill/commit/35b69da25c5bd17deff5a54b635e9150cb865cc0)) + +## [1.485.0](https://github.com/windmill-labs/windmill/compare/v1.484.0...v1.485.0) (2025-04-28) + + +### Features + +* add universal search to object viewer ([7254743](https://github.com/windmill-labs/windmill/commit/72547437fead0a071fceac27dae8628cdcae6a3e)) + + +### Bug Fixes + +* add svelte 5 boundaries to app components to contain errors ([1b16918](https://github.com/windmill-labs/windmill/commit/1b1691837a7e6b88afbacf7d88c14ca5e475b493)) +* Fix object handling on some MCP clients + better frontend for MCP ([#5663](https://github.com/windmill-labs/windmill/issues/5663)) ([12c3202](https://github.com/windmill-labs/windmill/commit/12c32026e5879a65fc0f1cc9f2481087c4b95111)) + +## [1.484.0](https://github.com/windmill-labs/windmill/compare/v1.483.2...v1.484.0) (2025-04-26) + + +### Features + +* Add MCP endpoints ([#5639](https://github.com/windmill-labs/windmill/issues/5639)) ([a34ac4f](https://github.com/windmill-labs/windmill/commit/a34ac4fa24c2a5482e45724e76316d57f64f7040)) +* Add MCP only mode ([#5661](https://github.com/windmill-labs/windmill/issues/5661)) ([1625524](https://github.com/windmill-labs/windmill/commit/162552431138d68002c7060cad4ae31f1ec4c69c)) +* Ansible improvements (vault, roles and git repos) ([#5655](https://github.com/windmill-labs/windmill/issues/5655)) ([fdd1642](https://github.com/windmill-labs/windmill/commit/fdd1642ce10866da1d8d373bda44f050e2e0f403)) + + +### Bug Fixes + +* check for valid teams_channel config when saving critical alerts settings ([#5660](https://github.com/windmill-labs/windmill/issues/5660)) ([dc5c8d8](https://github.com/windmill-labs/windmill/commit/dc5c8d8c5f8577b7ded3da1d684cdb735fa7a936)) +* Fix CI for MCP + optimization ([#5657](https://github.com/windmill-labs/windmill/issues/5657)) ([b199a77](https://github.com/windmill-labs/windmill/commit/b199a77d486c5bfd086ca73a58a14bb747e386b5)) +* fix token creation after mcp mode change to make it non workspace specific ([2b5dfcf](https://github.com/windmill-labs/windmill/commit/2b5dfcfb251471dcc39b04c54e25008d617cc34f)) +* improve full-scaleout of autoscaling event logging ([8435eb3](https://github.com/windmill-labs/windmill/commit/8435eb3adff8429a73db88b12204f7cf8f14d3d2)) +* improve skip failure on parallel branchall ([a7b2b51](https://github.com/windmill-labs/windmill/commit/a7b2b51444d757964560de3a89024b1c9b0fefe9)) + +## [1.483.2](https://github.com/windmill-labs/windmill/compare/v1.483.1...v1.483.2) (2025-04-23) + + +### Bug Fixes + +* batch reruns query missing workspace_id check in subquery ([#5652](https://github.com/windmill-labs/windmill/issues/5652)) ([444a6ab](https://github.com/windmill-labs/windmill/commit/444a6abad670114c52e44f3606bf6fefc5d3fd98)) +* **frontend:** fix validity check ([#5654](https://github.com/windmill-labs/windmill/issues/5654)) ([c41c1eb](https://github.com/windmill-labs/windmill/commit/c41c1eb587bf22364f1202310a0c64b6040ab968)) +* improve MySQL datetime parser timezone handling (WIN-1155) ([#5645](https://github.com/windmill-labs/windmill/issues/5645)) ([5bca8f6](https://github.com/windmill-labs/windmill/commit/5bca8f60e970cc67839edb5dc491685f36cf0499)) +* track relative imports in python and ts even if lockfile is provided ([e316dbd](https://github.com/windmill-labs/windmill/commit/e316dbd9bdd5c59e9aaba6a4472bb7d832834e84)) + ## [1.483.1](https://github.com/windmill-labs/windmill/compare/v1.483.0...v1.483.1) (2025-04-19) diff --git a/Dockerfile b/Dockerfile index 0c824dbb83..0073ecf8a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG DEBIAN_IMAGE=debian:bookworm-slim -ARG RUST_IMAGE=rust:1.85-slim-bookworm +ARG RUST_IMAGE=rust:1.86-slim-bookworm FROM ${RUST_IMAGE} AS rust_base diff --git a/README.md b/README.md index a733727825..27a5aaf296 100644 --- a/README.md +++ b/README.md @@ -363,6 +363,7 @@ you to have it being synced automatically everyday. | DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker | | DISABLE_RESPONSE_LOGS | false | Disable response logs | Server | | CREATE_WORKSPACE_REQUIRE_SUPERADMIN | true | If true, only superadmins can create new workspaces | Server | +| MIN_FREE_DISK_SPACE_MB | 15000 | Minimum amount of free space on worker. Sends critical alert if worker has less free space. | Worker | ## Run a local dev setup diff --git a/backend/.sqlx/query-f822df86bf511fe0d3044b69525a6ff2524167877929886adca9b0fa3d3fee45.json b/backend/.sqlx/query-00c4a602aa6a50f2f922851ce63b5216e915c7649698687a00d47da55c70349f.json similarity index 69% rename from backend/.sqlx/query-f822df86bf511fe0d3044b69525a6ff2524167877929886adca9b0fa3d3fee45.json rename to backend/.sqlx/query-00c4a602aa6a50f2f922851ce63b5216e915c7649698687a00d47da55c70349f.json index 3adc1f8cd5..a3ec5fbabe 100644 --- a/backend/.sqlx/query-f822df86bf511fe0d3044b69525a6ff2524167877929886adca9b0fa3d3fee45.json +++ b/backend/.sqlx/query-00c4a602aa6a50f2f922851ce63b5216e915c7649698687a00d47da55c70349f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n DELETE\n FROM parallel_monitor_lock\n WHERE last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval \n RETURNING parent_flow_id, job_id, last_ping, (SELECT workspace_id FROM v2_job_queue q\n WHERE q.id = parent_flow_id AND q.running = true AND q.canceled_by IS NULL\n ) AS workspace_id\n ", + "query": "\n DELETE\n FROM parallel_monitor_lock\n WHERE last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval\n RETURNING parent_flow_id, job_id, last_ping, (SELECT workspace_id FROM v2_job_queue q\n WHERE q.id = parent_flow_id AND q.running = true AND q.canceled_by IS NULL\n ) AS workspace_id\n ", "describe": { "columns": [ { @@ -36,5 +36,5 @@ null ] }, - "hash": "f822df86bf511fe0d3044b69525a6ff2524167877929886adca9b0fa3d3fee45" + "hash": "00c4a602aa6a50f2f922851ce63b5216e915c7649698687a00d47da55c70349f" } diff --git a/backend/.sqlx/query-06731936fb073169b3a1a8a9817f1e669b60edccd260625a95094b7244f5fb83.json b/backend/.sqlx/query-06731936fb073169b3a1a8a9817f1e669b60edccd260625a95094b7244f5fb83.json deleted file mode 100644 index c509986552..0000000000 --- a/backend/.sqlx/query-06731936fb073169b3a1a8a9817f1e669b60edccd260625a95094b7244f5fb83.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['retry'], $1)\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "06731936fb073169b3a1a8a9817f1e669b60edccd260625a95094b7244f5fb83" -} diff --git a/backend/.sqlx/query-06db0e720dd59a7c52c0a98ea7b316237eb1a547678858c1a1e45985035b3468.json b/backend/.sqlx/query-06db0e720dd59a7c52c0a98ea7b316237eb1a547678858c1a1e45985035b3468.json new file mode 100644 index 0000000000..aea8b8302b --- /dev/null +++ b/backend/.sqlx/query-06db0e720dd59a7c52c0a98ea7b316237eb1a547678858c1a1e45985035b3468.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = flow_status - 'retry'\n WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "06db0e720dd59a7c52c0a98ea7b316237eb1a547678858c1a1e45985035b3468" +} diff --git a/backend/.sqlx/query-75451b6d48e4c26812ae64981d0d968b8fb0bf4374a2fccc167fa879bad7078f.json b/backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json similarity index 87% rename from backend/.sqlx/query-75451b6d48e4c26812ae64981d0d968b8fb0bf4374a2fccc167fa879bad7078f.json rename to backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json index cf2b476448..7d24993172 100644 --- a/backend/.sqlx/query-75451b6d48e4c26812ae64981d0d968b8fb0bf4374a2fccc167fa879bad7078f.json +++ b/backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json @@ -1,12 +1,12 @@ { "db_name": "PostgreSQL", - "query": "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by from script where hash = $1 AND workspace_id = $2", + "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by, path from script where hash = $1 AND workspace_id = $2", "describe": { "columns": [ { "ordinal": 0, - "name": "path", - "type_info": "Varchar" + "name": "hash", + "type_info": "Int8" }, { "ordinal": 1, @@ -101,6 +101,11 @@ "ordinal": 13, "name": "created_by", "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "path", + "type_info": "Varchar" } ], "parameters": { @@ -123,8 +128,9 @@ true, true, true, + false, false ] }, - "hash": "75451b6d48e4c26812ae64981d0d968b8fb0bf4374a2fccc167fa879bad7078f" + "hash": "0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936" } diff --git a/backend/.sqlx/query-0aaec91ab06753e46c595d82469924a98f28b0dead245df7248a9ccb8a5f20c3.json b/backend/.sqlx/query-0aaec91ab06753e46c595d82469924a98f28b0dead245df7248a9ccb8a5f20c3.json new file mode 100644 index 0000000000..04f5ba5b84 --- /dev/null +++ b/backend/.sqlx/query-0aaec91ab06753e46c595d82469924a98f28b0dead245df7248a9ccb8a5f20c3.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2)\n WHERE id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "0aaec91ab06753e46c595d82469924a98f28b0dead245df7248a9ccb8a5f20c3" +} diff --git a/backend/.sqlx/query-0e296134f05593edc989c628c00cbb60a5446993217baffa83f843bc12a5ac73.json b/backend/.sqlx/query-0e296134f05593edc989c628c00cbb60a5446993217baffa83f843bc12a5ac73.json new file mode 100644 index 0000000000..5b086529a6 --- /dev/null +++ b/backend/.sqlx/query-0e296134f05593edc989c628c00cbb60a5446993217baffa83f843bc12a5ac73.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n value->'preprocessor_module' IS NOT NULL as has_preprocessor,\n value->'preprocessor_module'->'value'->'input_transforms'->'wm_trigger' IS NOT NULL as is_v1_preprocessor,\n schema as \"schema: _\"\n FROM flow \n WHERE workspace_id = $1 \n AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "has_preprocessor", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "is_v1_preprocessor", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "schema: _", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + true + ] + }, + "hash": "0e296134f05593edc989c628c00cbb60a5446993217baffa83f843bc12a5ac73" +} diff --git a/backend/.sqlx/query-1060c503cf8d4bb5cef9720c162b8192924b4a938d249fae92624cd55e44f488.json b/backend/.sqlx/query-1060c503cf8d4bb5cef9720c162b8192924b4a938d249fae92624cd55e44f488.json new file mode 100644 index 0000000000..39b7179e5c --- /dev/null +++ b/backend/.sqlx/query-1060c503cf8d4bb5cef9720c162b8192924b4a938d249fae92624cd55e44f488.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n flow_status,\n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT],\n $4\n )\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "1060c503cf8d4bb5cef9720c162b8192924b4a938d249fae92624cd55e44f488" +} diff --git a/backend/.sqlx/query-1182fe055306d7ea435d76b74d781e066915c8397e6bbc9e408ff3dda9fec27f.json b/backend/.sqlx/query-1182fe055306d7ea435d76b74d781e066915c8397e6bbc9e408ff3dda9fec27f.json deleted file mode 100644 index 767f5cba57..0000000000 --- a/backend/.sqlx/query-1182fe055306d7ea435d76b74d781e066915c8397e6bbc9e408ff3dda9fec27f.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE script SET ws_error_handler_muted = $3 WHERE workspace_id = $2 AND path = $1 AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "1182fe055306d7ea435d76b74d781e066915c8397e6bbc9e408ff3dda9fec27f" -} diff --git a/backend/.sqlx/query-1252ef3a652ffb99529c2ce84928197fa15efb9c78d68e3a191c01a04efe153f.json b/backend/.sqlx/query-1252ef3a652ffb99529c2ce84928197fa15efb9c78d68e3a191c01a04efe153f.json deleted file mode 100644 index d3d6dd84cf..0000000000 --- a/backend/.sqlx/query-1252ef3a652ffb99529c2ce84928197fa15efb9c78d68e3a191c01a04efe153f.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'iterator', 'index'],\n ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n )\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int4", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid", - "Text", - "Jsonb" - ] - }, - "nullable": [ - null - ] - }, - "hash": "1252ef3a652ffb99529c2ce84928197fa15efb9c78d68e3a191c01a04efe153f" -} diff --git a/backend/.sqlx/query-f4f6336fc671b00bed7835124892f7a4d3bbe673f7c48153819dab385a5cb357.json b/backend/.sqlx/query-1312b7fd622cc814a406c85dbbff61f003c29185c267642cfd898075ebda855d.json similarity index 74% rename from backend/.sqlx/query-f4f6336fc671b00bed7835124892f7a4d3bbe673f7c48153819dab385a5cb357.json rename to backend/.sqlx/query-1312b7fd622cc814a406c85dbbff61f003c29185c267642cfd898075ebda855d.json index b20ae56e30..582896cc65 100644 --- a/backend/.sqlx/query-f4f6336fc671b00bed7835124892f7a4d3bbe673f7c48153819dab385a5cb357.json +++ b/backend/.sqlx/query-1312b7fd622cc814a406c85dbbff61f003c29185c267642cfd898075ebda855d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n gcp_resource_path,\n subscription_id,\n topic_id,\n workspace_id,\n delivery_type AS \"delivery_type: _\",\n delivery_config AS \"delivery_config: _\",\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled\n FROM \n gcp_trigger\n WHERE \n workspace_id = $1 AND \n path = $2\n ", + "query": "\n SELECT\n gcp_resource_path,\n subscription_id,\n topic_id,\n workspace_id,\n delivery_type AS \"delivery_type: _\",\n delivery_config AS \"delivery_config: _\",\n subscription_mode AS \"subscription_mode: _\",\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled\n FROM \n gcp_trigger\n WHERE \n workspace_id = $1 AND \n path = $2\n ", "describe": { "columns": [ { @@ -45,56 +45,71 @@ }, { "ordinal": 6, + "name": "subscription_mode: _", + "type_info": { + "Custom": { + "name": "gcp_subscription_mode", + "kind": { + "Enum": [ + "create_update", + "existing" + ] + } + } + } + }, + { + "ordinal": 7, "name": "path", "type_info": "Varchar" }, { - "ordinal": 7, + "ordinal": 8, "name": "script_path", "type_info": "Varchar" }, { - "ordinal": 8, + "ordinal": 9, "name": "is_flow", "type_info": "Bool" }, { - "ordinal": 9, + "ordinal": 10, "name": "edited_by", "type_info": "Varchar" }, { - "ordinal": 10, + "ordinal": 11, "name": "email", "type_info": "Varchar" }, { - "ordinal": 11, + "ordinal": 12, "name": "edited_at", "type_info": "Timestamptz" }, { - "ordinal": 12, + "ordinal": 13, "name": "server_id", "type_info": "Varchar" }, { - "ordinal": 13, + "ordinal": 14, "name": "last_server_ping", "type_info": "Timestamptz" }, { - "ordinal": 14, + "ordinal": 15, "name": "extra_perms", "type_info": "Jsonb" }, { - "ordinal": 15, + "ordinal": 16, "name": "error", "type_info": "Text" }, { - "ordinal": 16, + "ordinal": 17, "name": "enabled", "type_info": "Bool" } @@ -118,6 +133,7 @@ false, false, false, + false, true, true, false, @@ -125,5 +141,5 @@ false ] }, - "hash": "f4f6336fc671b00bed7835124892f7a4d3bbe673f7c48153819dab385a5cb357" + "hash": "1312b7fd622cc814a406c85dbbff61f003c29185c267642cfd898075ebda855d" } diff --git a/backend/.sqlx/query-173fbfd3ca2344fd08f73af75524c917d27fdb6273a35a563292b1f0701dc6ed.json b/backend/.sqlx/query-173fbfd3ca2344fd08f73af75524c917d27fdb6273a35a563292b1f0701dc6ed.json deleted file mode 100644 index 214bf50c6e..0000000000 --- a/backend/.sqlx/query-173fbfd3ca2344fd08f73af75524c917d27fdb6273a35a563292b1f0701dc6ed.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['failure_module'], $1),\n ARRAY['step'],\n $2\n )\n WHERE id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "173fbfd3ca2344fd08f73af75524c917d27fdb6273a35a563292b1f0701dc6ed" -} diff --git a/backend/.sqlx/query-17851a0710b80ffd6bebe42012a354665dff01554549ea7bbbb9953c68231296.json b/backend/.sqlx/query-17851a0710b80ffd6bebe42012a354665dff01554549ea7bbbb9953c68231296.json deleted file mode 100644 index eefc009b5e..0000000000 --- a/backend/.sqlx/query-17851a0710b80ffd6bebe42012a354665dff01554549ea7bbbb9953c68231296.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n flow_status,\n ARRAY['modules', $1::TEXT, 'iterator', 'index'],\n ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n )\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int4", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "17851a0710b80ffd6bebe42012a354665dff01554549ea7bbbb9953c68231296" -} diff --git a/backend/.sqlx/query-1850552883e67da181d68ff5c4e1babaa2fe072900b57e78e461590a6dafb682.json b/backend/.sqlx/query-1850552883e67da181d68ff5c4e1babaa2fe072900b57e78e461590a6dafb682.json deleted file mode 100644 index 037e63ae49..0000000000 --- a/backend/.sqlx/query-1850552883e67da181d68ff5c4e1babaa2fe072900b57e78e461590a6dafb682.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1)\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "1850552883e67da181d68ff5c4e1babaa2fe072900b57e78e461590a6dafb682" -} diff --git a/backend/.sqlx/query-264da6a89d005aebf4d5cd5f05e70fcffccd5fae8e2e352de3e1eb104e7fc3a6.json b/backend/.sqlx/query-264da6a89d005aebf4d5cd5f05e70fcffccd5fae8e2e352de3e1eb104e7fc3a6.json deleted file mode 100644 index f43524a12a..0000000000 --- a/backend/.sqlx/query-264da6a89d005aebf4d5cd5f05e70fcffccd5fae8e2e352de3e1eb104e7fc3a6.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT jsonb_build_object(\n 'kind', jb.kind,\n 'script_path', jb.runnable_path,\n 'latest_schema', COALESCE(\n (SELECT DISTINCT ON (s.path) s.schema FROM script s WHERE s.path = jb.runnable_path AND jb.kind = 'script' ORDER BY s.path, s.created_at DESC),\n (SELECT flow_version.schema FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.path = jb.runnable_path AND jb.kind = 'flow')\n ),\n 'schemas', ARRAY(\n SELECT jsonb_build_object(\n 'script_hash', LPAD(TO_HEX(COALESCE(s.hash, f.id)), 16, '0'),\n 'job_ids', ARRAY_AGG(DISTINCT j.id),\n 'schema', (ARRAY_AGG(COALESCE(s.schema, f.schema)))[1]\n ) FROM v2_job j\n LEFT JOIN script s ON s.hash = j.runnable_id AND j.kind = 'script'\n LEFT JOIN flow_version f ON f.id = j.runnable_id AND j.kind = 'flow'\n WHERE j.id = ANY(ARRAY_AGG(jb.id))\n GROUP BY COALESCE(s.hash, f.id)\n )\n ) FROM v2_job jb\n WHERE (jb.kind = 'flow' OR jb.kind = 'script')\n AND jb.workspace_id = $1 AND jb.id = ANY($2)\n GROUP BY jb.kind, jb.runnable_path", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "jsonb_build_object", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text", - "UuidArray" - ] - }, - "nullable": [ - null - ] - }, - "hash": "264da6a89d005aebf4d5cd5f05e70fcffccd5fae8e2e352de3e1eb104e7fc3a6" -} diff --git a/backend/.sqlx/query-26761fbd7953416eb391de47b1694e0f4ab2bb96a6d838f1b1fdce4b58a8f5d4.json b/backend/.sqlx/query-26761fbd7953416eb391de47b1694e0f4ab2bb96a6d838f1b1fdce4b58a8f5d4.json new file mode 100644 index 0000000000..b14dab2503 --- /dev/null +++ b/backend/.sqlx/query-26761fbd7953416eb391de47b1694e0f4ab2bb96a6d838f1b1fdce4b58a8f5d4.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT jsonb_build_object(\n 'kind', jb.kind,\n 'script_path', jb.runnable_path,\n 'latest_schema', COALESCE(\n (SELECT DISTINCT ON (s.path) s.schema FROM script s WHERE s.workspace_id = $1 AND s.path = jb.runnable_path AND jb.kind = 'script' ORDER BY s.path, s.created_at DESC),\n (SELECT flow_version.schema FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.workspace_id = $1 AND flow.path = jb.runnable_path AND jb.kind = 'flow')\n ),\n 'schemas', ARRAY(\n SELECT jsonb_build_object(\n 'script_hash', LPAD(TO_HEX(COALESCE(s.hash, f.id)), 16, '0'),\n 'job_ids', ARRAY_AGG(DISTINCT j.id),\n 'schema', (ARRAY_AGG(COALESCE(s.schema, f.schema)))[1]\n ) FROM v2_job j\n LEFT JOIN script s ON s.hash = j.runnable_id AND j.kind = 'script'\n LEFT JOIN flow_version f ON f.id = j.runnable_id AND j.kind = 'flow'\n WHERE j.id = ANY(ARRAY_AGG(jb.id))\n GROUP BY COALESCE(s.hash, f.id)\n )\n ) FROM v2_job jb\n WHERE (jb.kind = 'flow' OR jb.kind = 'script')\n AND jb.workspace_id = $1 AND jb.id = ANY($2)\n GROUP BY jb.kind, jb.runnable_path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "jsonb_build_object", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "UuidArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "26761fbd7953416eb391de47b1694e0f4ab2bb96a6d838f1b1fdce4b58a8f5d4" +} diff --git a/backend/.sqlx/query-28b42ab9c3ce0c2f05cf385e81f3b72fa7c4b3c458d52a5891a61f9c53a49c6d.json b/backend/.sqlx/query-28b42ab9c3ce0c2f05cf385e81f3b72fa7c4b3c458d52a5891a61f9c53a49c6d.json deleted file mode 100644 index 817b993d8f..0000000000 --- a/backend/.sqlx/query-28b42ab9c3ce0c2f05cf385e81f3b72fa7c4b3c458d52a5891a61f9c53a49c6d.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n flow_status,\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'],\n ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb\n )\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int4", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "28b42ab9c3ce0c2f05cf385e81f3b72fa7c4b3c458d52a5891a61f9c53a49c6d" -} diff --git a/backend/.sqlx/query-ea2b88dc050aec038641ea37399d68d4385c5bdc721d1351609f27ca45e4dbdc.json b/backend/.sqlx/query-2a49e5b5486b650d96f3e9038cba8a5f2e75d3b12ee4718452e82c7318b1bcf4.json similarity index 63% rename from backend/.sqlx/query-ea2b88dc050aec038641ea37399d68d4385c5bdc721d1351609f27ca45e4dbdc.json rename to backend/.sqlx/query-2a49e5b5486b650d96f3e9038cba8a5f2e75d3b12ee4718452e82c7318b1bcf4.json index 1d30071300..77f61ccc47 100644 --- a/backend/.sqlx/query-ea2b88dc050aec038641ea37399d68d4385c5bdc721d1351609f27ca45e4dbdc.json +++ b/backend/.sqlx/query-2a49e5b5486b650d96f3e9038cba8a5f2e75d3b12ee4718452e82c7318b1bcf4.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 AND\n created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2))", + "query": "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1)", "describe": { "columns": [ { @@ -19,5 +19,5 @@ null ] }, - "hash": "ea2b88dc050aec038641ea37399d68d4385c5bdc721d1351609f27ca45e4dbdc" + "hash": "2a49e5b5486b650d96f3e9038cba8a5f2e75d3b12ee4718452e82c7318b1bcf4" } diff --git a/backend/.sqlx/query-15fbe481789a7817bf37415fb935f9ed537fd7d3b266d928af4d5d3dd8bb5c18.json b/backend/.sqlx/query-2dcff8b16da75740c362015b2293f578a21813f038d75184a5034c37d4daf36e.json similarity index 85% rename from backend/.sqlx/query-15fbe481789a7817bf37415fb935f9ed537fd7d3b266d928af4d5d3dd8bb5c18.json rename to backend/.sqlx/query-2dcff8b16da75740c362015b2293f578a21813f038d75184a5034c37d4daf36e.json index 5f76c31d8b..5622b3691e 100644 --- a/backend/.sqlx/query-15fbe481789a7817bf37415fb935f9ed537fd7d3b266d928af4d5d3dd8bb5c18.json +++ b/backend/.sqlx/query-2dcff8b16da75740c362015b2293f578a21813f038d75184a5034c37d4daf36e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n gcp_resource_path,\n subscription_id,\n topic_id,\n workspace_id,\n delivery_type AS \"delivery_type: _\",\n delivery_config AS \"delivery_config: _\",\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled\n FROM\n gcp_trigger\n WHERE\n delivery_type != 'push'::DELIVERY_MODE AND\n enabled IS TRUE\n AND (last_server_ping IS NULL OR\n last_server_ping < now() - interval '15 seconds'\n )\n ", + "query": "\n SELECT\n gcp_resource_path,\n subscription_id,\n topic_id,\n workspace_id,\n delivery_type AS \"delivery_type: _\",\n delivery_config AS \"delivery_config: _\",\n subscription_mode AS \"subscription_mode: _\",\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled\n FROM\n gcp_trigger\n WHERE\n delivery_type != 'push'::DELIVERY_MODE AND\n enabled IS TRUE\n AND (last_server_ping IS NULL OR\n last_server_ping < now() - interval '15 seconds'\n )\n ", "describe": { "columns": [ { @@ -45,56 +45,71 @@ }, { "ordinal": 6, + "name": "subscription_mode: _", + "type_info": { + "Custom": { + "name": "gcp_subscription_mode", + "kind": { + "Enum": [ + "create_update", + "existing" + ] + } + } + } + }, + { + "ordinal": 7, "name": "path", "type_info": "Varchar" }, { - "ordinal": 7, + "ordinal": 8, "name": "script_path", "type_info": "Varchar" }, { - "ordinal": 8, + "ordinal": 9, "name": "is_flow", "type_info": "Bool" }, { - "ordinal": 9, + "ordinal": 10, "name": "edited_by", "type_info": "Varchar" }, { - "ordinal": 10, + "ordinal": 11, "name": "email", "type_info": "Varchar" }, { - "ordinal": 11, + "ordinal": 12, "name": "edited_at", "type_info": "Timestamptz" }, { - "ordinal": 12, + "ordinal": 13, "name": "server_id", "type_info": "Varchar" }, { - "ordinal": 13, + "ordinal": 14, "name": "last_server_ping", "type_info": "Timestamptz" }, { - "ordinal": 14, + "ordinal": 15, "name": "extra_perms", "type_info": "Jsonb" }, { - "ordinal": 15, + "ordinal": 16, "name": "error", "type_info": "Text" }, { - "ordinal": 16, + "ordinal": 17, "name": "enabled", "type_info": "Bool" } @@ -115,6 +130,7 @@ false, false, false, + false, true, true, false, @@ -122,5 +138,5 @@ false ] }, - "hash": "15fbe481789a7817bf37415fb935f9ed537fd7d3b266d928af4d5d3dd8bb5c18" + "hash": "2dcff8b16da75740c362015b2293f578a21813f038d75184a5034c37d4daf36e" } diff --git a/backend/.sqlx/query-303c7e92ce23dc367d97d813415cd9aef958c15df1c0a7b02318a756cd3589e9.json b/backend/.sqlx/query-303c7e92ce23dc367d97d813415cd9aef958c15df1c0a7b02318a756cd3589e9.json deleted file mode 100644 index 58cfc98b09..0000000000 --- a/backend/.sqlx/query-303c7e92ce23dc367d97d813415cd9aef958c15df1c0a7b02318a756cd3589e9.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH job_result AS (\n SELECT result \n FROM v2_job_completed \n WHERE id = $1\n )\n UPDATE v2_job \n SET args = COALESCE(\n CASE \n WHEN job_result.result IS NULL THEN NULL\n WHEN jsonb_typeof(job_result.result) = 'object' \n THEN job_result.result\n WHEN jsonb_typeof(job_result.result) = 'null'\n THEN NULL\n ELSE jsonb_build_object('value', job_result.result)\n END, \n '{}'::jsonb\n ),\n preprocessed = TRUE\n FROM job_result\n WHERE v2_job.id = $2;\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "303c7e92ce23dc367d97d813415cd9aef958c15df1c0a7b02318a756cd3589e9" -} diff --git a/backend/.sqlx/query-3325f8ed245b1bce27c3d9f5e62ffc14b5c8aabf5ab53384f5f2b20eafd66cb3.json b/backend/.sqlx/query-3325f8ed245b1bce27c3d9f5e62ffc14b5c8aabf5ab53384f5f2b20eafd66cb3.json new file mode 100644 index 0000000000..6c02ca51d6 --- /dev/null +++ b/backend/.sqlx/query-3325f8ed245b1bce27c3d9f5e62ffc14b5c8aabf5ab53384f5f2b20eafd66cb3.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['retry'], $1)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "3325f8ed245b1bce27c3d9f5e62ffc14b5c8aabf5ab53384f5f2b20eafd66cb3" +} diff --git a/backend/.sqlx/query-ec1f31fd7628ea2e30995a0de1d8665831ee3e4ec3815e9ad90e886ffecba0f1.json b/backend/.sqlx/query-38b3945c04ae58aace881ed4d6aa4da72e61eeb8e081d2e3d1ab99a4fe450350.json similarity index 68% rename from backend/.sqlx/query-ec1f31fd7628ea2e30995a0de1d8665831ee3e4ec3815e9ad90e886ffecba0f1.json rename to backend/.sqlx/query-38b3945c04ae58aace881ed4d6aa4da72e61eeb8e081d2e3d1ab99a4fe450350.json index b5629e0425..f4ed338505 100644 --- a/backend/.sqlx/query-ec1f31fd7628ea2e30995a0de1d8665831ee3e4ec3815e9ad90e886ffecba0f1.json +++ b/backend/.sqlx/query-38b3945c04ae58aace881ed4d6aa4da72e61eeb8e081d2e3d1ab99a4fe450350.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE v2_job_runtime SET ping = NULL\n WHERE id = $1", + "query": "UPDATE v2_job_runtime SET ping = NULL\n WHERE id = $1", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "ec1f31fd7628ea2e30995a0de1d8665831ee3e4ec3815e9ad90e886ffecba0f1" + "hash": "38b3945c04ae58aace881ed4d6aa4da72e61eeb8e081d2e3d1ab99a4fe450350" } diff --git a/backend/.sqlx/query-3a534b4fc36171efaa7c647f48320b59bbc414cfb92e960c174dd63fc180e187.json b/backend/.sqlx/query-3a534b4fc36171efaa7c647f48320b59bbc414cfb92e960c174dd63fc180e187.json deleted file mode 100644 index 11d343ba4d..0000000000 --- a/backend/.sqlx/query-3a534b4fc36171efaa7c647f48320b59bbc414cfb92e960c174dd63fc180e187.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT tag, dedicated_worker from flow WHERE path = $1 and workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tag", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "dedicated_worker", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - true, - true - ] - }, - "hash": "3a534b4fc36171efaa7c647f48320b59bbc414cfb92e960c174dd63fc180e187" -} diff --git a/backend/.sqlx/query-3af32856235690827a8700bb2396f3ab44afe0d1a7c261a626d93fb44c00bdb7.json b/backend/.sqlx/query-3af32856235690827a8700bb2396f3ab44afe0d1a7c261a626d93fb44c00bdb7.json new file mode 100644 index 0000000000..73d2bc7c6a --- /dev/null +++ b/backend/.sqlx/query-3af32856235690827a8700bb2396f3ab44afe0d1a7c261a626d93fb44c00bdb7.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = flow_status - 'approval_conditions'\n WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "3af32856235690827a8700bb2396f3ab44afe0d1a7c261a626d93fb44c00bdb7" +} diff --git a/backend/.sqlx/query-3d58b5861c62f0b092b6b95c17ba1dac2cbcf00db116624bd2fe27a4d0dfb436.json b/backend/.sqlx/query-3d58b5861c62f0b092b6b95c17ba1dac2cbcf00db116624bd2fe27a4d0dfb436.json deleted file mode 100644 index 4f1fe255f9..0000000000 --- a/backend/.sqlx/query-3d58b5861c62f0b092b6b95c17ba1dac2cbcf00db116624bd2fe27a4d0dfb436.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['approval_conditions'], $1)\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "3d58b5861c62f0b092b6b95c17ba1dac2cbcf00db116624bd2fe27a4d0dfb436" -} diff --git a/backend/.sqlx/query-3e0cdd500dffc3bd1d8374ca3cc8fd60ce778c2fece27637d9985d4650778653.json b/backend/.sqlx/query-3e0cdd500dffc3bd1d8374ca3cc8fd60ce778c2fece27637d9985d4650778653.json deleted file mode 100644 index 9f5cb1d19d..0000000000 --- a/backend/.sqlx/query-3e0cdd500dffc3bd1d8374ca3cc8fd60ce778c2fece27637d9985d4650778653.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['cleanup_module', 'flow_jobs_to_clean'], COALESCE(flow_status->'cleanup_module'->'flow_jobs_to_clean', '[]'::jsonb) || $1)\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "3e0cdd500dffc3bd1d8374ca3cc8fd60ce778c2fece27637d9985d4650778653" -} diff --git a/backend/.sqlx/query-ebd25047d14bec8457465ba6ed9b6b15b8f2303157b4c6e80ac9e3d84a57d8b1.json b/backend/.sqlx/query-3e261911cc4c5289da49865f54350613f9e651540a279bd7d75e5e7d79f676a8.json similarity index 68% rename from backend/.sqlx/query-ebd25047d14bec8457465ba6ed9b6b15b8f2303157b4c6e80ac9e3d84a57d8b1.json rename to backend/.sqlx/query-3e261911cc4c5289da49865f54350613f9e651540a279bd7d75e5e7d79f676a8.json index 6c92e12f6a..321831d724 100644 --- a/backend/.sqlx/query-ebd25047d14bec8457465ba6ed9b6b15b8f2303157b4c6e80ac9e3d84a57d8b1.json +++ b/backend/.sqlx/query-3e261911cc4c5289da49865f54350613f9e651540a279bd7d75e5e7d79f676a8.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval \n AND running = true AND ping IS NULL AND same_worker = true AND worker IS NOT NULL GROUP BY worker", + "query": "SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval\n AND running = true AND ping IS NULL AND same_worker = true AND worker IS NOT NULL GROUP BY worker", "describe": { "columns": [ { @@ -22,5 +22,5 @@ null ] }, - "hash": "ebd25047d14bec8457465ba6ed9b6b15b8f2303157b4c6e80ac9e3d84a57d8b1" + "hash": "3e261911cc4c5289da49865f54350613f9e651540a279bd7d75e5e7d79f676a8" } diff --git a/backend/.sqlx/query-429aef2c320a152b16fe20c1ab84aab41142897db108dc8cbc2c51abb2e30c7c.json b/backend/.sqlx/query-429aef2c320a152b16fe20c1ab84aab41142897db108dc8cbc2c51abb2e30c7c.json deleted file mode 100644 index c40a18e4c9..0000000000 --- a/backend/.sqlx/query-429aef2c320a152b16fe20c1ab84aab41142897db108dc8cbc2c51abb2e30c7c.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2)\n WHERE id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "429aef2c320a152b16fe20c1ab84aab41142897db108dc8cbc2c51abb2e30c7c" -} diff --git a/backend/.sqlx/query-4455c7e8aa7616f3d547c5eb7d93c840d8aff3df4a6926f569792f69b2e9601f.json b/backend/.sqlx/query-4455c7e8aa7616f3d547c5eb7d93c840d8aff3df4a6926f569792f69b2e9601f.json new file mode 100644 index 0000000000..8c90c0c765 --- /dev/null +++ b/backend/.sqlx/query-4455c7e8aa7616f3d547c5eb7d93c840d8aff3df4a6926f569792f69b2e9601f.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "select hash from script where path = $1 AND workspace_id = $2 AND deleted = false AND lock IS not NULL AND lock_error_logs IS NULL ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "4455c7e8aa7616f3d547c5eb7d93c840d8aff3df4a6926f569792f69b2e9601f" +} diff --git a/backend/.sqlx/query-45dde0dc2da12fe46b5975ed53b520fc4160c008c3d29fe7d99e609c0b6e3e6d.json b/backend/.sqlx/query-45dde0dc2da12fe46b5975ed53b520fc4160c008c3d29fe7d99e609c0b6e3e6d.json new file mode 100644 index 0000000000..edb113673b --- /dev/null +++ b/backend/.sqlx/query-45dde0dc2da12fe46b5975ed53b520fc4160c008c3d29fe7d99e609c0b6e3e6d.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM metrics WHERE created_at < NOW() - INTERVAL '180 day'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "45dde0dc2da12fe46b5975ed53b520fc4160c008c3d29fe7d99e609c0b6e3e6d" +} diff --git a/backend/.sqlx/query-4622d28e2fa09bc60b9d0c79397efe0ca030638ded82c2ffd2155cacdf36ec11.json b/backend/.sqlx/query-4622d28e2fa09bc60b9d0c79397efe0ca030638ded82c2ffd2155cacdf36ec11.json new file mode 100644 index 0000000000..f30ab8d49d --- /dev/null +++ b/backend/.sqlx/query-4622d28e2fa09bc60b9d0c79397efe0ca030638ded82c2ffd2155cacdf36ec11.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['step'], $1)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "4622d28e2fa09bc60b9d0c79397efe0ca030638ded82c2ffd2155cacdf36ec11" +} diff --git a/backend/.sqlx/query-4c9cf8c3176abc2b8b9a1c3f671949e16830671a458d5a73762cd8545d26172d.json b/backend/.sqlx/query-4c9cf8c3176abc2b8b9a1c3f671949e16830671a458d5a73762cd8545d26172d.json deleted file mode 100644 index 0d108538dc..0000000000 --- a/backend/.sqlx/query-4c9cf8c3176abc2b8b9a1c3f671949e16830671a458d5a73762cd8545d26172d.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2),\n ARRAY['step'],\n $3\n )\n WHERE id = $4", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Jsonb", - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "4c9cf8c3176abc2b8b9a1c3f671949e16830671a458d5a73762cd8545d26172d" -} diff --git a/backend/.sqlx/query-4e07dcc1ba4396ada2f1080a400ad9fad00b1d77ea44c8639b7598c5953635ff.json b/backend/.sqlx/query-4e07dcc1ba4396ada2f1080a400ad9fad00b1d77ea44c8639b7598c5953635ff.json new file mode 100644 index 0000000000..506dce48dc --- /dev/null +++ b/backend/.sqlx/query-4e07dcc1ba4396ada2f1080a400ad9fad00b1d77ea44c8639b7598c5953635ff.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "4e07dcc1ba4396ada2f1080a400ad9fad00b1d77ea44c8639b7598c5953635ff" +} diff --git a/backend/.sqlx/query-7cba31d597215a343cb0bca5b204a15fbba193262f7895c2bba90feb4215d6f3.json b/backend/.sqlx/query-4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36.json similarity index 66% rename from backend/.sqlx/query-7cba31d597215a343cb0bca5b204a15fbba193262f7895c2bba90feb4215d6f3.json rename to backend/.sqlx/query-4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36.json index a313dd6076..15b107811b 100644 --- a/backend/.sqlx/query-7cba31d597215a343cb0bca5b204a15fbba193262f7895c2bba90feb4215d6f3.json +++ b/backend/.sqlx/query-4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT \n id, \n created_at, \n trigger_kind AS \"trigger_kind: _\",\n CASE \n WHEN pg_column_size(payload) < 40000 THEN payload \n ELSE '\"WINDMILL_TOO_BIG\"'::jsonb \n END AS \"payload!: _\",\n trigger_extra AS \"trigger_extra: _\"\n FROM \n capture\n WHERE \n workspace_id = $1 \n AND path = $2 \n AND is_flow = $3 \n AND ($4::trigger_kind IS NULL OR trigger_kind = $4)\n ORDER BY \n created_at DESC\n OFFSET $5\n LIMIT $6\n ", + "query": "\n SELECT \n id, \n created_at, \n trigger_kind AS \"trigger_kind: _\",\n CASE \n WHEN pg_column_size(main_args) < 40000 THEN main_args \n ELSE '\"WINDMILL_TOO_BIG\"'::jsonb \n END AS \"main_args!: _\",\n CASE\n WHEN pg_column_size(preprocessor_args) < 40000 THEN preprocessor_args\n ELSE '\"WINDMILL_TOO_BIG\"'::jsonb\n END AS \"preprocessor_args: _\"\n FROM \n capture\n WHERE \n workspace_id = $1 \n AND path = $2 \n AND is_flow = $3 \n AND ($4::trigger_kind IS NULL OR trigger_kind = $4)\n ORDER BY \n created_at DESC\n OFFSET $5\n LIMIT $6\n ", "describe": { "columns": [ { @@ -38,12 +38,12 @@ }, { "ordinal": 3, - "name": "payload!: _", + "name": "main_args!: _", "type_info": "Jsonb" }, { "ordinal": 4, - "name": "trigger_extra: _", + "name": "preprocessor_args: _", "type_info": "Jsonb" } ], @@ -80,8 +80,8 @@ false, false, null, - true + null ] }, - "hash": "7cba31d597215a343cb0bca5b204a15fbba193262f7895c2bba90feb4215d6f3" + "hash": "4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36" } diff --git a/backend/.sqlx/query-525a9ef57c7d9fac86cb1bf47868fa6fb782e9d589852e51530cdd1a38322a9d.json b/backend/.sqlx/query-525a9ef57c7d9fac86cb1bf47868fa6fb782e9d589852e51530cdd1a38322a9d.json new file mode 100644 index 0000000000..73c1a95797 --- /dev/null +++ b/backend/.sqlx/query-525a9ef57c7d9fac86cb1bf47868fa6fb782e9d589852e51530cdd1a38322a9d.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "525a9ef57c7d9fac86cb1bf47868fa6fb782e9d589852e51530cdd1a38322a9d" +} diff --git a/backend/.sqlx/query-52ad0c838d19cbd9e90b8368abe71dd12655179f41f43896e7d30fdfb3ae5939.json b/backend/.sqlx/query-52ad0c838d19cbd9e90b8368abe71dd12655179f41f43896e7d30fdfb3ae5939.json new file mode 100644 index 0000000000..1fbc39ba44 --- /dev/null +++ b/backend/.sqlx/query-52ad0c838d19cbd9e90b8368abe71dd12655179f41f43896e7d30fdfb3ae5939.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH job_result AS (\n SELECT result \n FROM v2_job_completed \n WHERE id = $1\n )\n UPDATE v2_job \n SET args = COALESCE(\n CASE \n WHEN job_result.result IS NULL THEN NULL\n WHEN jsonb_typeof(job_result.result) = 'object' \n THEN job_result.result\n WHEN jsonb_typeof(job_result.result) = 'null'\n THEN NULL\n ELSE jsonb_build_object('value', job_result.result)\n END, \n '{}'::jsonb\n ),\n preprocessed = TRUE\n FROM job_result\n WHERE v2_job.id = $2;\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "52ad0c838d19cbd9e90b8368abe71dd12655179f41f43896e7d30fdfb3ae5939" +} diff --git a/backend/.sqlx/query-553108ba3c0b8d579800bc8b5a4f887d79fb4c13b60b19c4913a8db18521958c.json b/backend/.sqlx/query-553108ba3c0b8d579800bc8b5a4f887d79fb4c13b60b19c4913a8db18521958c.json new file mode 100644 index 0000000000..e3d2e205f1 --- /dev/null +++ b/backend/.sqlx/query-553108ba3c0b8d579800bc8b5a4f887d79fb4c13b60b19c4913a8db18521958c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue q SET suspend = 0\n FROM v2_job j, v2_job_status f\n WHERE parent_job = $1\n AND f.id = j.id AND q.id = j.id\n AND suspend = $2 AND (f.flow_status->'step')::int = 0", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "553108ba3c0b8d579800bc8b5a4f887d79fb4c13b60b19c4913a8db18521958c" +} diff --git a/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json similarity index 50% rename from backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json rename to backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index c2dfed73a2..713ccb9dd3 100644 --- a/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH worker_ids AS (SELECT unnest($1::text[]) as worker) \n SELECT worker_ids.worker FROM worker_ids \n LEFT JOIN worker_ping ON worker_ids.worker = worker_ping.worker \n WHERE worker_ping.worker IS NULL OR worker_ping.ping_at < now() - ('60 seconds')::interval", + "query": "WITH worker_ids AS (SELECT unnest($1::text[]) as worker)\n SELECT worker_ids.worker FROM worker_ids\n LEFT JOIN worker_ping ON worker_ids.worker = worker_ping.worker\n WHERE worker_ping.worker IS NULL OR worker_ping.ping_at < now() - ('60 seconds')::interval", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927" + "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" } diff --git a/backend/.sqlx/query-5a4fa8ff2148d92946e6ac95f70585d082435e0e79a09821a2045e3b550c3276.json b/backend/.sqlx/query-5a4fa8ff2148d92946e6ac95f70585d082435e0e79a09821a2045e3b550c3276.json new file mode 100644 index 0000000000..bec0922efc --- /dev/null +++ b/backend/.sqlx/query-5a4fa8ff2148d92946e6ac95f70585d082435e0e79a09821a2045e3b550c3276.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1),\n ARRAY['step'],\n $2\n )\n WHERE id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "5a4fa8ff2148d92946e6ac95f70585d082435e0e79a09821a2045e3b550c3276" +} diff --git a/backend/.sqlx/query-6513c1433dbfe03f7c778963a05d964fda13a1091a8206ee174ed3a161248126.json b/backend/.sqlx/query-6513c1433dbfe03f7c778963a05d964fda13a1091a8206ee174ed3a161248126.json deleted file mode 100644 index e50d2f1154..0000000000 --- a/backend/.sqlx/query-6513c1433dbfe03f7c778963a05d964fda13a1091a8206ee174ed3a161248126.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT id\n FROM v2_job j JOIN v2_job_queue USING (id)\n WHERE j.workspace_id = $2 AND trigger_kind = 'schedule' AND trigger = $1 AND runnable_path = $4\n AND parent_job IS NULL\n AND j.id != $3\n AND running = true", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Uuid", - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "6513c1433dbfe03f7c778963a05d964fda13a1091a8206ee174ed3a161248126" -} diff --git a/backend/.sqlx/query-6664be80f0d72ea7b8b184c5348063db3bbfea67f4a056d6e3be1fc4255cfc06.json b/backend/.sqlx/query-6664be80f0d72ea7b8b184c5348063db3bbfea67f4a056d6e3be1fc4255cfc06.json deleted file mode 100644 index 0e7ae566fd..0000000000 --- a/backend/.sqlx/query-6664be80f0d72ea7b8b184c5348063db3bbfea67f4a056d6e3be1fc4255cfc06.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status\n SET flow_leaf_jobs = JSONB_SET(coalesce(flow_leaf_jobs, '{}'::jsonb), ARRAY[$1::TEXT], $2)\n WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $3), $3) = id", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "6664be80f0d72ea7b8b184c5348063db3bbfea67f4a056d6e3be1fc4255cfc06" -} diff --git a/backend/.sqlx/query-69db5305aadd911b06ecdc4eeb9610c0e233bca35edebe2ad1772a4a1a1d5bbe.json b/backend/.sqlx/query-69db5305aadd911b06ecdc4eeb9610c0e233bca35edebe2ad1772a4a1a1d5bbe.json new file mode 100644 index 0000000000..ec22d86a79 --- /dev/null +++ b/backend/.sqlx/query-69db5305aadd911b06ecdc4eeb9610c0e233bca35edebe2ad1772a4a1a1d5bbe.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n flow_status,\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'],\n ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb\n )\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "69db5305aadd911b06ecdc4eeb9610c0e233bca35edebe2ad1772a4a1a1d5bbe" +} diff --git a/backend/.sqlx/query-69ff806066a24c60188d7547a7776c160ee65eaa01b259c996f3f96ed44fcaaf.json b/backend/.sqlx/query-69ff806066a24c60188d7547a7776c160ee65eaa01b259c996f3f96ed44fcaaf.json deleted file mode 100644 index 75ef2996a0..0000000000 --- a/backend/.sqlx/query-69ff806066a24c60188d7547a7776c160ee65eaa01b259c996f3f96ed44fcaaf.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['step'], $1)\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "69ff806066a24c60188d7547a7776c160ee65eaa01b259c996f3f96ed44fcaaf" -} diff --git a/backend/.sqlx/query-1b58b90c184ca21d777ea4e264c79aecc2361134a4817c2b9580f2680425352d.json b/backend/.sqlx/query-6b0347da54d1b8646ece08a5ce78ed7e7c98780fef56f6ea6e1e1fd0458ae32f.json similarity index 71% rename from backend/.sqlx/query-1b58b90c184ca21d777ea4e264c79aecc2361134a4817c2b9580f2680425352d.json rename to backend/.sqlx/query-6b0347da54d1b8646ece08a5ce78ed7e7c98780fef56f6ea6e1e1fd0458ae32f.json index d0b582d17b..8259c07cd0 100644 --- a/backend/.sqlx/query-1b58b90c184ca21d777ea4e264c79aecc2361134a4817c2b9580f2680425352d.json +++ b/backend/.sqlx/query-6b0347da54d1b8646ece08a5ce78ed7e7c98780fef56f6ea6e1e1fd0458ae32f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT args AS \"args: Json>>\"\n FROM v2_job WHERE id = $1 AND workspace_id = $2", + "query": "SELECT args AS \"args: Json>>\"\n FROM v2_job WHERE id = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -19,5 +19,5 @@ true ] }, - "hash": "1b58b90c184ca21d777ea4e264c79aecc2361134a4817c2b9580f2680425352d" + "hash": "6b0347da54d1b8646ece08a5ce78ed7e7c98780fef56f6ea6e1e1fd0458ae32f" } diff --git a/backend/.sqlx/query-6d36da815795d5cac2e76b4d34a7f1f9f836bd5b6866d3db26b25428d39c0b23.json b/backend/.sqlx/query-6d36da815795d5cac2e76b4d34a7f1f9f836bd5b6866d3db26b25428d39c0b23.json deleted file mode 100644 index 009c8a0e26..0000000000 --- a/backend/.sqlx/query-6d36da815795d5cac2e76b4d34a7f1f9f836bd5b6866d3db26b25428d39c0b23.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT on_behalf_of_email, edited_by FROM flow WHERE path = $1 AND workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "on_behalf_of_email", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "edited_by", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - true, - false - ] - }, - "hash": "6d36da815795d5cac2e76b4d34a7f1f9f836bd5b6866d3db26b25428d39c0b23" -} diff --git a/backend/.sqlx/query-362419eb262c83d6a98a0200b116e831ada60399fe5f55a56d930cc69aff2675.json b/backend/.sqlx/query-726e956cfcd3ac7c07abeecdf92cf0996efe7fa7b671ac2b3b000ead0ea307de.json similarity index 82% rename from backend/.sqlx/query-362419eb262c83d6a98a0200b116e831ada60399fe5f55a56d930cc69aff2675.json rename to backend/.sqlx/query-726e956cfcd3ac7c07abeecdf92cf0996efe7fa7b671ac2b3b000ead0ea307de.json index 183062aa6d..4baf932023 100644 --- a/backend/.sqlx/query-362419eb262c83d6a98a0200b116e831ada60399fe5f55a56d930cc69aff2675.json +++ b/backend/.sqlx/query-726e956cfcd3ac7c07abeecdf92cf0996efe7fa7b671ac2b3b000ead0ea307de.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg \n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by created_at DESC", + "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg \n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC", "describe": { "columns": [ { @@ -25,5 +25,5 @@ true ] }, - "hash": "362419eb262c83d6a98a0200b116e831ada60399fe5f55a56d930cc69aff2675" + "hash": "726e956cfcd3ac7c07abeecdf92cf0996efe7fa7b671ac2b3b000ead0ea307de" } diff --git a/backend/.sqlx/query-7470e7067b948509d14828c24a8725da747e6b967554eb7e088ee3a018ec1f8b.json b/backend/.sqlx/query-7470e7067b948509d14828c24a8725da747e6b967554eb7e088ee3a018ec1f8b.json new file mode 100644 index 0000000000..a6979f19db --- /dev/null +++ b/backend/.sqlx/query-7470e7067b948509d14828c24a8725da747e6b967554eb7e088ee3a018ec1f8b.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id\n FROM v2_job j JOIN v2_job_queue USING (id)\n WHERE j.workspace_id = $2 AND trigger_kind = 'schedule' AND trigger = $1 AND runnable_path = $4\n AND parent_job IS NULL\n AND j.id != $3\n AND running = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Uuid", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7470e7067b948509d14828c24a8725da747e6b967554eb7e088ee3a018ec1f8b" +} diff --git a/backend/.sqlx/query-7b084617bf2de5ed9a657d2bd2cbc9de1246427bf0302c032f8bd26ff93313f7.json b/backend/.sqlx/query-7b084617bf2de5ed9a657d2bd2cbc9de1246427bf0302c032f8bd26ff93313f7.json deleted file mode 100644 index 3f72b7b760..0000000000 --- a/backend/.sqlx/query-7b084617bf2de5ed9a657d2bd2cbc9de1246427bf0302c032f8bd26ff93313f7.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['failure_module'], $1)\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "7b084617bf2de5ed9a657d2bd2cbc9de1246427bf0302c032f8bd26ff93313f7" -} diff --git a/backend/.sqlx/query-7bd7505b008954aae6152554c1f9bb0611d0ec451b48aa1a4de309dadeb53315.json b/backend/.sqlx/query-7bd7505b008954aae6152554c1f9bb0611d0ec451b48aa1a4de309dadeb53315.json deleted file mode 100644 index c7f3146594..0000000000 --- a/backend/.sqlx/query-7bd7505b008954aae6152554c1f9bb0611d0ec451b48aa1a4de309dadeb53315.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status\n SET flow_status = flow_status - 'approval_conditions'\n WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "7bd7505b008954aae6152554c1f9bb0611d0ec451b48aa1a4de309dadeb53315" -} diff --git a/backend/.sqlx/query-848c8371eeb17ebd4b36a33f7d8a61eb8f07c54d291bb857ddd41a549cbc88dd.json b/backend/.sqlx/query-848c8371eeb17ebd4b36a33f7d8a61eb8f07c54d291bb857ddd41a549cbc88dd.json new file mode 100644 index 0000000000..7751c14adb --- /dev/null +++ b/backend/.sqlx/query-848c8371eeb17ebd4b36a33f7d8a61eb8f07c54d291bb857ddd41a549cbc88dd.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE script \n SET ws_error_handler_muted = $3 \n WHERE ctid = (\n SELECT ctid FROM script\n WHERE path = $1 AND workspace_id = $2\n ORDER BY created_at DESC\n LIMIT 1\n )\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "848c8371eeb17ebd4b36a33f7d8a61eb8f07c54d291bb857ddd41a549cbc88dd" +} diff --git a/backend/.sqlx/query-d988e91087695742d75946100cf2b7593cb8eed2a97411697819849958c022b3.json b/backend/.sqlx/query-8780a8cd6781f86041ae8df58477913c01b5d8d27dd4251f0cabbe3e974c0b0a.json similarity index 70% rename from backend/.sqlx/query-d988e91087695742d75946100cf2b7593cb8eed2a97411697819849958c022b3.json rename to backend/.sqlx/query-8780a8cd6781f86041ae8df58477913c01b5d8d27dd4251f0cabbe3e974c0b0a.json index 15625b26c2..2e19aa5a0e 100644 --- a/backend/.sqlx/query-d988e91087695742d75946100cf2b7593cb8eed2a97411697819849958c022b3.json +++ b/backend/.sqlx/query-8780a8cd6781f86041ae8df58477913c01b5d8d27dd4251f0cabbe3e974c0b0a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT result AS \"result!: Json>\"\n FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + "query": "SELECT result AS \"result!: Json>\"\n FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -19,5 +19,5 @@ true ] }, - "hash": "d988e91087695742d75946100cf2b7593cb8eed2a97411697819849958c022b3" + "hash": "8780a8cd6781f86041ae8df58477913c01b5d8d27dd4251f0cabbe3e974c0b0a" } diff --git a/backend/.sqlx/query-89de3ff8ab32e545efcbcda05f994cb1a32c4991cbd25046282d34272587d2de.json b/backend/.sqlx/query-89de3ff8ab32e545efcbcda05f994cb1a32c4991cbd25046282d34272587d2de.json deleted file mode 100644 index 99c2dc90a2..0000000000 --- a/backend/.sqlx/query-89de3ff8ab32e545efcbcda05f994cb1a32c4991cbd25046282d34272587d2de.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by from script where path = $1 AND workspace_id = $2 AND\n created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "hash", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "tag", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "concurrency_key", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "concurrent_limit", - "type_info": "Int4" - }, - { - "ordinal": 4, - "name": "concurrency_time_window_s", - "type_info": "Int4" - }, - { - "ordinal": 5, - "name": "cache_ttl", - "type_info": "Int4" - }, - { - "ordinal": 6, - "name": "language: ScriptLang", - "type_info": { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java" - ] - } - } - } - }, - { - "ordinal": 7, - "name": "dedicated_worker", - "type_info": "Bool" - }, - { - "ordinal": 8, - "name": "priority", - "type_info": "Int2" - }, - { - "ordinal": 9, - "name": "delete_after_use", - "type_info": "Bool" - }, - { - "ordinal": 10, - "name": "timeout", - "type_info": "Int4" - }, - { - "ordinal": 11, - "name": "has_preprocessor", - "type_info": "Bool" - }, - { - "ordinal": 12, - "name": "on_behalf_of_email", - "type_info": "Text" - }, - { - "ordinal": 13, - "name": "created_by", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - true, - true, - true, - true, - true, - false, - true, - true, - true, - true, - true, - true, - false - ] - }, - "hash": "89de3ff8ab32e545efcbcda05f994cb1a32c4991cbd25046282d34272587d2de" -} diff --git a/backend/.sqlx/query-8be1ddb20ffd8c375b7d1ecb14bdb3a7c2f0c8f9308946b9262e14b8c584dd99.json b/backend/.sqlx/query-8be1ddb20ffd8c375b7d1ecb14bdb3a7c2f0c8f9308946b9262e14b8c584dd99.json deleted file mode 100644 index 60ca066b36..0000000000 --- a/backend/.sqlx/query-8be1ddb20ffd8c375b7d1ecb14bdb3a7c2f0c8f9308946b9262e14b8c584dd99.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status\n SET flow_status = flow_status - 'retry'\n WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "8be1ddb20ffd8c375b7d1ecb14bdb3a7c2f0c8f9308946b9262e14b8c584dd99" -} diff --git a/backend/.sqlx/query-8cb755510f2cfb23bdd0d1cf66b69949549a44855529f77a530f681a6e714646.json b/backend/.sqlx/query-8cb755510f2cfb23bdd0d1cf66b69949549a44855529f77a530f681a6e714646.json deleted file mode 100644 index 82b70ebe92..0000000000 --- a/backend/.sqlx/query-8cb755510f2cfb23bdd0d1cf66b69949549a44855529f77a530f681a6e714646.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n flow_status,\n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT],\n $4\n )\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Text", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "8cb755510f2cfb23bdd0d1cf66b69949549a44855529f77a530f681a6e714646" -} diff --git a/backend/.sqlx/query-8f3ed45a0290cd9989f40f34775de5e8c3762597e6f55f8b9575a54ccc31e085.json b/backend/.sqlx/query-8f3ed45a0290cd9989f40f34775de5e8c3762597e6f55f8b9575a54ccc31e085.json deleted file mode 100644 index 4fd9cd8a4f..0000000000 --- a/backend/.sqlx/query-8f3ed45a0290cd9989f40f34775de5e8c3762597e6f55f8b9575a54ccc31e085.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'],\n ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb\n )\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int4", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid", - "Text", - "Jsonb" - ] - }, - "nullable": [ - null - ] - }, - "hash": "8f3ed45a0290cd9989f40f34775de5e8c3762597e6f55f8b9575a54ccc31e085" -} diff --git a/backend/.sqlx/query-8fb2581a439c26391e66ae7fac32c6cd2932f28ab6490ace027ed3a790b2a0f7.json b/backend/.sqlx/query-8fb2581a439c26391e66ae7fac32c6cd2932f28ab6490ace027ed3a790b2a0f7.json deleted file mode 100644 index 0ae81ab6dd..0000000000 --- a/backend/.sqlx/query-8fb2581a439c26391e66ae7fac32c6cd2932f28ab6490ace027ed3a790b2a0f7.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT edited_by, on_behalf_of_email FROM flow WHERE path = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "on_behalf_of_email", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - true - ] - }, - "hash": "8fb2581a439c26391e66ae7fac32c6cd2932f28ab6490ace027ed3a790b2a0f7" -} diff --git a/backend/.sqlx/query-8fcf755b4a57ed4ebf10a57c0c82589075c240b16d872576a048349b56f468e5.json b/backend/.sqlx/query-8fcf755b4a57ed4ebf10a57c0c82589075c240b16d872576a048349b56f468e5.json new file mode 100644 index 0000000000..49153b05aa --- /dev/null +++ b/backend/.sqlx/query-8fcf755b4a57ed4ebf10a57c0c82589075c240b16d872576a048349b56f468e5.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET\n suspend = $1,\n suspend_until = now() + interval '14 day',\n running = true\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "8fcf755b4a57ed4ebf10a57c0c82589075c240b16d872576a048349b56f468e5" +} diff --git a/backend/.sqlx/query-fdedd3909a97db5d43d9c46ff77b800b8efd647121b538deb023f96dbaac3715.json b/backend/.sqlx/query-903cf23d6b620388c645d5b8ac7d106bb6eea8af03e350d4ba19a4aba2cb9625.json similarity index 72% rename from backend/.sqlx/query-fdedd3909a97db5d43d9c46ff77b800b8efd647121b538deb023f96dbaac3715.json rename to backend/.sqlx/query-903cf23d6b620388c645d5b8ac7d106bb6eea8af03e350d4ba19a4aba2cb9625.json index 905707255f..abef34ebc0 100644 --- a/backend/.sqlx/query-fdedd3909a97db5d43d9c46ff77b800b8efd647121b538deb023f96dbaac3715.json +++ b/backend/.sqlx/query-903cf23d6b620388c645d5b8ac7d106bb6eea8af03e350d4ba19a4aba2cb9625.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT args AS \"args: Json>>\"\n FROM v2_job WHERE id = $1", + "query": "SELECT args AS \"args: Json>>\"\n FROM v2_job WHERE id = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ true ] }, - "hash": "fdedd3909a97db5d43d9c46ff77b800b8efd647121b538deb023f96dbaac3715" + "hash": "903cf23d6b620388c645d5b8ac7d106bb6eea8af03e350d4ba19a4aba2cb9625" } diff --git a/backend/.sqlx/query-90635149190c59396ca557bf1670554a1e40d0ce9cc686ad09adca0904324cd8.json b/backend/.sqlx/query-90635149190c59396ca557bf1670554a1e40d0ce9cc686ad09adca0904324cd8.json deleted file mode 100644 index 9dce5f08b4..0000000000 --- a/backend/.sqlx/query-90635149190c59396ca557bf1670554a1e40d0ce9cc686ad09adca0904324cd8.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue q SET suspend = 0\n FROM v2_job j, v2_job_status f\n WHERE parent_job = $1\n AND f.id = j.id AND q.id = j.id\n AND suspend = $2 AND (f.flow_status->'step')::int = 0", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "90635149190c59396ca557bf1670554a1e40d0ce9cc686ad09adca0904324cd8" -} diff --git a/backend/.sqlx/query-c50b6a4a6739d6df087a3b37c209e5f4b72fc27578d988155b74b05ec5df30b9.json b/backend/.sqlx/query-92c7c961198e506426bf3f97a8ddbb34af450041c675b30b708fed3ef9e01d2d.json similarity index 73% rename from backend/.sqlx/query-c50b6a4a6739d6df087a3b37c209e5f4b72fc27578d988155b74b05ec5df30b9.json rename to backend/.sqlx/query-92c7c961198e506426bf3f97a8ddbb34af450041c675b30b708fed3ef9e01d2d.json index eef812552b..fcb9657c8a 100644 --- a/backend/.sqlx/query-c50b6a4a6739d6df087a3b37c209e5f4b72fc27578d988155b74b05ec5df30b9.json +++ b/backend/.sqlx/query-92c7c961198e506426bf3f97a8ddbb34af450041c675b30b708fed3ef9e01d2d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n kind AS \"job_kind!: JobKind\",\n runnable_id AS \"script_hash: ScriptHash\",\n flow_status AS \"flow_status!: Json>\",\n raw_flow AS \"raw_flow: Json>\"\n FROM v2_job INNER JOIN v2_job_status ON v2_job.id = v2_job_status.id WHERE v2_job.id = $1 AND v2_job.workspace_id = $2 LIMIT 1", + "query": "SELECT\n kind AS \"job_kind!: JobKind\",\n runnable_id AS \"script_hash: ScriptHash\",\n flow_status AS \"flow_status!: Json>\",\n raw_flow AS \"raw_flow: Json>\"\n FROM v2_job INNER JOIN v2_job_status ON v2_job.id = v2_job_status.id WHERE v2_job.id = $1 AND v2_job.workspace_id = $2 LIMIT 1", "describe": { "columns": [ { @@ -63,5 +63,5 @@ true ] }, - "hash": "c50b6a4a6739d6df087a3b37c209e5f4b72fc27578d988155b74b05ec5df30b9" + "hash": "92c7c961198e506426bf3f97a8ddbb34af450041c675b30b708fed3ef9e01d2d" } diff --git a/backend/.sqlx/query-33c1793e55b1127d88d2509aadd0eb04e042463200f237b4c2cb176612fa16fe.json b/backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json similarity index 51% rename from backend/.sqlx/query-33c1793e55b1127d88d2509aadd0eb04e042463200f237b4c2cb176612fa16fe.json rename to backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json index 8728e35a0c..7df22ca7b7 100644 --- a/backend/.sqlx/query-33c1793e55b1127d88d2509aadd0eb04e042463200f237b4c2cb176612fa16fe.json +++ b/backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)", + "query": "INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt)\n VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)\n ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", "describe": { "columns": [], "parameters": { @@ -17,5 +17,5 @@ }, "nullable": [] }, - "hash": "33c1793e55b1127d88d2509aadd0eb04e042463200f237b4c2cb176612fa16fe" + "hash": "92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b" } diff --git a/backend/.sqlx/query-94f11d70062eebce384fe0fde527f3d6cebca1aa84a6f792c2a962b798f8da22.json b/backend/.sqlx/query-94f11d70062eebce384fe0fde527f3d6cebca1aa84a6f792c2a962b798f8da22.json new file mode 100644 index 0000000000..79d7fdcf4d --- /dev/null +++ b/backend/.sqlx/query-94f11d70062eebce384fe0fde527f3d6cebca1aa84a6f792c2a962b798f8da22.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['failure_module'], $1)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "94f11d70062eebce384fe0fde527f3d6cebca1aa84a6f792c2a962b798f8da22" +} diff --git a/backend/.sqlx/query-96c0e34708bbba29db162e7289a942addd4581dddc88663b6c2cbae87ec205fc.json b/backend/.sqlx/query-96c0e34708bbba29db162e7289a942addd4581dddc88663b6c2cbae87ec205fc.json new file mode 100644 index 0000000000..0f6659264a --- /dev/null +++ b/backend/.sqlx/query-96c0e34708bbba29db162e7289a942addd4581dddc88663b6c2cbae87ec205fc.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'],\n ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb\n )\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text", + "Jsonb" + ] + }, + "nullable": [ + null + ] + }, + "hash": "96c0e34708bbba29db162e7289a942addd4581dddc88663b6c2cbae87ec205fc" +} diff --git a/backend/.sqlx/query-97048ce0bcabb9baecb80cde5ab3c989e1575fbd20ef22766d2887a86dce15e1.json b/backend/.sqlx/query-97048ce0bcabb9baecb80cde5ab3c989e1575fbd20ef22766d2887a86dce15e1.json deleted file mode 100644 index c8ab870c76..0000000000 --- a/backend/.sqlx/query-97048ce0bcabb9baecb80cde5ab3c989e1575fbd20ef22766d2887a86dce15e1.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT flow.versions[array_upper(flow.versions, 1)] AS \"version!: i64\"\n FROM flow WHERE path = $1 AND workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "version!: i64", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "97048ce0bcabb9baecb80cde5ab3c989e1575fbd20ef22766d2887a86dce15e1" -} diff --git a/backend/.sqlx/query-a0833b9899833166891c5de926f78632fae1123e736d728bf92cb2de004b6826.json b/backend/.sqlx/query-9b60fa8a1003015bc5a7cdbee9a4486b313d45347dfd9d4793d60e2760763ca3.json similarity index 69% rename from backend/.sqlx/query-a0833b9899833166891c5de926f78632fae1123e736d728bf92cb2de004b6826.json rename to backend/.sqlx/query-9b60fa8a1003015bc5a7cdbee9a4486b313d45347dfd9d4793d60e2760763ca3.json index a837158a37..f4251250be 100644 --- a/backend/.sqlx/query-a0833b9899833166891c5de926f78632fae1123e736d728bf92cb2de004b6826.json +++ b/backend/.sqlx/query-9b60fa8a1003015bc5a7cdbee9a4486b313d45347dfd9d4793d60e2760763ca3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by\n FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2", + "query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by, flow_version.id AS version\n FROM flow\n INNER JOIN flow_version\n ON flow_version.id = $3\n WHERE flow.path = $1 and flow.workspace_id = $2", "describe": { "columns": [ { @@ -32,12 +32,18 @@ "ordinal": 5, "name": "edited_by", "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "version", + "type_info": "Int8" } ], "parameters": { "Left": [ "Text", - "Text" + "Text", + "Int8" ] }, "nullable": [ @@ -46,8 +52,9 @@ null, null, true, + false, false ] }, - "hash": "a0833b9899833166891c5de926f78632fae1123e736d728bf92cb2de004b6826" + "hash": "9b60fa8a1003015bc5a7cdbee9a4486b313d45347dfd9d4793d60e2760763ca3" } diff --git a/backend/.sqlx/query-6781ba76dfce321dca4634566496ea5d698ac09d8264e35dfaa4cd8edc9e8414.json b/backend/.sqlx/query-9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf.json similarity index 76% rename from backend/.sqlx/query-6781ba76dfce321dca4634566496ea5d698ac09d8264e35dfaa4cd8edc9e8414.json rename to backend/.sqlx/query-9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf.json index 2e4638f1f7..c21c30c012 100644 --- a/backend/.sqlx/query-6781ba76dfce321dca4634566496ea5d698ac09d8264e35dfaa4cd8edc9e8414.json +++ b/backend/.sqlx/query-9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT \n id, \n created_at, \n trigger_kind AS \"trigger_kind: _\", \n payload AS \"payload!: _\", \n trigger_extra AS \"trigger_extra: _\"\n FROM \n capture\n WHERE \n id = $1 \n AND workspace_id = $2\n ", + "query": "\n SELECT \n id, \n created_at, \n trigger_kind AS \"trigger_kind: _\", \n main_args AS \"main_args!: _\", \n preprocessor_args AS \"preprocessor_args: _\"\n FROM \n capture\n WHERE \n id = $1 \n AND workspace_id = $2\n ", "describe": { "columns": [ { @@ -38,12 +38,12 @@ }, { "ordinal": 3, - "name": "payload!: _", + "name": "main_args!: _", "type_info": "Jsonb" }, { "ordinal": 4, - "name": "trigger_extra: _", + "name": "preprocessor_args: _", "type_info": "Jsonb" } ], @@ -61,5 +61,5 @@ true ] }, - "hash": "6781ba76dfce321dca4634566496ea5d698ac09d8264e35dfaa4cd8edc9e8414" + "hash": "9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf" } diff --git a/backend/.sqlx/query-9dec888d4b0666d1843fbbc4fb2475fd947f047a9965bc110d7338208b77783d.json b/backend/.sqlx/query-9dec888d4b0666d1843fbbc4fb2475fd947f047a9965bc110d7338208b77783d.json new file mode 100644 index 0000000000..7c49fe890e --- /dev/null +++ b/backend/.sqlx/query-9dec888d4b0666d1843fbbc4fb2475fd947f047a9965bc110d7338208b77783d.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT flow_version.id from flow\n INNER JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "9dec888d4b0666d1843fbbc4fb2475fd947f047a9965bc110d7338208b77783d" +} diff --git a/backend/.sqlx/query-a10ec229d7ed89f563b6b33e70e8ede5135a849e7b9108c37bfd90990a4be780.json b/backend/.sqlx/query-a10ec229d7ed89f563b6b33e70e8ede5135a849e7b9108c37bfd90990a4be780.json new file mode 100644 index 0000000000..9c49c0794c --- /dev/null +++ b/backend/.sqlx/query-a10ec229d7ed89f563b6b33e70e8ede5135a849e7b9108c37bfd90990a4be780.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n flow_status,\n ARRAY['modules', $1::TEXT, 'iterator', 'index'],\n ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n )\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a10ec229d7ed89f563b6b33e70e8ede5135a849e7b9108c37bfd90990a4be780" +} diff --git a/backend/.sqlx/query-30216cf02e972f961b7cc6054050fdc984be118df1ad68f7263c84e058bb1266.json b/backend/.sqlx/query-a3debece1a4171881431640f6af264d402d32e2b6ce925d1ebf1f60f3b688207.json similarity index 51% rename from backend/.sqlx/query-30216cf02e972f961b7cc6054050fdc984be118df1ad68f7263c84e058bb1266.json rename to backend/.sqlx/query-a3debece1a4171881431640f6af264d402d32e2b6ce925d1ebf1f60f3b688207.json index 3603ed5c5b..9448456dbb 100644 --- a/backend/.sqlx/query-30216cf02e972f961b7cc6054050fdc984be118df1ad68f7263c84e058bb1266.json +++ b/backend/.sqlx/query-a3debece1a4171881431640f6af264d402d32e2b6ce925d1ebf1f60f3b688207.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2)\n WHERE id = $3\n RETURNING flow_status AS \"flow_status: Json>\"", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2)\n WHERE id = $3\n RETURNING flow_status AS \"flow_status: Json>\"", "describe": { "columns": [ { @@ -20,5 +20,5 @@ true ] }, - "hash": "30216cf02e972f961b7cc6054050fdc984be118df1ad68f7263c84e058bb1266" + "hash": "a3debece1a4171881431640f6af264d402d32e2b6ce925d1ebf1f60f3b688207" } diff --git a/backend/.sqlx/query-a3f315fdae54e51b56b0681fab2bbff779a4a62d129916dd4c3054b45e0b654e.json b/backend/.sqlx/query-a3f315fdae54e51b56b0681fab2bbff779a4a62d129916dd4c3054b45e0b654e.json new file mode 100644 index 0000000000..dc0b37ca8c --- /dev/null +++ b/backend/.sqlx/query-a3f315fdae54e51b56b0681fab2bbff779a4a62d129916dd4c3054b45e0b654e.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'approvers'], $2)\n WHERE id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "a3f315fdae54e51b56b0681fab2bbff779a4a62d129916dd4c3054b45e0b654e" +} diff --git a/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json b/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json new file mode 100644 index 0000000000..a2a19a6c35 --- /dev/null +++ b/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json @@ -0,0 +1,70 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT has_preprocessor, language as \"language: _\", content, schema as \"schema: _\" FROM script WHERE workspace_id = $1 AND hash = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "has_preprocessor", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "language: _", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java" + ] + } + } + } + }, + { + "ordinal": 2, + "name": "content", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "schema: _", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + true, + false, + false, + true + ] + }, + "hash": "a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b" +} diff --git a/backend/.sqlx/query-4507c3907bf49f93f6c17956d9cf9495f4538b20ce0299acde7578386db4278c.json b/backend/.sqlx/query-aa4ecf6b7ab078544c280957ad6614c4ccd1bc9b4695ca56f56ba0899b23562f.json similarity index 66% rename from backend/.sqlx/query-4507c3907bf49f93f6c17956d9cf9495f4538b20ce0299acde7578386db4278c.json rename to backend/.sqlx/query-aa4ecf6b7ab078544c280957ad6614c4ccd1bc9b4695ca56f56ba0899b23562f.json index dc57aefcdc..c231ff2bd0 100644 --- a/backend/.sqlx/query-4507c3907bf49f93f6c17956d9cf9495f4538b20ce0299acde7578386db4278c.json +++ b/backend/.sqlx/query-aa4ecf6b7ab078544c280957ad6614c4ccd1bc9b4695ca56f56ba0899b23562f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO parallel_monitor_lock (parent_flow_id, job_id)\n VALUES ($1, $2)", + "query": "INSERT INTO parallel_monitor_lock (parent_flow_id, job_id)\n VALUES ($1, $2)", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "4507c3907bf49f93f6c17956d9cf9495f4538b20ce0299acde7578386db4278c" + "hash": "aa4ecf6b7ab078544c280957ad6614c4ccd1bc9b4695ca56f56ba0899b23562f" } diff --git a/backend/.sqlx/query-aed8bd751c3e988f422216e74acfb77dc03469355d2a0da0b2d6b4aeeea37d3e.json b/backend/.sqlx/query-aed8bd751c3e988f422216e74acfb77dc03469355d2a0da0b2d6b4aeeea37d3e.json new file mode 100644 index 0000000000..d8f93f74c4 --- /dev/null +++ b/backend/.sqlx/query-aed8bd751c3e988f422216e74acfb77dc03469355d2a0da0b2d6b4aeeea37d3e.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_leaf_jobs = JSONB_SET(coalesce(flow_leaf_jobs, '{}'::jsonb), ARRAY[$1::TEXT], $2)\n WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $3), $3) = id", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "aed8bd751c3e988f422216e74acfb77dc03469355d2a0da0b2d6b4aeeea37d3e" +} diff --git a/backend/.sqlx/query-af925931f3217bbd32313678989ad1a66bbd8dacd12dea36608cc20197df358f.json b/backend/.sqlx/query-af925931f3217bbd32313678989ad1a66bbd8dacd12dea36608cc20197df358f.json deleted file mode 100644 index b90300787b..0000000000 --- a/backend/.sqlx/query-af925931f3217bbd32313678989ad1a66bbd8dacd12dea36608cc20197df358f.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1),\n ARRAY['step'],\n $2\n )\n WHERE id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "af925931f3217bbd32313678989ad1a66bbd8dacd12dea36608cc20197df358f" -} diff --git a/backend/.sqlx/query-b01160fe44d69834ac08bbf60feacb3e3caa02a04b084da44cdcb9103794b39e.json b/backend/.sqlx/query-b01160fe44d69834ac08bbf60feacb3e3caa02a04b084da44cdcb9103794b39e.json new file mode 100644 index 0000000000..0e87ff0632 --- /dev/null +++ b/backend/.sqlx/query-b01160fe44d69834ac08bbf60feacb3e3caa02a04b084da44cdcb9103794b39e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['cleanup_module', 'flow_jobs_to_clean'], COALESCE(flow_status->'cleanup_module'->'flow_jobs_to_clean', '[]'::jsonb) || $1)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "b01160fe44d69834ac08bbf60feacb3e3caa02a04b084da44cdcb9103794b39e" +} diff --git a/backend/.sqlx/query-47e6b25cc092ec8718a6581c76aca10b275653e10ea4aa17a8ef5091ca09294a.json b/backend/.sqlx/query-b1c96c527c4b263b5155d689eb88894ea93f0eaba37874f828a733062af17640.json similarity index 67% rename from backend/.sqlx/query-47e6b25cc092ec8718a6581c76aca10b275653e10ea4aa17a8ef5091ca09294a.json rename to backend/.sqlx/query-b1c96c527c4b263b5155d689eb88894ea93f0eaba37874f828a733062af17640.json index 3206f22bc5..8fd4c311c5 100644 --- a/backend/.sqlx/query-47e6b25cc092ec8718a6581c76aca10b275653e10ea4aa17a8ef5091ca09294a.json +++ b/backend/.sqlx/query-b1c96c527c4b263b5155d689eb88894ea93f0eaba37874f828a733062af17640.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT result, id\n FROM v2_job_completed\n WHERE id = ANY($1) AND workspace_id = $2", + "query": "SELECT result, id\n FROM v2_job_completed\n WHERE id = ANY($1) AND workspace_id = $2", "describe": { "columns": [ { @@ -25,5 +25,5 @@ false ] }, - "hash": "47e6b25cc092ec8718a6581c76aca10b275653e10ea4aa17a8ef5091ca09294a" + "hash": "b1c96c527c4b263b5155d689eb88894ea93f0eaba37874f828a733062af17640" } diff --git a/backend/.sqlx/query-597b148ff09a1e0f369bb04781ee4e429ebce64a4d5c16b0f136142ad213cdb1.json b/backend/.sqlx/query-b41fa341e65ee348f468ed04ac1160770b19c0a00cd333abc48b29c54f863149.json similarity index 52% rename from backend/.sqlx/query-597b148ff09a1e0f369bb04781ee4e429ebce64a4d5c16b0f136142ad213cdb1.json rename to backend/.sqlx/query-b41fa341e65ee348f468ed04ac1160770b19c0a00cd333abc48b29c54f863149.json index 4a704e58d8..f37bea0531 100644 --- a/backend/.sqlx/query-597b148ff09a1e0f369bb04781ee4e429ebce64a4d5c16b0f136142ad213cdb1.json +++ b/backend/.sqlx/query-b41fa341e65ee348f468ed04ac1160770b19c0a00cd333abc48b29c54f863149.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n args AS \"args: Json>>\"\n FROM v2_job\n WHERE id = $1", + "query": "SELECT\n args AS \"args: Json>>\"\n FROM v2_job\n WHERE id = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ true ] }, - "hash": "597b148ff09a1e0f369bb04781ee4e429ebce64a4d5c16b0f136142ad213cdb1" + "hash": "b41fa341e65ee348f468ed04ac1160770b19c0a00cd333abc48b29c54f863149" } diff --git a/backend/.sqlx/query-a17260a1f1ee02e786690994d98c84ddf81e2eeb883f895c9cfc47e144d422cb.json b/backend/.sqlx/query-b5860f6a7672a368d740dcd367a8d5ab98fa93e0382a57a698564695db6c40ac.json similarity index 62% rename from backend/.sqlx/query-a17260a1f1ee02e786690994d98c84ddf81e2eeb883f895c9cfc47e144d422cb.json rename to backend/.sqlx/query-b5860f6a7672a368d740dcd367a8d5ab98fa93e0382a57a698564695db6c40ac.json index 5f8386be37..a08c31e743 100644 --- a/backend/.sqlx/query-a17260a1f1ee02e786690994d98c84ddf81e2eeb883f895c9cfc47e144d422cb.json +++ b/backend/.sqlx/query-b5860f6a7672a368d740dcd367a8d5ab98fa93e0382a57a698564695db6c40ac.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT content FROM script WHERE path = $1 AND workspace_id = $2 AND\n created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND archived = false AND workspace_id = $2)", + "query": "SELECT content FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1", "describe": { "columns": [ { @@ -19,5 +19,5 @@ false ] }, - "hash": "a17260a1f1ee02e786690994d98c84ddf81e2eeb883f895c9cfc47e144d422cb" + "hash": "b5860f6a7672a368d740dcd367a8d5ab98fa93e0382a57a698564695db6c40ac" } diff --git a/backend/.sqlx/query-bbc2c0769bf833f4e95bfc7908897ecbfe662efb13ffdd8ee3f1930bff4cd9c4.json b/backend/.sqlx/query-bbc2c0769bf833f4e95bfc7908897ecbfe662efb13ffdd8ee3f1930bff4cd9c4.json new file mode 100644 index 0000000000..e8f8de3ef8 --- /dev/null +++ b/backend/.sqlx/query-bbc2c0769bf833f4e95bfc7908897ecbfe662efb13ffdd8ee3f1930bff4cd9c4.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2),\n ARRAY['step'],\n $3\n )\n WHERE id = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Jsonb", + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "bbc2c0769bf833f4e95bfc7908897ecbfe662efb13ffdd8ee3f1930bff4cd9c4" +} diff --git a/backend/.sqlx/query-bcfe877749ff7b944fef302ea37481b170c221349a793c9608c4ccd52ba8a5af.json b/backend/.sqlx/query-bcfe877749ff7b944fef302ea37481b170c221349a793c9608c4ccd52ba8a5af.json new file mode 100644 index 0000000000..4460ab7b36 --- /dev/null +++ b/backend/.sqlx/query-bcfe877749ff7b944fef302ea37481b170c221349a793c9608c4ccd52ba8a5af.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['approval_conditions'], $1)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "bcfe877749ff7b944fef302ea37481b170c221349a793c9608c4ccd52ba8a5af" +} diff --git a/backend/.sqlx/query-c202f6fbae6a727f88f3ac692985c70e6ebc68e4a16d02e4e36b79f3cfb1c661.json b/backend/.sqlx/query-c202f6fbae6a727f88f3ac692985c70e6ebc68e4a16d02e4e36b79f3cfb1c661.json deleted file mode 100644 index e1094f38ef..0000000000 --- a/backend/.sqlx/query-c202f6fbae6a727f88f3ac692985c70e6ebc68e4a16d02e4e36b79f3cfb1c661.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH suspend AS (\n UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + $3\n WHERE id = $4\n RETURNING id\n ) UPDATE v2_job_status SET flow_status = JSONB_SET(\n flow_status,\n ARRAY['modules', flow_status->>'step'::TEXT],\n $1\n ) WHERE id = (SELECT id FROM suspend)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Int4", - "Interval", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "c202f6fbae6a727f88f3ac692985c70e6ebc68e4a16d02e4e36b79f3cfb1c661" -} diff --git a/backend/.sqlx/query-0102308ffa1c0dbfba54d29246535bb81146a4cfae0ec408435570e0813a3bef.json b/backend/.sqlx/query-c2925a28212265bd9ac8e2d498b3588fc040c7a29ae6d9bab1f05af0b535e2ac.json similarity index 73% rename from backend/.sqlx/query-0102308ffa1c0dbfba54d29246535bb81146a4cfae0ec408435570e0813a3bef.json rename to backend/.sqlx/query-c2925a28212265bd9ac8e2d498b3588fc040c7a29ae6d9bab1f05af0b535e2ac.json index 06390d0ce9..eec208dbbb 100644 --- a/backend/.sqlx/query-0102308ffa1c0dbfba54d29246535bb81146a4cfae0ec408435570e0813a3bef.json +++ b/backend/.sqlx/query-c2925a28212265bd9ac8e2d498b3588fc040c7a29ae6d9bab1f05af0b535e2ac.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n gcp_resource_path,\n subscription_id,\n topic_id,\n workspace_id,\n delivery_type AS \"delivery_type: _\",\n delivery_config AS \"delivery_config: _\",\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled\n FROM \n gcp_trigger\n WHERE \n workspace_id = $1\n ", + "query": "\n SELECT\n gcp_resource_path,\n subscription_id,\n topic_id,\n workspace_id,\n delivery_type AS \"delivery_type: _\",\n delivery_config AS \"delivery_config: _\",\n subscription_mode AS \"subscription_mode: _\",\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled\n FROM \n gcp_trigger\n WHERE \n workspace_id = $1\n ", "describe": { "columns": [ { @@ -45,56 +45,71 @@ }, { "ordinal": 6, + "name": "subscription_mode: _", + "type_info": { + "Custom": { + "name": "gcp_subscription_mode", + "kind": { + "Enum": [ + "create_update", + "existing" + ] + } + } + } + }, + { + "ordinal": 7, "name": "path", "type_info": "Varchar" }, { - "ordinal": 7, + "ordinal": 8, "name": "script_path", "type_info": "Varchar" }, { - "ordinal": 8, + "ordinal": 9, "name": "is_flow", "type_info": "Bool" }, { - "ordinal": 9, + "ordinal": 10, "name": "edited_by", "type_info": "Varchar" }, { - "ordinal": 10, + "ordinal": 11, "name": "email", "type_info": "Varchar" }, { - "ordinal": 11, + "ordinal": 12, "name": "edited_at", "type_info": "Timestamptz" }, { - "ordinal": 12, + "ordinal": 13, "name": "server_id", "type_info": "Varchar" }, { - "ordinal": 13, + "ordinal": 14, "name": "last_server_ping", "type_info": "Timestamptz" }, { - "ordinal": 14, + "ordinal": 15, "name": "extra_perms", "type_info": "Jsonb" }, { - "ordinal": 15, + "ordinal": 16, "name": "error", "type_info": "Text" }, { - "ordinal": 16, + "ordinal": 17, "name": "enabled", "type_info": "Bool" } @@ -117,6 +132,7 @@ false, false, false, + false, true, true, false, @@ -124,5 +140,5 @@ false ] }, - "hash": "0102308ffa1c0dbfba54d29246535bb81146a4cfae0ec408435570e0813a3bef" + "hash": "c2925a28212265bd9ac8e2d498b3588fc040c7a29ae6d9bab1f05af0b535e2ac" } diff --git a/backend/.sqlx/query-c71e12ec9d0054dd5605a4ea2ef77fa54ce686da9c93790420a91a0735589ac7.json b/backend/.sqlx/query-c71e12ec9d0054dd5605a4ea2ef77fa54ce686da9c93790420a91a0735589ac7.json deleted file mode 100644 index ecc62b1682..0000000000 --- a/backend/.sqlx/query-c71e12ec9d0054dd5605a4ea2ef77fa54ce686da9c93790420a91a0735589ac7.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(JSONB_SET(flow_status, ARRAY['retry'], $1), ARRAY['modules', $3::TEXT, 'failed_retries'], $4)\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Uuid", - "Text", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "c71e12ec9d0054dd5605a4ea2ef77fa54ce686da9c93790420a91a0735589ac7" -} diff --git a/backend/.sqlx/query-c794ff5e14429a1bc56f5e70cfd10e2160d1b7e123c7ab226dc77d1789f164f9.json b/backend/.sqlx/query-c794ff5e14429a1bc56f5e70cfd10e2160d1b7e123c7ab226dc77d1789f164f9.json deleted file mode 100644 index c4810efe68..0000000000 --- a/backend/.sqlx/query-c794ff5e14429a1bc56f5e70cfd10e2160d1b7e123c7ab226dc77d1789f164f9.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT tag, dedicated_worker, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by\n FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tag", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "dedicated_worker", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "has_preprocessor", - "type_info": "Bool" - }, - { - "ordinal": 3, - "name": "on_behalf_of_email", - "type_info": "Text" - }, - { - "ordinal": 4, - "name": "edited_by", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - true, - true, - null, - true, - false - ] - }, - "hash": "c794ff5e14429a1bc56f5e70cfd10e2160d1b7e123c7ab226dc77d1789f164f9" -} diff --git a/backend/.sqlx/query-cd79f4dc6a426f1c4c19c2a86dc877a6d5bc5771b27d9e208a219a63add200f3.json b/backend/.sqlx/query-cd79f4dc6a426f1c4c19c2a86dc877a6d5bc5771b27d9e208a219a63add200f3.json new file mode 100644 index 0000000000..279ff511bd --- /dev/null +++ b/backend/.sqlx/query-cd79f4dc6a426f1c4c19c2a86dc877a6d5bc5771b27d9e208a219a63add200f3.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['failure_module'], $1),\n ARRAY['step'],\n $2\n )\n WHERE id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "cd79f4dc6a426f1c4c19c2a86dc877a6d5bc5771b27d9e208a219a63add200f3" +} diff --git a/backend/.sqlx/query-cf2a6ad6471a40b6298775cda9300aeecdd75503bed59d80cd62091d1642d1ec.json b/backend/.sqlx/query-cf2a6ad6471a40b6298775cda9300aeecdd75503bed59d80cd62091d1642d1ec.json new file mode 100644 index 0000000000..4e17af3ce2 --- /dev/null +++ b/backend/.sqlx/query-cf2a6ad6471a40b6298775cda9300aeecdd75503bed59d80cd62091d1642d1ec.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg \n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "deployment_msg", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "cf2a6ad6471a40b6298775cda9300aeecdd75503bed59d80cd62091d1642d1ec" +} diff --git a/backend/.sqlx/query-d6db1103fee4bad6831656d77a28254c9a832b4f660ec755f4fa14f6f7bb3353.json b/backend/.sqlx/query-d6db1103fee4bad6831656d77a28254c9a832b4f660ec755f4fa14f6f7bb3353.json new file mode 100644 index 0000000000..09f24968f3 --- /dev/null +++ b/backend/.sqlx/query-d6db1103fee4bad6831656d77a28254c9a832b4f660ec755f4fa14f6f7bb3353.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'iterator', 'index'],\n ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n )\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text", + "Jsonb" + ] + }, + "nullable": [ + null + ] + }, + "hash": "d6db1103fee4bad6831656d77a28254c9a832b4f660ec755f4fa14f6f7bb3353" +} diff --git a/backend/.sqlx/query-abfc6b530565f125bd0b8ac99cd67fd22f14f8fce89e59890ba21e87fe323af5.json b/backend/.sqlx/query-daf9674838fb3e3653a356c7434c719616a614d77e726433737e5f5d9bd60134.json similarity index 76% rename from backend/.sqlx/query-abfc6b530565f125bd0b8ac99cd67fd22f14f8fce89e59890ba21e87fe323af5.json rename to backend/.sqlx/query-daf9674838fb3e3653a356c7434c719616a614d77e726433737e5f5d9bd60134.json index 55039055c8..2e2a9ba027 100644 --- a/backend/.sqlx/query-abfc6b530565f125bd0b8ac99cd67fd22f14f8fce89e59890ba21e87fe323af5.json +++ b/backend/.sqlx/query-daf9674838fb3e3653a356c7434c719616a614d77e726433737e5f5d9bd60134.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH to_update AS (\n SELECT q.id, q.workspace_id, r.ping, COALESCE(zjc.counter, 0) as counter\n FROM v2_job_queue q\n JOIN v2_job j ON j.id = q.id\n JOIN v2_job_runtime r ON r.id = j.id\n LEFT JOIN zombie_job_counter zjc ON zjc.job_id = q.id\n WHERE ping < now() - ($1 || ' seconds')::interval\n AND running = true\n AND kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow')\n AND same_worker = false\n AND (zjc.counter IS NULL OR zjc.counter <= $2)\n FOR UPDATE of q SKIP LOCKED\n ),\n zombie_jobs AS (\n UPDATE v2_job_queue q\n SET running = false, started_at = null\n FROM to_update tu\n WHERE q.id = tu.id AND (tu.counter IS NULL OR tu.counter < $2)\n RETURNING q.id, q.workspace_id, ping, tu.counter\n ),\n update_ping AS (\n UPDATE v2_job_runtime r\n SET ping = null\n FROM zombie_jobs zj\n WHERE r.id = zj.id\n ),\n increment_counter AS (\n INSERT INTO zombie_job_counter (job_id, counter)\n SELECT id, 1 FROM to_update WHERE counter < $2\n ON CONFLICT (job_id) DO UPDATE \n SET counter = zombie_job_counter.counter + 1\n ),\n update_concurrency AS (\n UPDATE concurrency_counter cc\n SET job_uuids = job_uuids - zj.id::text\n FROM zombie_jobs zj\n INNER JOIN concurrency_key ck ON ck.job_id = zj.id\n WHERE cc.concurrency_id = ck.key\n )\n SELECT id AS \"id!\", workspace_id AS \"workspace_id!\", ping, counter + 1 AS counter FROM to_update", + "query": "WITH to_update AS (\n SELECT q.id, q.workspace_id, r.ping, COALESCE(zjc.counter, 0) as counter\n FROM v2_job_queue q\n JOIN v2_job j ON j.id = q.id\n JOIN v2_job_runtime r ON r.id = j.id\n LEFT JOIN zombie_job_counter zjc ON zjc.job_id = q.id\n WHERE ping < now() - ($1 || ' seconds')::interval\n AND running = true\n AND kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow')\n AND same_worker = false\n AND (zjc.counter IS NULL OR zjc.counter <= $2)\n FOR UPDATE of q SKIP LOCKED\n ),\n zombie_jobs AS (\n UPDATE v2_job_queue q\n SET running = false, started_at = null\n FROM to_update tu\n WHERE q.id = tu.id AND (tu.counter IS NULL OR tu.counter < $2)\n RETURNING q.id, q.workspace_id, ping, tu.counter\n ),\n update_ping AS (\n UPDATE v2_job_runtime r\n SET ping = null\n FROM zombie_jobs zj\n WHERE r.id = zj.id\n ),\n increment_counter AS (\n INSERT INTO zombie_job_counter (job_id, counter)\n SELECT id, 1 FROM to_update WHERE counter < $2\n ON CONFLICT (job_id) DO UPDATE\n SET counter = zombie_job_counter.counter + 1\n ),\n update_concurrency AS (\n UPDATE concurrency_counter cc\n SET job_uuids = job_uuids - zj.id::text\n FROM zombie_jobs zj\n INNER JOIN concurrency_key ck ON ck.job_id = zj.id\n WHERE cc.concurrency_id = ck.key\n )\n SELECT id AS \"id!\", workspace_id AS \"workspace_id!\", ping, counter + 1 AS counter FROM to_update", "describe": { "columns": [ { @@ -37,5 +37,5 @@ null ] }, - "hash": "abfc6b530565f125bd0b8ac99cd67fd22f14f8fce89e59890ba21e87fe323af5" + "hash": "daf9674838fb3e3653a356c7434c719616a614d77e726433737e5f5d9bd60134" } diff --git a/backend/.sqlx/query-de2a213bc5a08d08bcd52ad630559cc0f16d7286c3f07c27d962258022f0e04d.json b/backend/.sqlx/query-de2a213bc5a08d08bcd52ad630559cc0f16d7286c3f07c27d962258022f0e04d.json new file mode 100644 index 0000000000..f3ef9ff937 --- /dev/null +++ b/backend/.sqlx/query-de2a213bc5a08d08bcd52ad630559cc0f16d7286c3f07c27d962258022f0e04d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(JSONB_SET(flow_status, ARRAY['retry'], $1), ARRAY['modules', $3::TEXT, 'failed_retries'], $4)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "de2a213bc5a08d08bcd52ad630559cc0f16d7286c3f07c27d962258022f0e04d" +} diff --git a/backend/.sqlx/query-defd99dd2427cdc54bb662d1ba3a1aea7f410ef204ec3465f4fb6c9acd256c95.json b/backend/.sqlx/query-defd99dd2427cdc54bb662d1ba3a1aea7f410ef204ec3465f4fb6c9acd256c95.json deleted file mode 100644 index 6aab7bc049..0000000000 --- a/backend/.sqlx/query-defd99dd2427cdc54bb662d1ba3a1aea7f410ef204ec3465f4fb6c9acd256c95.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue SET\n suspend = $1,\n suspend_until = now() + interval '14 day',\n running = true\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int4", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "defd99dd2427cdc54bb662d1ba3a1aea7f410ef204ec3465f4fb6c9acd256c95" -} diff --git a/backend/.sqlx/query-e3f545460bf317c3e2f34f4cbd12740141eb8b5ed07c090ae9356a970caeadd0.json b/backend/.sqlx/query-e3f545460bf317c3e2f34f4cbd12740141eb8b5ed07c090ae9356a970caeadd0.json new file mode 100644 index 0000000000..34df271cc8 --- /dev/null +++ b/backend/.sqlx/query-e3f545460bf317c3e2f34f4cbd12740141eb8b5ed07c090ae9356a970caeadd0.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH suspend AS (\n UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + $3\n WHERE id = $4\n RETURNING id\n ) UPDATE v2_job_status SET flow_status = JSONB_SET(\n flow_status,\n ARRAY['modules', flow_status->>'step'::TEXT],\n $1\n ) WHERE id = (SELECT id FROM suspend)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Int4", + "Interval", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "e3f545460bf317c3e2f34f4cbd12740141eb8b5ed07c090ae9356a970caeadd0" +} diff --git a/backend/.sqlx/query-f9f7f31390f8ea1f4facd8e6a888886ea136f2327169bf93f126e2ef130d0946.json b/backend/.sqlx/query-eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423.json similarity index 75% rename from backend/.sqlx/query-f9f7f31390f8ea1f4facd8e6a888886ea136f2327169bf93f126e2ef130d0946.json rename to backend/.sqlx/query-eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423.json index a97a131baa..8890104678 100644 --- a/backend/.sqlx/query-f9f7f31390f8ea1f4facd8e6a888886ea136f2327169bf93f126e2ef130d0946.json +++ b/backend/.sqlx/query-eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO \n capture (\n workspace_id, path, is_flow, trigger_kind, payload, trigger_extra, created_by\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7\n )\n ", + "query": "\n INSERT INTO \n capture (\n workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7\n )\n ", "describe": { "columns": [], "parameters": { @@ -34,5 +34,5 @@ }, "nullable": [] }, - "hash": "f9f7f31390f8ea1f4facd8e6a888886ea136f2327169bf93f126e2ef130d0946" + "hash": "eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423" } diff --git a/backend/.sqlx/query-ec7836df5f9056ec70015800b7f4feaeb1b671120f5f8c98fca8c89c6587fc35.json b/backend/.sqlx/query-ec7836df5f9056ec70015800b7f4feaeb1b671120f5f8c98fca8c89c6587fc35.json deleted file mode 100644 index 73ac1919c1..0000000000 --- a/backend/.sqlx/query-ec7836df5f9056ec70015800b7f4feaeb1b671120f5f8c98fca8c89c6587fc35.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "select tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, on_behalf_of_email, created_by from script where hash = $1 AND workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tag", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "concurrency_key", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "concurrent_limit", - "type_info": "Int4" - }, - { - "ordinal": 3, - "name": "concurrency_time_window_s", - "type_info": "Int4" - }, - { - "ordinal": 4, - "name": "cache_ttl", - "type_info": "Int4" - }, - { - "ordinal": 5, - "name": "language: ScriptLang", - "type_info": { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java" - ] - } - } - } - }, - { - "ordinal": 6, - "name": "dedicated_worker", - "type_info": "Bool" - }, - { - "ordinal": 7, - "name": "priority", - "type_info": "Int2" - }, - { - "ordinal": 8, - "name": "delete_after_use", - "type_info": "Bool" - }, - { - "ordinal": 9, - "name": "timeout", - "type_info": "Int4" - }, - { - "ordinal": 10, - "name": "on_behalf_of_email", - "type_info": "Text" - }, - { - "ordinal": 11, - "name": "created_by", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Int8", - "Text" - ] - }, - "nullable": [ - true, - true, - true, - true, - true, - false, - true, - true, - true, - true, - true, - false - ] - }, - "hash": "ec7836df5f9056ec70015800b7f4feaeb1b671120f5f8c98fca8c89c6587fc35" -} diff --git a/backend/.sqlx/query-f3c78cb67379f9407f1f32ce3387184d7f4fcb04c07a8e2f07c5effc10f8fd5c.json b/backend/.sqlx/query-f3c78cb67379f9407f1f32ce3387184d7f4fcb04c07a8e2f07c5effc10f8fd5c.json deleted file mode 100644 index 0cf878b15c..0000000000 --- a/backend/.sqlx/query-f3c78cb67379f9407f1f32ce3387184d7f4fcb04c07a8e2f07c5effc10f8fd5c.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'approvers'], $2)\n WHERE id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "f3c78cb67379f9407f1f32ce3387184d7f4fcb04c07a8e2f07c5effc10f8fd5c" -} diff --git a/backend/.sqlx/query-f44f1a794ab217be05a9feb21e57b3409d9166a028701981cfc280aaab3c4436.json b/backend/.sqlx/query-f44f1a794ab217be05a9feb21e57b3409d9166a028701981cfc280aaab3c4436.json deleted file mode 100644 index 79ef441a8a..0000000000 --- a/backend/.sqlx/query-f44f1a794ab217be05a9feb21e57b3409d9166a028701981cfc280aaab3c4436.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT tag, dedicated_worker, on_behalf_of_email, edited_by from flow WHERE path = $1 and workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tag", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "dedicated_worker", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "on_behalf_of_email", - "type_info": "Text" - }, - { - "ordinal": 3, - "name": "edited_by", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - true, - true, - true, - false - ] - }, - "hash": "f44f1a794ab217be05a9feb21e57b3409d9166a028701981cfc280aaab3c4436" -} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 247f2cc5ba..97203b8559 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -109,23 +109,23 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", "once_cell", "version_check", ] [[package]] name = "ahash" -version = "0.8.11" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", "const-random", - "getrandom 0.2.15", + "getrandom 0.3.3", "once_cell", "version_check", - "zerocopy 0.7.35", + "zerocopy", ] [[package]] @@ -229,6 +229,15 @@ version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arc-swap" version = "1.7.1" @@ -279,9 +288,9 @@ dependencies = [ [[package]] name = "arrow" -version = "52.2.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05048a8932648b63f21c37d88b552ccc8a65afb6dfe9fc9f30ce79174c2e7a85" +checksum = "3095aaf545942ff5abd46654534f15b03a90fba78299d661e045e5d587222f0d" dependencies = [ "arrow-arith", "arrow-array", @@ -300,41 +309,40 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "52.2.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d8a57966e43bfe9a3277984a14c24ec617ad874e4c0e1d2a1b083a39cfbf22c" +checksum = "00752064ff47cee746e816ddb8450520c3a52cbad1e256f6fa861a35f86c45e7" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "half", "num", ] [[package]] name = "arrow-array" -version = "52.2.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f4a9468c882dc66862cef4e1fd8423d47e67972377d85d80e022786427768c" +checksum = "cebfe926794fbc1f49ddd0cdaf898956ca9f6e79541efce62dabccfd81380472" dependencies = [ - "ahash 0.8.11", + "ahash 0.8.12", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "chrono-tz 0.9.0", + "chrono-tz", "half", - "hashbrown 0.14.5", + "hashbrown 0.15.3", "num", ] [[package]] name = "arrow-buffer" -version = "52.2.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c975484888fc95ec4a632cdc98be39c085b1bb518531b0c80c5d462063e5daa1" +checksum = "0303c7ec4cf1a2c60310fc4d6bbc3350cd051a17bf9e9c0a8e47b4db79277824" dependencies = [ "bytes", "half", @@ -343,9 +351,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "52.2.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da26719e76b81d8bc3faad1d4dbdc1bcc10d14704e63dc17fc9f3e7e1e567c8e" +checksum = "335f769c5a218ea823d3760a743feba1ef7857cba114c01399a891c2fff34285" dependencies = [ "arrow-array", "arrow-buffer", @@ -364,28 +372,25 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "52.2.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c13c36dc5ddf8c128df19bab27898eea64bf9da2b555ec1cd17a8ff57fba9ec2" +checksum = "510db7dfbb4d5761826516cc611d97b3a68835d0ece95b034a052601109c0b1b" dependencies = [ "arrow-array", - "arrow-buffer", "arrow-cast", - "arrow-data", "arrow-schema", "chrono", "csv", "csv-core", "lazy_static", - "lexical-core", "regex", ] [[package]] name = "arrow-data" -version = "52.2.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd9d6f18c65ef7a2573ab498c374d8ae364b4a4edf67105357491c031f716ca5" +checksum = "e8affacf3351a24039ea24adab06f316ded523b6f8c3dbe28fbac5f18743451b" dependencies = [ "arrow-buffer", "arrow-schema", @@ -395,13 +400,12 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "52.2.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e786e1cdd952205d9a8afc69397b317cfbb6e0095e445c69cda7e8da5c1eeb0f" +checksum = "69880a9e6934d9cba2b8630dd08a3463a91db8693b16b499d54026b6137af284" dependencies = [ "arrow-array", "arrow-buffer", - "arrow-cast", "arrow-data", "arrow-schema", "flatbuffers", @@ -410,9 +414,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "52.2.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb22284c5a2a01d73cebfd88a33511a3234ab45d66086b2ca2d1228c3498e445" +checksum = "d8dafd17a05449e31e0114d740530e0ada7379d7cb9c338fd65b09a8130960b0" dependencies = [ "arrow-array", "arrow-buffer", @@ -423,33 +427,32 @@ dependencies = [ "half", "indexmap 2.9.0", "lexical-core", + "memchr", "num", "serde", "serde_json", + "simdutf8", ] [[package]] name = "arrow-ord" -version = "52.2.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42745f86b1ab99ef96d1c0bcf49180848a64fe2c7a7a0d945bc64fa2b21ba9bc" +checksum = "895644523af4e17502d42c3cb6b27cb820f0cb77954c22d75c23a85247c849e1" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", - "half", - "num", ] [[package]] name = "arrow-row" -version = "52.2.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd09a518c602a55bd406bcc291a967b284cfa7a63edfbf8b897ea4748aad23c" +checksum = "9be8a2a4e5e7d9c822b2b8095ecd77010576d824f654d347817640acfc97d229" dependencies = [ - "ahash 0.8.11", "arrow-array", "arrow-buffer", "arrow-data", @@ -459,17 +462,17 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "52.2.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e972cd1ff4a4ccd22f86d3e53e835c2ed92e0eea6a3e8eadb72b4f1ac802cf8" +checksum = "7450c76ab7c5a6805be3440dc2e2096010da58f7cab301fdc996a4ee3ee74e49" [[package]] name = "arrow-select" -version = "52.2.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "600bae05d43483d216fb3494f8c32fdbefd8aa4e1de237e790dbb3d9f44690a3" +checksum = "aa5f5a93c75f46ef48e4001535e7b6c922eeb0aa20b73cf58d09e13d057490d8" dependencies = [ - "ahash 0.8.11", + "ahash 0.8.12", "arrow-array", "arrow-buffer", "arrow-data", @@ -479,9 +482,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "52.2.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dc1985b67cb45f6606a248ac2b4a288849f196bab8c657ea5589f47cdd55e6" +checksum = "6e7005d858d84b56428ba2a98a107fe88c0132c61793cf6b8232a1f9bfc0452b" dependencies = [ "arrow-array", "arrow-buffer", @@ -551,7 +554,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -572,7 +575,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" dependencies = [ "brotli 7.0.0", - "bzip2 0.5.2", + "bzip2", "flate2", "futures-core", "futures-io", @@ -634,7 +637,7 @@ dependencies = [ "serde", "serde-aux", "serde_json", - "sha2 0.10.8", + "sha2 0.10.9", "thiserror 1.0.69", "url", ] @@ -653,7 +656,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -675,7 +678,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -686,7 +689,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -741,9 +744,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.6.1" +version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c39646d1a6b51240a1a23bb57ea4eebede7e16fbc237fdc876980233dcecb4f" +checksum = "b6fcc63c9860579e4cb396239570e979376e70aab79e496621748a09913f8b36" dependencies = [ "aws-credential-types", "aws-runtime", @@ -771,9 +774,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4471bef4c22a06d2c7a1b6492493d3fdf24a805323109d6874f9c94d5906ac14" +checksum = "687bc16bc431a8533fe0097c7f0182874767f920989d7260950172ae8e3c4465" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -783,9 +786,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b756939cb2f8dc900aa6dcd505e6e2428e9cae7ff7b028c49e3946efa70878" +checksum = "93fcc8f365936c834db5514fc45aee5b1202d677e6b40e48468aaaa8183ca8c7" dependencies = [ "aws-lc-sys", "zeroize", @@ -793,9 +796,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ddeb19ee86cb16ecfc871e5b0660aff6285760957aaedda6284cf0e790d3769" +checksum = "61b1d86e7705efe1be1b569bab41d4fa1e14e220b60a160f78de2db687add079" dependencies = [ "bindgen 0.69.5", "cc", @@ -806,9 +809,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.6" +version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aff45ffe35196e593ea3b9dd65b320e51e2dda95aff4390bc459e461d09c6ad" +checksum = "6c4063282c69991e57faab9e5cb21ae557e59f5b0fb285c196335243df8dc25c" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -822,7 +825,6 @@ dependencies = [ "fastrand", "http 0.2.12", "http-body 0.4.6", - "once_cell", "percent-encoding", "pin-project-lite", "tracing", @@ -831,9 +833,9 @@ dependencies = [ [[package]] name = "aws-sdk-sqs" -version = "1.64.0" +version = "1.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514d007ac4d5b156b408d8dd623a57b37ae77425810e0fedcffab57b0dabaded" +checksum = "c6f15bedfb1c4385fccc474f0fe46dffb0335d0b3d6b4413df06fb30d90caba8" dependencies = [ "aws-credential-types", "aws-runtime", @@ -854,9 +856,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.64.0" +version = "1.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d4bdb0e5f80f0689e61c77ab678b2b9304af329616af38aef5b6b967b8e736" +checksum = "0d4863da26489d1e6da91d7e12b10c17e86c14f94c53f416bd10e0a9c34057ba" dependencies = [ "aws-credential-types", "aws-runtime", @@ -877,9 +879,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.65.0" +version = "1.68.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbbb3ce8da257aedbccdcb1aadafbbb6a5fe9adf445db0e1ea897bdc7e22d08" +checksum = "95caa3998d7237789b57b95a8e031f60537adab21fa84c91e35bef9455c652e4" dependencies = [ "aws-credential-types", "aws-runtime", @@ -900,9 +902,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.65.0" +version = "1.68.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96a78a8f50a1630db757b60f679c8226a8a70ee2ab5f5e6e51dc67f6c61c7cfd" +checksum = "4939f6f449a37308a78c5a910fd91265479bd2bb11d186f0b8fc114d89ec828d" dependencies = [ "aws-credential-types", "aws-runtime", @@ -924,9 +926,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d03c3c05ff80d54ff860fe38c726f6f494c639ae975203a101335f223386db" +checksum = "3503af839bd8751d0bdc5a46b9cac93a003a353e635b0c12cf2376b5b53e41ea" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -938,9 +940,8 @@ dependencies = [ "hmac", "http 0.2.12", "http 1.3.1", - "once_cell", "percent-encoding", - "sha2 0.10.8", + "sha2 0.10.9", "time", "tracing", ] @@ -958,9 +959,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.62.0" +version = "0.62.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5949124d11e538ca21142d1fba61ab0a2a2c1bc3ed323cdb3e4b878bfb83166" +checksum = "99335bec6cdc50a346fda1437f9fefe33abf8c99060739a546a16457f2862ca9" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -970,7 +971,6 @@ dependencies = [ "http 0.2.12", "http 1.3.1", "http-body 0.4.6", - "once_cell", "percent-encoding", "pin-project-lite", "pin-utils", @@ -979,14 +979,14 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aff1159006441d02e57204bf57a1b890ba68bedb6904ffd2873c1c4c11c546b" +checksum = "7e44697a9bded898dcd0b1cb997430d949b87f4f8940d91023ae9062bf218250" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "h2 0.4.9", + "h2 0.4.10", "http 0.2.12", "http 1.3.1", "http-body 0.4.6", @@ -997,7 +997,7 @@ dependencies = [ "hyper-util", "pin-project-lite", "rustls 0.21.12", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -1016,12 +1016,11 @@ dependencies = [ [[package]] name = "aws-smithy-observability" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445d065e76bc1ef54963db400319f1dd3ebb3e0a74af20f7f7630625b0cc7cc0" +checksum = "9364d5989ac4dd918e5cc4c4bdcc61c9be17dcd2586ea7f69e348fc7c6cab393" dependencies = [ "aws-smithy-runtime-api", - "once_cell", ] [[package]] @@ -1036,9 +1035,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.8.1" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0152749e17ce4d1b47c7747bdfec09dac1ccafdcbc741ebf9daa2a373356730f" +checksum = "14302f06d1d5b7d333fd819943075b13d27c7700b414f574c3c35859bfb55d5e" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -1052,7 +1051,6 @@ dependencies = [ "http 1.3.1", "http-body 0.4.6", "http-body 1.0.1", - "once_cell", "pin-project-lite", "pin-utils", "tokio", @@ -1061,9 +1059,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.7.4" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da37cf5d57011cb1753456518ec76e31691f1f474b73934a284eb2a1c76510f" +checksum = "a1e5d9e3a80a18afa109391fb5ad09c3daf887b516c6fd805a157c6ea7994a57" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -1078,9 +1076,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836155caafba616c0ff9b07944324785de2ab016141c3550bd1c07882f8cee8f" +checksum = "40076bd09fadbc12d5e026ae080d0930defa606856186e31d83ccc6a255eeaf3" dependencies = [ "base64-simd 0.8.0", "bytes", @@ -1113,9 +1111,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.6" +version = "1.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3873f8deed8927ce8d04487630dc9ff73193bab64742a61d050e57a68dec4125" +checksum = "8a322fec39e4df22777ed3ad8ea868ac2f94cd15e1a55f6ee8d8d6305057689a" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -1200,9 +1198,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.74" +version = "0.3.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" dependencies = [ "addr2line", "cfg-if", @@ -1318,7 +1316,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.100", + "syn 2.0.101", "which 4.4.2", ] @@ -1339,7 +1337,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1419,9 +1417,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389a099b34312839e16420d499a9cad9650541715937ffbdd40d36f49e77eeb3" +checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" dependencies = [ "arrayref", "arrayvec", @@ -1544,7 +1542,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1554,7 +1552,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17d4f95e880cfd28c4ca5a006cf7f6af52b4bcb7b5866f573b2faa126fb7affb" dependencies = [ "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1652,9 +1650,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.22.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" +checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" dependencies = [ "bytemuck_derive", ] @@ -1667,7 +1665,7 @@ checksum = "7ecc273b49b3205b83d648f0690daa588925572cc5063745bfe547fe7ec8e1a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1701,16 +1699,6 @@ version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659" -[[package]] -name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - [[package]] name = "bzip2" version = "0.5.2" @@ -1738,30 +1726,31 @@ checksum = "1bf2a5fb3207c12b5d208ebc145f967fea5cac41a021c37417ccc31ba40f39ee" [[package]] name = "candle-core" -version = "0.3.3" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db8659ea87ee8197d2fc627348916cce0561330ee7ae3874e771691d3cecb2f" +checksum = "a9f51e2ecf6efe9737af8f993433c839f956d2b6ed4fd2dd4a7c6d8b0fa667ff" dependencies = [ "byteorder", - "gemm", + "gemm 0.17.1", "half", "memmap2 0.9.5", "num-traits", "num_cpus", - "rand 0.8.5", - "rand_distr", + "rand 0.9.0", + "rand_distr 0.5.1", "rayon", "safetensors", "thiserror 1.0.69", - "yoke", + "ug", + "yoke 0.7.5", "zip", ] [[package]] name = "candle-nn" -version = "0.3.3" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ddce8312032760a6791d6adc9c56dc54fd7c1be38d85dcc4862f1c75228bbc7" +checksum = "c1980d53280c8f9e2c6cbe1785855d7ff8010208b46e21252b978badf13ad69d" dependencies = [ "candle-core", "half", @@ -1774,21 +1763,21 @@ dependencies = [ [[package]] name = "candle-transformers" -version = "0.3.3" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68834a0cacb7e002d1f4abfe26a7cd1237e2ba342fddcf2e30913c4edb96409d" +checksum = "186cb80045dbe47e0b387ea6d3e906f02fb3056297080d9922984c90e90a72b0" dependencies = [ "byteorder", "candle-core", "candle-nn", + "fancy-regex 0.13.0", "num-traits", - "rand 0.8.5", + "rand 0.9.0", "rayon", "serde", "serde_json", "serde_plain", "tracing", - "wav", ] [[package]] @@ -1819,7 +1808,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b4a6cae9efc04cc6cbb8faf338d2c497c165c83e74509cf4dbedea948bbf6e5" dependencies = [ "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1833,9 +1822,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.19" +version = "1.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" +checksum = "32db95edf998450acc7881c932f94cd9b05c87b4b2599e8bab064753da4acfd1" dependencies = [ "jobserver", "libc", @@ -1877,9 +1866,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.39" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1888,7 +1877,7 @@ dependencies = [ "pure-rust-locales", "serde", "wasm-bindgen", - "windows-targets 0.52.6", + "windows-link", ] [[package]] @@ -1900,17 +1889,6 @@ dependencies = [ "chrono", ] -[[package]] -name = "chrono-tz" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" -dependencies = [ - "chrono", - "chrono-tz-build 0.3.0", - "phf", -] - [[package]] name = "chrono-tz" version = "0.10.3" @@ -1918,21 +1896,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efdce149c370f133a071ca8ef6ea340b7b88748ab0810097a9e2976eaa34b4f3" dependencies = [ "chrono", - "chrono-tz-build 0.4.1", + "chrono-tz-build", "phf", ] -[[package]] -name = "chrono-tz-build" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" -dependencies = [ - "parse-zoneinfo", - "phf", - "phf_codegen", -] - [[package]] name = "chrono-tz-build" version = "0.4.1" @@ -1970,14 +1937,14 @@ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", - "libloading 0.8.6", + "libloading 0.8.7", ] [[package]] name = "clap" -version = "4.5.37" +version = "4.5.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eccb054f56cbd38340b380d4a8e69ef1f02f1af43db2f0cc817a4774d80ae071" +checksum = "ed93b9805f8ba930df42c2590f05453d5ec36cbb85d018868a5b24d31f6ac000" dependencies = [ "clap_builder", "clap_derive", @@ -1985,9 +1952,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.37" +version = "4.5.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd9466fac8543255d3b1fcad4762c5e116ffe808c8a3043d4263cd4fd4862a2" +checksum = "379026ff283facf611b0ea629334361c4211d1b12ee01024eec1591133b04120" dependencies = [ "anstream", "anstyle", @@ -2004,7 +1971,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2059,7 +2026,7 @@ dependencies = [ "nom 7.1.3", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2133,7 +2100,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", "once_cell", "tiny-keccak", ] @@ -2255,9 +2222,9 @@ dependencies = [ [[package]] name = "crc" -version = "3.2.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" dependencies = [ "crc-catalog", ] @@ -2306,19 +2273,6 @@ dependencies = [ "chrono", ] -[[package]] -name = "crossbeam" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" -dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-epoch", - "crossbeam-queue", - "crossbeam-utils", -] - [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -2454,7 +2408,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2464,7 +2418,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b28bfe653d79bd16c77f659305b195b82bb5ce0c0eb2a4846b82ddbd77586813" dependencies = [ "bitflags 2.9.0", - "libloading 0.8.6", + "libloading 0.8.7", "winapi", ] @@ -2537,7 +2491,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2570,7 +2524,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2586,6 +2540,20 @@ dependencies = [ "parking_lot_core 0.9.10", ] +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core 0.9.10", +] + [[package]] name = "data-encoding" version = "2.9.0" @@ -2600,52 +2568,53 @@ checksum = "5c297a1c74b71ae29df00c3e22dd9534821d60eb9af5a0192823fa2acea70c2a" [[package]] name = "datafusion" -version = "39.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f92d2d7a9cba4580900b32b009848d9eb35f1028ac84cdd6ddcf97612cd0068" +checksum = "ffe060b978f74ab446be722adb8a274e052e005bf6dfd171caadc3abaad10080" dependencies = [ - "ahash 0.8.11", "arrow", - "arrow-array", "arrow-ipc", "arrow-schema", - "async-compression", "async-trait", "bytes", - "bzip2 0.4.4", + "bzip2", "chrono", - "dashmap", + "datafusion-catalog", + "datafusion-catalog-listing", "datafusion-common", "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", "datafusion-execution", "datafusion-expr", + "datafusion-expr-common", "datafusion-functions", "datafusion-functions-aggregate", - "datafusion-functions-array", + "datafusion-functions-nested", + "datafusion-functions-table", + "datafusion-functions-window", + "datafusion-macros", "datafusion-optimizer", "datafusion-physical-expr", "datafusion-physical-expr-common", + "datafusion-physical-optimizer", "datafusion-physical-plan", + "datafusion-session", "datafusion-sql", "flate2", "futures", - "glob", - "half", - "hashbrown 0.14.5", - "indexmap 2.9.0", - "itertools 0.12.1", + "itertools 0.14.0", "log", - "num_cpus", "object_store", "parking_lot 0.12.3", "parquet", - "paste", - "pin-project-lite", "rand 0.8.5", + "regex", "sqlparser", "tempfile", "tokio", - "tokio-util", "url", "uuid", "xz2", @@ -2653,49 +2622,223 @@ dependencies = [ ] [[package]] -name = "datafusion-common" -version = "39.0.0" +name = "datafusion-catalog" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "effed030d2c1667eb1e11df5372d4981eaf5d11a521be32220b3985ae5ba6971" +checksum = "61fe34f401bd03724a1f96d12108144f8cd495a3cdda2bf5e091822fb80b7e66" dependencies = [ - "ahash 0.8.11", "arrow", - "arrow-array", - "arrow-buffer", - "arrow-schema", - "chrono", - "half", - "hashbrown 0.14.5", - "instant", - "libc", - "num_cpus", + "async-trait", + "dashmap 6.1.0", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-session", + "datafusion-sql", + "futures", + "itertools 0.14.0", + "log", "object_store", - "parquet", - "sqlparser", -] - -[[package]] -name = "datafusion-common-runtime" -version = "39.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0091318129dad1359f08e4c6c71f855163c35bba05d1dbf983196f727857894" -dependencies = [ + "parking_lot 0.12.3", "tokio", ] [[package]] -name = "datafusion-execution" -version = "39.0.0" +name = "datafusion-catalog-listing" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8385aba84fc4a06d3ebccfbcbf9b4f985e80c762fac634b49079f7cc14933fb1" +checksum = "a4411b8e3bce5e0fc7521e44f201def2e2d5d1b5f176fb56e8cdc9942c890f00" dependencies = [ "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "log", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-common" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0734015d81c8375eb5d4869b7f7ecccc2ee8d6cb81948ef737cd0e7b743bd69c" +dependencies = [ + "ahash 0.8.12", + "arrow", + "arrow-ipc", + "base64 0.22.1", + "half", + "hashbrown 0.14.5", + "indexmap 2.9.0", + "libc", + "log", + "object_store", + "parquet", + "paste", + "recursive", + "sqlparser", + "tokio", + "web-time", +] + +[[package]] +name = "datafusion-common-runtime" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5167bb1d2ccbb87c6bc36c295274d7a0519b14afcfdaf401d53cbcaa4ef4968b" +dependencies = [ + "futures", + "log", + "tokio", +] + +[[package]] +name = "datafusion-datasource" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e602dcdf2f50c2abf297cc2203c73531e6f48b29516af7695d338cf2a778b1" +dependencies = [ + "arrow", + "async-compression", + "async-trait", + "bytes", + "bzip2", "chrono", - "dashmap", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "flate2", + "futures", + "glob", + "itertools 0.14.0", + "log", + "object_store", + "parquet", + "rand 0.8.5", + "tempfile", + "tokio", + "tokio-util", + "url", + "xz2", + "zstd", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bb2253952dc32296ed5b84077cb2e0257fea4be6373e1c376426e17ead4ef6" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-json" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8c7f47a5d2fe03bfa521ec9bafdb8a5c82de8377f60967c3663f00c8790352" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "serde_json", + "tokio", +] + +[[package]] +name = "datafusion-datasource-parquet" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27d15868ea39ed2dc266728b554f6304acd473de2142281ecfa1294bb7415923" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot 0.12.3", + "parquet", + "rand 0.8.5", + "tokio", +] + +[[package]] +name = "datafusion-doc" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a91f8c2c5788ef32f48ff56c68e5b545527b744822a284373ac79bba1ba47292" + +[[package]] +name = "datafusion-execution" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06f004d100f49a3658c9da6fb0c3a9b760062d96cd4ad82ccc3b7b69a9fb2f84" +dependencies = [ + "arrow", + "dashmap 6.1.0", "datafusion-common", "datafusion-expr", "futures", - "hashbrown 0.14.5", "log", "object_store", "parking_lot 0.12.3", @@ -2706,160 +2849,258 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "39.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebb192f0055d2ce64e38ac100abc18e4e6ae9734d3c28eee522bbbd6a32108a3" +checksum = "7a4e4ce3802609be38eeb607ee72f6fe86c3091460de9dbfae9e18db423b3964" dependencies = [ - "ahash 0.8.11", "arrow", - "arrow-array", - "arrow-buffer", "chrono", "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", + "indexmap 2.9.0", "paste", + "recursive", "serde_json", "sqlparser", - "strum 0.26.3", - "strum_macros 0.26.4", +] + +[[package]] +name = "datafusion-expr-common" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "422ac9cf3b22bbbae8cdf8ceb33039107fde1b5492693168f13bd566b1bcc839" +dependencies = [ + "arrow", + "datafusion-common", + "indexmap 2.9.0", + "itertools 0.14.0", + "paste", ] [[package]] name = "datafusion-functions" -version = "39.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c081ae5b7edd712b92767fb8ed5c0e32755682f8075707666cd70835807c0b" +checksum = "2ddf0a0a2db5d2918349c978d42d80926c6aa2459cd8a3c533a84ec4bb63479e" dependencies = [ "arrow", + "arrow-buffer", "base64 0.22.1", "blake2", "blake3", "chrono", "datafusion-common", + "datafusion-doc", "datafusion-execution", "datafusion-expr", - "datafusion-physical-expr", - "hashbrown 0.14.5", + "datafusion-expr-common", + "datafusion-macros", "hex", - "itertools 0.12.1", + "itertools 0.14.0", "log", "md-5 0.10.6", "rand 0.8.5", "regex", - "sha2 0.10.8", + "sha2 0.10.9", "unicode-segmentation", "uuid", ] [[package]] name = "datafusion-functions-aggregate" -version = "39.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "feb28a4ea52c28a26990646986a27c4052829a2a2572386258679e19263f8b78" +checksum = "408a05dafdc70d05a38a29005b8b15e21b0238734dab1e98483fcb58038c5aba" dependencies = [ - "ahash 0.8.11", + "ahash 0.8.12", "arrow", - "arrow-schema", "datafusion-common", + "datafusion-doc", "datafusion-execution", "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", "datafusion-physical-expr-common", + "half", "log", "paste", - "sqlparser", ] [[package]] -name = "datafusion-functions-array" -version = "39.0.0" +name = "datafusion-functions-aggregate-common" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b17c02a74cdc87380a56758ec27e7d417356bf806f33062700908929aedb8a" +checksum = "756d21da2dd6c9bef97af1504970ff56cbf35d03fbd4ffd62827f02f4d2279d4" +dependencies = [ + "ahash 0.8.12", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-functions-nested" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d8d50f6334b378930d992d801a10ac5b3e93b846b39e4a05085742572844537" dependencies = [ "arrow", - "arrow-array", - "arrow-buffer", "arrow-ord", - "arrow-schema", "datafusion-common", + "datafusion-doc", "datafusion-execution", "datafusion-expr", "datafusion-functions", - "itertools 0.12.1", + "datafusion-functions-aggregate", + "datafusion-macros", + "datafusion-physical-expr-common", + "itertools 0.14.0", "log", "paste", ] +[[package]] +name = "datafusion-functions-table" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9a97220736c8fff1446e936be90d57216c06f28969f9ffd3b72ac93c958c8a" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot 0.12.3", + "paste", +] + +[[package]] +name = "datafusion-functions-window" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cefc2d77646e1aadd1d6a9c40088937aedec04e68c5f0465939912e1291f8193" +dependencies = [ + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd4aff082c42fa6da99ce0698c85addd5252928c908eb087ca3cfa64ff16b313" +dependencies = [ + "datafusion-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-macros" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df6f88d7ee27daf8b108ba910f9015176b36fbc72902b1ca5c2a5f1d1717e1a1" +dependencies = [ + "datafusion-expr", + "quote", + "syn 2.0.101", +] + [[package]] name = "datafusion-optimizer" -version = "39.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12172f2a6c9eb4992a51e62d709eeba5dedaa3b5369cce37ff6c2260e100ba76" +checksum = "084d9f979c4b155346d3c34b18f4256e6904ded508e9554d90fed416415c3515" dependencies = [ "arrow", - "async-trait", "chrono", "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "hashbrown 0.14.5", "indexmap 2.9.0", - "itertools 0.12.1", + "itertools 0.14.0", "log", + "recursive", + "regex", "regex-syntax 0.8.5", ] [[package]] name = "datafusion-physical-expr" -version = "39.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a3fce531b623e94180f6cd33d620ef01530405751b6ddd2fd96250cdbd78e2e" +checksum = "64c536062b0076f4e30084065d805f389f9fe38af0ca75bcbac86bc5e9fbab65" dependencies = [ - "ahash 0.8.11", + "ahash 0.8.12", "arrow", - "arrow-array", - "arrow-buffer", - "arrow-ord", - "arrow-schema", - "arrow-string", - "base64 0.22.1", - "chrono", "datafusion-common", - "datafusion-execution", "datafusion-expr", - "datafusion-functions-aggregate", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", "hashbrown 0.14.5", - "hex", "indexmap 2.9.0", - "itertools 0.12.1", + "itertools 0.14.0", "log", "paste", - "petgraph 0.6.5", - "regex", + "petgraph", ] [[package]] name = "datafusion-physical-expr-common" -version = "39.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046400b6a2cc3ed57a7c576f5ae6aecc77804ac8e0186926b278b189305b2a77" +checksum = "f8a92b53b3193fac1916a1c5b8e3f4347c526f6822e56b71faa5fb372327a863" +dependencies = [ + "ahash 0.8.12", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "hashbrown 0.14.5", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fa0a5ac94c7cf3da97bedabd69d6bbca12aef84b9b37e6e9e8c25286511b5e2" dependencies = [ "arrow", "datafusion-common", + "datafusion-execution", "datafusion-expr", - "rand 0.8.5", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "itertools 0.14.0", + "log", + "recursive", ] [[package]] name = "datafusion-physical-plan" -version = "39.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4aed47f5a2ad8766260befb375b201592e86a08b260256e168ae4311426a2bff" +checksum = "690c615db468c2e5fe5085b232d8b1c088299a6c63d87fd960a354a71f7acb55" dependencies = [ - "ahash 0.8.11", + "ahash 0.8.12", "arrow", - "arrow-array", - "arrow-buffer", "arrow-ord", "arrow-schema", "async-trait", @@ -2868,37 +3109,59 @@ dependencies = [ "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", - "datafusion-functions-aggregate", + "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", "futures", "half", "hashbrown 0.14.5", "indexmap 2.9.0", - "itertools 0.12.1", + "itertools 0.14.0", "log", - "once_cell", "parking_lot 0.12.3", "pin-project-lite", - "rand 0.8.5", + "tokio", +] + +[[package]] +name = "datafusion-session" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad229a134c7406c057ece00c8743c0c34b97f4e72f78b475fe17b66c5e14fa4f" +dependencies = [ + "arrow", + "async-trait", + "dashmap 6.1.0", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-sql", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot 0.12.3", "tokio", ] [[package]] name = "datafusion-sql" -version = "39.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fa92bb1fd15e46ce5fb6f1c85f3ac054592560f294429a28e392b5f9cd4255e" +checksum = "64f6ab28b72b664c21a27b22a2ff815fd390ed224c26e89a93b5a8154a4e8607" dependencies = [ "arrow", - "arrow-array", - "arrow-schema", + "bigdecimal", "datafusion-common", "datafusion-expr", + "indexmap 2.9.0", "log", + "recursive", "regex", "sqlparser", - "strum 0.26.3", ] [[package]] @@ -2931,7 +3194,7 @@ dependencies = [ "once_cell", "percent-encoding", "serde", - "sourcemap 9.1.2", + "sourcemap 9.2.0", "swc_atoms", "swc_common", "swc_config", @@ -2984,7 +3247,7 @@ dependencies = [ "deno_error", "rusqlite", "serde", - "sha2 0.10.8", + "sha2 0.10.9", "thiserror 2.0.12", "tokio", ] @@ -3012,7 +3275,7 @@ dependencies = [ "parking_lot 0.12.3", "serde", "serde_json", - "sha2 0.10.8", + "sha2 0.10.9", "sys_traits", "thiserror 1.0.69", "url", @@ -3160,7 +3423,7 @@ dependencies = [ "serde", "serde_bytes", "sha1", - "sha2 0.10.8", + "sha2 0.10.9", "signature", "spki", "thiserror 2.0.12", @@ -3191,7 +3454,7 @@ checksum = "babccedee31ce7e57c3e6dff2cb3ab8d68c49d0df8222fe0d11d628e65192790" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -3211,7 +3474,7 @@ dependencies = [ "deno_tls", "dyn-clone", "error_reporter", - "h2 0.4.9", + "h2 0.4.10", "hickory-resolver", "http 1.3.1", "http-body-util", @@ -3492,7 +3755,7 @@ dependencies = [ "elliptic-curve", "errno", "faster-hex", - "h2 0.4.9", + "h2 0.4.10", "hkdf", "http 1.3.1", "http-body-util", @@ -3529,7 +3792,7 @@ dependencies = [ "sec1", "serde", "sha1", - "sha2 0.10.8", + "sha2 0.10.9", "sha3", "signature", "simd-json", @@ -3541,12 +3804,12 @@ dependencies = [ "tokio", "tokio-eld", "url", - "webpki-root-certs", + "webpki-root-certs 0.26.11", "winapi", "windows-sys 0.59.0", "x25519-dalek", "x509-parser", - "yoke", + "yoke 0.7.5", ] [[package]] @@ -3580,9 +3843,9 @@ dependencies = [ "proc-macro2", "quote", "stringcase", - "strum 0.25.0", - "strum_macros 0.25.3", - "syn 2.0.100", + "strum", + "strum_macros", + "syn 2.0.101", "thiserror 2.0.12", ] @@ -3702,7 +3965,7 @@ dependencies = [ "async-trait", "base32", "boxed_error", - "dashmap", + "dashmap 5.5.3", "deno_cache_dir", "deno_config", "deno_error", @@ -3856,14 +4119,14 @@ dependencies = [ "deno_core", "deno_error", "deno_native_certs", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pemfile 2.2.0", "rustls-tokio-stream", "rustls-webpki 0.102.8", "serde", "thiserror 2.0.12", "tokio", - "webpki-roots", + "webpki-roots 0.26.11", ] [[package]] @@ -3948,7 +4211,7 @@ dependencies = [ "deno_permissions", "deno_tls", "fastwebsockets", - "h2 0.4.9", + "h2 0.4.10", "http 1.3.1", "http-body-util", "hyper 1.6.0", @@ -4084,7 +4347,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -4097,6 +4360,17 @@ dependencies = [ "serde", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "derive_builder" version = "0.12.0" @@ -4130,15 +4404,15 @@ dependencies = [ [[package]] name = "derive_more" -version = "0.99.19" +version = "0.99.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da29a38df43d6f156149c9b43ded5e018ddff2a855cf2cfd62e8cd7d079c69f" +checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" dependencies = [ "convert_case 0.4.0", "proc-macro2", "quote", "rustc_version 0.4.1", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -4243,7 +4517,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -4278,15 +4552,9 @@ checksum = "f2b99bf03862d7f545ebc28ddd33a665b50865f4dfd84031a393823879bd4c54" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] -[[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - [[package]] name = "document-features" version = "0.2.11" @@ -4340,7 +4608,7 @@ dependencies = [ "num-traits", "pkcs8", "rfc6979", - "sha2 0.10.8", + "sha2 0.10.9", "signature", "zeroize", ] @@ -4373,6 +4641,15 @@ dependencies = [ "reborrow", ] +[[package]] +name = "dyn-stack" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490bd48eb68fffcfed519b4edbfd82c69cbe741d175b84f0e0cbe8c57cbe0bdd" +dependencies = [ + "bytemuck", +] + [[package]] name = "dynasm" version = "1.2.3" @@ -4424,9 +4701,9 @@ dependencies = [ [[package]] name = "ecow" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef5eeffa816451f3a6a4cce9cd796e3e5ba6018638d3ce5cc7f87b73bababf60" +checksum = "b92b481eb5d59fd8e80e92ff11d057d1ca8d144b2cd8c66cc8d5bd177a3c0dc5" dependencies = [ "serde", ] @@ -4451,7 +4728,7 @@ dependencies = [ "ed25519", "rand_core 0.6.4", "serde", - "sha2 0.10.8", + "sha2 0.10.9", "signature", "subtle", "zeroize", @@ -4532,7 +4809,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -4552,7 +4829,7 @@ checksum = "fc4caf64a58d7a6d65ab00639b046ff54399a39f5f2554728895ace4b297cd79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -4583,9 +4860,9 @@ dependencies = [ [[package]] name = "error-code" -version = "3.3.1" +version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d9305ccc6942a704f4335694ecd3de2ea531b114ac2d51f5f843750787a92f" +checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" [[package]] name = "error_reporter" @@ -4648,6 +4925,17 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fancy-regex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" +dependencies = [ + "bit-set 0.5.3", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", +] + [[package]] name = "fancy-regex" version = "0.14.0" @@ -4707,7 +4995,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", - "rustix 1.0.5", + "rustix 1.0.7", "windows-sys 0.59.0", ] @@ -4754,12 +5042,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - [[package]] name = "fixedbitset" version = "0.5.7" @@ -4768,11 +5050,11 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flatbuffers" -version = "24.12.23" +version = "25.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" +checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "rustc_version 0.4.1", ] @@ -4783,6 +5065,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", + "libz-rs-sys", "libz-sys", "miniz_oxide 0.8.8", ] @@ -4847,7 +5130,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -4885,7 +5168,7 @@ checksum = "32016f1242eb82af5474752d00fd8ebcd9004bd69b462b1c91de833972d08ed4" dependencies = [ "proc-macro2", "swc_macros_common", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -5020,7 +5303,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -5084,17 +5367,37 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ab24cc62135b40090e31a76a9b2766a501979f3070fa27f689c27ec04377d32" dependencies = [ - "dyn-stack", - "gemm-c32", - "gemm-c64", - "gemm-common", - "gemm-f16", - "gemm-f32", - "gemm-f64", + "dyn-stack 0.10.0", + "gemm-c32 0.17.1", + "gemm-c64 0.17.1", + "gemm-common 0.17.1", + "gemm-f16 0.17.1", + "gemm-f32 0.17.1", + "gemm-f64 0.17.1", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 10.7.0", + "seq-macro", +] + +[[package]] +name = "gemm" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab96b703d31950f1aeddded248bc95543c9efc7ac9c4a21fda8703a83ee35451" +dependencies = [ + "dyn-stack 0.13.0", + "gemm-c32 0.18.2", + "gemm-c64 0.18.2", + "gemm-common 0.18.2", + "gemm-f16 0.18.2", + "gemm-f32 0.18.2", + "gemm-f64 0.18.2", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 11.5.0", "seq-macro", ] @@ -5104,12 +5407,27 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9c030d0b983d1e34a546b86e08f600c11696fde16199f971cd46c12e67512c0" dependencies = [ - "dyn-stack", - "gemm-common", + "dyn-stack 0.10.0", + "gemm-common 0.17.1", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 10.7.0", + "seq-macro", +] + +[[package]] +name = "gemm-c32" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6db9fd9f40421d00eea9dd0770045a5603b8d684654816637732463f4073847" +dependencies = [ + "dyn-stack 0.13.0", + "gemm-common 0.18.2", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 11.5.0", "seq-macro", ] @@ -5119,12 +5437,27 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbb5f2e79fefb9693d18e1066a557b4546cd334b226beadc68b11a8f9431852a" dependencies = [ - "dyn-stack", - "gemm-common", + "dyn-stack 0.10.0", + "gemm-common 0.17.1", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 10.7.0", + "seq-macro", +] + +[[package]] +name = "gemm-c64" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfcad8a3d35a43758330b635d02edad980c1e143dc2f21e6fd25f9e4eada8edf" +dependencies = [ + "dyn-stack 0.13.0", + "gemm-common 0.18.2", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 11.5.0", "seq-macro", ] @@ -5135,17 +5468,38 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8" dependencies = [ "bytemuck", - "dyn-stack", + "dyn-stack 0.10.0", "half", "num-complex", "num-traits", "once_cell", "paste", - "pulp", - "raw-cpuid", + "pulp 0.18.22", + "raw-cpuid 10.7.0", "rayon", "seq-macro", - "sysctl", + "sysctl 0.5.5", +] + +[[package]] +name = "gemm-common" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a352d4a69cbe938b9e2a9cb7a3a63b7e72f9349174a2752a558a8a563510d0f3" +dependencies = [ + "bytemuck", + "dyn-stack 0.13.0", + "half", + "libm", + "num-complex", + "num-traits", + "once_cell", + "paste", + "pulp 0.21.5", + "raw-cpuid 11.5.0", + "rayon", + "seq-macro", + "sysctl 0.6.0", ] [[package]] @@ -5154,14 +5508,32 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ca4c06b9b11952071d317604acb332e924e817bd891bec8dfb494168c7cedd4" dependencies = [ - "dyn-stack", - "gemm-common", - "gemm-f32", + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "gemm-f32 0.17.1", "half", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 10.7.0", + "rayon", + "seq-macro", +] + +[[package]] +name = "gemm-f16" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff95ae3259432f3c3410eaa919033cd03791d81cebd18018393dc147952e109" +dependencies = [ + "dyn-stack 0.13.0", + "gemm-common 0.18.2", + "gemm-f32 0.18.2", + "half", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 11.5.0", "rayon", "seq-macro", ] @@ -5172,12 +5544,27 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9a69f51aaefbd9cf12d18faf273d3e982d9d711f60775645ed5c8047b4ae113" dependencies = [ - "dyn-stack", - "gemm-common", + "dyn-stack 0.10.0", + "gemm-common 0.17.1", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 10.7.0", + "seq-macro", +] + +[[package]] +name = "gemm-f32" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc8d3d4385393304f407392f754cd2dc4b315d05063f62cf09f47b58de276864" +dependencies = [ + "dyn-stack 0.13.0", + "gemm-common 0.18.2", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 11.5.0", "seq-macro", ] @@ -5187,12 +5574,27 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa397a48544fadf0b81ec8741e5c0fba0043008113f71f2034def1935645d2b0" dependencies = [ - "dyn-stack", - "gemm-common", + "dyn-stack 0.10.0", + "gemm-common 0.17.1", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 10.7.0", + "seq-macro", +] + +[[package]] +name = "gemm-f64" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35b2a4f76ce4b8b16eadc11ccf2e083252d8237c1b589558a49b0183545015bd" +dependencies = [ + "dyn-stack 0.13.0", + "gemm-common 0.18.2", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 11.5.0", "seq-macro", ] @@ -5241,9 +5643,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "js-sys", @@ -5254,9 +5656,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", "js-sys", @@ -5299,7 +5701,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -5460,7 +5862,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5eb37859fda6792e95231aef1c5838f4043ec0ee352d8313421e311c606df612" dependencies = [ "anyhow", - "strum 0.25.0", + "strum", "thiserror 1.0.69", "unic-ucd-category", ] @@ -5492,7 +5894,7 @@ checksum = "dcf29e94d6d243368b7a56caa16bc213e4f9f8ed38c4d9557069527b5d5281ca" dependencies = [ "bitflags 2.9.0", "gpu-descriptor-types", - "hashbrown 0.15.2", + "hashbrown 0.15.3", ] [[package]] @@ -5545,9 +5947,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75249d144030531f8dee69fe9cea04d3edf809a017ae445e2abdff6629e86633" +checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5" dependencies = [ "atomic-waker", "bytes", @@ -5564,16 +5966,16 @@ dependencies = [ [[package]] name = "half" -version = "2.4.1" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" dependencies = [ "bytemuck", "cfg-if", "crunchy", "num-traits", - "rand 0.8.5", - "rand_distr", + "rand 0.9.0", + "rand_distr 0.5.1", ] [[package]] @@ -5601,15 +6003,15 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "ahash 0.8.11", + "ahash 0.8.12", "allocator-api2", ] [[package]] name = "hashbrown" -version = "0.15.2" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3" dependencies = [ "allocator-api2", "equivalent", @@ -5624,7 +6026,7 @@ checksum = "f208758247e68e239acaa059e72e4ce1f30f2a4b6523f19c1b923d25b7e9cceb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -5642,7 +6044,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" dependencies = [ - "hashbrown 0.15.2", + "hashbrown 0.15.3", ] [[package]] @@ -5792,17 +6194,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "hostname" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65" -dependencies = [ - "cfg-if", - "libc", - "windows-link", -] - [[package]] name = "hstr" version = "0.2.17" @@ -5930,7 +6321,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.9", + "h2 0.4.10", "http 1.3.1", "http-body 1.0.1", "httparse", @@ -5983,13 +6374,13 @@ dependencies = [ "http 1.3.1", "hyper 1.6.0", "hyper-util", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", "tokio-rustls 0.26.2", "tower-service", - "webpki-roots", + "webpki-roots 0.26.11", ] [[package]] @@ -6095,21 +6486,22 @@ dependencies = [ [[package]] name = "icu_collections" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" dependencies = [ "displaydoc", - "yoke", + "potential_utf", + "yoke 0.8.0", "zerofrom", "zerovec", ] [[package]] -name = "icu_locid" -version = "1.5.0" +name = "icu_locale_core" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" dependencies = [ "displaydoc", "litemap", @@ -6118,31 +6510,11 @@ dependencies = [ "zerovec", ] -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" - [[package]] name = "icu_normalizer" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" dependencies = [ "displaydoc", "icu_collections", @@ -6150,67 +6522,54 @@ dependencies = [ "icu_properties", "icu_provider", "smallvec", - "utf16_iter", - "utf8_iter", - "write16", "zerovec", ] [[package]] name = "icu_normalizer_data" -version = "1.5.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" -version = "1.5.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +checksum = "2549ca8c7241c82f59c80ba2a6f415d931c5b58d24fb8412caa1a1f02c49139a" dependencies = [ "displaydoc", "icu_collections", - "icu_locid_transform", + "icu_locale_core", "icu_properties_data", "icu_provider", - "tinystr", + "potential_utf", + "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "1.5.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" +checksum = "8197e866e47b68f8f7d95249e172903bec06004b18b2937f1095d40a0c57de04" [[package]] name = "icu_provider" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" dependencies = [ "displaydoc", - "icu_locid", - "icu_provider_macros", + "icu_locale_core", "stable_deref_trait", "tinystr", "writeable", - "yoke", + "yoke 0.8.0", "zerofrom", + "zerotrie", "zerovec", ] -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - [[package]] name = "ident_case" version = "1.0.1" @@ -6230,9 +6589,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" dependencies = [ "icu_normalizer", "icu_properties", @@ -6308,7 +6667,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", - "hashbrown 0.15.2", + "hashbrown 0.15.3", "serde", ] @@ -6418,7 +6777,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -6496,7 +6855,7 @@ version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ - "getrandom 0.3.2", + "getrandom 0.3.3", "libc", ] @@ -6568,7 +6927,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "once_cell", - "sha2 0.10.8", + "sha2 0.10.9", "signature", ] @@ -6597,7 +6956,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76" dependencies = [ "libc", - "libloading 0.8.6", + "libloading 0.8.7", "pkg-config", ] @@ -6624,9 +6983,9 @@ checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" [[package]] name = "kqueue" -version = "1.0.8" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c" +checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" dependencies = [ "kqueue-sys", "libc", @@ -6668,7 +7027,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -6694,9 +7053,9 @@ checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" [[package]] name = "lexical-core" -version = "0.8.5" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +checksum = "b765c31809609075565a70b4b71402281283aeda7ecaf4818ac14a7b2ade8958" dependencies = [ "lexical-parse-float", "lexical-parse-integer", @@ -6707,9 +7066,9 @@ dependencies = [ [[package]] name = "lexical-parse-float" -version = "0.8.5" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" dependencies = [ "lexical-parse-integer", "lexical-util", @@ -6718,9 +7077,9 @@ dependencies = [ [[package]] name = "lexical-parse-integer" -version = "0.8.6" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" dependencies = [ "lexical-util", "static_assertions", @@ -6728,18 +7087,18 @@ dependencies = [ [[package]] name = "lexical-util" -version = "0.8.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" dependencies = [ "static_assertions", ] [[package]] name = "lexical-write-float" -version = "0.8.5" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +checksum = "c5afc668a27f460fb45a81a757b6bf2f43c2d7e30cb5a2dcd3abf294c78d62bd" dependencies = [ "lexical-util", "lexical-write-integer", @@ -6748,9 +7107,9 @@ dependencies = [ [[package]] name = "lexical-write-integer" -version = "0.8.5" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +checksum = "629ddff1a914a836fb245616a7888b62903aae58fa771e1d83943035efa0f978" dependencies = [ "lexical-util", "static_assertions", @@ -6793,19 +7152,19 @@ dependencies = [ [[package]] name = "libloading" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +checksum = "6a793df0d7afeac54f95b471d3af7f0d4fb975699f972341a4b76988d49cdf0c" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.53.0", ] [[package]] name = "libm" -version = "0.2.11" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "libproc" @@ -6826,7 +7185,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.9.0", "libc", - "redox_syscall 0.5.11", + "redox_syscall 0.5.12", ] [[package]] @@ -6851,6 +7210,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +dependencies = [ + "zlib-rs", +] + [[package]] name = "libz-sys" version = "1.1.22" @@ -6883,9 +7251,9 @@ checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" [[package]] name = "litemap" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "litrs" @@ -6938,9 +7306,24 @@ version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.15.2", + "hashbrown 0.15.3", ] +[[package]] +name = "lru" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f8cc7106155f10bdf99a6f379688f543ad6596a415375b36a59a054ceda1198" +dependencies = [ + "hashbrown 0.15.3", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "lscolors" version = "0.17.0" @@ -7041,12 +7424,12 @@ dependencies = [ "base64 0.22.1", "gethostname", "mail-builder", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pki-types", "smtp-proto", "tokio", "tokio-rustls 0.26.2", - "webpki-roots", + "webpki-roots 0.26.11", ] [[package]] @@ -7240,9 +7623,9 @@ dependencies = [ [[package]] name = "miette" -version = "7.5.0" +version = "7.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a955165f87b37fd1862df2a59547ac542c77ef6d17c666f619d1ad22dd89484" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" dependencies = [ "cfg-if", "miette-derive", @@ -7252,19 +7635,18 @@ dependencies = [ "supports-unicode", "terminal_size", "textwrap", - "thiserror 1.0.69", "unicode-width 0.1.14", ] [[package]] name = "miette-derive" -version = "7.5.0" +version = "7.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf45bf44ab49be92fd1227a3be6fc6f617f1a337c06af54981048574d8783147" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -7384,7 +7766,7 @@ checksum = "c402a4092d5e204f32c9e155431046831fa712637043c58cb73bc6bc6c9663b5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -7406,9 +7788,9 @@ dependencies = [ [[package]] name = "multimap" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" [[package]] name = "murmurhash32" @@ -7418,9 +7800,9 @@ checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" [[package]] name = "mysql-common-derive" -version = "0.32.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deb6d9ff4094f6d58d3f892fc558e60048476213dd17dcf904b62202e9029da6" +checksum = "66f62cad7623a9cb6f8f64037f0c4f69c8db8e82914334a83c9788201c2c1bfa" dependencies = [ "darling 0.20.11", "heck 0.5.0", @@ -7429,31 +7811,30 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", "termcolor", - "thiserror 1.0.69", + "thiserror 2.0.12", ] [[package]] name = "mysql_async" -version = "0.35.1" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d14cf024116ba8fef4a7fec5abf0bd5de89b9fb29a7e55818a119ac5ec745077" +checksum = "277ce2f2459b2af4cc6d0a0b7892381f80800832f57c533f03e2845f4ea331ea" dependencies = [ "bytes", - "crossbeam", + "crossbeam-queue", "flate2", "futures-core", "futures-sink", "futures-util", "keyed_priority_queue", - "lru", + "lru 0.14.0", "mysql_common", "native-tls", "pem 3.0.5", "percent-encoding", - "pin-project", - "rand 0.8.5", + "rand 0.9.0", "serde", "serde_json", "socket2", @@ -7467,35 +7848,30 @@ dependencies = [ [[package]] name = "mysql_common" -version = "0.34.1" +version = "0.35.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34a9141e735d5bb02414a7ac03add09522466d4db65bdd827069f76ae0850e58" +checksum = "6e0ec195e788c95f36b7cf88127d538465fc2f7773e6e47af01834738eab0aee" dependencies = [ "base64 0.22.1", "bitflags 2.9.0", "btoi", "byteorder", "bytes", - "cc", - "cmake", "crc32fast", "flate2", - "lazy_static", + "getrandom 0.3.3", "mysql-common-derive", "num-bigint", "num-traits", - "rand 0.8.5", "regex", "rust_decimal", "saturating", "serde", "serde_json", "sha1", - "sha2 0.10.8", - "subprocess", - "thiserror 1.0.69", + "sha2 0.10.9", + "thiserror 2.0.12", "uuid", - "zstd", ] [[package]] @@ -7526,7 +7902,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", ] [[package]] @@ -7538,7 +7914,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -7624,7 +8000,7 @@ dependencies = [ "data-encoding", "ed25519", "ed25519-dalek", - "getrandom 0.2.15", + "getrandom 0.2.16", "log", "rand 0.8.5", "signatory", @@ -7639,7 +8015,7 @@ dependencies = [ "anyhow", "async-trait", "boxed_error", - "dashmap", + "dashmap 5.5.3", "deno_error", "deno_media_type", "deno_package_json", @@ -7733,7 +8109,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -7796,11 +8172,11 @@ dependencies = [ "chrono-humanize", "dirs 5.0.1", "dirs-sys 0.4.1", - "fancy-regex", + "fancy-regex 0.14.0", "heck 0.5.0", "indexmap 2.9.0", "log", - "lru", + "lru 0.12.5", "miette", "nix 0.29.0", "nu-derive-value", @@ -7841,7 +8217,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d1468fa8e6e12d9d53c90b44f3d11a37d87502d7a30d145f122341c5b33745" dependencies = [ "crossterm_winapi", - "fancy-regex", + "fancy-regex 0.14.0", "log", "lscolors", "nix 0.29.0", @@ -8001,7 +8377,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -8018,14 +8394,14 @@ checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ "base64 0.22.1", "chrono", - "getrandom 0.2.15", + "getrandom 0.2.16", "http 1.3.1", "rand 0.8.5", "reqwest 0.12.15", "serde", "serde_json", "serde_path_to_error", - "sha2 0.10.8", + "sha2 0.10.9", "thiserror 1.0.69", "url", ] @@ -8050,32 +8426,38 @@ dependencies = [ [[package]] name = "object_store" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6da452820c715ce78221e8202ccc599b4a52f3e1eb3eedb487b680c81a8e3f3" +version = "0.12.0" +source = "git+https://github.com/apache/arrow-rs-object-store?rev=36752c975d4f29e20b57c91f81a10872dcd48ae7#36752c975d4f29e20b57c91f81a10872dcd48ae7" dependencies = [ "async-trait", "base64 0.22.1", "bytes", "chrono", + "form_urlencoded", "futures", + "http 1.3.1", + "http-body-util", + "httparse", "humantime", "hyper 1.6.0", - "itertools 0.13.0", + "itertools 0.14.0", "md-5 0.10.6", "parking_lot 0.12.3", "percent-encoding", - "quick-xml 0.36.2", - "rand 0.8.5", + "quick-xml 0.37.5", + "rand 0.9.0", "reqwest 0.12.15", "ring 0.17.14", "serde", "serde_json", - "snafu", + "serde_urlencoded", + "thiserror 2.0.12", "tokio", "tracing", "url", "walkdir", + "wasm-bindgen-futures", + "web-time", ] [[package]] @@ -8178,7 +8560,7 @@ dependencies = [ "serde_path_to_error", "serde_plain", "serde_with", - "sha2 0.10.8", + "sha2 0.10.9", "subtle", "thiserror 1.0.69", "url", @@ -8207,7 +8589,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -8227,9 +8609,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.107" +version = "0.9.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8288979acd84749c744a9014b4382d42b8f7b2592847b5afb2ed29e5d16ede07" +checksum = "e145e1651e858e820e4860f7b9c5e169bc1d8ce1c86043be79fa7b7634821847" dependencies = [ "cc", "libc", @@ -8432,7 +8814,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "sha2 0.10.8", + "sha2 0.10.9", ] [[package]] @@ -8444,7 +8826,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "sha2 0.10.8", + "sha2 0.10.9", ] [[package]] @@ -8456,7 +8838,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "sha2 0.10.8", + "sha2 0.10.9", ] [[package]] @@ -8470,7 +8852,7 @@ dependencies = [ "elliptic-curve", "primeorder", "rand_core 0.6.4", - "sha2 0.10.8", + "sha2 0.10.9", ] [[package]] @@ -8522,18 +8904,18 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.11", + "redox_syscall 0.5.12", "smallvec", "windows-targets 0.52.6", ] [[package]] name = "parquet" -version = "52.2.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e977b9066b4d3b03555c22bdc442f3fadebd96a39111249113087d0edb2691cd" +checksum = "cd31a8290ac5b19f09ad77ee7a1e6a541f1be7674ad410547d5f1eef6eef4a9c" dependencies = [ - "ahash 0.8.11", + "ahash 0.8.12", "arrow-array", "arrow-buffer", "arrow-cast", @@ -8542,25 +8924,25 @@ dependencies = [ "arrow-schema", "arrow-select", "base64 0.22.1", - "brotli 6.0.0", + "brotli 7.0.0", "bytes", "chrono", "flate2", "futures", "half", - "hashbrown 0.14.5", + "hashbrown 0.15.3", "lz4_flex", "num", "num-bigint", "object_store", "paste", "seq-macro", + "simdutf8", "snap", "thrift", "tokio", - "twox-hash 1.6.3", + "twox-hash 2.1.0", "zstd", - "zstd-sys", ] [[package]] @@ -8645,23 +9027,13 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "petgraph" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" -dependencies = [ - "fixedbitset 0.4.2", - "indexmap 2.9.0", -] - [[package]] name = "petgraph" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ - "fixedbitset 0.5.7", + "fixedbitset", "indexmap 2.9.0", ] @@ -8714,7 +9086,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -8755,7 +9127,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -8792,7 +9164,7 @@ dependencies = [ "der", "pbkdf2", "scrypt", - "sha2 0.10.8", + "sha2 0.10.9", "spki", ] @@ -8881,7 +9253,7 @@ dependencies = [ "md-5 0.10.6", "memchr", "rand 0.8.5", - "sha2 0.10.8", + "sha2 0.10.9", "stringprep", ] @@ -8899,7 +9271,7 @@ dependencies = [ "md-5 0.10.6", "memchr", "rand 0.9.0", - "sha2 0.10.8", + "sha2 0.10.9", "stringprep", ] @@ -8930,6 +9302,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -8942,7 +9323,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.8.24", + "zerocopy", ] [[package]] @@ -8958,7 +9339,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" dependencies = [ "proc-macro2", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -8976,7 +9357,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" dependencies = [ - "toml_edit 0.22.24", + "toml_edit 0.22.26", ] [[package]] @@ -9022,7 +9403,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -9033,7 +9414,7 @@ checksum = "07c277e4e643ef00c1233393c673f655e3672cf7eb3ba08a00bdd0ea59139b5f" dependencies = [ "proc-macro-rules-macros", "proc-macro2", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -9045,7 +9426,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -9137,12 +9518,12 @@ dependencies = [ "log", "multimap", "once_cell", - "petgraph 0.7.1", + "petgraph", "prettyplease", "prost", "prost-types", "regex", - "syn 2.0.100", + "syn 2.0.101", "tempfile", ] @@ -9156,7 +9537,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -9170,9 +9551,9 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f58e5423e24c18cc840e1c98370b3993c6649cd1678b4d24318bcf0a083cbe88" +checksum = "6e944464ec8536cd1beb0bbfd96987eb5e3b72f2ecdafdc5c769a37f1fa2ae1f" dependencies = [ "cc", ] @@ -9221,6 +9602,20 @@ dependencies = [ "reborrow", ] +[[package]] +name = "pulp" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b86df24f0a7ddd5e4b95c94fc9ed8a98f1ca94d3b01bdce2824097e7835907" +dependencies = [ + "bytemuck", + "cfg-if", + "libm", + "num-complex", + "reborrow", + "version_check", +] + [[package]] name = "pure-rust-locales" version = "0.8.1" @@ -9249,9 +9644,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.36.2" +version = "0.37.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" dependencies = [ "memchr", "serde", @@ -9259,21 +9654,21 @@ dependencies = [ [[package]] name = "quick_cache" -version = "0.6.13" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "287e56aac5a2b4fb25a6fb050961d157635924c8696305a5c937a76f29841a0f" +checksum = "6b450dad8382b1b95061d5ca1eb792081fb082adf48c678791fe917509596d5f" dependencies = [ - "ahash 0.8.11", + "ahash 0.8.12", "equivalent", - "hashbrown 0.15.2", + "hashbrown 0.15.3", "parking_lot 0.12.3", ] [[package]] name = "quinn" -version = "0.11.7" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3bd15a6f2967aef83887dcb9fec0014580467e33720d073560cf015a5683012" +checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" dependencies = [ "bytes", "cfg_aliases 0.2.1", @@ -9281,7 +9676,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls 0.23.26", + "rustls 0.23.27", "socket2", "thiserror 2.0.12", "tokio", @@ -9291,16 +9686,17 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.10" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc" +checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" dependencies = [ "bytes", - "getrandom 0.3.2", + "getrandom 0.3.3", + "lru-slab", "rand 0.9.0", "ring 0.17.14", "rustc-hash 2.1.1", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pki-types", "slab", "thiserror 2.0.12", @@ -9311,9 +9707,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "541d0f57c6ec747a90738a52741d3221f7960e8ac2f0ff4b1a63680e033b4ab5" +checksum = "ee4e529991f949c5e25755532370b8af5d114acae52326361d68d47af64aa842" dependencies = [ "cfg_aliases 0.2.1", "libc", @@ -9373,7 +9769,7 @@ checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", - "zerocopy 0.8.24", + "zerocopy", ] [[package]] @@ -9402,7 +9798,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", ] [[package]] @@ -9411,7 +9807,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.2", + "getrandom 0.3.3", ] [[package]] @@ -9424,6 +9820,16 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.0", +] + [[package]] name = "range-alloc" version = "0.1.4" @@ -9439,6 +9845,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "raw-cpuid" +version = "11.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146" +dependencies = [ + "bitflags 2.9.0", +] + [[package]] name = "raw-window-handle" version = "0.6.2" @@ -9514,6 +9929,26 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn 2.0.101", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -9534,9 +9969,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" +checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" dependencies = [ "bitflags 2.9.0", ] @@ -9547,7 +9982,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", "libredox", "thiserror 1.0.69", ] @@ -9569,7 +10004,7 @@ checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -9691,7 +10126,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.4.9", + "h2 0.4.10", "http 1.3.1", "http-body 1.0.1", "http-body-util", @@ -9708,7 +10143,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-native-certs 0.8.1", "rustls-pemfile 2.2.0", "rustls-pki-types", @@ -9728,7 +10163,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", + "webpki-roots 0.26.11", "windows-registry", ] @@ -9756,7 +10191,7 @@ dependencies = [ "anyhow", "async-trait", "futures", - "getrandom 0.2.15", + "getrandom 0.2.16", "http 1.3.1", "hyper 1.6.0", "parking_lot 0.11.2", @@ -9771,12 +10206,9 @@ dependencies = [ [[package]] name = "resolv-conf" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48375394603e3dd4b2d64371f7148fd8c7baa2680e28741f2cb8d23b59e3d4c4" -dependencies = [ - "hostname", -] +checksum = "fc7c8f7f733062b66dc1c63f9db168ac0b97a9210e247fa90fdc9ad08f51b302" [[package]] name = "retry-policies" @@ -9797,12 +10229,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "riff" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9b1a3d5f46d53f4a3478e2be4a5a5ce5108ea58b100dcd139830eae7f79a3a1" - [[package]] name = "ring" version = "0.16.20" @@ -9826,7 +10252,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.15", + "getrandom 0.2.16", "libc", "untrusted 0.9.0", "windows-sys 0.52.0", @@ -9870,6 +10296,40 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rmcp" +version = "0.1.5" +source = "git+https://github.com/windmill-labs/rust-sdk#9142b40202e49ca0b6530fa49a3abc8bd1b2fcf0" +dependencies = [ + "async-stream", + "axum", + "base64 0.21.7", + "chrono", + "futures", + "paste", + "pin-project-lite", + "rand 0.9.0", + "rmcp-macros", + "schemars", + "serde", + "serde_json", + "thiserror 2.0.12", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", +] + +[[package]] +name = "rmcp-macros" +version = "0.1.5" +source = "git+https://github.com/windmill-labs/rust-sdk#9142b40202e49ca0b6530fa49a3abc8bd1b2fcf0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "ron" version = "0.8.1" @@ -9957,7 +10417,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.100", + "syn 2.0.101", "walkdir", ] @@ -9967,7 +10427,7 @@ version = "7.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d38ff6bf570dc3bb7100fce9f7b60c33fa71d80e88da3f2580df4ff2bdded74" dependencies = [ - "sha2 0.10.8", + "sha2 0.10.9", "walkdir", ] @@ -10058,9 +10518,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.5" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" dependencies = [ "bitflags 2.9.0", "errno", @@ -10097,16 +10557,16 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.26" +version = "0.23.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0" +checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" dependencies = [ "aws-lc-rs", "log", "once_cell", "ring 0.17.14", "rustls-pki-types", - "rustls-webpki 0.103.1", + "rustls-webpki 0.103.3", "subtle", "zeroize", ] @@ -10168,11 +10628,12 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ "web-time", + "zeroize", ] [[package]] @@ -10182,7 +10643,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22557157d7395bc30727745b365d923f1ecc230c4c80b176545f3f4f08c46e33" dependencies = [ "futures", - "rustls 0.23.26", + "rustls 0.23.27", "socket2", "tokio", ] @@ -10210,9 +10671,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.1" +version = "0.103.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" +checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" dependencies = [ "aws-lc-rs", "ring 0.17.14", @@ -10418,7 +10879,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -10442,7 +10903,7 @@ dependencies = [ "password-hash", "pbkdf2", "salsa20", - "sha2 0.10.8", + "sha2 0.10.9", ] [[package]] @@ -10550,9 +11011,9 @@ dependencies = [ [[package]] name = "serde-aux" -version = "4.6.0" +version = "4.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5290c39c5f6992b9dddbda28541d965dba46468294e6018a408fa297e6c602de" +checksum = "207f67b28fe90fb596503a9bf0bf1ea5e831e21307658e177c5dfcdfc3ab8a0a" dependencies = [ "chrono", "serde", @@ -10598,7 +11059,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -10609,7 +11070,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -10661,7 +11122,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -10726,7 +11187,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -10765,9 +11226,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -10810,9 +11271,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" dependencies = [ "libc", "signal-hook-registry", @@ -10820,9 +11281,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.2" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" dependencies = [ "libc", ] @@ -10870,7 +11331,7 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa2bcf6c6e164e81bc7a5d49fc6988b3d515d9e8c07457d7b74ffb9324b9cd40" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", "halfbrown", "ref-cast", "serde", @@ -10909,6 +11370,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" +[[package]] +name = "size" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6709c7b6754dca1311b3c73e79fcce40dd414c782c66d88e8823030093b02b" + [[package]] name = "sketches-ddsketch" version = "0.2.2" @@ -10971,28 +11438,6 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7d3950ab75b03c52f2f13fd52aab91c9d62698b231b67240e85c3ef5301e63e" -[[package]] -name = "snafu" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" -dependencies = [ - "doc-comment", - "snafu-derive", -] - -[[package]] -name = "snafu-derive" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "snap" version = "1.1.1" @@ -11030,17 +11475,16 @@ dependencies = [ [[package]] name = "sourcemap" -version = "9.1.2" +version = "9.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c4ea7042fd1a155ad95335b5d505ab00d5124ea0332a06c8390d200bb1a76a" +checksum = "dd430118acc9fdd838557649b9b43fd0a78e3834d84a283b466f8e84720d6101" dependencies = [ - "base64-simd 0.7.0", + "base64-simd 0.8.0", "bitvec", "data-encoding", "debugid", "if_chain", - "rustc-hash 1.1.0", - "rustc_version 0.2.3", + "rustc-hash 2.1.1", "serde", "serde_json", "unicode-id-start", @@ -11111,23 +11555,24 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.47.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25" +checksum = "c4521174166bac1ff04fe16ef4524c70144cd29682a45978978ca3d7f4e0be11" dependencies = [ "log", + "recursive", "sqlparser_derive", ] [[package]] name = "sqlparser_derive" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" +checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -11161,17 +11606,17 @@ dependencies = [ "futures-intrusive", "futures-io", "futures-util", - "hashbrown 0.15.2", + "hashbrown 0.15.3", "hashlink 0.10.0", "indexmap 2.9.0", "log", "memchr", "once_cell", "percent-encoding", - "rustls 0.23.26", + "rustls 0.23.27", "serde", "serde_json", - "sha2 0.10.8", + "sha2 0.10.9", "smallvec", "thiserror 2.0.12", "tokio", @@ -11179,7 +11624,7 @@ dependencies = [ "tracing", "url", "uuid", - "webpki-roots", + "webpki-roots 0.26.11", ] [[package]] @@ -11192,7 +11637,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -11210,12 +11655,12 @@ dependencies = [ "quote", "serde", "serde_json", - "sha2 0.10.8", + "sha2 0.10.9", "sqlx-core", "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.100", + "syn 2.0.101", "tempfile", "tokio", "url", @@ -11256,7 +11701,7 @@ dependencies = [ "rsa", "serde", "sha1", - "sha2 0.10.8", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", @@ -11297,7 +11742,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "sha2 0.10.8", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", @@ -11341,9 +11786,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stacker" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601f9201feb9b09c00266478bf459952b9ef9a6b94edb2f21eba14ab681a60a9" +checksum = "cddb07e32ddb770749da91081d8d0ac3a16f1a569a18b20348cd371f5dead06b" dependencies = [ "cc", "cfg-if", @@ -11367,7 +11812,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -11414,16 +11859,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" dependencies = [ - "strum_macros 0.25.3", -] - -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros 0.26.4", + "strum_macros", ] [[package]] @@ -11436,30 +11872,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.100", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.100", -] - -[[package]] -name = "subprocess" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086" -dependencies = [ - "libc", - "winapi", + "syn 2.0.101", ] [[package]] @@ -11520,9 +11933,9 @@ version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83406221c501860fce9c27444f44125eafe9e598b8b81be7563d7036784cd05c" dependencies = [ - "ahash 0.8.11", + "ahash 0.8.12", "anyhow", - "dashmap", + "dashmap 5.5.3", "once_cell", "regex", "serde", @@ -11545,7 +11958,7 @@ dependencies = [ "rustc-hash 1.1.0", "serde", "siphasher 0.3.11", - "sourcemap 9.1.2", + "sourcemap 9.2.0", "swc_allocator", "swc_atoms", "swc_eq_ignore_macros", @@ -11578,7 +11991,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -11609,7 +12022,7 @@ dependencies = [ "num-bigint", "once_cell", "serde", - "sourcemap 9.1.2", + "sourcemap 9.2.0", "swc_allocator", "swc_atoms", "swc_common", @@ -11627,7 +12040,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -11712,7 +12125,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -11742,7 +12155,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76c76d8b9792ce51401d38da0fa62158d61f6d80d16d68fe5b03ce4bf5fba383" dependencies = [ "base64 0.21.7", - "dashmap", + "dashmap 5.5.3", "indexmap 2.9.0", "once_cell", "serde", @@ -11819,7 +12232,7 @@ checksum = "63db0adcff29d220c3d151c5b25c0eabe7e32dd936212b84cdaa1392e3130497" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -11830,7 +12243,7 @@ checksum = "f486687bfb7b5c560868f69ed2d458b880cebc9babebcb67e49f31b55c5bf847" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -11853,7 +12266,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -11869,9 +12282,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.100" +version = "2.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" dependencies = [ "proc-macro2", "quote", @@ -11907,13 +12320,13 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -11949,6 +12362,20 @@ dependencies = [ "walkdir", ] +[[package]] +name = "sysctl" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" +dependencies = [ + "bitflags 2.9.0", + "byteorder", + "enum-as-inner", + "libc", + "thiserror 1.0.69", + "walkdir", +] + [[package]] name = "sysinfo" version = "0.32.1" @@ -12005,6 +12432,20 @@ dependencies = [ "libc", ] +[[package]] +name = "systemstat" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668a4db78b439df482c238f559e4ea869017f9e62ef0a059c8bfcd841a4df544" +dependencies = [ + "bytesize", + "lazy_static", + "libc", + "nom 7.1.3", + "time", + "winapi", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -12033,7 +12474,7 @@ dependencies = [ "itertools 0.12.1", "levenshtein_automata", "log", - "lru", + "lru 0.12.5", "lz4_flex", "measure_time", "memmap2 0.9.5", @@ -12139,7 +12580,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8" dependencies = [ "murmurhash32", - "rand_distr", + "rand_distr 0.4.3", "tantivy-common", ] @@ -12171,14 +12612,14 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.19.1" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" dependencies = [ "fastrand", - "getrandom 0.3.2", + "getrandom 0.3.3", "once_cell", - "rustix 1.0.5", + "rustix 1.0.7", "windows-sys 0.59.0", ] @@ -12197,7 +12638,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45c6481c4829e4cc63825e62c49186a34538b7b2750b73b266581ffb612fb5ed" dependencies = [ - "rustix 1.0.5", + "rustix 1.0.7", "windows-sys 0.59.0", ] @@ -12246,7 +12687,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -12257,7 +12698,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -12394,9 +12835,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.7.6" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" dependencies = [ "displaydoc", "zerovec", @@ -12442,7 +12883,7 @@ dependencies = [ "clap", "derive_builder", "esaxx-rs", - "getrandom 0.2.15", + "getrandom 0.2.16", "indicatif", "itertools 0.11.0", "lazy_static", @@ -12467,9 +12908,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.44.2" +version = "1.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" +checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165" dependencies = [ "backtrace", "bytes", @@ -12502,7 +12943,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -12615,7 +13056,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.26", + "rustls 0.23.27", "tokio", ] @@ -12673,16 +13114,16 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", "futures-io", "futures-sink", "futures-util", - "hashbrown 0.14.5", + "hashbrown 0.15.3", "pin-project-lite", "slab", "tokio", @@ -12723,9 +13164,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3" dependencies = [ "serde", ] @@ -12745,13 +13186,13 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.24" +version = "0.22.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" +checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e" dependencies = [ "indexmap 2.9.0", "toml_datetime", - "winnow 0.7.6", + "winnow 0.7.10", ] [[package]] @@ -12766,7 +13207,7 @@ dependencies = [ "base64 0.22.1", "bytes", "flate2", - "h2 0.4.9", + "h2 0.4.10", "http 1.3.1", "http-body 1.0.1", "http-body-util", @@ -12786,7 +13227,7 @@ dependencies = [ "tower-layer", "tower-service", "tracing", - "webpki-roots", + "webpki-roots 0.26.11", ] [[package]] @@ -12844,9 +13285,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.2" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" +checksum = "0fdb0c213ca27a9f57ab69ddb290fd80d970922355b83ae380b395d3986b8a2e" dependencies = [ "async-compression", "bitflags 2.9.0", @@ -12907,7 +13348,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -13151,7 +13592,28 @@ checksum = "35f5380909ffc31b4de4f4bdf96b877175a016aa2ca98cee39fcfd8c4d53d952" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", +] + +[[package]] +name = "ug" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b70b37e9074642bc5f60bb23247fd072a84314ca9e71cdf8527593406a0dd3" +dependencies = [ + "gemm 0.18.2", + "half", + "libloading 0.8.7", + "memmap2 0.9.5", + "num", + "num-traits", + "num_cpus", + "rayon", + "safetensors", + "serde", + "thiserror 1.0.69", + "tracing", + "yoke 0.7.5", ] [[package]] @@ -13374,12 +13836,12 @@ dependencies = [ "log", "native-tls", "once_cell", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pki-types", "serde", "serde_json", "url", - "webpki-roots", + "webpki-roots 0.26.11", ] [[package]] @@ -13418,12 +13880,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - [[package]] name = "utf8-ranges" version = "1.0.5" @@ -13454,8 +13910,10 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" dependencies = [ - "getrandom 0.3.2", + "getrandom 0.3.3", + "js-sys", "serde", + "wasm-bindgen", ] [[package]] @@ -13597,7 +14055,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", "wasm-bindgen-shared", ] @@ -13632,7 +14090,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -13667,7 +14125,7 @@ checksum = "17d5042cc5fa009658f9a7333ef24291b1291a25b6382dd68862a7f3b969f69b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -13708,15 +14166,6 @@ dependencies = [ "thiserror 2.0.12", ] -[[package]] -name = "wav" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d97402f69875b579ec37f2aa52d1f455a1d6224251edba32e8c18a5da2698d" -dependencies = [ - "riff", -] - [[package]] name = "web-sys" version = "0.3.77" @@ -13739,18 +14188,36 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "0.26.8" +version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09aed61f5e8d2c18344b3faa33a4c837855fe56642757754775548fee21386c4" +checksum = "75c7f0ef91146ebfb530314f5f1d24528d7f0767efbfd31dce919275413e393e" +dependencies = [ + "webpki-root-certs 1.0.0", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a83f7e1a9f8712695c03eabe9ed3fbca0feff0152f33f12593e5a6303cb1a4" dependencies = [ "rustls-pki-types", ] [[package]] name = "webpki-roots" -version = "0.26.8" +version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.0", +] + +[[package]] +name = "webpki-roots" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2853738d1cc4f2da3a225c18ec6c3721abb31961096e9dbf5ab35fa88b19cfdb" dependencies = [ "rustls-pki-types", ] @@ -13806,7 +14273,7 @@ dependencies = [ "js-sys", "khronos-egl", "libc", - "libloading 0.8.6", + "libloading 0.8.7", "log", "metal", "naga", @@ -13868,7 +14335,7 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" dependencies = [ - "redox_syscall 0.5.11", + "redox_syscall 0.5.12", "wasite", "web-sys", ] @@ -13912,7 +14379,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "axum", @@ -13932,12 +14399,14 @@ dependencies = [ "quote", "rand 0.9.0", "reqwest 0.12.15", - "rustls 0.23.26", + "rustls 0.23.27", "serde", "serde_json", "sha1", - "sha2 0.10.8", + "sha2 0.10.9", + "size", "sqlx", + "systemstat", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", "tikv-jemallocator", @@ -13959,7 +14428,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "argon2", @@ -13980,7 +14449,7 @@ dependencies = [ "candle-nn", "candle-transformers", "chrono", - "chrono-tz 0.10.3", + "chrono-tz", "const_format", "constant_time_eq", "cookie 0.17.0", @@ -13992,7 +14461,6 @@ dependencies = [ "git-version", "google-cloud-googleapis", "google-cloud-pubsub", - "half", "hex", "hf-hub", "hmac", @@ -14020,6 +14488,7 @@ dependencies = [ "rdkafka", "regex", "reqwest 0.12.15", + "rmcp", "rsa", "rumqttc", "rust-embed", @@ -14029,7 +14498,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sha1", - "sha2 0.10.8", + "sha2 0.10.9", "sql-builder", "sqlx", "tempfile", @@ -14068,7 +14537,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.483.1" +version = "1.490.0" dependencies = [ "base64 0.22.1", "chrono", @@ -14083,7 +14552,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.483.1" +version = "1.490.0" dependencies = [ "chrono", "serde", @@ -14096,7 +14565,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "serde", @@ -14110,7 +14579,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "async-stream", @@ -14120,11 +14589,12 @@ dependencies = [ "backon", "bytes", "chrono", - "chrono-tz 0.10.3", + "chrono-tz", "const_format", "crc", "cron", "croner", + "datafusion", "futures", "futures-core", "gethostname", @@ -14155,13 +14625,17 @@ dependencies = [ "semver 1.0.26", "serde", "serde_json", - "sha2 0.10.8", + "sha2 0.10.9", + "size", "sqlx", + "systemstat", "tar", "tempfile", "thiserror 2.0.12", "tikv-jemalloc-ctl", "tokio", + "tokio-stream", + "tokio-util", "tonic", "tracing", "tracing-appender", @@ -14170,11 +14644,12 @@ dependencies = [ "tracing-subscriber", "uuid", "windmill-macros", + "windmill-parser-sql", ] [[package]] name = "windmill-git-sync" -version = "1.483.1" +version = "1.490.0" dependencies = [ "regex", "serde", @@ -14188,7 +14663,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "bytes", @@ -14211,19 +14686,19 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.483.1" +version = "1.490.0" dependencies = [ "itertools 0.14.0", "lazy_static", "proc-macro2", "quote", "regex", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] name = "windmill-parser" -version = "1.483.1" +version = "1.490.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -14232,7 +14707,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "lazy_static", @@ -14244,7 +14719,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "serde_json", @@ -14256,7 +14731,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "gosyn", @@ -14268,7 +14743,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "lazy_static", @@ -14280,7 +14755,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "serde_json", @@ -14292,7 +14767,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "nu-parser", @@ -14303,7 +14778,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14314,7 +14789,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14325,7 +14800,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "async-recursion", @@ -14345,7 +14820,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -14355,14 +14830,14 @@ dependencies = [ "quote", "regex", "serde_json", - "syn 2.0.100", + "syn 2.0.101", "toml", "windmill-parser", ] [[package]] name = "windmill-parser-sql" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "lazy_static", @@ -14374,7 +14849,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "lazy_static", @@ -14392,10 +14867,10 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", - "getrandom 0.2.15", + "getrandom 0.2.16", "serde_json", "wasm-bindgen", "wasm-bindgen-test", @@ -14416,7 +14891,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "serde_json", @@ -14426,14 +14901,14 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "async-recursion", "axum", "backon", "chrono", - "chrono-tz 0.10.3", + "chrono-tz", "cron", "futures", "futures-core", @@ -14459,7 +14934,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.483.1" +version = "1.490.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -14469,10 +14944,11 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.483.1" +version = "1.490.0" dependencies = [ "anyhow", "async-recursion", + "async-stream", "backon", "base64 0.22.1", "bit-vec 0.6.3", @@ -14523,7 +14999,7 @@ dependencies = [ "rust_decimal", "serde", "serde_json", - "sha2 0.10.8", + "sha2 0.10.9", "sqlx", "tar", "tiberius", @@ -14645,7 +15121,7 @@ checksum = "f6fc35f58ecd95a9b71c4f2329b911016e6bec66b3f2e6a4aad86bd2e99e2f9b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -14656,7 +15132,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -14667,7 +15143,7 @@ checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -14678,7 +15154,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -14689,7 +15165,7 @@ checksum = "08990546bf4edef8f431fa6326e032865f27138718c587dc21bc0265bbcb57cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -14700,7 +15176,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -14711,7 +15187,7 @@ checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -14722,7 +15198,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -15029,9 +15505,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.6" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10" +checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec" dependencies = [ "memchr", ] @@ -15061,17 +15537,11 @@ dependencies = [ "bitflags 2.9.0", ] -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - [[package]] name = "writeable" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" [[package]] name = "wtf8" @@ -15124,7 +15594,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d65cbf2f12c15564212d48f4e3dfb87923d25d611f2aed18f4cb23f0413d89e" dependencies = [ "libc", - "rustix 1.0.5", + "rustix 1.0.7", ] [[package]] @@ -15171,7 +15641,19 @@ checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" dependencies = [ "serde", "stable_deref_trait", - "yoke-derive", + "yoke-derive 0.7.5", + "zerofrom", +] + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive 0.8.0", "zerofrom", ] @@ -15183,48 +15665,40 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", - "synstructure 0.13.1", + "syn 2.0.101", + "synstructure 0.13.2", +] + +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", + "synstructure 0.13.2", ] [[package]] name = "zerocopy" -version = "0.7.35" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" dependencies = [ - "zerocopy-derive 0.7.35", -] - -[[package]] -name = "zerocopy" -version = "0.8.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" -dependencies = [ - "zerocopy-derive 0.8.24", + "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.35" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -15244,8 +15718,8 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", - "synstructure 0.13.1", + "syn 2.0.101", + "synstructure 0.13.2", ] [[package]] @@ -15265,42 +15739,63 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", +] + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke 0.8.0", + "zerofrom", ] [[package]] name = "zerovec" -version = "0.10.4" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" dependencies = [ - "yoke", + "yoke 0.8.0", "zerofrom", "zerovec-derive", ] [[package]] name = "zerovec-derive" -version = "0.10.3" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] name = "zip" -version = "0.6.6" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +checksum = "9cc23c04387f4da0374be4533ad1208cbb091d5c11d070dfef13676ad6497164" dependencies = [ - "byteorder", + "arbitrary", "crc32fast", "crossbeam-utils", + "displaydoc", + "indexmap 2.9.0", + "num_enum", + "thiserror 1.0.69", ] +[[package]] +name = "zlib-rs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" + [[package]] name = "zstd" version = "0.13.3" @@ -15312,18 +15807,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.1" +version = "7.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.12+zstd.1.5.6" +version = "2.0.15+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" dependencies = [ "cc", "pkg-config", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 52f6f39ce6..f2a92433ae 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.483.1" +version = "1.490.0" authors.workspace = true edition.workspace = true @@ -32,7 +32,7 @@ members = [ ] [workspace.package] -version = "1.483.1" +version = "1.490.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -72,6 +72,7 @@ dind = ["windmill-worker/dind"] websocket = ["windmill-api/websocket"] http_trigger = ["windmill-api/http_trigger"] postgres_trigger = ["windmill-api/postgres_trigger"] +mcp = ["windmill-api/mcp"] mqtt_trigger = ["windmill-api/mqtt_trigger"] sqs_trigger = ["windmill-api/sqs_trigger", "windmill-common/aws_auth", "windmill-api/openidconnect"] gcp_trigger = ["windmill-api/gcp_trigger"] @@ -95,6 +96,9 @@ java = ["windmill-worker/java"] all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "mssql", "bigquery", "csharp", "nu", "php", "java"] +[patch.crates-io] +object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7" } + [dependencies] anyhow.workspace = true tokio.workspace = true @@ -131,6 +135,8 @@ quote.workspace = true memchr.workspace = true v8 = { workspace = true, optional = true } rustls.workspace = true +systemstat.workspace = true +size.workspace = true [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = { optional = true, workspace = true } @@ -191,7 +197,7 @@ serde_json = { version = "^1", features = ["preserve_order", "raw_value"] } uuid = { version = "^1", features = ["serde", "v4"] } thiserror = "^2" anyhow = "^1" -chrono = { version = "=0.4.39", features = ["serde"] } +chrono = { version = "^0.4", features = ["serde"] } chrono-tz = "^0.10.1" tracing = "^0" tracing-subscriber = { version = "^0", features = ["env-filter", "json"] } @@ -314,9 +320,9 @@ nix = { version = "0.27.1", features = ["process", "signal"] } tinyvector = { git = "https://github.com/windmill-labs/tinyvector", rev = "20823b94c20f2b9093f318badd24026cf54dcc85" } hf-hub = "0.3.2" tokenizers = "0.14.1" -candle-core = "0.3.0" -candle-transformers = "0.3.0" -candle-nn = "0.3.0" +candle-core = "0.9.1" +candle-transformers = "0.9.1" +candle-nn = "0.9.1" tiberius = { version = "0.12.3", default-features = false, features = ["rustls", "tds73", "chrono", "sql-browser-tokio"]} pin-project = "1" indexmap = { version = "2.2.5", features = ["serde"]} @@ -330,8 +336,8 @@ async-nats = "0.38.0" nkeys = "0.4.4" nu-parser = { version = "0.101.0", default-features = false } -datafusion = "39.0.0" -object_store = { version = "0.10.0", features = ["aws", "azure"] } +datafusion = "47.0.0" +object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7", features = ["aws", "azure"] } openidconnect = { version = "4.0.0-rc.1" } aws-config = "^1" aws-sdk-sqs = "1.57.0" @@ -354,9 +360,6 @@ bollard = "0.18.1" tonic = { version = "=0.12.3", features = ["tls-native-roots"] } byteorder = "1.5.0" -# todo remove -half = "=2.4.1" - tikv-jemallocator = { version = "0.5" } tikv-jemalloc-sys = { version = "^0.5" } tikv-jemalloc-ctl = { version = "^0.5" } @@ -367,6 +370,8 @@ pin-project-lite = "^0" tantivy = "0.22.0" backon = "1.3.0" +systemstat = "0.2.4" +size = "0.5.0" flume = { version = "0.11.1", features = ["async"] } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 668c3a7485..9600e384bf 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -4c4db954d9de775905a2e207a5969d5feff23455 \ No newline at end of file +4dc1f25f4fcc013334d4cc1d07cbe60a22b56d1f \ No newline at end of file diff --git a/backend/migrations/20250414082127_update_capture_format.down.sql b/backend/migrations/20250414082127_update_capture_format.down.sql new file mode 100644 index 0000000000..3739642e41 --- /dev/null +++ b/backend/migrations/20250414082127_update_capture_format.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE capture RENAME COLUMN preprocessor_args to trigger_extra; +ALTER TABLE capture RENAME COLUMN main_args to payload; diff --git a/backend/migrations/20250414082127_update_capture_format.up.sql b/backend/migrations/20250414082127_update_capture_format.up.sql new file mode 100644 index 0000000000..dc1653315e --- /dev/null +++ b/backend/migrations/20250414082127_update_capture_format.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE capture RENAME COLUMN trigger_extra to preprocessor_args; +ALTER TABLE capture RENAME COLUMN payload to main_args; \ No newline at end of file diff --git a/backend/migrations/20250420144035_add-new-column-to-gcp-trigger.down.sql b/backend/migrations/20250420144035_add-new-column-to-gcp-trigger.down.sql new file mode 100644 index 0000000000..076f1891b6 --- /dev/null +++ b/backend/migrations/20250420144035_add-new-column-to-gcp-trigger.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +ALTER TABLE gcp_trigger DROP COLUMN subscription_mode; +DROP TYPE GCP_SUBSCRIPTION_MODE; \ No newline at end of file diff --git a/backend/migrations/20250420144035_add-new-column-to-gcp-trigger.up.sql b/backend/migrations/20250420144035_add-new-column-to-gcp-trigger.up.sql new file mode 100644 index 0000000000..529d9580db --- /dev/null +++ b/backend/migrations/20250420144035_add-new-column-to-gcp-trigger.up.sql @@ -0,0 +1,3 @@ +-- Add up migration script here +CREATE TYPE GCP_SUBSCRIPTION_MODE AS ENUM ('create_update', 'existing'); +ALTER TABLE gcp_trigger ADD COLUMN subscription_mode GCP_SUBSCRIPTION_MODE NOT NULL DEFAULT 'create_update'::GCP_SUBSCRIPTION_MODE; \ No newline at end of file diff --git a/backend/migrations/20250421120705_alter_url_column_length_on_websocket_triggers.down.sql b/backend/migrations/20250421120705_alter_url_column_length_on_websocket_triggers.down.sql new file mode 100644 index 0000000000..18b1b65140 --- /dev/null +++ b/backend/migrations/20250421120705_alter_url_column_length_on_websocket_triggers.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +ALTER TABLE websocket_trigger +ALTER COLUMN url TYPE VARCHAR(255); \ No newline at end of file diff --git a/backend/migrations/20250421120705_alter_url_column_length_on_websocket_triggers.up.sql b/backend/migrations/20250421120705_alter_url_column_length_on_websocket_triggers.up.sql new file mode 100644 index 0000000000..51da384e02 --- /dev/null +++ b/backend/migrations/20250421120705_alter_url_column_length_on_websocket_triggers.up.sql @@ -0,0 +1,3 @@ +-- Add up migration script here +ALTER TABLE websocket_trigger +ALTER COLUMN url TYPE VARCHAR(1000); \ No newline at end of file diff --git a/backend/migrations/20250424144434_critical_alerts_on_db_oversize.down.sql b/backend/migrations/20250424144434_critical_alerts_on_db_oversize.down.sql new file mode 100644 index 0000000000..5eceb2c095 --- /dev/null +++ b/backend/migrations/20250424144434_critical_alerts_on_db_oversize.down.sql @@ -0,0 +1 @@ +DELETE FROM global_settings WHERE name = 'critical_alerts_on_db_oversize'; diff --git a/backend/migrations/20250424144434_critical_alerts_on_db_oversize.up.sql b/backend/migrations/20250424144434_critical_alerts_on_db_oversize.up.sql new file mode 100644 index 0000000000..cb1868dbba --- /dev/null +++ b/backend/migrations/20250424144434_critical_alerts_on_db_oversize.up.sql @@ -0,0 +1,3 @@ +INSERT INTO global_settings (name, value) +VALUES ('critical_alerts_on_db_oversize', '{}') +ON CONFLICT (name) DO NOTHING; diff --git a/backend/migrations/20250428170426_mcp_mode_log.down.sql b/backend/migrations/20250428170426_mcp_mode_log.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20250428170426_mcp_mode_log.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20250428170426_mcp_mode_log.up.sql b/backend/migrations/20250428170426_mcp_mode_log.up.sql new file mode 100644 index 0000000000..038bb8234a --- /dev/null +++ b/backend/migrations/20250428170426_mcp_mode_log.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TYPE log_mode ADD VALUE 'mcp'; \ No newline at end of file diff --git a/backend/migrations/20250429211554_create_indices_on_queue.down.sql b/backend/migrations/20250429211554_create_indices_on_queue.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20250429211554_create_indices_on_queue.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20250429211554_create_indices_on_queue.up.sql b/backend/migrations/20250429211554_create_indices_on_queue.up.sql new file mode 100644 index 0000000000..0aafb20d34 --- /dev/null +++ b/backend/migrations/20250429211554_create_indices_on_queue.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +CREATE INDEX IF NOT EXISTS idx_metrics_id_created_at ON public.metrics (id, created_at DESC) WHERE id LIKE 'queue_%'; \ No newline at end of file diff --git a/backend/migrations/20250429214657_create_indices_on_job_stats.down.sql b/backend/migrations/20250429214657_create_indices_on_job_stats.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20250429214657_create_indices_on_job_stats.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20250429214657_create_indices_on_job_stats.up.sql b/backend/migrations/20250429214657_create_indices_on_job_stats.up.sql new file mode 100644 index 0000000000..b201db5dbe --- /dev/null +++ b/backend/migrations/20250429214657_create_indices_on_job_stats.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +CREATE INDEX IF NOT EXISTS job_stats_id ON job_stats (job_id); \ No newline at end of file diff --git a/backend/migrations/20250506092215_runnable_version_notify.down.sql b/backend/migrations/20250506092215_runnable_version_notify.down.sql new file mode 100644 index 0000000000..b94db9dfdc --- /dev/null +++ b/backend/migrations/20250506092215_runnable_version_notify.down.sql @@ -0,0 +1,4 @@ +-- Add down migration script here +DROP TRIGGER script_update_trigger ON script; +DROP TRIGGER flow_update_trigger ON flow_version; +DROP FUNCTION notify_runnable_version_change(); diff --git a/backend/migrations/20250506092215_runnable_version_notify.up.sql b/backend/migrations/20250506092215_runnable_version_notify.up.sql new file mode 100644 index 0000000000..a8d11142f4 --- /dev/null +++ b/backend/migrations/20250506092215_runnable_version_notify.up.sql @@ -0,0 +1,22 @@ +-- Add up migration script here +CREATE OR REPLACE FUNCTION notify_runnable_version_change() +RETURNS TRIGGER AS $$ +DECLARE + source_type TEXT; +BEGIN + source_type := TG_ARGV[0]; + + PERFORM pg_notify('notify_runnable_version_change', NEW.workspace_id || ':' || source_type || ':' || NEW.path); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER script_update_trigger +AFTER UPDATE OF lock ON script +FOR EACH ROW +EXECUTE FUNCTION notify_runnable_version_change('script'); + +CREATE TRIGGER flow_update_trigger +AFTER INSERT ON flow_version +FOR EACH ROW +EXECUTE FUNCTION notify_runnable_version_change('flow'); diff --git a/backend/migrations/20250506151818_orderby_refactor.down.sql b/backend/migrations/20250506151818_orderby_refactor.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20250506151818_orderby_refactor.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20250506151818_orderby_refactor.up.sql b/backend/migrations/20250506151818_orderby_refactor.up.sql new file mode 100644 index 0000000000..a983639760 --- /dev/null +++ b/backend/migrations/20250506151818_orderby_refactor.up.sql @@ -0,0 +1,3 @@ +-- Add up migration script here +DROP INDEX IF EXISTS index_script_on_path_created_at; +CREATE INDEX IF NOT EXISTS index_script_on_path_created_at ON script (workspace_id, path, created_at DESC); diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index 2f18b74d6b..3b06a491d5 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -44,6 +44,7 @@ fn replace_full_import(x: &str) -> Option { lazy_static! { static ref RE: Regex = Regex::new(r"^\#\s?(\S+)\s*$").unwrap(); static ref PIN_RE: Regex = Regex::new(r"(?:\s*#\s*(pin|repin):\s*)(\S*)").unwrap(); + static ref PKG_RE: Regex = Regex::new(r"^([^!=<>]+)(?:[!=<>]|$)").unwrap(); } fn process_import(module: Option, path: &str, level: usize) -> Vec { @@ -167,30 +168,28 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result> .take_while(|e| *e != '\n') .collect::(); - if hs.trim_start().is_empty(){ + if hs.trim_start().is_empty() { return None; } - PIN_RE - .captures(&hs) - .and_then(|x| { - x.get(1).zip(x.get(2)).and_then(|(ty_m, pkg_m)| { - let pkg = pkg_m.as_str().to_owned(); - if ty_m.as_str() == "pin" { - Some(vec![NImport::Pin { - pins: vec![ImportPin { pkg, path: path.to_owned() }], - key, - }]) - } else if ty_m.as_str() == "repin" { - Some(vec![NImport::Repin { - pin: ImportPin { pkg, path: path.to_owned() }, - key, - }]) - } else { - None - } - }) + PIN_RE.captures(&hs).and_then(|x| { + x.get(1).zip(x.get(2)).and_then(|(ty_m, pkg_m)| { + let pkg = pkg_m.as_str().to_owned(); + if ty_m.as_str() == "pin" { + Some(vec![NImport::Pin { + pins: vec![ImportPin { pkg, path: path.to_owned() }], + key, + }]) + } else if ty_m.as_str() == "repin" { + Some(vec![NImport::Repin { + pin: ImportPin { pkg, path: path.to_owned() }, + key, + }]) + } else { + None + } }) + }) }; let mut nimports: Vec = ast @@ -268,7 +267,14 @@ pub async fn parse_python_imports( Ok(p.pkg) }).collect_vec(), NImportResolved::Repin { pin: ImportPin { pkg, .. }, .. } => vec![Ok(pkg)], - NImportResolved::Auto { pkg, ..} => vec![Ok(pkg)], + NImportResolved::Auto { pkg, key } => vec![ + + if let Some(key) = key { + Ok(format!("{pkg} # (mapped from {key})")) + } else { + Ok(pkg) + } + ], }) .flatten() .collect::>>()? @@ -284,6 +290,13 @@ pub async fn parse_python_imports( Ok((imports, compile_error_hint)) } +fn extract_pkg_name(requirement: &str) -> String { + PKG_RE + .captures(requirement) + .map(|x| x.get(1).map(|m| m.as_str().to_string()).unwrap_or_default()) + .unwrap_or_default() +} + #[async_recursion] async fn parse_python_imports_inner( code: &str, @@ -343,11 +356,15 @@ async fn parse_python_imports_inner( RE.captures(x).and_then(|x| { x.get(1).map(|m| { let requirement = m.as_str().to_string(); + let key = extract_pkg_name(&requirement); requirements.insert( - requirement.clone(), - NImportResolved::Repin { - pin: ImportPin { pkg: requirement, path: Default::default() }, - key: Default::default(), + key.clone(), + NImportResolved::Pin { + pins: vec![ImportPin { + pkg: requirement.clone(), + path: Default::default(), + }], + key, }, ); }) @@ -368,9 +385,16 @@ async fn parse_python_imports_inner( RE.captures(x).and_then(|x| { x.get(1).map(|m| { let requirement = m.as_str().to_string(); + let key = extract_pkg_name(&requirement); imports.insert( - requirement.clone(), - NImportResolved::Auto { key: None, pkg: requirement }, + key.clone(), + NImportResolved::Pin { + pins: vec![ImportPin { + pkg: requirement, + path: Default::default(), + }], + key, + }, ); }) }) diff --git a/backend/parsers/windmill-parser-py-imports/src/mapping.rs b/backend/parsers/windmill-parser-py-imports/src/mapping.rs index 11677ffbfd..1bd0688951 100644 --- a/backend/parsers/windmill-parser-py-imports/src/mapping.rs +++ b/backend/parsers/windmill-parser-py-imports/src/mapping.rs @@ -142,6 +142,8 @@ pub static FULL_IMPORTS_MAP: PyMap = phf_map! { "google.cloud.dns" => "google-cloud-dns", "google.cloud.runtimeconfig" => "google-cloud-runtimeconfig", "google.cloud.iot" => "google-cloud-iot", + "google.generativeai" => "google-generativeai", + "google.genai" => "google-genai", // Azure "azure.mgmt.hybridkubernetes" => "azure-mgmt-hybridkubernetes", "azure.mgmt.sql" => "azure-mgmt-sql", @@ -325,6 +327,7 @@ pub static FULL_IMPORTS_MAP: PyMap = phf_map! { "azure.mgmt.nspkg" => "azure-mgmt-nspkg", "azure.keyvault.secrets" => "azure-keyvault-secrets", "azure.storage.blob" => "azure-storage-blob", + "azure.storage.filedatalake" => "azure-storage-file-datalake", // Add new entry here ^ }; @@ -376,5 +379,6 @@ pub static SHORT_IMPORTS_MAP: PyMap = phf_map! { "socks" => "PySocks", "taiga" => "python-taiga", "docx" => "python-docx", + "vt" => "vt-py", // Add new entry here ^ }; diff --git a/backend/parsers/windmill-parser-py-imports/tests/tests.rs b/backend/parsers/windmill-parser-py-imports/tests/tests.rs index d734cb8ead..9fee9b21c9 100644 --- a/backend/parsers/windmill-parser-py-imports/tests/tests.rs +++ b/backend/parsers/windmill-parser-py-imports/tests/tests.rs @@ -29,7 +29,15 @@ def main(): ) .await?; // println!("{}", serde_json::to_string(&r)?); - assert_eq!(r, vec!["matplotlib", "wmill", "zanzibar"]); + assert_eq!( + r, + vec![ + "matplotlib # (mapped from matplotlib.pyplot)", + "wmill", + "zanzibar # (mapped from zanzibar.estonie)" + ] + ); + Ok(()) } diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index 5b9e1ddd6f..21ebe45f7b 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -120,6 +120,60 @@ pub fn parse_db_resource(code: &str) -> Option { cap.map(|x| x.get(1).map(|x| x.as_str().to_string()).unwrap()) } +#[derive(Clone, Copy, Debug)] +pub enum S3ModeFormat { + Json, + Csv, + Parquet, +} +pub fn s3_mode_extension(format: S3ModeFormat) -> &'static str { + match format { + S3ModeFormat::Json => "json", + S3ModeFormat::Csv => "csv", + S3ModeFormat::Parquet => "parquet", + } +} +pub struct S3ModeArgs { + pub prefix: Option, + pub storage: Option, + pub format: S3ModeFormat, +} +pub fn parse_s3_mode(code: &str) -> anyhow::Result> { + let cap = match RE_S3_MODE.captures(code) { + Some(x) => x, + None => return Ok(None), + }; + let args_str = cap + .get(1) + .map(|x| x.as_str().to_string()) + .unwrap_or_default(); + + let mut prefix = None; + let mut storage = None; + let mut format = S3ModeFormat::Json; + + for kv in args_str.split(' ').map(|kv| kv.trim()) { + if kv.is_empty() { + continue; + } + let mut it = kv.split('='); + let (Some(key), Some(value)) = (it.next(), it.next()) else { + return Err(anyhow!("Invalid S3 mode argument: {}", kv)); + }; + match (key.trim(), value.trim()) { + ("prefix", _) => prefix = Some(value.to_string()), + ("storage", _) => storage = Some(value.to_string()), + ("format", "json") => format = S3ModeFormat::Json, + ("format", "parquet") => format = S3ModeFormat::Parquet, + ("format", "csv") => format = S3ModeFormat::Csv, + ("format", format) => return Err(anyhow!("Invalid S3 mode format: {}", format)), + (_, _) => return Err(anyhow!("Invalid S3 mode argument: {}", kv)), + } + } + + Ok(Some(S3ModeArgs { prefix, storage, format })) +} + pub fn parse_sql_blocks(code: &str) -> Vec<&str> { let mut blocks = vec![]; let mut last_idx = 0; @@ -147,6 +201,7 @@ lazy_static::lazy_static! { static ref RE_NONEMPTY_SQL_BLOCK: Regex = Regex::new(r#"(?m)^\s*[^\s](?:[^-]|$)"#).unwrap(); static ref RE_DB: Regex = Regex::new(r#"(?m)^-- database (\S+) *(?:\r|\n|$)"#).unwrap(); + static ref RE_S3_MODE: Regex = Regex::new(r#"(?m)^-- s3( (.+))? *(?:\r|\n|$)"#).unwrap(); // -- $1 name (type) = default static ref RE_ARG_MYSQL: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index d4d34cecd6..2024f4880c 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -506,7 +506,7 @@ fn one_of_label(members: &Vec) -> Option { let Expr::Ident(Ident { sym, .. }) = &**key else { return None; }; - if sym != "label" { + if sym != "label" && sym != "kind" { return None; } diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index 2f4bc83c1f..cd280e7af5 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -17,7 +17,7 @@ fn wrap_sig(r: anyhow::Result) -> String { #[cfg(feature = "ts-parser")] #[wasm_bindgen] -pub fn parse_deno(code: &str, main_override: Option, skip_params: Option) -> String { +pub fn parse_deno(code: &str, main_override: Option) -> String { wrap_sig(windmill_parser_ts::parse_deno_signature( code, false, diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index c2d417eb6c..06adac1ee3 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use anyhow::anyhow; use serde_json::json; use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ}; @@ -208,15 +210,52 @@ pub struct AnsibleInventory { resource_type: Option, pub pinned_resource: Option, } + +#[derive(Debug, Clone)] +pub struct GitRepo { + pub url: String, + pub commit: Option, + pub branch: Option, + pub target_path: String, +} + #[derive(Debug, Clone)] pub struct AnsibleRequirements { pub python_reqs: Vec, - pub collections: Option, + pub roles_and_collections: Option, pub file_resources: Vec, pub inventories: Vec, pub vars: Vec<(String, String)>, pub resources: Vec<(String, String)>, pub options: AnsiblePlaybookOptions, + pub vault_password: Option, + pub vault_id: Vec, + pub git_repos: Vec, + pub git_ssh_identity: Vec, +} + +impl Default for AnsibleRequirements { + fn default() -> Self { + Self { + python_reqs: vec![], + roles_and_collections: None, + file_resources: vec![], + inventories: vec![], + vars: vec![], + resources: vec![], + options: AnsiblePlaybookOptions { + verbosity: None, + forks: None, + timeout: None, + flush_cache: None, + force_handlers: None, + }, + vault_password: None, + vault_id: vec![], + git_repos: vec![], + git_ssh_identity: vec![], + } + } } fn parse_inventories(inventory_yaml: &Yaml) -> anyhow::Result> { @@ -275,22 +314,7 @@ pub fn parse_ansible_reqs( return Ok((logs, None, inner_content.to_string())); } - let opts = AnsiblePlaybookOptions { - verbosity: None, - forks: None, - timeout: None, - flush_cache: None, - force_handlers: None, - }; - let mut ret = AnsibleRequirements { - python_reqs: vec![], - collections: None, - file_resources: vec![], - inventories: vec![], - vars: vec![], - resources: vec![], - options: opts, - }; + let mut ret = AnsibleRequirements::default(); if let Yaml::Hash(doc) = &docs[0] { for (key, value) in doc { @@ -303,7 +327,7 @@ pub fn parse_ansible_reqs( let mut out_str = String::new(); let mut emitter = YamlEmitter::new(&mut out_str); emitter.dump(galaxy_requirements)?; - ret.collections = Some(out_str); + ret.roles_and_collections = Some(out_str); } if let Some(Yaml::Array(py_reqs)) = deps.get(&Yaml::String("python".to_string())) @@ -345,11 +369,60 @@ pub fn parse_ansible_reqs( Yaml::String(key) if key == "inventory" => { ret.inventories = parse_inventories(value)?; } + Yaml::String(key) if key == "vault_password" => { + let Yaml::String(filename) = value else { + return Err(anyhow!( + "Vault Password File expects a String containing the file name" + )); + }; + ret.vault_password = Some(filename.to_string()); + } + Yaml::String(key) if key == "vault_id" => { + let Yaml::Array(filenames) = value else { + return Err(anyhow!("Vault ID field expects an array of strings in the format: `label@filename`")); + }; + + for f in filenames { + let Yaml::String(filename) = f else { + return Err(anyhow!("The elements of the vault_id field should be strings in the format: `label@filename`")); + }; + ret.vault_id.push(filename.to_string()); + } + } Yaml::String(key) if key == "options" => { if let Yaml::Array(opts) = &value { ret.options = parse_ansible_options(opts); } } + Yaml::String(key) if key == "git_repos" => { + let Yaml::Array(repos) = &value else { + return Err(anyhow!("git_repos field expects an array of repos")); + }; + + for r in repos { + ret.git_repos.push( + parse_git_repo(r) + .map_err(|e| anyhow!("Failed to parse git repo: {e}"))?, + ); + } + } + Yaml::String(key) if key == "git_ssh_identity" => { + let Yaml::Array(indentities) = &value else { + return Err(anyhow!( + "git_ssh_identity expects an array of windmill variables (or secrets) containing ssh IDs" + )); + }; + + for r in indentities { + let Yaml::String(file_name) = r else { + return Err(anyhow!( + "Git ssh identity file must be a string path to a Windmill variable/secret" + )); + }; + + ret.git_ssh_identity.push(file_name.clone()); + } + } Yaml::String(key) => logs.push_str(&format!("\nUnknown field `{}`. Ignoring", key)), _ => (), } @@ -364,6 +437,38 @@ pub fn parse_ansible_reqs( Ok((logs, Some(ret), out_str)) } +fn parse_git_repo(r: &Yaml) -> anyhow::Result { + let Yaml::Hash(repo) = r else { + return Err(anyhow!("Should be a Map")); + }; + + let url = repo + .get(&Yaml::String("url".to_string())) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .ok_or(anyhow!("Expected `url` field"))?; + + let target_path = repo + .get(&Yaml::String("target".to_string())) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .ok_or(anyhow!( + "Expected `target` field (target directory for cloning the repo)" + ))?; + + let branch = repo + .get(&Yaml::String("branch".to_string())) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + + let commit = repo + .get(&Yaml::String("commit".to_string())) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + + Ok(GitRepo { url, commit, branch, target_path }) +} + fn parse_ansible_options(opts: &Vec) -> AnsiblePlaybookOptions { let mut ret = AnsiblePlaybookOptions { verbosity: None, @@ -529,3 +634,78 @@ fn yaml_to_json(yaml: &Yaml) -> serde_json::Value { _ => serde_json::Value::Null, } } + +fn update_versions( + section: &str, + yaml: &mut Yaml, + versions: &HashMap, +) -> anyhow::Result { + let mut logs = String::new(); + + let Yaml::Hash(ref mut m) = yaml else { + return Err(anyhow!("{section} dependency should be a map")); + }; + + if let Some(Yaml::Array(elements)) = m.get_mut(&Yaml::String(section.to_string())) { + for el in elements { + let Yaml::Hash(ref mut h) = el else { + return Err(anyhow!("{section} dependency element should be a map")); + }; + + if let Some(name) = h + .get(&Yaml::String("name".to_string())) + .and_then(|n| n.as_str()) + { + if let Some(version) = versions.get(name) { + h.insert( + Yaml::String("version".to_string()), + Yaml::String(version.to_string()), + ); + } else { + logs.push_str(&format!("WARNING: {section} dependency `{name}` has no locked version, using the latest or system installed version.\n")); + } + } else { + return Err(anyhow!( + "{section} dependency element: missing or invalid `name` field" + )); + } + } + } + + Ok(logs) +} + +pub fn add_versions_to_requirements_yaml( + input: &str, + role_versions: &HashMap, + collection_versions: &HashMap, +) -> anyhow::Result<(String,String)> { + let mut docs = + YamlLoader::load_from_str(input).map_err(|e| anyhow!("YAML parse error: {}", e))?; + let doc = &mut docs[0]; + + let mut logs = String::new(); + + logs.push_str( + &update_versions("roles", doc, role_versions) + .map_err(|e| anyhow!("Error updating role versions: {e}"))?, + ); + logs.push_str( + &update_versions("collections", doc, collection_versions) + .map_err(|e| anyhow!("Error updating collection versions: {e}"))?, + ); + + if !logs.is_empty() { + logs.push_str("WARNING: You might want to try adding manual versions for these, otherwise there could be breaking changes on deployed scripts\n"); + } + + let mut out_str = String::new(); + { + let mut emitter = YamlEmitter::new(&mut out_str); + emitter + .dump(doc) + .map_err(|e| anyhow!("YAML emit error: {}", e))?; + } + + Ok((out_str, logs)) +} diff --git a/backend/src/main.rs b/backend/src/main.rs index 7f4bcda628..1e4f8010eb 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -8,11 +8,12 @@ use anyhow::Context; use monitor::{ - load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config, + load_base_url, load_otel, reload_critical_alerts_on_db_oversize, + reload_delete_logs_periodically_setting, reload_indexer_config, reload_instance_python_version_setting, reload_maven_repos_setting, reload_no_default_maven_setting, reload_nuget_config_setting, reload_timeout_wait_result_setting, send_current_log_file_to_object_store, - send_logs_to_object_store, + send_logs_to_object_store, WORKERS_NAMES, }; use rand::Rng; use sqlx::postgres::PgListener; @@ -33,15 +34,16 @@ use windmill_common::{ agent_workers::build_agent_http_client, get_database_url, global_settings::{ - BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, - CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, - DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, - EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - HUB_BASE_URL_SETTING, INDEXER_SETTING, INSTANCE_PYTHON_VERSION_SETTING, - JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, - LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, - NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, - OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, + CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, + DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, + ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, + INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, + KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, + MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, + NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, + PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, @@ -265,6 +267,8 @@ async fn windmill_main() -> anyhow::Result<()> { if mode == Mode::Standalone { println!("Running in standalone mode"); + } else if mode == Mode::MCP { + println!("Running in MCP mode"); } #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] @@ -297,7 +301,7 @@ async fn windmill_main() -> anyhow::Result<()> { } #[allow(unused_mut)] - let mut num_workers = if mode == Mode::Server || mode == Mode::Indexer { + let mut num_workers = if mode == Mode::Server || mode == Mode::Indexer || mode == Mode::MCP { 0 } else { std::env::var("NUM_WORKERS") @@ -319,8 +323,9 @@ async fn windmill_main() -> anyhow::Result<()> { && (mode == Mode::Server || mode == Mode::Standalone); let indexer_mode = mode == Mode::Indexer; + let mcp_mode = mode == Mode::MCP; - let server_bind_address: IpAddr = if server_mode || indexer_mode { + let server_bind_address: IpAddr = if server_mode || indexer_mode || mcp_mode { std::env::var("SERVER_BIND_ADDR") .ok() .and_then(|x| x.parse().ok()) @@ -380,7 +385,7 @@ async fn windmill_main() -> anyhow::Result<()> { .is_some_and(|x| x == "1" || x == "true"); if let Some(db) = conn.as_sql() { - if !is_agent && !indexer_mode { + if !is_agent && !indexer_mode && !mcp_mode { let skip_migration = std::env::var("SKIP_MIGRATION") .map(|val| val == "true") .unwrap_or(false); @@ -443,7 +448,7 @@ Windmill Community Edition {GIT_VERSION} if !valid_key && !server_mode { tracing::error!("Invalid license key, workers require a valid license key"); } - if server_mode { + if server_mode || mcp_mode { if let Some(db) = conn.as_sql() { // only force renewal if invalid but not empty (= expired) let renewed_now = maybe_renew_license_key_on_start( @@ -463,10 +468,10 @@ Windmill Community Edition {GIT_VERSION} } } - if server_mode || worker_mode || indexer_mode { + if server_mode || worker_mode || indexer_mode || mcp_mode { let port_var = std::env::var("PORT").ok().and_then(|x| x.parse().ok()); - let port = if server_mode || indexer_mode { + let port = if server_mode || indexer_mode || mcp_mode { port_var.unwrap_or(DEFAULT_PORT as u16) } else { port_var.unwrap_or(0) @@ -647,6 +652,7 @@ Windmill Community Edition {GIT_VERSION} server_killpill_rx, base_internal_tx, server_mode, + mcp_mode, base_internal_url.clone(), ) .await?; @@ -781,6 +787,29 @@ Windmill Community Edition {GIT_VERSION} tracing::info!("Workspace premium change detected, invalidating workspace premium cache: {}", workspace_id); windmill_common::workspaces::IS_PREMIUM_CACHE.remove(workspace_id); }, + "notify_runnable_version_change" => { + let payload = n.payload(); + tracing::info!("Runnable version change detected: {}", payload); + match payload.split(':').collect::>().as_slice() { + [workspace_id, source_type, path] => { + let key = (workspace_id.to_string(), path.to_string()); + match source_type { + &"script" => { + windmill_common::DEPLOYED_SCRIPT_HASH_CACHE.remove(&key); + } + &"flow" => { + windmill_common::FLOW_VERSION_CACHE.remove(&key); + }, + _ => { + tracing::warn!("Unknown runnable version change payload: {}", payload); + } + } + }, + _ => { + tracing::warn!("Unknown runnable version change payload: {}", payload); + } + } + }, "notify_global_setting_change" => { tracing::info!("Global setting change detected: {}", n.payload()); match n.payload() { @@ -913,6 +942,12 @@ Windmill Community Edition {GIT_VERSION} tracing::error!(error = %e, "Could not reload critical error emails setting"); } }, + CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING => { + if let Err(e) = reload_critical_alerts_on_db_oversize(&db).await { + tracing::error!(error = %e, "Could not reload critical alerts on db oversize setting"); + } + + }, JWT_SECRET_SETTING => { if let Err(e) = reload_jwt_secret_setting(&db).await { tracing::error!(error = %e, "Could not reload jwt secret setting"); @@ -1049,15 +1084,19 @@ Windmill Community Edition {GIT_VERSION} } } - futures::try_join!( - shutdown_signal, - workers_f, - monitor_f, - server_f, - metrics_f, - indexer_f, - log_indexer_f - )?; + if mcp_mode { + futures::try_join!(shutdown_signal, workers_f, server_f)?; + } else { + futures::try_join!( + shutdown_signal, + workers_f, + monitor_f, + server_f, + metrics_f, + indexer_f, + log_indexer_f + )?; + } } else { tracing::info!("Nothing to do, exiting."); } @@ -1092,6 +1131,7 @@ async fn listen_pg(url: &str) -> Option { "notify_global_setting_change", "notify_webhook_change", "notify_workspace_envs_change", + "notify_runnable_version_change", ]; #[cfg(feature = "cloud")] channels.push("notify_workspace_premium_change"); @@ -1206,10 +1246,12 @@ pub async fn run_workers( "Starting {num_workers} workers and SLEEP_QUEUE={}ms", *windmill_worker::SLEEP_QUEUE ); + for i in 1..(num_workers + 1) { let wk_conf = &workers[i as usize - 1]; let conn1 = wk_conf.conn.clone(); let worker_name = wk_conf.worker_name.clone(); + WORKERS_NAMES.write().await.push(worker_name.clone()); let ip = ip.clone(); let rx = killpill_rxs.pop().unwrap(); let tx = tx.clone(); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 8dae3f8c5e..cf04ff8c32 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -13,7 +13,7 @@ use std::{ use chrono::{NaiveDateTime, Utc}; use futures::{stream::FuturesUnordered, StreamExt}; -use serde::de::DeserializeOwned; +use serde::{de::DeserializeOwned, Deserialize}; use sqlx::{Pool, Postgres}; use tokio::{ join, @@ -28,30 +28,52 @@ use windmill_api::{ SCIM_TOKEN, }; +#[cfg(feature = "enterprise")] +use windmill_common::ee::low_disk_alerts; #[cfg(feature = "enterprise")] use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts}; #[cfg(feature = "oauth2")] use windmill_common::global_settings::OAUTH_SETTING; use windmill_common::{ - utils::empty_string_as_none, - agent_workers::DECODED_AGENT_TOKEN, auth::create_token_for_owner, ee::CriticalErrorChannel, error, flow_status::{FlowStatus, FlowStatusModule}, global_settings::{ - BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, - CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, - DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, - EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, - JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, - LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, - NUGET_CONFIG_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + agent_workers::DECODED_AGENT_TOKEN, + auth::create_token_for_owner, + ee::CriticalErrorChannel, + error, + flow_status::{FlowStatus, FlowStatusModule}, + global_settings::{ + BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, + CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, + DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, + EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, + HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, + JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, + MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, + OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, - }, indexer::load_indexer_config, jobs::QueuedJob, jwt::JWT_SECRET, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, server::load_smtp_config, tracing_init::JSON_FMT, users::truncate_token, utils::{now_from_db, rd_string, report_critical_error, Mode}, worker::{ - load_worker_config, reload_custom_tags_setting, store_pull_query, store_suspended_pull_query, update_min_version, Connection, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, TMP_DIR, WORKER_CONFIG, WORKER_GROUP - }, KillpillSender, BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS, + }, + indexer::load_indexer_config, + jobs::QueuedJob, + jwt::JWT_SECRET, + oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, + server::load_smtp_config, + tracing_init::JSON_FMT, + users::truncate_token, + utils::{empty_as_none, now_from_db, rd_string, report_critical_error, Mode}, + worker::{ + load_env_vars, load_init_bash_from_env, load_whitelist_env_vars_from_env, load_worker_config, reload_custom_tags_setting, store_pull_query, store_suspended_pull_query, update_min_version, Connection, WorkerConfig, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, TMP_DIR, WORKER_CONFIG, WORKER_GROUP + }, + KillpillSender, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERT_MUTE_UI_ENABLED, + CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, + METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, + OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS, }; use windmill_queue::{cancel_job, MiniPulledJob, SameWorkerPayload}; use windmill_worker::{ - handle_job_error, AuthedClient, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL + handle_job_error, AuthedClient, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES, + INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, + NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, }; #[cfg(feature = "parquet")] @@ -111,7 +133,7 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(false); - + pub static ref WORKERS_NAMES: Arc>> = Arc::new(RwLock::new(Vec::new())); static ref QUEUE_COUNT_TAGS: Arc>> = Arc::new(RwLock::new(Vec::new())); static ref DISABLE_CONCURRENCY_LIMIT: bool = std::env::var("DISABLE_CONCURRENCY_LIMIT").is_ok_and(|s| s == "true"); @@ -135,7 +157,6 @@ pub async fn initial_load( } } - if let Err(e) = load_metrics_enabled(conn).await { tracing::error!("Error loading expose metrics: {e:#}"); } @@ -161,6 +182,12 @@ pub async fn initial_load( if server_mode { if let Some(db) = conn.as_sql() { load_require_preexisting_user(db).await; + if let Err(e) = reload_critical_alerts_on_db_oversize(db).await { + tracing::error!( + "Error reloading critical alerts on db oversize setting: {:?}", + e + ) + } } } @@ -172,11 +199,26 @@ pub async fn initial_load( } Connection::Http(_) => { // TODO: reload worker config from http - WORKER_CONFIG.write().await.worker_tags = DECODED_AGENT_TOKEN.as_ref().map(|x| x.tags.clone()).unwrap_or_default(); + let mut config = WORKER_CONFIG.write().await; + *config = WorkerConfig { + worker_tags: DECODED_AGENT_TOKEN + .as_ref() + .map(|x| x.tags.clone()) + .unwrap_or_default(), + env_vars: load_env_vars( + load_whitelist_env_vars_from_env(), + &std::collections::HashMap::new(), + ), + priority_tags_sorted: vec![], + dedicated_worker: None, + init_bash: load_init_bash_from_env(), + cache_clear: None, + additional_python_paths: None, + pip_local_dependencies: None, + }; } } } - if let Err(e) = reload_hub_base_url_setting(conn, server_mode).await { tracing::error!("Error reloading hub base url: {:?}", e) @@ -190,7 +232,6 @@ pub async fn initial_load( if let Err(e) = reload_custom_tags_setting(db).await { tracing::error!("Error reloading custom tags: {:?}", e) } - } #[cfg(feature = "parquet")] @@ -225,7 +266,8 @@ pub async fn initial_load( } pub async fn load_metrics_enabled(conn: &Connection) -> error::Result<()> { - let metrics_enabled = load_value_from_global_settings_with_conn(conn, EXPOSE_METRICS_SETTING, true).await; + let metrics_enabled = + load_value_from_global_settings_with_conn(conn, EXPOSE_METRICS_SETTING, true).await; match metrics_enabled { Ok(Some(serde_json::Value::Bool(t))) => METRICS_ENABLED.store(t, Ordering::Relaxed), _ => (), @@ -238,13 +280,13 @@ struct OtelSetting { metrics_enabled: Option, logs_enabled: Option, tracing_enabled: Option, - #[serde(default, deserialize_with = "empty_string_as_none")] + #[serde(default, deserialize_with = "empty_as_none")] otel_exporter_otlp_endpoint: Option, - #[serde(default, deserialize_with = "empty_string_as_none")] + #[serde(default, deserialize_with = "empty_as_none")] otel_exporter_otlp_headers: Option, - #[serde(default, deserialize_with = "empty_string_as_none")] + #[serde(default, deserialize_with = "empty_as_none")] otel_exporter_otlp_protocol: Option, - #[serde(default, deserialize_with = "empty_string_as_none")] + #[serde(default, deserialize_with = "empty_as_none")] otel_exporter_otlp_compression: Option, } @@ -345,13 +387,13 @@ pub async fn reload_critical_alert_mute_ui_setting(conn: &Connection) -> error:: load_value_from_global_settings_with_conn(conn, CRITICAL_ALERT_MUTE_UI_SETTING, true).await { CRITICAL_ALERT_MUTE_UI_ENABLED.store(t, Ordering::Relaxed); - } Ok(()) } pub async fn load_metrics_debug_enabled(conn: &Connection) -> error::Result<()> { - let metrics_enabled = load_value_from_global_settings_with_conn(conn, EXPOSE_DEBUG_METRICS_SETTING, true).await; + let metrics_enabled = + load_value_from_global_settings_with_conn(conn, EXPOSE_DEBUG_METRICS_SETTING, true).await; match metrics_enabled { Ok(Some(serde_json::Value::Bool(t))) => { METRICS_DEBUG_ENABLED.store(t, Ordering::Relaxed); @@ -575,7 +617,9 @@ async fn send_log_file_to_object_store( }; let exists = LAST_LOG_FILE_SENT.lock().map(|last_log_file_sent| { - last_log_file_sent.map(|last_log_file_sent| last_log_file_sent >= ts).unwrap_or(false) + last_log_file_sent + .map(|last_log_file_sent| last_log_file_sent >= ts) + .unwrap_or(false) }); if exists.unwrap_or(false) { @@ -612,7 +656,9 @@ async fn send_log_file_to_object_store( let (ok_lines, err_lines) = read_log_counters(ts_str); if let Some(db) = conn.as_sql() { - if let Err(e) = sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)", + if let Err(e) = sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) + VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8) + ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", hostname, mode.to_string(), worker_group.clone(), ts, highest_file, ok_lines as i64, err_lines as i64, *JSON_FMT) .execute(db) .await { @@ -1002,11 +1048,21 @@ pub async fn reload_nuget_config_setting(conn: &Connection) { .await; } pub async fn reload_maven_repos_setting(conn: &Connection) { - reload_option_setting_with_tracing(conn, windmill_common::global_settings::MAVEN_REPOS_SETTING, "MAVEN_REPOS", MAVEN_REPOS.clone()) - .await; + reload_option_setting_with_tracing( + conn, + windmill_common::global_settings::MAVEN_REPOS_SETTING, + "MAVEN_REPOS", + MAVEN_REPOS.clone(), + ) + .await; } pub async fn reload_no_default_maven_setting(conn: &Connection) { - let value = load_value_from_global_settings_with_conn(conn, windmill_common::global_settings::NO_DEFAULT_MAVEN_SETTING, true).await; + let value = load_value_from_global_settings_with_conn( + conn, + windmill_common::global_settings::NO_DEFAULT_MAVEN_SETTING, + true, + ) + .await; match value { Ok(Some(serde_json::Value::Bool(t))) => NO_DEFAULT_MAVEN.store(t, Ordering::Relaxed), Err(e) => { @@ -1175,7 +1231,6 @@ pub async fn load_value_from_global_settings( Ok(r) } - pub async fn load_value_from_global_settings_with_conn( conn: &Connection, setting_name: &str, @@ -1185,14 +1240,18 @@ pub async fn load_value_from_global_settings_with_conn( Connection::Sql(db) => Ok(load_value_from_global_settings(db, setting_name).await?), Connection::Http(client) => { if load_from_http { - client.get::>(&format!("/api/agent_workers/get_global_setting/{}", setting_name)).await - .map_err(|e| anyhow::anyhow!("Error loading setting {}: {}", setting_name, e)) + client + .get::>(&format!( + "/api/agent_workers/get_global_setting/{}", + setting_name + )) + .await + .map_err(|e| anyhow::anyhow!("Error loading setting {}: {}", setting_name, e)) } else { Ok(None) } } } - } pub async fn reload_option_setting( @@ -1314,12 +1373,12 @@ pub async fn monitor_db( let zombie_jobs_f = async { if server_mode && !initial_load && !*DISABLE_ZOMBIE_JOBS_MONITORING { if let Some(db) = conn.as_sql() { - handle_zombie_jobs(db, base_internal_url, "server").await; - match handle_zombie_flows(db).await { - Err(err) => { - tracing::error!("Error handling zombie flows: {:?}", err); - }, - _ => {} + handle_zombie_jobs(db, base_internal_url, "server").await; + match handle_zombie_flows(db).await { + Err(err) => { + tracing::error!("Error handling zombie flows: {:?}", err); + } + _ => {} } } } @@ -1327,7 +1386,7 @@ pub async fn monitor_db( let expired_items_f = async { if server_mode && !initial_load { if let Some(db) = conn.as_sql() { - delete_expired_items(&db).await; + delete_expired_items(&db).await; } } }; @@ -1365,6 +1424,23 @@ pub async fn monitor_db( } }; + let low_disk_alerts_f = async { + #[cfg(feature = "enterprise")] + if let Some(db) = conn.as_sql() { + low_disk_alerts( + &db, + server_mode, + _worker_mode, + WORKERS_NAMES.read().await.clone(), + ) + .await; + } + #[cfg(not(feature = "enterprise"))] + { + () + } + }; + let apply_autoscaling_f = async { #[cfg(feature = "enterprise")] if server_mode && !initial_load { @@ -1387,6 +1463,7 @@ pub async fn monitor_db( verify_license_key_f, worker_groups_alerts_f, jobs_waiting_alerts_f, + low_disk_alerts_f, apply_autoscaling_f, update_min_worker_version_f, ); @@ -1496,11 +1573,7 @@ pub async fn reload_indexer_config(db: &Pool) { } } -pub async fn reload_worker_config( - db: &DB, - tx: KillpillSender, - kill_if_change: bool, -) { +pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: bool) { let config = load_worker_config(db, tx.clone()).await; if let Err(e) = config { tracing::error!("Error reloading worker config: {:?}", e) @@ -1543,7 +1616,8 @@ pub async fn reload_worker_config( } pub async fn load_base_url(conn: &Connection) -> error::Result { - let q_base_url = load_value_from_global_settings_with_conn(conn, BASE_URL_SETTING, false).await?; + let q_base_url = + load_value_from_global_settings_with_conn(conn, BASE_URL_SETTING, false).await?; let std_base_url = std::env::var("BASE_URL") .ok() @@ -1574,10 +1648,9 @@ pub async fn load_base_url(conn: &Connection) -> error::Result { } pub async fn reload_base_url_setting(conn: &Connection) -> error::Result<()> { - #[cfg(feature = "oauth2")] let oauths = if let Some(db) = conn.as_sql() { - let q_oauth = load_value_from_global_settings (db, OAUTH_SETTING).await?; + let q_oauth = load_value_from_global_settings(db, OAUTH_SETTING).await?; if let Some(q) = q_oauth { if let Ok(v) = serde_json::from_value::< @@ -1652,7 +1725,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker increment_counter AS ( INSERT INTO zombie_job_counter (job_id, counter) SELECT id, 1 FROM to_update WHERE counter < $2 - ON CONFLICT (job_id) DO UPDATE + ON CONFLICT (job_id) DO UPDATE SET counter = zombie_job_counter.counter + 1 ), update_concurrency AS ( @@ -1744,7 +1817,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker let same_worker_timeout_jobs = { let long_same_worker_jobs = sqlx::query!( - "SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval + "SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval AND running = true AND ping IS NULL AND same_worker = true AND worker IS NOT NULL GROUP BY worker", ) .fetch_all(db) @@ -1758,9 +1831,9 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker .collect::>(); let long_dead_workers: std::collections::HashSet = sqlx::query_scalar!( - "WITH worker_ids AS (SELECT unnest($1::text[]) as worker) - SELECT worker_ids.worker FROM worker_ids - LEFT JOIN worker_ping ON worker_ids.worker = worker_ping.worker + "WITH worker_ids AS (SELECT unnest($1::text[]) as worker) + SELECT worker_ids.worker FROM worker_ids + LEFT JOIN worker_ping ON worker_ids.worker = worker_ping.worker WHERE worker_ping.worker IS NULL OR worker_ping.ping_at < now() - ('60 seconds')::interval", &worker_ids[..] ) @@ -1804,7 +1877,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker let non_restartable_jobs = if *RESTART_ZOMBIE_JOBS { vec![] } else { - sqlx::query_as::<_, QueuedJob>("SELECT * FROM v2_as_queue WHERE last_ping < now() - ($1 || ' seconds')::interval + sqlx::query_as::<_, QueuedJob>("SELECT * FROM v2_as_queue WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow') AND same_worker = false") .bind(ZOMBIE_JOB_TIMEOUT.as_str()) .fetch_all(db) @@ -1863,7 +1936,8 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker mpsc::channel::(1); let same_worker_tx_never_used = SameWorkerSender(same_worker_tx_never_used, Arc::new(AtomicU16::new(0))); - let (send_result_never_used, _send_result_rx_never_used) = JobCompletedSender::new_never_used(); + let (send_result_never_used, _send_result_rx_never_used) = + JobCompletedSender::new_never_used(); let label = if job.permissioned_as != format!("u/{}", job.created_by) && job.permissioned_as != job.created_by @@ -2001,7 +2075,7 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { } ); report_critical_error(reason.clone(), db.clone(), Some(&flow.workspace_id), None).await; - cancel_zombie_flow_job(db, flow.id, &flow.workspace_id, + cancel_zombie_flow_job(db, flow.id, &flow.workspace_id, format!(r#"{reason} This would happen if a worker was interrupted, killed or crashed while doing a state transition at the end of a job which is always an unexpected behavior that should never happen. Please check your worker logs for more details and feel free to report it to the Windmill team on our Discord or support@windmill.dev (response for non EE customers will be best effort) with as much context as possible, ideally: @@ -2018,7 +2092,7 @@ Please check your worker logs for more details and feel free to report it to the r#" DELETE FROM parallel_monitor_lock - WHERE last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval + WHERE last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval RETURNING parent_flow_id, job_id, last_ping, (SELECT workspace_id FROM v2_job_queue q WHERE q.id = parent_flow_id AND q.running = true AND q.canceled_by IS NULL ) AS workspace_id @@ -2073,8 +2147,12 @@ async fn cancel_zombie_flow_job( Ok(()) } -pub async fn reload_hub_base_url_setting(conn: &Connection, server_mode: bool) -> error::Result<()> { - let hub_base_url = load_value_from_global_settings_with_conn(conn, HUB_BASE_URL_SETTING, true).await?; +pub async fn reload_hub_base_url_setting( + conn: &Connection, + server_mode: bool, +) -> error::Result<()> { + let hub_base_url = + load_value_from_global_settings_with_conn(conn, HUB_BASE_URL_SETTING, true).await?; let base_url = if let Some(q) = hub_base_url { if let Ok(v) = serde_json::from_value::(q.clone()) { @@ -2126,7 +2204,7 @@ pub async fn reload_critical_error_channels_setting(conn: &DB) -> error::Result< v } else { tracing::error!( - "Could not parse critical_error_emails setting as an array of channels, found: {:#?}", + "Could not parse critical_error_channels setting as an array of channels, found: {:#?}", &q ); vec![] @@ -2141,6 +2219,39 @@ pub async fn reload_critical_error_channels_setting(conn: &DB) -> error::Result< Ok(()) } +pub async fn reload_critical_alerts_on_db_oversize(conn: &DB) -> error::Result<()> { + #[derive(Deserialize)] + struct DBOversize { + #[serde(default)] + enabled: bool, + #[serde(default)] + value: f32, + } + let db_oversize_value = + load_value_from_global_settings(conn, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING).await?; + + let db_oversize = if let Some(q) = db_oversize_value { + match serde_json::from_value::(q.clone()) { + Ok(DBOversize { enabled: true, value }) => Some(value), + Ok(_) => None, + Err(q) => { + tracing::error!( + "Could not parse critical_alerts_on_db_oversize setting, found: {:#?}", + &q + ); + None + } + } + } else { + None + }; + + let mut l = CRITICAL_ALERTS_ON_DB_OVERSIZE.write().await; + *l = db_oversize; + + Ok(()) +} + async fn generate_and_save_jwt_secret(db: &DB) -> error::Result { let secret = rd_string(32); sqlx::query!( diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 182ed60f2e..8b7509bb84 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -16,16 +16,20 @@ use tokio::sync::RwLock; #[cfg(feature = "enterprise")] use tokio::time::{timeout, Duration}; +#[cfg(feature = "python")] use windmill_api_client::types::{CreateFlowBody, RawScript}; #[cfg(feature = "enterprise")] use windmill_api_client::types::{EditSchedule, NewSchedule, ScriptArgs}; use windmill_api_client::types::{NewScript, ScriptLang as NewScriptLanguage}; use serde::Serialize; +#[cfg(feature = "deno_core")] +use windmill_common::flows::InputTransform; use windmill_common::worker::WORKER_CONFIG; + use windmill_common::{ flow_status::{FlowStatus, FlowStatusModule, RestartedFrom}, - flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform}, + flows::{FlowModule, FlowModuleValue, FlowValue}, jobs::{JobKind, JobPayload, RawCode}, jwt::JWT_SECRET, scripts::{ScriptHash, ScriptLang}, @@ -137,10 +141,11 @@ impl ApiServer { rx, port_tx, false, + false, format!("http://localhost:{}", addr.port()), )); - _port_rx.await.unwrap(); + _port_rx.await.expect("failed to receive port"); // clear the cache between tests windmill_common::cache::clear(); @@ -166,6 +171,7 @@ impl ApiServer { // Ok(()) // } +#[cfg(feature = "python")] fn get_module(cjob: &CompletedJob, id: &str) -> Option { cjob.flow_status.clone().and_then(|fs| { find_module_in_vec( @@ -175,6 +181,7 @@ fn get_module(cjob: &CompletedJob, id: &str) -> Option { }) } +#[cfg(feature = "python")] fn find_module_in_vec(modules: Vec, id: &str) -> Option { modules.into_iter().find(|s| s.id() == id) } @@ -283,6 +290,7 @@ mod suspend_resume { .unwrap() } + #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test(db: Pool) { initialize_tracing().await; @@ -365,6 +373,7 @@ mod suspend_resume { ); } + #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn cancel_from_job(db: Pool) { initialize_tracing().await; @@ -390,6 +399,7 @@ mod suspend_resume { ); } + #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn cancel_after_suspend(db: Pool) { initialize_tracing().await; @@ -563,6 +573,7 @@ def main(last, port): .unwrap() } + #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_pass(db: Pool) { initialize_tracing().await; @@ -608,6 +619,7 @@ def main(last, port): assert_eq!(json!([3, 5, 7, 9]), result); } + #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_fail_step_zero(db: Pool) { initialize_tracing().await; @@ -651,6 +663,7 @@ def main(last, port): ); } + #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_fail_step_one(db: Pool) { initialize_tracing().await; @@ -692,6 +705,7 @@ def main(last, port): .contains("index out of range")); } + #[cfg(feature = "python")] #[sqlx::test(fixtures("base"))] async fn test_with_failure_module(db: Pool) { initialize_tracing().await; @@ -768,6 +782,7 @@ def main(error, port): } } +#[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_iteration(db: Pool) { initialize_tracing().await; @@ -826,6 +841,7 @@ async fn test_iteration(db: Pool) { .contains("2")); } +#[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_iteration_parallel(db: Pool) { initialize_tracing().await; @@ -1104,6 +1120,7 @@ trait StreamFind: futures::Stream + Unpin + Sized { impl StreamFind for T {} +#[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_deno_flow(db: Pool) { initialize_tracing().await; @@ -1222,6 +1239,7 @@ async fn test_deno_flow(db: Pool) { } } +#[cfg(feature = "python")] #[sqlx::test(fixtures("base"))] async fn test_identity(db: Pool) { initialize_tracing().await; @@ -1259,6 +1277,7 @@ async fn test_identity(db: Pool) { assert_eq!(result, serde_json::json!(42)); } +#[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_deno_flow_same_worker(db: Pool) { initialize_tracing().await; @@ -1534,6 +1553,7 @@ async fn test_flow_result_by_id(db: Pool) { assert_eq!(result, serde_json::json!([[42]])); } +#[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_stop_after_if(db: Pool) { initialize_tracing().await; @@ -1587,6 +1607,7 @@ async fn test_stop_after_if(db: Pool) { assert_eq!(json!(-123), result); } +#[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_stop_after_if_nested(db: Pool) { initialize_tracing().await; @@ -1645,6 +1666,7 @@ async fn test_stop_after_if_nested(db: Pool) { assert_eq!(json!([-123]), result); } +#[cfg(all(feature = "deno_core", feature = "python"))] #[sqlx::test(fixtures("base"))] async fn test_python_flow(db: Pool) { initialize_tracing().await; @@ -1702,6 +1724,7 @@ async fn test_python_flow(db: Pool) { } } +#[cfg(feature = "python")] #[sqlx::test(fixtures("base"))] async fn test_python_flow_2(db: Pool) { initialize_tracing().await; @@ -1776,6 +1799,7 @@ func main(derp string) (string, error) { assert_eq!(result, serde_json::json!("hello world")); } +#[cfg(feature = "rust")] #[sqlx::test(fixtures("base"))] async fn test_rust_job(db: Pool) { initialize_tracing().await; @@ -2031,6 +2055,7 @@ public class Main { assert_eq!(job.json_result(), Some(json!("hello world"))); } +#[cfg(feature = "python")] #[sqlx::test(fixtures("base"))] async fn test_python_job(db: Pool) { initialize_tracing().await; @@ -2064,6 +2089,7 @@ def main(): assert_eq!(result, serde_json::json!("hello world")); } +#[cfg(feature = "python")] #[sqlx::test(fixtures("base"))] async fn test_python_job_heavy_dep(db: Pool) { initialize_tracing().await; @@ -2100,6 +2126,7 @@ def main(): assert_eq!(result, serde_json::json!(3)); } +#[cfg(feature = "python")] #[sqlx::test(fixtures("base"))] async fn test_python_job_with_imports(db: Pool) { initialize_tracing().await; @@ -2203,6 +2230,7 @@ export async function main(a: Date) { assert_eq!(result, serde_json::json!("object")); } +#[cfg(feature = "python")] #[sqlx::test(fixtures("base"))] async fn test_python_job_datetime_and_bytes(db: Pool) { initialize_tracing().await; @@ -2238,6 +2266,7 @@ def main(a: datetime, b: bytes): assert_eq!(result, serde_json::json!([true, true])); } +#[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_empty_loop_1(db: Pool) { initialize_tracing().await; @@ -2294,6 +2323,7 @@ async fn test_empty_loop_1(db: Pool) { assert_eq!(result, serde_json::json!(0)); } +#[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_invalid_first_step(db: Pool) { initialize_tracing().await; @@ -2374,6 +2404,7 @@ async fn test_empty_loop_2(db: Pool) { assert_eq!(result, serde_json::json!([])); } +#[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_step_after_loop(db: Pool) { initialize_tracing().await; @@ -2497,6 +2528,7 @@ async fn test_branchone_simple(db: Pool) { assert_eq!(result, serde_json::json!([1, 2])); } +#[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_branchone_with_cond(db: Pool) { initialize_tracing().await; @@ -2533,6 +2565,7 @@ async fn test_branchone_with_cond(db: Pool) { assert_eq!(result, serde_json::json!([1, 3])); } +#[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_branchall_sequential(db: Pool) { initialize_tracing().await; @@ -2571,6 +2604,7 @@ async fn test_branchall_sequential(db: Pool) { assert_eq!(result, serde_json::json!([[1, 2], [1, 3]])); } +#[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_branchall_simple(db: Pool) { initialize_tracing().await; @@ -2698,6 +2732,7 @@ async fn test_branchall_skip_failure(db: Pool) { ); } +#[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_branchone_nested(db: Pool) { initialize_tracing().await; @@ -2817,6 +2852,7 @@ async fn test_branchall_nested(db: Pool) { ); } +#[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_failure_module(db: Pool) { initialize_tracing().await; @@ -2929,6 +2965,7 @@ async fn test_failure_module(db: Pool) { assert_eq!(json!({ "l": [0, 1, 2] }), result); } +#[cfg(feature = "python")] #[sqlx::test(fixtures("base"))] async fn test_flow_lock_all(db: Pool) { use futures::StreamExt; @@ -3067,6 +3104,7 @@ async fn test_flow_lock_all(db: Pool) { }); } +#[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_complex_flow_restart(db: Pool) { @@ -3751,6 +3789,7 @@ export async function main() { run_preview_relative_imports(&db, content, ScriptLang::Bun).await; } +#[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base", "relative_bun"))] async fn test_nested_imports_bun(db: Pool) { let content = r#" @@ -3799,6 +3838,7 @@ export async function main() { run_preview_relative_imports(&db, content, ScriptLang::Deno).await; } +#[cfg(feature = "python")] #[sqlx::test(fixtures("base", "relative_python"))] async fn test_relative_imports_python(db: Pool) { let content = r#" @@ -3816,6 +3856,7 @@ def main(): run_preview_relative_imports(&db, content, ScriptLang::Python3).await; } +#[cfg(feature = "python")] #[sqlx::test(fixtures("base", "relative_python"))] async fn test_nested_imports_python(db: Pool) { let content = r#" @@ -3831,6 +3872,7 @@ def main(): run_preview_relative_imports(&db, content, ScriptLang::Python3).await; } +#[cfg(feature = "python")] async fn assert_lockfile( db: &Pool, script_content: String, @@ -3923,6 +3965,8 @@ async fn assert_lockfile( ) .await; } + +#[cfg(feature = "python")] #[sqlx::test(fixtures("base", "lockfile_python"))] async fn test_requirements_python(db: Pool) { let content = r#" @@ -3948,6 +3992,8 @@ def main(): ) .await; } + +#[cfg(feature = "python")] #[sqlx::test(fixtures("base", "lockfile_python"))] async fn test_extra_requirements_python(db: Pool) { { @@ -3975,9 +4021,10 @@ def main(): .await; } } + +#[cfg(feature = "python")] #[sqlx::test(fixtures("base", "lockfile_python"))] async fn test_extra_requirements_python2(db: Pool) { - let content = r#" # py311 # extra_requirements: @@ -3993,26 +4040,23 @@ def main(): &db, content, ScriptLang::Python3, - vec![ - "# py311", - "simplejson==3.20.1", - "tiny==0.1.3" - ], + vec!["# py311", "simplejson==3.20.1", "tiny==0.1.3"], ) .await; - } +#[cfg(feature = "python")] #[sqlx::test(fixtures("base", "lockfile_python"))] async fn test_pins_python(db: Pool) { let content = r#" # py311 # extra_requirements: # tiny==0.1.3 +# bottle==0.13.2 import f.system.requirements import f.system.pins -import tiny # repin: bottle==0.13.0 +import tiny # repin: tiny==0.1.3 import simplejson def main(): @@ -4026,10 +4070,10 @@ def main(): ScriptLang::Python3, vec![ "# py311", - "bottle==0.13.0", + "bottle==0.13.2", "microdot==2.2.0", "simplejson==3.19.3", - "tiny==0.1.3" + "tiny==0.1.3", ], ) .await; @@ -4285,6 +4329,7 @@ mod job_payload { ]; } + #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base", "hello"))] async fn test_script_hash_payload(db: Pool) { initialize_tracing().await; @@ -4445,6 +4490,7 @@ mod job_payload { test_for_versions(VERSION_FLAGS.iter().cloned(), test).await; } + #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base", "hello"))] async fn test_flow_node_payload(db: Pool) { initialize_tracing().await; @@ -4629,6 +4675,7 @@ mod job_payload { test_for_versions(VERSION_FLAGS.iter().cloned(), test).await; } + #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base", "hello"))] async fn test_flow_payload(db: Pool) { initialize_tracing().await; @@ -4640,6 +4687,7 @@ mod job_payload { path: "f/system/hello_with_nodes_flow".to_string(), dedicated_worker: None, apply_preprocessor: false, + version: 1443253234253454, }) .run_until_complete(&db, port) .await @@ -4671,6 +4719,7 @@ mod job_payload { test_for_versions(VERSION_FLAGS.iter().cloned(), test).await; } + #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base", "hello"))] async fn test_flow_payload_with_preprocessor(db: Pool) { initialize_tracing().await; @@ -4683,6 +4732,7 @@ mod job_payload { path: "f/system/hello_with_preprocessor".to_string(), dedicated_worker: None, apply_preprocessor: true, + version: 1443253234253456, }) .run_until_complete_with(db, port, |id| async move { let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id) @@ -4738,6 +4788,7 @@ mod job_payload { test_for_versions(VERSION_FLAGS.iter().cloned(), test).await; } + #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base", "hello"))] async fn test_restarted_flow_payload(db: Pool) { initialize_tracing().await; @@ -4749,6 +4800,7 @@ mod job_payload { path: "f/system/hello_with_nodes_flow".to_string(), dedicated_worker: None, apply_preprocessor: true, + version: 1443253234253454, }) .run_until_complete(&db, port) .await @@ -4790,6 +4842,7 @@ mod job_payload { test_for_versions(VERSION_FLAGS.iter().cloned(), test).await; } + #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base", "hello"))] async fn test_raw_flow_payload(db: Pool) { initialize_tracing().await; @@ -4836,6 +4889,7 @@ mod job_payload { test_for_versions(VERSION_FLAGS.iter().cloned(), test).await; } + #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base", "hello"))] async fn test_raw_flow_payload_with_restarted_from(db: Pool) { initialize_tracing().await; diff --git a/backend/windmill-api-client/codegen.rs b/backend/windmill-api-client/codegen.rs index ec1a363f88..202145159e 100644 --- a/backend/windmill-api-client/codegen.rs +++ b/backend/windmill-api-client/codegen.rs @@ -2750,6 +2750,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK pub expr: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub skip_if_stopped: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_message: Option } impl From<&FlowModuleStopAfterAllItersIf> for FlowModuleStopAfterAllItersIf { fn from(value: &FlowModuleStopAfterAllItersIf) -> Self { @@ -2761,6 +2763,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK pub expr: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub skip_if_stopped: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_message: Option } impl From<&FlowModuleStopAfterIf> for FlowModuleStopAfterIf { fn from(value: &FlowModuleStopAfterIf) -> Self { diff --git a/backend/windmill-api-client/src/codegen.rs b/backend/windmill-api-client/src/codegen.rs index 2ee220f87d..9bc26262d5 100644 --- a/backend/windmill-api-client/src/codegen.rs +++ b/backend/windmill-api-client/src/codegen.rs @@ -1824,6 +1824,8 @@ the execution of this script will be permissioned_as and by extension its DT_TOK pub expr: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub skip_if_stopped: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_message: Option } impl From<&FlowModuleStopAfterIf> for FlowModuleStopAfterIf { fn from(value: &FlowModuleStopAfterIf) -> Self { diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index c6702b8ac3..233fbeec54 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -15,7 +15,7 @@ stripe = [] agent_worker_server = [] enterprise_saml = ["dep:samael", "dep:libxml"] benchmark = [] -embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn", "dep:half"] +embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn"] parquet = ["dep:datafusion", "dep:object_store", "dep:url", "windmill-common/parquet", "windmill-worker/parquet"] prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker/prometheus"] openidconnect = ["dep:openidconnect"] @@ -35,8 +35,10 @@ sqs_trigger = ["dep:aws-sdk-sqs", "dep:thiserror", "dep:aws-config"] deno_core = ["dep:deno_core", "dep:deno_error"] gcp_trigger = ["dep:thiserror", "dep:google-cloud-pubsub", "dep:google-cloud-googleapis", "dep:tonic"] cloud = ["windmill-common/cloud"] +mcp = ["dep:rmcp"] [dependencies] +rmcp = { git = "https://github.com/windmill-labs/rust-sdk", features = ["transport-sse-server"], optional = true } windmill-queue.workspace = true windmill-common = { workspace = true, default-features = false } windmill-audit.workspace = true @@ -52,7 +54,6 @@ tokio-stream.workspace = true anyhow.workspace = true argon2.workspace = true axum.workspace = true -half = { workspace = true, optional = true} futures.workspace = true git-version.workspace = true tower.workspace = true diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 329cd80968..544b1348d1 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.483.1 + version: 1.490.0 title: Windmill API contact: @@ -9539,7 +9539,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/NewGcpTrigger" + $ref: "#/components/schemas/GcpTriggerData" responses: "201": description: gcp trigger created @@ -9563,7 +9563,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/EditGcpTrigger" + $ref: "#/components/schemas/GcpTriggerData" responses: "200": description: gcp trigger updated @@ -9757,7 +9757,7 @@ paths: type: array items: type: string - + /w/{workspace}/gcp_triggers/subscriptions/list/{path}: post: summary: list all subscription of a give topic from google cloud service @@ -14132,7 +14132,7 @@ components: csharp, nu, java - # for related places search: ADD_NEW_LANG + # for related places search: ADD_NEW_LANG ] Preview: @@ -15045,8 +15045,6 @@ components: type: string authenticate: type: boolean - base_endpoint: - type: string required: - authenticate - base_endpoint @@ -15068,6 +15066,8 @@ components: $ref: "#/components/schemas/DeliveryType" delivery_config: $ref: "#/components/schemas/PushConfig" + subscription_mode: + $ref: "#/components/schemas/SubscriptionMode" last_server_ping: type: string format: date-time @@ -15081,7 +15081,9 @@ components: - subscription_id - enabled - delivery_type - + - subscription_mode + + SubscriptionMode: type: string enum: @@ -15089,55 +15091,24 @@ components: - create_update description: "The mode of subscription. 'existing' means using an existing GCP subscription, while 'create_update' involves creating or updating a new subscription." - GcpExistingSubscription: + + GcpTriggerData: type: object properties: + gcp_resource_path: + type: string + subscription_mode: + $ref: "#/components/schemas/SubscriptionMode" + topic_id: + type: string subscription_id: type: string base_endpoint: type: string - required: - - subscription_id - - base_endpoint - - GcpCreateUpdateSubscription: - type: object - properties: - subscription_id: - type: string delivery_type: $ref: "#/components/schemas/DeliveryType" delivery_config: $ref: "#/components/schemas/PushConfig" - required: - - delivery_type - - - GcpSubscriptionModeConfig: - type: object - properties: - subscription_mode: - $ref: "#/components/schemas/SubscriptionMode" - required: - - subscription_mode - allOf: - - oneOf: - - $ref: "#/components/schemas/GcpExistingSubscription" - - $ref: "#/components/schemas/GcpCreateUpdateSubscription" - description: | - "This is a union type representing the subscription mode. - - 'existing': Represents an existing GCP subscription, and should be accompanied by an 'ExistingGcpSubscription' object. - - 'create_update': Represents a new or updated GCP subscription, and should be accompanied by a 'CreateUpdateConfig' object." - - NewGcpTrigger: - type: object - properties: - gcp_resource_path: - type: string - topic_id: - type: string - subscription_mode: - $ref: "#/components/schemas/GcpSubscriptionModeConfig" path: type: string script_path: @@ -15154,34 +15125,6 @@ components: - topic_id - subscription_mode - EditGcpTrigger: - type: object - properties: - gcp_resource_path: - type: string - topic_id: - type: string - subscription_mode: - $ref: "#/components/schemas/GcpSubscriptionModeConfig" - path: - type: string - script_path: - type: string - is_flow: - type: boolean - enabled: - type: boolean - required: - - path - - script_path - - is_flow - - enabled - - mqtt_resource_path - - subscription_id - - delivery_type - - topic_id - - subscription_mode - GetAllTopicSubscription: type: object properties: @@ -15204,7 +15147,7 @@ components: enum: - oidc - credentials - + SqsTrigger: allOf: - $ref: "#/components/schemas/TriggerExtraProperty" @@ -16596,8 +16539,8 @@ components: properties: trigger_kind: $ref: "#/components/schemas/CaptureTriggerKind" - payload: {} - trigger_extra: {} + main_args: {} + preprocessor_args: {} id: type: integer created_at: @@ -16605,7 +16548,8 @@ components: format: date-time required: - trigger_kind - - payload + - main_args + - preprocessor_args - id - created_at CaptureConfig: @@ -16761,4 +16705,29 @@ components: presigned: type: string required: - - s3 \ No newline at end of file + - s3 + + TeamsChannel: + type: object + required: + - team_id + - team_name + - channel_id + - channel_name + properties: + team_id: + type: string + description: Microsoft Teams team ID + minLength: 1 + team_name: + type: string + description: Microsoft Teams team name + minLength: 1 + channel_id: + type: string + description: Microsoft Teams channel ID + minLength: 1 + channel_name: + type: string + description: Microsoft Teams channel name + minLength: 1 \ No newline at end of file diff --git a/backend/windmill-api/src/agent_workers_ee.rs b/backend/windmill-api/src/agent_workers_ee.rs index af31e61229..4ffc0550ef 100644 --- a/backend/windmill-api/src/agent_workers_ee.rs +++ b/backend/windmill-api/src/agent_workers_ee.rs @@ -21,8 +21,8 @@ pub fn workspaced_service( _base_internal_url: String, ) -> ( Router, - Option>, - windmill_worker::JobCompletedSender, + Vec>, + Option, ) { use windmill_common::worker::Connection; use windmill_worker::JobCompletedSender; @@ -32,7 +32,7 @@ pub fn workspaced_service( let router = Router::new(); - (router, None, job_completed_tx) + (router, vec![], Some(job_completed_tx)) } #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/backend/windmill-api/src/args.rs b/backend/windmill-api/src/args.rs index fd2ceefabe..9b929804e6 100644 --- a/backend/windmill-api/src/args.rs +++ b/backend/windmill-api/src/args.rs @@ -1,7 +1,5 @@ use std::collections::HashMap; -#[cfg(feature = "parquet")] -use crate::job_helpers_ee::get_workspace_s3_resource; use axum::{ extract::{FromRequest, FromRequestParts, Multipart, Query, Request}, http::{HeaderMap, Uri}, @@ -9,139 +7,318 @@ use axum::{ }; use bytes::Bytes; use http::{header::CONTENT_TYPE, request::Parts, StatusCode}; -#[cfg(feature = "parquet")] -use object_store::{Attribute, Attributes}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use sqlx::types::JsonRawValue; -#[cfg(feature = "parquet")] -use windmill_common::s3_helpers::build_object_store_client; use windmill_common::{error::Error, worker::to_raw_value, DB}; -use windmill_queue::PushArgsOwned; +use windmill_queue::{PushArgsOwned, TriggerKind}; -use crate::db::ApiAuthed; -#[cfg(feature = "parquet")] -use crate::job_helpers_ee::{get_random_file_name, upload_file_internal}; +use crate::{ + db::ApiAuthed, + trigger_helpers::{get_runnable_format, RunnableFormat, RunnableFormatVersion, RunnableId}, +}; -#[derive(Debug, Default)] -pub struct WebhookArgs { - pub args: PushArgsOwned, - pub multipart: Option, - pub wrap_body: Option, +#[derive(Debug)] +pub enum RawBody { + Json(String), + CEJson(String), + Text(String), + Xml(String), + UrlEncoded(Bytes), + Multipart(Multipart), + Empty, } -impl WebhookArgs { +#[derive(Clone, Serialize)] +#[serde(untagged)] +pub enum Body { + HashMap(HashMap>), + NoHashMap(Box), +} + +#[derive(Clone, Default)] +pub struct WebhookArgsMetadata { + pub raw_string: Option, + pub headers: HashMap>, + pub method: http::Method, + pub query: HashMap>, + pub query_wrap_body: bool, + pub query_use_raw: bool, +} + +pub struct RawWebhookArgs { + pub body: RawBody, + pub metadata: WebhookArgsMetadata, +} + +#[derive(Clone)] +pub struct WebhookArgs { + pub body: Body, + pub metadata: WebhookArgsMetadata, +} + +// capture +// + +impl RawWebhookArgs { #[cfg(not(feature = "parquet"))] - pub async fn to_push_args_owned( - self, + pub async fn process_multipart( + _multipart: Multipart, _authed: &ApiAuthed, _db: &DB, _w_id: &str, - ) -> Result { - if self.multipart.is_some() { - return Err(Error::BadRequest(format!( - "multipart/form-data requires the parquet feature" - ))); - } - - Ok(self.args) + ) -> Result>, Error> { + return Err(Error::BadRequest(format!( + "multipart/form-data requires the parquet feature" + ))); } #[cfg(feature = "parquet")] - pub async fn to_push_args_owned( - mut self, + async fn process_multipart( + mut multipart: Multipart, + authed: &ApiAuthed, + db: &DB, + w_id: &str, + ) -> Result>, Error> { + use crate::job_helpers_ee::{ + get_random_file_name, get_workspace_s3_resource, upload_file_internal, + }; + use futures::TryStreamExt; + use object_store::{Attribute, Attributes}; + use windmill_common::s3_helpers::build_object_store_client; + + let (_, s3_resource) = get_workspace_s3_resource(authed, db, None, "", w_id, None).await?; + + if let Some(s3_resource) = s3_resource { + let s3_client = build_object_store_client(&s3_resource).await?; + + let mut body = HashMap::new(); + let mut files = HashMap::new(); + + while let Some(field) = multipart.next_field().await.map_err(|e| { + Error::BadRequest(format!("Error reading multipart field: {}", e.body_text())) + })? { + if let Some(name) = field.name().map(|x| x.to_string()) { + if let Some(content_type) = field.content_type() { + let ext = field + .file_name() + .map(|x| x.split('.').last()) + .flatten() + .map(|x| x.to_string()); + + let file_key = get_random_file_name(ext); + + let options = Attributes::from_iter(vec![ + (Attribute::ContentType, content_type.to_string()), + ( + Attribute::ContentDisposition, + if let Some(filename) = field.file_name() { + format!("inline; filename=\"{}\"", filename) + } else { + "inline".to_string() + }, + ), + ]) + .into(); + + let bytes_stream = field + .into_stream() + .map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err)); + + upload_file_internal(s3_client.clone(), &file_key, bytes_stream, options) + .await?; + + files.entry(name).or_insert(vec![]).push(serde_json::json!({ + "s3": &file_key + })); + } else { + body.insert(name, to_raw_value(&field.text().await.unwrap_or_default())); + } + } + } + + for (k, v) in files { + body.insert(k, to_raw_value(&v)); + } + + Ok(body) + } else { + Err(Error::BadRequest(format!( + "You need to connect your workspace to an S3 bucket to use multipart/form-data" + ))) + } + } + + pub async fn process_args( + self, + authed: &ApiAuthed, + db: &DB, + w_id: &str, + force_use_raw: Option, + ) -> Result { + let use_raw = force_use_raw.unwrap_or(self.metadata.query_use_raw); + + match self.body { + RawBody::Multipart(multipart) => { + let body = Self::process_multipart(multipart, authed, db, w_id).await?; + Ok(WebhookArgs { body: Body::HashMap(body), metadata: self.metadata }) + } + RawBody::Empty => { + let mut metadata = self.metadata; + if use_raw { + metadata.raw_string = Some("".to_string()); + } + Ok(WebhookArgs { body: Body::HashMap(HashMap::new()), metadata }) + } + RawBody::Text(s) | RawBody::Xml(s) => Ok(WebhookArgs { + body: Body::HashMap(HashMap::new()), + metadata: WebhookArgsMetadata { raw_string: Some(s), ..self.metadata }, + }), + RawBody::UrlEncoded(bytes) => { + let mut metadata = self.metadata; + if use_raw { + let raw_string = String::from_utf8(bytes.to_vec()) + .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)))?; + metadata.raw_string = Some(raw_string); + } + let payload: HashMap> = serde_urlencoded::from_bytes(&bytes) + .map_err(|e| Error::BadRequest(format!("invalid urlencoded data: {}", e)))?; + let payload = payload + .into_iter() + .map(|(k, v)| (k, to_raw_value(&v))) + .collect::>(); + + Ok(WebhookArgs { body: Body::HashMap(payload), metadata }) + } + RawBody::Json(s) => WebhookArgs::from_json(self.metadata, use_raw, s).await, + RawBody::CEJson(s) => WebhookArgs::from_ce_json(self.metadata, use_raw, s).await, + } + } + + pub async fn to_main_args( + self, authed: &ApiAuthed, db: &DB, w_id: &str, ) -> Result { - use futures::TryStreamExt; + let args = self.process_args(authed, db, w_id, None).await?; + args.to_main_args() + } - if let Some(mut multipart) = self.multipart { - { - let (_, s3_resource) = - get_workspace_s3_resource(authed, db, None, "", w_id, None).await?; + pub async fn to_args_from_runnable( + self, + authed: &ApiAuthed, + db: &DB, + w_id: &str, + runnable_id: RunnableId, + skip_preprocessor: Option, + ) -> Result { + let args = self.process_args(authed, db, w_id, None).await?; + args.to_args_from_runnable(db, w_id, runnable_id, skip_preprocessor) + .await + } +} - if let Some(s3_resource) = s3_resource { - let s3_client = build_object_store_client(&s3_resource).await?; +#[derive(Serialize)] +struct WebhookPreprocessorEvent { + kind: String, + body: Box, + raw_string: Option, + headers: HashMap>, + query: HashMap>, +} - let mut body = HashMap::new(); - let mut files = HashMap::new(); +impl WebhookArgs { + pub fn to_main_args(self) -> Result { + self.to_args_from_format(RunnableFormat { + has_preprocessor: false, + version: RunnableFormatVersion::V2, + }) + } - while let Some(field) = multipart.next_field().await.map_err(|e| { - Error::BadRequest(format!( - "Error reading multipart field: {}", - e.body_text() - )) - })? { - if let Some(name) = field.name().map(|x| x.to_string()) { - if let Some(content_type) = field.content_type() { - let ext = field - .file_name() - .map(|x| x.split('.').last()) - .flatten() - .map(|x| x.to_string()); + pub async fn to_args_from_runnable( + self, + db: &DB, + w_id: &str, + runnable_id: RunnableId, + skip_preprocessor: Option, + ) -> Result { + if skip_preprocessor.unwrap_or(false) { + self.to_main_args() + } else { + let runnable_format = + get_runnable_format(runnable_id, w_id, db, &TriggerKind::Webhook).await?; - let file_key = get_random_file_name(ext); + self.to_args_from_format(runnable_format) + } + } - let options = Attributes::from_iter(vec![ - (Attribute::ContentType, content_type.to_string()), - ( - Attribute::ContentDisposition, - if let Some(filename) = field.file_name() { - format!("inline; filename=\"{}\"", filename) - } else { - "inline".to_string() - }, - ), - ]) - .into(); + pub fn to_args_from_format( + self, + runnable_format: RunnableFormat, + ) -> Result { + match runnable_format { + RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V2 } => { + let mut args = HashMap::new(); - let bytes_stream = field.into_stream().map_err(|err| { - std::io::Error::new(std::io::ErrorKind::Other, err) - }); + args.insert( + "event".to_string(), + to_raw_value(&WebhookPreprocessorEvent { + kind: "webhook".to_string(), + body: to_raw_value(&self.body), + raw_string: self.metadata.raw_string, + headers: self.metadata.headers, + query: self.metadata.query, + }), + ); - upload_file_internal( - s3_client.clone(), - &file_key, - bytes_stream, - options, - ) - .await?; + Ok(PushArgsOwned { args, extra: None }) + } + RunnableFormat { has_preprocessor, .. } => { + let mut extra = HashMap::new(); - files.entry(name).or_insert(vec![]).push(serde_json::json!({ - "s3": &file_key - })); - } else { - body.insert( - name, - to_raw_value(&field.text().await.unwrap_or_default()), - ); - } + let WebhookArgsMetadata { query, query_wrap_body, headers, raw_string, .. } = + self.metadata; + + for (k, v) in headers { + extra.insert(k, v); + } + + for (k, v) in query { + extra.insert(k, v); + } + + if let Some(raw_string) = raw_string { + extra.insert("raw_string".to_string(), to_raw_value(&raw_string)); + } + + if has_preprocessor { + // if has preprocessor, it has to be v1 + extra.insert( + "wm_trigger".to_string(), + to_raw_value(&serde_json::json!({ + "kind": "webhook", + })), + ); + } + + let extra = if extra.is_empty() { None } else { Some(extra) }; + + match self.body { + Body::HashMap(mut body) => { + if query_wrap_body { + body = HashMap::from([("body".to_string(), to_raw_value(&body))]); } + Ok(PushArgsOwned { args: body, extra }) } - - for (k, v) in files { - body.insert(k, to_raw_value(&v)); + Body::NoHashMap(args) => { + let mut hm = HashMap::new(); + hm.insert("body".to_string(), args); + Ok(PushArgsOwned { args: hm, extra }) } - - if self.wrap_body.unwrap_or(false) { - self.args - .args - .insert("body".to_string(), to_raw_value(&body)); - } else { - self.args.args.extend(body); - } - - return Ok(self.args); } } - - return Err(Error::BadRequest(format!( - "You need to connect your workspace to an S3 bucket to use multipart/form-data" - ))); } - - Ok(self.args) } } @@ -166,26 +343,36 @@ async fn req_to_string( pub async fn try_from_request_body( request: Request, _state: &S, - use_raw: Option, - wrap_body: Option, -) -> Result + is_http_trigger: bool, +) -> Result where S: Send + Sync, { - let (content_type, mut extra, use_raw, wrap_body) = { + let (content_type, metadata) = { let headers_map = request.headers(); let content_type_header = headers_map.get(CONTENT_TYPE); let content_type = content_type_header.and_then(|value| value.to_str().ok()); let uri = request.uri(); - let query = Query::::try_from_uri(uri).unwrap().0; - let mut extra = build_extra(&headers_map, query.include_header); - let query_decode = DecodeQueries::from_uri(uri); + let request_query = Query::::try_from_uri(uri).unwrap().0; + let headers = build_headers(&headers_map, request_query.include_header, is_http_trigger); + let query_decode = DecodeQueries::from_uri(uri, is_http_trigger); + let mut query = HashMap::new(); if let Some(DecodeQueries(queries)) = query_decode { - extra.extend(queries); + query.extend(queries); } - let raw = query.raw.unwrap_or(use_raw.unwrap_or(false)); - let wrap_body = query.wrap_body.unwrap_or(wrap_body.unwrap_or(false)); - (content_type, extra, raw, wrap_body) + let raw = !is_http_trigger && request_query.raw.unwrap_or(false); + let wrap_body = !is_http_trigger && request_query.wrap_body.unwrap_or(false); + ( + content_type, + WebhookArgsMetadata { + headers, + query, + method: request.method().clone(), + raw_string: None, + query_wrap_body: wrap_body, + query_use_raw: raw, + }, + ) }; let no_content_type = content_type.is_none(); @@ -194,33 +381,19 @@ where .await .map_err(IntoResponse::into_response)?; if no_content_type && bytes.is_empty() { - if use_raw { - extra.insert("raw_string".to_string(), to_raw_value(&"".to_string())); - } - let mut args = HashMap::new(); - if wrap_body { - args.insert("body".to_string(), to_raw_value(&serde_json::json!({}))); - } - return Ok(WebhookArgs { - args: PushArgsOwned { extra: Some(extra), args: args }, - ..Default::default() - }); + Ok(RawWebhookArgs { body: RawBody::Empty, metadata }) + } else { + let str = String::from_utf8(bytes.to_vec()) + .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?; + Ok(RawWebhookArgs { body: RawBody::Json(str), metadata }) } - let str = String::from_utf8(bytes.to_vec()) - .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?; - - PushArgsOwned::from_json(extra, use_raw, wrap_body, str) - .await - .map(|args| WebhookArgs { args, ..Default::default() }) } else if content_type .unwrap() .starts_with("application/cloudevents+json") { let str = req_to_string(request, _state).await?; - PushArgsOwned::from_ce_json(extra, use_raw, str) - .await - .map(|args| WebhookArgs { args, ..Default::default() }) + Ok(RawWebhookArgs { body: RawBody::CEJson(str), metadata }) } else if content_type .unwrap() .starts_with("application/cloudevents-batch+json") @@ -231,11 +404,7 @@ where ) } else if content_type.unwrap().starts_with("text/plain") { let str = req_to_string(request, _state).await?; - extra.insert("raw_string".to_string(), to_raw_value(&str)); - Ok(WebhookArgs { - args: PushArgsOwned { extra: Some(extra), args: HashMap::new() }, - ..Default::default() - }) + Ok(RawWebhookArgs { body: RawBody::Text(str), metadata }) } else if content_type .unwrap() .starts_with("application/x-www-form-urlencoded") @@ -244,58 +413,32 @@ where .await .map_err(IntoResponse::into_response)?; - if use_raw { - let raw_string = String::from_utf8(bytes.to_vec()) - .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?; - extra.insert("raw_string".to_string(), to_raw_value(&raw_string)); - } - - let payload: HashMap> = serde_urlencoded::from_bytes(&bytes) - .map_err(|e| { - Error::BadRequest(format!("invalid urlencoded data: {}", e)).into_response() - })?; - let payload = payload - .into_iter() - .map(|(k, v)| (k, to_raw_value(&v))) - .collect::>(); - - return Ok(WebhookArgs { - args: PushArgsOwned { extra: Some(extra), args: payload }, - ..Default::default() - }); + Ok(RawWebhookArgs { body: RawBody::UrlEncoded(bytes), metadata }) } else if content_type.unwrap().starts_with("application/xml") || content_type.unwrap().starts_with("text/xml") { let str = req_to_string(request, _state).await?; - extra.insert("raw_string".to_string(), to_raw_value(&str)); - Ok(WebhookArgs { - args: PushArgsOwned { extra: Some(extra), args: HashMap::new() }, - ..Default::default() - }) + Ok(RawWebhookArgs { body: RawBody::Xml(str), metadata }) } else if content_type.unwrap().starts_with("multipart/form-data") { let multipart = Multipart::from_request(request, _state) .await .map_err(IntoResponse::into_response)?; - Ok(WebhookArgs { - args: PushArgsOwned { extra: Some(extra), args: HashMap::new() }, - multipart: Some(multipart), - wrap_body: Some(wrap_body), - }) + Ok(RawWebhookArgs { body: RawBody::Multipart(multipart), metadata }) } else { Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response()) } } #[axum::async_trait] -impl FromRequest for WebhookArgs +impl FromRequest for RawWebhookArgs where S: Send + Sync, { type Rejection = Response; async fn from_request(request: Request, _state: &S) -> Result { - let args = try_from_request_body(request, _state, None, None).await?; + let args = try_from_request_body(request, _state, false).await?; Ok(args) } @@ -309,27 +452,37 @@ lazy_static::lazy_static! { .collect()).unwrap_or_default(); } -pub fn build_extra( +pub fn build_headers( headers: &HeaderMap, include_header: Option, + is_http_trigger: bool, ) -> HashMap> { - let mut args = HashMap::new(); - let whitelist = include_header - .map(|s| s.split(",").map(|s| s.to_string()).collect::>()) - .unwrap_or_default(); + let mut selected_headers = HashMap::new(); - whitelist - .iter() - .chain(INCLUDE_HEADERS.iter()) - .for_each(|h| { - if let Some(v) = headers.get(h) { - args.insert( - h.to_string().to_lowercase().replace('-', "_"), - to_raw_value(&v.to_str().unwrap().to_string()), - ); - } - }); - args + if is_http_trigger { + for (k, v) in headers.iter() { + selected_headers.insert( + k.to_string(), + to_raw_value(&v.to_str().unwrap_or("").to_string()), + ); + } + } else { + let whitelist = include_header + .map(|s| s.split(",").map(|s| s.to_string()).collect::>()) + .unwrap_or_default(); + whitelist + .iter() + .chain(INCLUDE_HEADERS.iter()) + .for_each(|h| { + if let Some(v) = headers.get(h) { + selected_headers.insert( + h.to_string().to_lowercase().replace('-', "_"), + to_raw_value(&v.to_str().unwrap_or("").to_string()), + ); + } + }); + } + selected_headers } #[derive(Deserialize)] @@ -347,37 +500,49 @@ where type Rejection = Response; async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { - Ok(DecodeQueries::from_uri(&parts.uri).unwrap_or_else(|| DecodeQueries(HashMap::new()))) + Ok(DecodeQueries::from_uri(&parts.uri, false) + .unwrap_or_else(|| DecodeQueries(HashMap::new()))) } } impl DecodeQueries { - pub fn from_uri(uri: &Uri) -> Option { + pub fn from_uri(uri: &Uri, is_http_trigger: bool) -> Option { let query = uri.query(); if query.is_none() { return None; } let query = query.unwrap(); - let include_query = serde_urlencoded::from_str::(query) - .map(|x| x.include_query) - .ok() - .flatten() - .unwrap_or_default(); - let parse_query_args = include_query - .split(",") - .map(|s| s.to_string()) - .collect::>(); - let mut args = HashMap::new(); - if !parse_query_args.is_empty() { + if is_http_trigger { let queries = serde_urlencoded::from_str::>(query).unwrap_or_default(); - parse_query_args.iter().for_each(|h| { - if let Some(v) = queries.get(h) { - args.insert(h.to_string(), to_raw_value(v)); - } - }); + Some(DecodeQueries( + queries + .into_iter() + .map(|(k, v)| (k, to_raw_value(&v))) + .collect(), + )) + } else { + let include_query = serde_urlencoded::from_str::(query) + .map(|x| x.include_query) + .ok() + .flatten() + .unwrap_or_default(); + let parse_query_args = include_query + .split(",") + .map(|s| s.to_string()) + .collect::>(); + let mut args = HashMap::new(); + if !parse_query_args.is_empty() { + let queries = serde_urlencoded::from_str::>(query) + .unwrap_or_default(); + parse_query_args.iter().for_each(|h| { + if let Some(v) = queries.get(h) { + args.insert(h.to_string(), to_raw_value(v)); + } + }); + } + Some(DecodeQueries(args)) } - Some(DecodeQueries(args)) } } @@ -414,69 +579,50 @@ fn restructure_cloudevents_metadata( } } -trait PushArgsOwnedExt: Sized { +impl WebhookArgs { async fn from_json( - extra: HashMap>, - use_raw: bool, - force_wrap_body: bool, - str: String, - ) -> Result; - - async fn from_ce_json( - extra: HashMap>, + mut metadata: WebhookArgsMetadata, use_raw: bool, str: String, - ) -> Result; -} - -impl PushArgsOwnedExt for PushArgsOwned { - async fn from_json( - mut extra: HashMap>, - use_raw: bool, - force_wrap_body: bool, - str: String, - ) -> Result { + ) -> Result { if use_raw { - extra.insert("raw_string".to_string(), to_raw_value(&str)); + metadata.raw_string = Some(str.clone()); } - let wrap_body = force_wrap_body || str.len() > 0 && str.chars().next().unwrap() != '{'; + let no_hashmap = str.len() > 0 && str.chars().next().unwrap() != '{'; - if wrap_body { + if no_hashmap { let args = serde_json::from_str::>>(&str) - .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())? + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); - let mut hm = HashMap::new(); - hm.insert("body".to_string(), args); - Ok(PushArgsOwned { extra: Some(extra), args: hm }) + + Ok(Self { body: Body::NoHashMap(args), metadata }) } else { let hm = serde_json::from_str::>>>(&str) - .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())? + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? .unwrap_or_else(HashMap::new); - Ok(PushArgsOwned { extra: Some(extra), args: hm }) + Ok(Self { body: Body::HashMap(hm), metadata }) } } async fn from_ce_json( - mut extra: HashMap>, + mut metadata: WebhookArgsMetadata, use_raw: bool, str: String, - ) -> Result { + ) -> Result { if use_raw { - extra.insert("raw_string".to_string(), to_raw_value(&str)); + metadata.raw_string = Some(str.clone()); } - let hm = serde_json::from_str::>>(&str).map_err(|e| { - Error::BadRequest(format!("invalid cloudevents+json: {}", e)).into_response() - })?; - let hm = restructure_cloudevents_metadata(hm).map_err(|e| e.into_response())?; - Ok(PushArgsOwned { extra: Some(extra), args: hm }) + let hm = serde_json::from_str::>>(&str) + .map_err(|e| Error::BadRequest(format!("invalid cloudevents+json: {}", e)))?; + let hm = restructure_cloudevents_metadata(hm)?; + Ok(Self { body: Body::HashMap(hm), metadata }) } } #[cfg(test)] mod tests { - use std::collections::HashMap; use super::*; @@ -514,24 +660,35 @@ mod tests { "data" : 1.5 } "#; - let extra = HashMap::new(); + let metadata = WebhookArgsMetadata::default(); - let a1 = PushArgsOwned::from_ce_json(extra.clone(), false, r1.to_string()) + let a1 = WebhookArgs::from_ce_json(metadata.clone(), false, r1.to_string()) .await .expect("Failed to parse the cloudevent"); - let a2 = PushArgsOwned::from_ce_json(extra.clone(), false, r2.to_string()) + let a2 = WebhookArgs::from_ce_json(metadata.clone(), false, r2.to_string()) .await .expect("Failed to parse the cloudevent"); - a1.args.get("WEBHOOK__METADATA__").expect( - "CloudEvents should generate a neighboring `webhook-metadata` field in PushArgs", - ); - assert_eq!( - a2.args - .get("body") - .expect("Cloud events with a data field with no wrapping curly brackets should be inside of a `body` field in PushArgs") - .to_string(), - "1.5" - ); + match a1.body { + Body::HashMap(body) => { + body.get("WEBHOOK__METADATA__").expect( + "CloudEvents should generate a neighboring `webhook-metadata` field in PushArgs", + ); + } + _ => panic!("Expected a HashMap"), + } + + match a2.body { + Body::HashMap(body) => { + assert_eq!( + body + .get("body") + .expect("Cloud events with a data field with no wrapping curly brackets should be inside of a `body` field in PushArgs") + .to_string(), + "1.5" + ); + } + _ => panic!("Expected a HashMap"), + } } } diff --git a/backend/windmill-api/src/auth.rs b/backend/windmill-api/src/auth.rs index 8175e0eae5..2936b74957 100644 --- a/backend/windmill-api/src/auth.rs +++ b/backend/windmill-api/src/auth.rs @@ -511,6 +511,13 @@ where let path_vec: Vec<&str> = original_uri.path().split("/").collect(); let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" { Some(path_vec[3].to_owned()) + } else if path_vec.len() >= 5 + && path_vec[0] == "" + && path_vec[1] == "api" + && path_vec[2] == "mcp" + && path_vec[3] == "w" + { + Some(path_vec[4].to_string()) } else { if path_vec.len() >= 5 && path_vec[0] == "" diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index 1247dd78a7..0c3840adc4 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -8,17 +8,15 @@ #[cfg(feature = "http_trigger")] use { - crate::{ - args::try_from_request_body, - http_triggers::{build_http_trigger_extra, HttpMethod}, - }, + crate::http_trigger_args::{HttpMethod, RawHttpTriggerArgs}, axum::response::{IntoResponse, Response}, std::collections::HashMap, }; #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] use crate::gcp_triggers_ee::{ - manage_google_subscription, process_google_push_request, validate_jwt_token, SubscriptionMode, + manage_google_subscription, process_google_push_request, validate_jwt_token, + CreateUpdateConfig, SubscriptionMode, }; #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] @@ -29,8 +27,10 @@ use windmill_common::auth::aws::AwsAuthResourceType; all(feature = "enterprise", feature = "gcp_trigger") ))] use { - axum::extract::Request, http::HeaderMap, serde::de::DeserializeOwned, - windmill_common::error::Error, + axum::extract::Request, + http::HeaderMap, + serde::de::DeserializeOwned, + windmill_common::{error::Error, utils::empty_as_none}, }; #[cfg(all(feature = "enterprise", feature = "kafka"))] @@ -53,8 +53,9 @@ use { }; use crate::{ - args::WebhookArgs, + args::RawWebhookArgs, db::{ApiAuthed, DB}, + trigger_helpers::{RunnableFormat, RunnableFormatVersion}, users::fetch_api_authed, utils::RunnableKind, }; @@ -160,8 +161,13 @@ pub struct SqsTriggerConfig { #[derive(Debug, Serialize, Deserialize)] pub struct GcpTriggerConfig { pub gcp_resource_path: String, - #[serde(flatten)] pub subscription_mode: SubscriptionMode, + #[serde(default, deserialize_with = "empty_as_none")] + pub subscription_id: Option, + #[serde(default, deserialize_with = "empty_as_none")] + pub base_endpoint: Option, + #[serde(flatten)] + pub create_update: Option, pub topic_id: String, } @@ -370,11 +376,15 @@ async fn set_gcp_trigger_config( &gcp_config.gcp_resource_path, &capture_config.path, &gcp_config.topic_id, + &mut gcp_config.subscription_id, + &mut gcp_config.base_endpoint, gcp_config.subscription_mode, + gcp_config.create_update, + false, ) .await?; - - gcp_config.subscription_mode = SubscriptionMode::CreateUpdate(config); + gcp_config.create_update = Some(config); + gcp_config.subscription_mode = SubscriptionMode::CreateUpdate; capture_config.trigger_config = Some(TriggerConfig::Gcp(gcp_config)); Ok(capture_config) @@ -484,8 +494,8 @@ struct Capture { id: i64, created_at: chrono::DateTime, trigger_kind: TriggerKind, - payload: SqlxJson>, - trigger_extra: Option>>, + main_args: SqlxJson>, + preprocessor_args: Option>>, } #[derive(Deserialize)] @@ -513,10 +523,13 @@ async fn list_captures( created_at, trigger_kind AS "trigger_kind: _", CASE - WHEN pg_column_size(payload) < 40000 THEN payload + WHEN pg_column_size(main_args) < 40000 THEN main_args ELSE '"WINDMILL_TOO_BIG"'::jsonb - END AS "payload!: _", - trigger_extra AS "trigger_extra: _" + END AS "main_args!: _", + CASE + WHEN pg_column_size(preprocessor_args) < 40000 THEN preprocessor_args + ELSE '"WINDMILL_TOO_BIG"'::jsonb + END AS "preprocessor_args: _" FROM capture WHERE @@ -558,8 +571,8 @@ async fn get_capture( id, created_at, trigger_kind AS "trigger_kind: _", - payload AS "payload!: _", - trigger_extra AS "trigger_extra: _" + main_args AS "main_args!: _", + preprocessor_args AS "preprocessor_args: _" FROM capture WHERE @@ -808,15 +821,15 @@ pub async fn insert_capture_payload( path: &str, is_flow: bool, trigger_kind: &TriggerKind, - payload: PushArgsOwned, - trigger_extra: Option>, + main_args: PushArgsOwned, + preprocessor_args: PushArgsOwned, owner: &str, ) -> Result<()> { sqlx::query!( r#" INSERT INTO capture ( - workspace_id, path, is_flow, trigger_kind, payload, trigger_extra, created_by + workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by ) VALUES ( $1, $2, $3, $4, $5, $6, $7 @@ -826,11 +839,9 @@ pub async fn insert_capture_payload( path, is_flow, trigger_kind as &TriggerKind, - SqlxJson(to_raw_value(&PushArgs { - args: &payload.args, - extra: payload.extra - })) as SqlxJson>, - trigger_extra.map(SqlxJson) as Option>>, + SqlxJson(PushArgs { args: &main_args.args, extra: main_args.extra }) as SqlxJson, + SqlxJson(PushArgs { args: &preprocessor_args.args, extra: preprocessor_args.extra }) + as SqlxJson, owner, ) .execute(db) @@ -844,7 +855,7 @@ pub async fn insert_capture_payload( async fn webhook_payload( Extension(db): Extension, Path((w_id, runnable_kind, path)): Path<(String, RunnableKind, StripPath)>, - args: WebhookArgs, + args: RawWebhookArgs, ) -> Result { let (owner, email) = get_active_capture_owner_and_email( &db, @@ -856,7 +867,15 @@ async fn webhook_payload( .await?; let authed = fetch_api_authed(owner.clone(), email, &w_id, &db, None).await?; - let args = args.to_push_args_owned(&authed, &db, &w_id).await?; + + let args = args.process_args(&authed, &db, &w_id, None).await?; + + let preprocessor_args = args.clone().to_args_from_format(RunnableFormat { + has_preprocessor: true, + version: RunnableFormatVersion::V2, + })?; + + let main_args = args.to_main_args()?; insert_capture_payload( &db, @@ -864,12 +883,8 @@ async fn webhook_payload( &path.to_path(), matches!(runnable_kind, RunnableKind::Flow), &TriggerKind::Webhook, - args, - Some(to_raw_value(&serde_json::json!({ - "wm_trigger": { - "kind": "webhook", - } - }))), + main_args, + preprocessor_args, &owner, ) .await?; @@ -885,13 +900,15 @@ async fn gcp_payload( headers: HeaderMap, request: Request, ) -> Result { + use crate::{gcp_triggers_ee::GcpTrigger, trigger_helpers::TriggerJobArgs}; + let is_flow = matches!(runnable_kind, RunnableKind::Flow); let (gcp_trigger_config, owner, email): (GcpTriggerConfig, _, _) = get_capture_trigger_config_and_owner(&db, &w_id, &path, is_flow, &TriggerKind::Gcp).await?; let authed = fetch_api_authed(owner.clone(), email, &w_id, &db, None).await?; - let SubscriptionMode::CreateUpdate(config) = &gcp_trigger_config.subscription_mode else { + let Some(config) = &gcp_trigger_config.create_update else { return Err(Error::BadConfig("Bad config".to_string())); }; @@ -906,9 +923,9 @@ async fn gcp_payload( ) .await?; - let (args, extra) = process_google_push_request(headers, request).await?; + let (payload, gcp) = process_google_push_request(headers, request).await?; - let payload = PushArgsOwned { args, extra: None }; + let (main_args, preprocessor_args) = GcpTrigger::build_capture_payloads(payload, gcp); let _ = insert_capture_payload( &db, @@ -916,8 +933,8 @@ async fn gcp_payload( &path, is_flow, &TriggerKind::Gcp, - payload, - Some(to_raw_value(&extra)), + main_args, + preprocessor_args, &owner, ) .await?; @@ -929,10 +946,7 @@ async fn gcp_payload( async fn http_payload( Extension(db): Extension, Path((w_id, runnable_kind, path, route_path)): Path<(String, RunnableKind, String, StripPath)>, - Query(query): Query>, - method: http::Method, - headers: HeaderMap, - request: Request, + args: RawHttpTriggerArgs, ) -> std::result::Result { let path = path.replace(".", "/"); let is_flow = matches!(runnable_kind, RunnableKind::Flow); @@ -942,20 +956,17 @@ async fn http_payload( .await .map_err(|e| e.into_response())?; - let args = try_from_request_body( - request, - &(), - http_trigger_config.raw_string, - http_trigger_config.wrap_body, - ) - .await - .map_err(|e| e.into_response())?; - let authed = fetch_api_authed(owner.clone(), email, &w_id, &db, None) .await .map_err(|e| e.into_response())?; - let mut args = args - .to_push_args_owned(&authed, &db, &w_id) + + let args = args + .process_args( + &authed, + &db, + &w_id, + http_trigger_config.raw_string.unwrap_or(false), + ) .await .map_err(|e| e.into_response())?; @@ -973,31 +984,23 @@ async fn http_payload( .map(|(k, v)| (k.to_string(), v.to_string())) .collect(); - let extra = args.extra.get_or_insert_with(HashMap::new); + let preprocessor_args = args + .clone() + .to_v2_preprocessor_args(&http_trigger_config.route_path, &route_path, ¶ms) + .map_err(|e| e.into_response())?; - extra.insert( - "wm_trigger".to_string(), - build_http_trigger_extra( - &http_trigger_config.route_path, - route_path, - &method, - ¶ms, - &query, - &headers, - ) - .await, - ); + let main_args = args + .to_main_args(http_trigger_config.wrap_body.unwrap_or(false)) + .map_err(|e| e.into_response())?; - let extra = Some(to_raw_value(&extra)); - args.extra = None; insert_capture_payload( &db, &w_id, &path, is_flow, &TriggerKind::Http, - args, - extra, + main_args, + preprocessor_args, &owner, ) .await diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index 1ecef03fc5..557981d214 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -48,6 +48,9 @@ lazy_static::lazy_static! { (20250102145420, include_str!( "../../migrations/20250102145420_more_captures.up.sql" ).replace("public.", "")), + (20250429211554, include_str!( + "../../migrations/20250429211554_create_indices_on_queue.up.sql" + ).replace("public.", "")), (20241006144414, include_str!( "../../custom_migrations/grant_all_current_schema.sql" ).to_string()), @@ -231,7 +234,7 @@ pub async fn migrate(db: &DB) -> Result>, Error> { { Ok(_) => Ok(()), Err(sqlx::migrate::MigrateError::VersionMissing(e)) => { - tracing::error!("Database had been applied more migrations than this container. + tracing::error!("Database had been applied more migrations than this container. This usually mean than another container on a more recent version migrated the database and this one is on an earlier version. Please update the container to latest. Not critical, but may cause issues if migration introduced a breaking change. Version missing: {e:#}"); custom_migrator.unlock().await?; @@ -433,7 +436,7 @@ async fn v2_finalize(db: &DB) -> Result<(), Error> { r#" LOCK TABLE v2_job_completed IN ACCESS EXCLUSIVE MODE; DROP FUNCTION IF EXISTS v2_job_completed_before_insert CASCADE; - DROP FUNCTION IF EXISTS v2_job_completed_before_update CASCADE; + DROP FUNCTION IF EXISTS v2_job_completed_before_update CASCADE; "#, ) .await?; @@ -445,7 +448,7 @@ async fn v2_finalize(db: &DB) -> Result<(), Error> { LOCK TABLE v2_job_queue IN ACCESS EXCLUSIVE MODE; DROP FUNCTION IF EXISTS v2_job_queue_after_insert CASCADE; DROP FUNCTION IF EXISTS v2_job_queue_before_insert CASCADE; - DROP FUNCTION IF EXISTS v2_job_queue_before_update CASCADE; + DROP FUNCTION IF EXISTS v2_job_queue_before_update CASCADE; "#, ) .await?; @@ -456,7 +459,7 @@ async fn v2_finalize(db: &DB) -> Result<(), Error> { r#" LOCK TABLE v2_job_runtime IN ACCESS EXCLUSIVE MODE; DROP FUNCTION IF EXISTS v2_job_runtime_before_insert CASCADE; - DROP FUNCTION IF EXISTS v2_job_runtime_before_update CASCADE; + DROP FUNCTION IF EXISTS v2_job_runtime_before_update CASCADE; "#, ) .await?; @@ -467,7 +470,7 @@ async fn v2_finalize(db: &DB) -> Result<(), Error> { r#" LOCK TABLE v2_job_status IN ACCESS EXCLUSIVE MODE; DROP FUNCTION IF EXISTS v2_job_status_before_insert CASCADE; - DROP FUNCTION IF EXISTS v2_job_status_before_update CASCADE; + DROP FUNCTION IF EXISTS v2_job_status_before_update CASCADE; "#, ) .await?; diff --git a/backend/windmill-api/src/embeddings.rs b/backend/windmill-api/src/embeddings.rs index 30415156ce..aba907ec16 100644 --- a/backend/windmill-api/src/embeddings.rs +++ b/backend/windmill-api/src/embeddings.rs @@ -253,7 +253,7 @@ impl ModelInstance { let token_ids = Tensor::new(&tokens[..], &Device::Cpu)?.unsqueeze(0)?; let token_type_ids = token_ids.zeros_like()?; - let embedding = self.model.forward(&token_ids, &token_type_ids)?; + let embedding = self.model.forward(&token_ids, &token_type_ids, None)?; let embedding = (embedding.sum(1)? / embedding.dim(1)? as f64)?; let embedding = normalize_l2(&embedding)?; diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 95a2145bcf..e34b2cacaf 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -1345,7 +1345,7 @@ mod tests { }), stop_after_if: Some(StopAfterIf { expr: "foo = 'bar'".to_string(), - skip_if_stopped: false, + ..Default::default() }), stop_after_all_iters_if: None, summary: None, @@ -1374,7 +1374,7 @@ mod tests { }), stop_after_if: Some(StopAfterIf { expr: "previous.isEmpty()".to_string(), - skip_if_stopped: false, + ..Default::default() }), stop_after_all_iters_if: None, summary: None, @@ -1402,7 +1402,7 @@ mod tests { .into(), stop_after_if: Some(StopAfterIf { expr: "previous.isEmpty()".to_string(), - skip_if_stopped: false, + ..Default::default() }), stop_after_all_iters_if: None, summary: None, @@ -1452,7 +1452,8 @@ mod tests { }, "stop_after_if": { "expr": "foo = 'bar'", - "skip_if_stopped": false + "skip_if_stopped": false, + "error_message": null } }, { @@ -1474,6 +1475,7 @@ mod tests { "stop_after_if": { "expr": "previous.isEmpty()", "skip_if_stopped": false, + "error_message": null } } ], @@ -1486,7 +1488,8 @@ mod tests { }, "stop_after_if": { "expr": "previous.isEmpty()", - "skip_if_stopped": false + "skip_if_stopped": false, + "error_message": null } }, }); diff --git a/backend/windmill-api/src/gcp_triggers_ee.rs b/backend/windmill-api/src/gcp_triggers_ee.rs index ec0a6ec115..bdf98c9bb9 100644 --- a/backend/windmill-api/src/gcp_triggers_ee.rs +++ b/backend/windmill-api/src/gcp_triggers_ee.rs @@ -1,4 +1,5 @@ use crate::db::{ApiAuthed, DB}; +use crate::trigger_helpers::TriggerJobArgs; use axum::{extract::Request, Router}; use http::HeaderMap; use serde::{Deserialize, Serialize}; @@ -7,10 +8,12 @@ use sqlx::prelude::FromRow; use sqlx::types::Json as SqlxJson; use std::collections::HashMap; use windmill_common::db::UserDB; +use windmill_common::worker::to_raw_value; use windmill_common::{ error::{Error as WindmillError, Result as WindmillResult}, - utils::empty_string_as_none, + utils::empty_as_none, }; +use windmill_queue::TriggerKind; #[derive(sqlx::Type, Debug, Deserialize, Serialize)] #[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] @@ -30,9 +33,9 @@ impl Default for DeliveryType { #[derive(FromRow, Deserialize, Serialize, Debug)] #[allow(unused)] pub struct PushConfig { - #[serde(deserialize_with = "empty_string_as_none")] + #[serde(deserialize_with = "empty_as_none")] route_path: Option, - #[serde(deserialize_with = "empty_string_as_none")] + #[serde(deserialize_with = "empty_as_none")] audience: Option, authenticate: bool, base_endpoint: String, @@ -41,7 +44,7 @@ pub struct PushConfig { #[allow(unused)] pub struct CreateUpdateConfig { pub delivery_type: DeliveryType, - #[serde(default, deserialize_with = "empty_string_as_none")] + #[serde(default, deserialize_with = "empty_as_none")] pub subscription_id: Option, pub delivery_config: Option>, } @@ -52,11 +55,12 @@ pub struct ExistingGcpSubscription { pub base_endpoint: String, } -#[derive(Debug, Deserialize, Serialize)] -#[serde(tag = "subscription_mode", rename_all = "snake_case")] +#[derive(Debug, Deserialize, Serialize, sqlx::Type)] +#[serde(rename_all = "snake_case")] +#[sqlx(type_name = "GCP_SUBSCRIPTION_MODE", rename_all = "snake_case")] pub enum SubscriptionMode { - Existing(ExistingGcpSubscription), - CreateUpdate(CreateUpdateConfig), + Existing, + CreateUpdate, } pub fn workspaced_service() -> Router { @@ -77,7 +81,11 @@ pub async fn manage_google_subscription( _gcp_resource_path: &str, _path: &str, _topic_id: &str, + _subscription_id: &mut Option, + _base_endpoint: &mut Option, _subscription_mode: SubscriptionMode, + _create_update_config: Option, + _trigger_mode: bool, ) -> WindmillResult { Ok(CreateUpdateConfig::default()) } @@ -85,14 +93,8 @@ pub async fn manage_google_subscription( pub async fn process_google_push_request( _headers: HeaderMap, _request: Request, -) -> Result< - ( - HashMap>, - Option>>, - ), - WindmillError, -> { - Ok((HashMap::new(), None)) +) -> Result<(String, HashMap>), WindmillError> { + Ok((String::new(), HashMap::new())) } pub async fn validate_jwt_token( @@ -117,6 +119,7 @@ pub struct GcpTrigger { pub subscription_id: String, pub delivery_type: DeliveryType, pub delivery_config: Option>, + pub subscription_mode: SubscriptionMode, pub topic_id: String, pub path: String, pub script_path: String, @@ -131,3 +134,13 @@ pub struct GcpTrigger { pub last_server_ping: Option>, pub enabled: bool, } + +impl TriggerJobArgs for GcpTrigger { + fn v1_payload_fn(payload: String) -> HashMap> { + HashMap::from([("payload".to_string(), to_raw_value(&payload))]) + } + + fn trigger_kind() -> TriggerKind { + TriggerKind::Gcp + } +} diff --git a/backend/windmill-api/src/http_trigger_args.rs b/backend/windmill-api/src/http_trigger_args.rs new file mode 100644 index 0000000000..e825f4fce8 --- /dev/null +++ b/backend/windmill-api/src/http_trigger_args.rs @@ -0,0 +1,206 @@ +use std::collections::HashMap; + +use axum::{ + extract::{FromRequest, Request}, + response::Response, +}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use windmill_common::{error::Error, worker::to_raw_value, DB}; +use windmill_queue::PushArgsOwned; + +use crate::{ + args::{try_from_request_body, Body, RawWebhookArgs, WebhookArgs, WebhookArgsMetadata}, + db::ApiAuthed, + trigger_helpers::{RunnableFormat, RunnableFormatVersion}, +}; + +pub struct RawHttpTriggerArgs(pub RawWebhookArgs); + +#[derive(Serialize, Deserialize, sqlx::Type, Debug)] +#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum HttpMethod { + Get, + Post, + Put, + Delete, + Patch, +} + +impl TryFrom<&http::Method> for HttpMethod { + type Error = Error; + fn try_from(method: &http::Method) -> Result { + match method { + &http::Method::GET => Ok(HttpMethod::Get), + &http::Method::POST => Ok(HttpMethod::Post), + &http::Method::PUT => Ok(HttpMethod::Put), + &http::Method::DELETE => Ok(HttpMethod::Delete), + &http::Method::PATCH => Ok(HttpMethod::Patch), + _ => Err(Error::BadRequest("Invalid HTTP method".to_string())), + } + } +} + +#[axum::async_trait] +impl FromRequest for RawHttpTriggerArgs +where + S: Send + Sync, +{ + type Rejection = Response; + + async fn from_request(request: Request, _state: &S) -> Result { + let args = try_from_request_body(request, _state, true).await?; + + Ok(Self(args)) + } +} + +#[derive(Clone)] +pub struct HttpTriggerArgs(pub WebhookArgs); + +impl RawHttpTriggerArgs { + pub async fn process_args( + self, + authed: &ApiAuthed, + db: &DB, + w_id: &str, + use_raw: bool, + ) -> Result { + if self.0.metadata.query_use_raw || self.0.metadata.query_wrap_body { + return Err(Error::BadRequest( + "Specifying use raw or wrap body with query args is not supported anymore on http routes, please set it in the trigger config".to_string(), + ) + .into()); + } + + let args = self.0.process_args(authed, db, w_id, Some(use_raw)).await?; + + Ok(HttpTriggerArgs(args)) + } +} + +#[derive(Serialize)] +struct HttpTriggerPreprocessorEvent<'a> { + kind: String, + route: &'a str, + path: &'a str, + body: Box, + raw_string: Option, + params: &'a HashMap, + headers: HashMap>, + query: HashMap>, + method: HttpMethod, +} + +#[derive(Serialize)] +struct HttpTriggerWmTrigger<'a> { + route: &'a str, + path: &'a str, + params: &'a HashMap, + query: &'a HashMap>, + headers: &'a HashMap>, + method: HttpMethod, +} + +impl HttpTriggerArgs { + pub fn to_main_args(self, wrap_body: bool) -> Result { + let mut extra = HashMap::new(); + + let WebhookArgsMetadata { raw_string, .. } = self.0.metadata; + + if let Some(raw_string) = raw_string { + extra.insert("raw_string".to_string(), to_raw_value(&raw_string)); + } + + let extra = if extra.is_empty() { None } else { Some(extra) }; + + match self.0.body { + Body::HashMap(mut body) => { + if wrap_body { + body = HashMap::from([("body".to_string(), to_raw_value(&body))]); + } + Ok(PushArgsOwned { args: body, extra }) + } + Body::NoHashMap(args) => { + let mut hm = HashMap::new(); + hm.insert("body".to_string(), args); + Ok(PushArgsOwned { args: hm, extra }) + } + } + } + + pub fn to_args_from_format( + self, + route_path: &str, + called_path: &str, + params: &HashMap, + format: RunnableFormat, + wrap_body: bool, + ) -> Result { + match format { + RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V2 } => { + // we don't care about wrap_body in v2 + self.to_v2_preprocessor_args(route_path, called_path, params) + } + RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V1 } => { + self.to_v1_preprocessor_args(route_path, called_path, params, wrap_body) + } + RunnableFormat { has_preprocessor: false, .. } => self.to_main_args(wrap_body), + } + } + + fn to_v1_preprocessor_args( + self, + route_path: &str, + called_path: &str, + params: &HashMap, + wrap_body: bool, + ) -> Result { + let mut extra = HashMap::new(); + let mut wm_trigger = HashMap::new(); + wm_trigger.insert("kind".to_string(), to_raw_value(&"http".to_string())); + wm_trigger.insert( + "http".to_string(), + to_raw_value(&HttpTriggerWmTrigger { + route: route_path, + path: called_path, + method: (&self.0.metadata.method).try_into()?, + params, + query: &self.0.metadata.query, + headers: &self.0.metadata.headers, + }), + ); + extra.insert("wm_trigger".to_string(), to_raw_value(&wm_trigger)); + + let mut args = self.to_main_args(wrap_body)?; + + args.extra.get_or_insert_default().extend(extra); + + Ok(args) + } + + pub fn to_v2_preprocessor_args( + self, + route_path: &str, + called_path: &str, + params: &HashMap, + ) -> Result { + let mut args = HashMap::new(); + args.insert( + "event".to_string(), + to_raw_value(&HttpTriggerPreprocessorEvent { + kind: "http".to_string(), + body: to_raw_value(&self.0.body), + raw_string: self.0.metadata.raw_string, + headers: self.0.metadata.headers, + query: self.0.metadata.query, + method: (&self.0.metadata.method).try_into()?, + route: route_path, + path: called_path, + params, + }), + ); + Ok(PushArgsOwned { args, extra: None }) + } +} diff --git a/backend/windmill-api/src/http_triggers.rs b/backend/windmill-api/src/http_triggers.rs index fa26406a13..ec295404b4 100644 --- a/backend/windmill-api/src/http_triggers.rs +++ b/backend/windmill-api/src/http_triggers.rs @@ -1,10 +1,11 @@ -use crate::http_trigger_auth::{self}; +#[cfg(feature = "http_trigger")] +use crate::http_trigger_args::{HttpMethod, RawHttpTriggerArgs}; #[cfg(feature = "parquet")] use crate::job_helpers_ee::get_workspace_s3_resource; use crate::resources::try_get_resource_from_db_as; +use crate::trigger_helpers::{get_runnable_format, RunnableId}; use crate::utils::non_empty_str; use crate::{ - args::try_from_request_body, auth::{AuthCache, OptTokened}, db::{ApiAuthed, DB}, jobs::{ @@ -15,7 +16,7 @@ use crate::{ }; use axum::response::Response; use axum::{ - extract::{Path, Query, Request}, + extract::{Path, Query}, response::IntoResponse, routing::{delete, get, post}, Extension, Json, Router, @@ -38,8 +39,9 @@ use windmill_common::{ error::{self, JsonResult}, s3_helpers::S3Object, utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, - worker::{to_raw_value, CLOUD_HOSTED}, + worker::CLOUD_HOSTED, }; +use windmill_queue::TriggerKind; lazy_static::lazy_static! { static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"/?:[-\w]+").unwrap(); @@ -81,31 +83,6 @@ pub fn workspaced_service() -> Router { .route("/route_exists", post(exists_route)) } -#[derive(Serialize, Deserialize, sqlx::Type, Debug)] -#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum HttpMethod { - Get, - Post, - Put, - Delete, - Patch, -} - -impl TryFrom<&http::Method> for HttpMethod { - type Error = error::Error; - fn try_from(method: &http::Method) -> Result { - match method { - &http::Method::GET => Ok(HttpMethod::Get), - &http::Method::POST => Ok(HttpMethod::Post), - &http::Method::PUT => Ok(HttpMethod::Put), - &http::Method::DELETE => Ok(HttpMethod::Delete), - &http::Method::PATCH => Ok(HttpMethod::Patch), - _ => Err(error::Error::BadRequest("Invalid HTTP method".to_string())), - } - } -} - #[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone, Copy)] #[sqlx(type_name = "AUTHENTICATION_METHOD", rename_all = "snake_case")] #[serde(rename_all(serialize = "snake_case", deserialize = "snake_case"))] @@ -902,42 +879,14 @@ async fn get_http_route_trigger( Ok((trigger, route_path.0, params, authed)) } -pub async fn build_http_trigger_extra( - route_path: &str, - called_path: &str, - method: &http::Method, - params: &HashMap, - query: &HashMap, - headers: &HeaderMap, -) -> Box { - let headers = headers - .iter() - .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string())) - .collect::>(); - - to_raw_value(&serde_json::json!({ - "kind": "http", - "http": { - "route": route_path, - "path": called_path, - "method": method.to_string().to_lowercase(), - "params": params, - "query": query, - "headers": headers - }, - })) -} - async fn route_job( Extension(db): Extension, Extension(user_db): Extension, Extension(auth_cache): Extension>, OptTokened { token }: OptTokened, Path(route_path): Path, - Query(query): Query>, - method: http::Method, headers: HeaderMap, - request: Request, + args: RawHttpTriggerArgs, ) -> Result { let route_path = route_path.to_path().trim_end_matches("/"); let (trigger, called_path, params, authed) = get_http_route_trigger( @@ -946,25 +895,13 @@ async fn route_job( token.as_ref(), &db, user_db.clone(), - &method, + &args.0.metadata.method, ) .await .map_err(|e| e.into_response())?; - let args = try_from_request_body( - request, - &(), - Some(match trigger.authentication_method { - AuthenticationMethod::CustomScript | AuthenticationMethod::Signature => true, - _ => trigger.raw_string, - }), - Some(trigger.wrap_body), - ) - .await - .map_err(|e| e.into_response())?; - - let mut args = args - .to_push_args_owned(&authed, &db, &trigger.workspace_id) + let args = args + .process_args(&authed, &db, &trigger.workspace_id, trigger.raw_string) .await .map_err(|e| e.into_response())?; @@ -984,7 +921,7 @@ async fn route_job( }; let authentication_method = - try_get_resource_from_db_as::( + try_get_resource_from_db_as::( authed.clone(), Some(user_db.clone()), &db, @@ -995,14 +932,11 @@ async fn route_job( .map_err(|e| e.into_response())?; let raw_payload = args - .extra + .0 + .metadata + .raw_string .as_ref() - .and_then(|extra| { - extra - .get("raw_string") - .and_then(|value| Some(value.to_string())) - .and_then(|raw_payload| Some(serde_json::from_str::(&raw_payload))) - }) + .map(|raw_payload| serde_json::from_str::(raw_payload)) .transpose() .map_err(|e| { windmill_common::error::Error::SerdeJson { location: e.to_string(), error: e } @@ -1130,20 +1064,28 @@ async fn route_job( } } - let extra = args.extra.get_or_insert_with(HashMap::new); + let runnable_format = get_runnable_format( + if trigger.is_flow { + RunnableId::from_flow_path(&trigger.script_path) + } else { + RunnableId::from_script_path(&trigger.script_path) + }, + &trigger.workspace_id, + &db, + &TriggerKind::Http, + ) + .await + .map_err(|e| e.into_response())?; - extra.insert( - "wm_trigger".to_string(), - build_http_trigger_extra( + let args = args + .to_args_from_format( &trigger.route_path, &called_path, - &method, ¶ms, - &query, - &headers, + runnable_format, + trigger.wrap_body, ) - .await, - ); + .map_err(|e| e.into_response())?; let run_query = RunJobQuery::default(); @@ -1157,7 +1099,6 @@ async fn route_job( StripPath(trigger.script_path.to_owned()), run_query, args, - None, ) .await .into_response() @@ -1170,7 +1111,6 @@ async fn route_job( user_db, args, trigger.workspace_id.clone(), - None, ) .await .into_response() @@ -1185,7 +1125,6 @@ async fn route_job( StripPath(trigger.script_path.to_owned()), run_query, args, - None, ) .await .into_response() @@ -1198,7 +1137,6 @@ async fn route_job( user_db, trigger.workspace_id.clone(), args, - None, ) .await .into_response() diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 0948e44497..ff9cd24f63 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -7,6 +7,7 @@ */ use axum::body::Body; +use axum::extract::Request; use axum::http::HeaderValue; #[cfg(feature = "deno_core")] use deno_core::{op2, serde_v8, v8, JsRuntime, OpState}; @@ -37,10 +38,11 @@ use crate::add_webhook_allowed_origin; use crate::concurrency_groups::join_concurrency_key; use crate::db::ApiAuthed; +use crate::trigger_helpers::RunnableId; use crate::users::get_scope_tags; use crate::utils::content_plain; use crate::{ - args::{DecodeQueries, WebhookArgs}, + args::{self, RawWebhookArgs}, db::DB, users::{check_scopes, require_owner_of_path, OptAuthed}, utils::require_super_admin, @@ -55,7 +57,7 @@ use axum::{ use base64::Engine; use chrono::Utc; use hmac::Mac; -use hyper::{Request, StatusCode}; +use hyper::StatusCode; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use sql_builder::prelude::*; use sqlx::types::JsonRawValue; @@ -86,7 +88,10 @@ use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS; #[cfg(feature = "prometheus")] use windmill_common::{METRICS_DEBUG_ENABLED, METRICS_ENABLED}; -use windmill_common::{get_latest_deployed_hash_for_path, BASE_URL}; +use windmill_common::{ + get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, + get_script_info_for_hash, FlowVersionInfo, ScriptHashInfo, BASE_URL, +}; use windmill_queue::{ cancel_job, get_result_and_success_by_id_from_flow, job_is_complete, push, PushArgs, PushArgsOwned, PushIsolationLevel, @@ -545,58 +550,6 @@ async fn force_cancel( } } -pub async fn get_path_tag_limits_cache_for_hash( - mut tx: Transaction<'_, Postgres>, - w_id: &str, - hash: i64, -) -> error::Result<( - String, - Option, - Option, - Option, - Option, - Option, - ScriptLang, - Option, - Option, - Option, - Option, - Option, - Option, - String, -)> { - let script = sqlx::query!( - "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by from script where hash = $1 AND workspace_id = $2", - hash, - w_id - ) - .fetch_optional(&mut *tx) - .await - .map_err(|e| { - Error::internal_err(format!( - "querying getting path for hash {hash} in {w_id}: {e:#}" - )) - })?.ok_or_else(|| Error::NotFound(format!( - "deployed script not found at hash {hash} in workspace {w_id}" - )))?; - Ok(( - script.path, - script.tag, - script.concurrency_key, - script.concurrent_limit, - script.concurrency_time_window_s, - script.cache_ttl, - script.language, - script.dedicated_worker, - script.priority, - script.delete_after_use, - script.timeout, - script.has_preprocessor, - script.on_behalf_of_email, - script.created_by, - )) -} - async fn get_flow_job_debug_info( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, @@ -674,8 +627,8 @@ async fn list_selected_job_groups( 'kind', jb.kind, 'script_path', jb.runnable_path, 'latest_schema', COALESCE( - (SELECT DISTINCT ON (s.path) s.schema FROM script s WHERE s.path = jb.runnable_path AND jb.kind = 'script' ORDER BY s.path, s.created_at DESC), - (SELECT flow_version.schema FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.path = jb.runnable_path AND jb.kind = 'flow') + (SELECT DISTINCT ON (s.path) s.schema FROM script s WHERE s.workspace_id = $1 AND s.path = jb.runnable_path AND jb.kind = 'script' ORDER BY s.path, s.created_at DESC), + (SELECT flow_version.schema FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.workspace_id = $1 AND flow.path = jb.runnable_path AND jb.kind = 'flow') ), 'schemas', ARRAY( SELECT jsonb_build_object( @@ -3466,7 +3419,6 @@ async fn batch_rerun_handle_job( StripPath(job.script_path.clone()), RunJobQuery { ..Default::default() }, PushArgsOwned { extra: None, args }, - None, ) .await; if let Ok((_, uuid)) = result { @@ -3483,7 +3435,6 @@ async fn batch_rerun_handle_job( StripPath(job.script_path.clone()), RunJobQuery { ..Default::default() }, PushArgsOwned { extra: None, args }, - None, ) .await } else { @@ -3495,7 +3446,6 @@ async fn batch_rerun_handle_job( job.script_hash, RunJobQuery { ..Default::default() }, PushArgsOwned { extra: None, args }, - None, ) .await }; @@ -3516,11 +3466,19 @@ pub async fn run_flow_by_path( Extension(user_db): Extension, Path((w_id, flow_path)): Path<(String, StripPath)>, Query(run_query): Query, - args: WebhookArgs, + args: RawWebhookArgs, ) -> error::Result<(StatusCode, String)> { - let args = args.to_push_args_owned(&authed, &db, &w_id).await?; + let args = args + .to_args_from_runnable( + &authed, + &db, + &w_id, + RunnableId::from_flow_path(&flow_path.0), + run_query.skip_preprocessor, + ) + .await?; - run_flow_by_path_inner(authed, db, user_db, w_id, flow_path, run_query, args, None).await + run_flow_by_path_inner(authed, db, user_db, w_id, flow_path, run_query, args).await } pub async fn run_flow_by_path_inner( @@ -3531,7 +3489,6 @@ pub async fn run_flow_by_path_inner( flow_path: StripPath, run_query: RunJobQuery, args: PushArgsOwned, - label_prefix: Option, ) -> error::Result<(StatusCode, String)> { #[cfg(feature = "enterprise")] check_license_key_valid().await?; @@ -3539,23 +3496,17 @@ pub async fn run_flow_by_path_inner( check_scopes(&authed, || format!("run:flow/{flow_path}"))?; let mut tx = user_db.clone().begin(&authed).await?; - let (tag, dedicated_worker, has_preprocessor, on_behalf_of_email, edited_by) = sqlx::query!( - "SELECT tag, dedicated_worker, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by - FROM flow - LEFT JOIN flow_version - ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] - WHERE flow.path = $1 and flow.workspace_id = $2", - flow_path, - w_id - ) - .fetch_optional(&mut *tx) - .await? - .map(|x| (x.tag, x.dedicated_worker, x.has_preprocessor, x.on_behalf_of_email, x.edited_by)) - .ok_or_else(|| { - Error::NotFound(format!( - "flow not found at path {flow_path} in workspace {w_id}" - )) - })?; + + let FlowVersionInfo { + version, + tag, + dedicated_worker, + has_preprocessor, + on_behalf_of_email, + edited_by, + .. + } = get_latest_flow_version_info_for_path(&mut *tx, &w_id, &flow_path, true).await?; + drop(tx); let tag = run_query.tag.clone().or(tag); @@ -3587,13 +3538,12 @@ pub async fn run_flow_by_path_inner( JobPayload::Flow { path: flow_path.to_string(), dedicated_worker, + version, apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) && has_preprocessor.unwrap_or(false), }, PushArgs { args: &args.args, extra: args.extra }, - &label_prefix - .map(|x| x + authed.display_username()) - .unwrap_or_else(|| authed.display_username().to_string()), + authed.display_username(), email, permissioned_as, scheduled_for, @@ -3715,20 +3665,19 @@ pub async fn run_script_by_path( Extension(user_db): Extension, Path((w_id, script_path)): Path<(String, StripPath)>, Query(run_query): Query, - args: WebhookArgs, + args: RawWebhookArgs, ) -> error::Result<(StatusCode, String)> { - let args = args.to_push_args_owned(&authed, &db, &w_id).await?; - run_script_by_path_inner( - authed, - db, - user_db, - w_id, - script_path, - run_query, - args, - None, - ) - .await + let args = args + .to_args_from_runnable( + &authed, + &db, + &w_id, + RunnableId::from_script_path(&script_path.0), + run_query.skip_preprocessor, + ) + .await?; + + run_script_by_path_inner(authed, db, user_db, w_id, script_path, run_query, args).await } pub async fn run_script_by_path_inner( @@ -3739,7 +3688,6 @@ pub async fn run_script_by_path_inner( script_path: StripPath, run_query: RunJobQuery, args: PushArgsOwned, - label_prefix: Option, ) -> error::Result<(StatusCode, String)> { #[cfg(feature = "enterprise")] check_license_key_valid().await?; @@ -3780,9 +3728,7 @@ pub async fn run_script_by_path_inner( &w_id, job_payload, PushArgs { args: &args.args, extra: args.extra }, - &label_prefix - .map(|x| x + authed.display_username()) - .unwrap_or_else(|| authed.display_username().to_string()), + authed.display_username(), email, permissioned_as, scheduled_for, @@ -4384,7 +4330,7 @@ pub async fn run_wait_result_job_by_path_get( Extension(db): Extension, Path((w_id, script_path)): Path<(String, StripPath)>, Query(run_query): Query, - DecodeQueries(queries): DecodeQueries, + args: RawWebhookArgs, ) -> error::Result { #[cfg(feature = "enterprise")] check_license_key_valid().await?; @@ -4396,17 +4342,23 @@ pub async fn run_wait_result_job_by_path_get( x.map_err(|e| Error::internal_err(format!("Impossible to decode query payload: {e:#?}"))) }); - let mut payload_args = if let Some(payload) = payload_r { + let payload_args = if let Some(payload) = payload_r { payload? } else { HashMap::new() }; - queries.iter().for_each(|(k, v)| { - payload_args.insert(k.to_string(), v.clone()); - }); - let inner_args: HashMap> = HashMap::new(); - let args = PushArgs { extra: Some(payload_args), args: &inner_args }; + let mut args = args.process_args(&authed, &db, &w_id, None).await?; + args.body = args::Body::HashMap(payload_args); + + let args = args + .to_args_from_runnable( + &db, + &w_id, + RunnableId::from_script_path(&script_path.0), + run_query.skip_preprocessor, + ) + .await?; check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?; let script_path = script_path.to_path(); @@ -4478,7 +4430,7 @@ pub async fn run_wait_result_flow_by_path_get( Extension(db): Extension, Path((w_id, flow_path)): Path<(String, StripPath)>, Query(run_query): Query, - DecodeQueries(queries): DecodeQueries, + args: RawWebhookArgs, ) -> error::Result { #[cfg(feature = "enterprise")] check_license_key_valid().await?; @@ -4492,22 +4444,26 @@ pub async fn run_wait_result_flow_by_path_get( }) }); - let mut payload_args = if let Some(payload) = payload_r { + let payload_args = if let Some(payload) = payload_r { payload? } else { HashMap::new() }; - queries.iter().for_each(|(k, v)| { - payload_args.insert(k.to_string(), v.clone()); - }); + let mut args = args.process_args(&authed, &db, &w_id, None).await?; + args.body = args::Body::HashMap(payload_args); - let args = PushArgsOwned { extra: Some(payload_args), args: HashMap::new() }; + let args = args + .to_args_from_runnable( + &db, + &w_id, + RunnableId::from_flow_path(&flow_path.0), + run_query.skip_preprocessor, + ) + .await?; - run_wait_result_flow_by_path_internal( - db, run_query, flow_path, authed, user_db, args, w_id, None, - ) - .await + run_wait_result_flow_by_path_internal(db, run_query, flow_path, authed, user_db, args, w_id) + .await } pub async fn run_wait_result_script_by_path( @@ -4516,24 +4472,23 @@ pub async fn run_wait_result_script_by_path( Extension(db): Extension, Path((w_id, script_path)): Path<(String, StripPath)>, Query(run_query): Query, - args: WebhookArgs, + args: RawWebhookArgs, ) -> error::Result { #[cfg(feature = "enterprise")] check_license_key_valid().await?; - let args = args.to_push_args_owned(&authed, &db, &w_id).await?; + let args = args + .to_args_from_runnable( + &authed, + &db, + &w_id, + RunnableId::from_script_path(&script_path.0), + run_query.skip_preprocessor, + ) + .await?; - run_wait_result_script_by_path_internal( - db, - run_query, - script_path, - authed, - user_db, - w_id, - args, - None, - ) - .await + run_wait_result_script_by_path_internal(db, run_query, script_path, authed, user_db, w_id, args) + .await } pub async fn run_wait_result_script_by_path_internal( @@ -4544,7 +4499,6 @@ pub async fn run_wait_result_script_by_path_internal( user_db: UserDB, w_id: String, args: PushArgsOwned, - label_prefix: Option, ) -> error::Result { check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?; let script_path = script_path.to_path(); @@ -4580,9 +4534,7 @@ pub async fn run_wait_result_script_by_path_internal( &w_id, job_payload, PushArgs { args: &args.args, extra: args.extra }, - &label_prefix - .map(|x| x + authed.display_username()) - .unwrap_or_else(|| authed.display_username().to_string()), + authed.display_username(), email, permissioned_as, None, @@ -4616,20 +4568,29 @@ pub async fn run_wait_result_script_by_hash( Extension(db): Extension, Path((w_id, script_hash)): Path<(String, ScriptHash)>, Query(run_query): Query, - args: WebhookArgs, + args: RawWebhookArgs, ) -> error::Result { #[cfg(feature = "enterprise")] check_license_key_valid().await?; - let args = args.to_push_args_owned(&authed, &db, &w_id).await?; + let args = args + .to_args_from_runnable( + &authed, + &db, + &w_id, + RunnableId::from_script_hash(script_hash), + run_query.skip_preprocessor, + ) + .await?; check_queue_too_long(&db, run_query.queue_limit).await?; let hash = script_hash.0; - let ( + let mut tx = user_db.clone().begin(&authed).await?; + let ScriptHashInfo { path, tag, - custom_concurrency_key, + concurrency_key, concurrent_limit, concurrency_time_window_s, mut cache_ttl, @@ -4641,8 +4602,8 @@ pub async fn run_wait_result_script_by_hash( has_preprocessor, on_behalf_of_email, created_by, - ) = get_path_tag_limits_cache_for_hash(user_db.clone().begin(&authed).await?, &w_id, hash) - .await?; + .. + } = get_script_info_for_hash(&mut *tx, &w_id, hash).await?; if let Some(run_query_cache_ttl) = run_query.cache_ttl { cache_ttl = Some(run_query_cache_ttl); } @@ -4675,7 +4636,7 @@ pub async fn run_wait_result_script_by_hash( JobPayload::ScriptHash { hash: ScriptHash(hash), path: path, - custom_concurrency_key, + custom_concurrency_key: concurrency_key, concurrent_limit: concurrent_limit, concurrency_time_window_s: concurrency_time_window_s, cache_ttl, @@ -4720,17 +4681,23 @@ pub async fn run_wait_result_flow_by_path( Extension(db): Extension, Path((w_id, flow_path)): Path<(String, StripPath)>, Query(run_query): Query, - args: WebhookArgs, + args: RawWebhookArgs, ) -> error::Result { #[cfg(feature = "enterprise")] check_license_key_valid().await?; - let args = args.to_push_args_owned(&authed, &db, &w_id).await?; + let args = args + .to_args_from_runnable( + &authed, + &db, + &w_id, + RunnableId::from_flow_path(&flow_path.0), + run_query.skip_preprocessor, + ) + .await?; - run_wait_result_flow_by_path_internal( - db, run_query, flow_path, authed, user_db, args, w_id, None, - ) - .await + run_wait_result_flow_by_path_internal(db, run_query, flow_path, authed, user_db, args, w_id) + .await } pub async fn run_wait_result_flow_by_path_internal( @@ -4741,7 +4708,6 @@ pub async fn run_wait_result_flow_by_path_internal( user_db: UserDB, args: PushArgsOwned, w_id: String, - label_prefix: Option, ) -> error::Result { check_queue_too_long(&db, run_query.queue_limit).await?; @@ -4751,23 +4717,16 @@ pub async fn run_wait_result_flow_by_path_internal( let scheduled_for = run_query.get_scheduled_for(&db).await?; let mut tx = user_db.clone().begin(&authed).await?; - let (tag, dedicated_worker, early_return, has_preprocessor, on_behalf_of_email, edited_by) = sqlx::query!( - "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by - FROM flow - LEFT JOIN flow_version - ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] - WHERE flow.path = $1 and flow.workspace_id = $2", - flow_path, - w_id - ) - .fetch_optional(&mut *tx) - .await? - .map(|x| (x.tag, x.dedicated_worker, x.early_return, x.has_preprocessor, x.on_behalf_of_email, x.edited_by)) - .ok_or_else(|| { - Error::NotFound(format!( - "flow not found at path {flow_path} in workspace {w_id}" - )) - })?; + + let FlowVersionInfo { + tag, + dedicated_worker, + early_return, + has_preprocessor, + on_behalf_of_email, + edited_by, + version, + } = get_latest_flow_version_info_for_path(&mut *tx, &w_id, &flow_path, true).await?; let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&w_id, &tag, &authed).await?; @@ -4796,13 +4755,12 @@ pub async fn run_wait_result_flow_by_path_internal( JobPayload::Flow { path: flow_path.to_string(), dedicated_worker, + version, apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) && has_preprocessor.unwrap_or(false), }, PushArgs { args: &args.args, extra: args.extra }, - &label_prefix - .map(|x| x + authed.display_username()) - .unwrap_or_else(|| authed.display_username().to_string()), + authed.display_username(), email, permissioned_as, scheduled_for, @@ -5256,29 +5214,23 @@ async fn add_batch_jobs( "script" => { if let Some(path) = batch_info.path { let mut tx = user_db.clone().begin(&authed).await?; - let ( - script_hash, - _tag, - custom_concurrency_key, + let ScriptHashInfo { + hash: script_hash, + concurrency_key, concurrent_limit, concurrency_time_window_s, - _cache_ttl, language, dedicated_worker, - _priority, - _delete_after_use, timeout, - _, - _, // TODO: consider on_behalf_of_email and created_by for batch jobs - _, // ------------------------------------------ - ) = get_latest_deployed_hash_for_path(&mut *tx, &w_id, &path).await?; + .. // TODO: consider on_behalf_of_email and created_by for batch jobs + } = get_latest_deployed_hash_for_path(&mut *tx, &w_id, &path).await?; ( Some(script_hash), Some(path), JobKind::Script, Some(language), dedicated_worker, - custom_concurrency_key, + concurrency_key, concurrent_limit, concurrency_time_window_s, timeout, @@ -5418,7 +5370,7 @@ async fn add_batch_jobs( raw_lock, raw_flow.map(sqlx::types::Json) as Option>, tag, - hash.map(|h| h.0), + hash, path, job_kind.clone() as JobKind, language as ScriptLang, @@ -5563,20 +5515,19 @@ pub async fn run_job_by_hash( Extension(user_db): Extension, Path((w_id, script_hash)): Path<(String, ScriptHash)>, Query(run_query): Query, - args: WebhookArgs, + args: RawWebhookArgs, ) -> error::Result<(StatusCode, String)> { - let args = args.to_push_args_owned(&authed, &db, &w_id).await?; - run_job_by_hash_inner( - authed, - db, - user_db, - w_id, - script_hash, - run_query, - args, - None, - ) - .await + let args = args + .to_args_from_runnable( + &authed, + &db, + &w_id, + RunnableId::from_script_hash(script_hash), + run_query.skip_preprocessor, + ) + .await?; + + run_job_by_hash_inner(authed, db, user_db, w_id, script_hash, run_query, args).await } pub async fn run_job_by_hash_inner( @@ -5587,29 +5538,28 @@ pub async fn run_job_by_hash_inner( script_hash: ScriptHash, run_query: RunJobQuery, args: PushArgsOwned, - label_prefix: Option, ) -> error::Result<(StatusCode, String)> { #[cfg(feature = "enterprise")] check_license_key_valid().await?; let hash = script_hash.0; - let ( + let mut tx = user_db.clone().begin(&authed).await?; + let ScriptHashInfo { path, tag, - custom_concurrency_key, + concurrency_key, concurrent_limit, concurrency_time_window_s, mut cache_ttl, language, dedicated_worker, priority, - _delete_after_use, // not taken into account in async endpoints timeout, has_preprocessor, on_behalf_of_email, created_by, - ) = get_path_tag_limits_cache_for_hash(user_db.clone().begin(&authed).await?, &w_id, hash) - .await?; + .. // delete_after_use not taken into account in async endpoints + } = get_script_info_for_hash(&mut *tx, &w_id, hash).await?; check_scopes(&authed, || format!("run:script/{path}"))?; if let Some(run_query_cache_ttl) = run_query.cache_ttl { cache_ttl = Some(run_query_cache_ttl); @@ -5643,7 +5593,7 @@ pub async fn run_job_by_hash_inner( JobPayload::ScriptHash { hash: ScriptHash(hash), path: path, - custom_concurrency_key, + custom_concurrency_key: concurrency_key, concurrent_limit: concurrent_limit, concurrency_time_window_s: concurrency_time_window_s, cache_ttl, @@ -5654,9 +5604,7 @@ pub async fn run_job_by_hash_inner( && has_preprocessor.unwrap_or(false), }, PushArgs { args: &args.args, extra: args.extra }, - &label_prefix - .map(|x| x + authed.display_username()) - .unwrap_or_else(|| authed.display_username().to_string()), + authed.display_username(), email, permissioned_as, scheduled_for, @@ -5839,7 +5787,7 @@ pub fn filter_list_completed_query( if let Some(label) = &lq.label { if lq.allow_wildcards.unwrap_or(false) { let wh = format!( - "EXISTS (SELECT 1 FROM jsonb_array_elements_text(result->'wm_labels') label WHERE label LIKE '{}')", + "EXISTS (SELECT 1 FROM jsonb_array_elements_text(result->'wm_labels') label WHERE jsonb_typeof(result->'wm_labels') = 'array' AND label LIKE '{}')", &label.replace("*", "%").replace("'", "''") ); sqlb.and_where("result ? 'wm_labels'"); diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index ece80c64b1..d7d9498fab 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -18,6 +18,8 @@ use crate::oauth2_ee::SlackVerifier; #[cfg(feature = "smtp")] use crate::smtp_server_ee::SmtpServer; +#[cfg(feature = "mcp")] +use crate::mcp::{setup_mcp_server, Runner as McpRunner}; use crate::tracing_init::MyOnFailure; use crate::{ tracing_init::{MyMakeSpan, MyOnResponse}, @@ -27,6 +29,7 @@ use crate::{ #[cfg(feature = "agent_worker_server")] use agent_workers_ee::AgentCache; + use anyhow::Context; use argon2::Argon2; use axum::extract::DefaultBodyLimit; @@ -78,6 +81,8 @@ mod folders; mod granular_acls; mod groups; #[cfg(feature = "http_trigger")] +mod http_trigger_args; +#[cfg(feature = "http_trigger")] mod http_trigger_auth; #[cfg(feature = "http_trigger")] mod http_triggers; @@ -119,6 +124,7 @@ mod slack_approvals; mod smtp_server_ee; #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] mod sqs_triggers_ee; +mod trigger_helpers; mod static_assets; #[cfg(all(feature = "stripe", feature = "enterprise"))] @@ -139,6 +145,9 @@ mod workspaces_ee; mod workspaces_export; mod workspaces_extra; +#[cfg(feature = "mcp")] +mod mcp; + pub const DEFAULT_BODY_LIMIT: usize = 2097152 * 100; // 200MB lazy_static::lazy_static! { @@ -219,6 +228,7 @@ pub async fn run_server( mut killpill_rx: tokio::sync::broadcast::Receiver<()>, port_tx: tokio::sync::oneshot::Sender, server_mode: bool, + mcp_mode: bool, _base_internal_url: String, ) -> anyhow::Result<()> { let user_db = UserDB::new(db.clone()); @@ -404,7 +414,7 @@ pub async fn run_server( Router::new() }; - if !*CLOUD_HOSTED && server_mode { + if !*CLOUD_HOSTED && server_mode && !mcp_mode { #[cfg(feature = "websocket")] { let ws_killpill_rx = killpill_rx.resubscribe(); @@ -448,9 +458,41 @@ pub async fn run_server( } } + let listener = tokio::net::TcpListener::bind(addr) + .await + .context("binding main windmill server")?; + let port = listener.local_addr().map(|x| x.port()).unwrap_or(8000); + let ip = listener + .local_addr() + .map(|x| x.ip().to_string()) + .unwrap_or("localhost".to_string()); + + // Setup MCP server + #[allow(unused_variables)] + let (mcp_router, mcp_main_ct, mcp_service_ct) = { + #[cfg(feature = "mcp")] + if server_mode || mcp_mode { + let (mcp_sse_server, mcp_router) = setup_mcp_server(addr, "/api/mcp/w/:workspace_id")?; + #[cfg(feature = "mcp")] + let mcp_main_ct = mcp_sse_server.config.ct.clone(); // Token to signal shutdown *to* MCP + #[cfg(feature = "mcp")] + let mcp_service_ct = mcp_sse_server.with_service(McpRunner::new); // Token to wait for MCP *service* shutdown + (mcp_router, Some(mcp_main_ct), Some(mcp_service_ct)) + } else { + (Router::new(), None, None) + } + + #[cfg(not(feature = "mcp"))] + (Router::new(), None::<()>, None::<()>) + }; + #[cfg(feature = "agent_worker_server")] let (agent_workers_router, agent_workers_bg_processor, agent_workers_killpill_tx) = - agent_workers_ee::workspaced_service(db.clone(), _base_internal_url.clone()); + if server_mode { + agent_workers_ee::workspaced_service(db.clone(), _base_internal_url.clone()) + } else { + (Router::new(), vec![], None) + }; #[cfg(feature = "agent_worker_server")] let agent_cache = Arc::new(AgentCache::new()); @@ -531,26 +573,6 @@ pub async fn run_server( .nest("/ai", ai::global_service()) .route_layer(from_extractor::()) .route_layer(from_extractor::()) - .nest("/agent_workers", { - #[cfg(feature = "agent_worker_server")] - { - agent_workers_ee::global_service().layer(Extension(agent_cache.clone())) - } - #[cfg(not(feature = "agent_worker_server"))] - { - Router::new() - } - }) - .nest("/w/:workspace_id/agent_workers", { - #[cfg(feature = "agent_worker_server")] - { - agent_workers_router.layer(Extension(agent_cache.clone())) - } - #[cfg(not(feature = "agent_worker_server"))] - { - Router::new() - } - }) .nest("/jobs", jobs::global_root_service()) .nest( "/srch/w/:workspace_id/index", @@ -586,6 +608,28 @@ pub async fn run_server( .layer(from_extractor::()) .layer(cors.clone()), ) + .nest("/mcp/w/:workspace_id", mcp_router) + .layer(from_extractor::()) + .nest("/agent_workers", { + #[cfg(feature = "agent_worker_server")] + { + agent_workers_ee::global_service().layer(Extension(agent_cache.clone())) + } + #[cfg(not(feature = "agent_worker_server"))] + { + Router::new() + } + }) + .nest("/w/:workspace_id/agent_workers", { + #[cfg(feature = "agent_worker_server")] + { + agent_workers_router.layer(Extension(agent_cache.clone())) + } + #[cfg(not(feature = "agent_worker_server"))] + { + Router::new() + } + }) .nest( "/w/:workspace_id/jobs_u", jobs::workspace_unauthed_service().layer(cors.clone()), @@ -694,14 +738,6 @@ pub async fn run_server( .on_failure(MyOnFailure {}), ) }; - let listener = tokio::net::TcpListener::bind(addr) - .await - .context("binding main windmill server")?; - let port = listener.local_addr().map(|x| x.port()).unwrap_or(8000); - let ip = listener - .local_addr() - .map(|x| x.ip().to_string()) - .unwrap_or("localhost".to_string()); let server = axum::serve(listener, app.into_make_service()); @@ -719,14 +755,28 @@ pub async fn run_server( let server = server.with_graceful_shutdown(async move { killpill_rx.recv().await.ok(); #[cfg(feature = "agent_worker_server")] - if let Err(e) = agent_workers_killpill_tx.kill().await { - tracing::error!("Error killing agent workers: {e:#}"); + if let Some(agent_workers_killpill_tx) = agent_workers_killpill_tx { + if let Err(e) = agent_workers_killpill_tx.kill().await { + tracing::error!("Error killing agent workers: {e:#}"); + } } tracing::info!("Graceful shutdown of server"); + + #[cfg(feature = "mcp")] + { + if let Some(mcp_main_ct) = mcp_main_ct { + tracing::info!("Received shutdown signal, cancelling MCP server..."); + mcp_main_ct.cancel(); + } + if let Some(mcp_service_ct) = mcp_service_ct { + tracing::info!("Waiting for MCP service cancellation..."); + mcp_service_ct.cancelled().await; + tracing::info!("MCP service cancelled."); + } + } }); server.await?; - #[cfg(feature = "agent_worker_server")] for (i, bg_processor) in agent_workers_bg_processor.into_iter().enumerate() { tracing::info!("server off. shutting down agent worker bg processor {i}"); diff --git a/backend/windmill-api/src/mcp.rs b/backend/windmill-api/src/mcp.rs new file mode 100644 index 0000000000..431e121ba8 --- /dev/null +++ b/backend/windmill-api/src/mcp.rs @@ -0,0 +1,1141 @@ +use std::borrow::Cow; +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::Arc; + +use axum::body::to_bytes; +use axum::Router; +use rmcp::transport::sse_server::{SseServer, SseServerConfig}; +use rmcp::{ + handler::server::ServerHandler, + model::*, + service::{RequestContext, RoleServer}, + Error, +}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sql_builder::prelude::*; +use sqlx::FromRow; +use tokio::try_join; +use tokio_util::sync::CancellationToken; +use windmill_common::db::UserDB; +use windmill_common::worker::to_raw_value; +use windmill_common::{DB, HUB_BASE_URL}; + +use windmill_common::scripts::{get_full_hub_script_by_path, Schema}; + +use crate::db::ApiAuthed; +use crate::jobs::{ + run_wait_result_flow_by_path_internal, run_wait_result_script_by_path_internal, RunJobQuery, +}; +use crate::HTTP_CLIENT; +use windmill_common::utils::{query_elems_from_hub, StripPath}; + +/// Transforms the path for workspace scripts/flows. +/// +/// This function takes a path and a type string. +/// It then formats the transformed path with the type prefix. +/// This is used when listing, because we can't have names with slashes. +/// Because we replace slashes with underscores, we also need to escape underscores. +/// +/// # Parameters +/// - `path`: The path to transform. +/// - `type_str`: The type of the item (script or flow). +/// +/// # Returns +/// - `String`: The transformed path. +fn transform_path(path: &str, type_str: &str) -> String { + // Only apply special underscore escaping for paths starting with "f/" + let transformed = if path.starts_with("f/") { + let escaped_path = path.replace('_', "__"); + escaped_path.replace('/', "_") + } else { + path.replace('/', "_") + }; + + // first letter of type_str is used as prefix, only one letter to avoid reaching 60 char name limit + format!("{}-{}", &type_str[..1], transformed) +} + +fn convert_schema_to_schema_type(schema: Option) -> SchemaType { + let schema_obj = if let Some(ref s) = schema { + match serde_json::from_str::(s.0.get()) { + Ok(val) => val, + Err(_) => SchemaType::default(), + } + } else { + SchemaType::default() + }; + schema_obj +} + +trait ToolableItem { + fn get_path_or_id(&self) -> String; + fn get_summary(&self) -> &str; + fn get_description(&self) -> &str; + fn get_schema(&self) -> SchemaType; + fn is_hub(&self) -> bool; + fn item_type(&self) -> &'static str; + fn get_integration_type(&self) -> Option; +} + +impl ToolableItem for ScriptInfo { + fn get_path_or_id(&self) -> String { + transform_path(&self.path, "script") + } + fn get_summary(&self) -> &str { + self.summary.as_deref().unwrap_or("No summary") + } + fn get_description(&self) -> &str { + self.description.as_deref().unwrap_or("No description") + } + fn get_schema(&self) -> SchemaType { + convert_schema_to_schema_type(self.schema.clone()) + } + fn is_hub(&self) -> bool { + false + } + fn item_type(&self) -> &'static str { + "script" + } + fn get_integration_type(&self) -> Option { + None + } +} + +impl ToolableItem for FlowInfo { + fn get_path_or_id(&self) -> String { + transform_path(&self.path, "flow") + } + fn get_summary(&self) -> &str { + self.summary.as_deref().unwrap_or("No summary") + } + fn get_description(&self) -> &str { + self.description.as_deref().unwrap_or("No description") + } + fn get_schema(&self) -> SchemaType { + convert_schema_to_schema_type(self.schema.clone()) + } + fn is_hub(&self) -> bool { + false + } + fn item_type(&self) -> &'static str { + "flow" + } + fn get_integration_type(&self) -> Option { + None + } +} + +impl ToolableItem for HubScriptInfo { + fn get_path_or_id(&self) -> String { + let id = self.version_id; + let summary = self.summary.as_deref().unwrap_or("No summary"); + format!("hs-{}-{}", id, summary.replace(" ", "_")) + } + fn get_summary(&self) -> &str { + self.summary.as_deref().unwrap_or("No summary") + } + fn get_description(&self) -> &str { + self.description.as_deref().unwrap_or("No description") + } + fn get_schema(&self) -> SchemaType { + match serde_json::from_value::(self.schema.clone().unwrap_or_default()) { + Ok(schema_type) => schema_type, + Err(_) => SchemaType::default(), + } + } + fn is_hub(&self) -> bool { + true + } + fn item_type(&self) -> &'static str { + "script" + } + fn get_integration_type(&self) -> Option { + self.app.clone() + } +} + +#[derive(Clone)] +pub struct Runner {} + +#[derive(Serialize, Deserialize, Debug)] +struct HubResponse { + asks: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +struct HubScriptInfo { + version_id: u64, + summary: Option, + description: Option, + schema: Option, + app: Option, +} + +#[derive(Serialize, FromRow, Deserialize, Debug, Clone)] +struct SchemaType { + r#type: String, + properties: std::collections::HashMap, + required: Vec, +} + +impl Default for SchemaType { + fn default() -> Self { + Self { + r#type: "object".to_string(), + properties: std::collections::HashMap::new(), + required: vec![], + } + } +} + +#[derive(Serialize, FromRow, Debug)] +struct ScriptInfo { + path: String, + summary: Option, + description: Option, + schema: Option, +} + +#[derive(Serialize, FromRow)] +struct ItemSchema { + schema: Option, +} + +#[derive(Serialize, FromRow, Debug)] +struct FlowInfo { + path: String, + summary: Option, + description: Option, + schema: Option, +} + +#[derive(Serialize, FromRow, Debug)] +struct ResourceInfo { + path: String, + description: Option, + resource_type: String, +} + +#[derive(Serialize, FromRow, Debug, Clone)] +struct ResourceType { + name: String, + description: Option, +} + +impl Runner { + pub fn new() -> Self { + Self {} + } + + async fn get_item_schema( + path: &str, + user_db: &UserDB, + authed: &ApiAuthed, + workspace_id: &str, + item_type: &str, + ) -> Result, Error> { + let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type)); + sqlb.fields(&["o.schema"]); + sqlb.and_where("o.path = ?".bind(&path)); + sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)); + sqlb.and_where("o.archived = false"); + sqlb.and_where("o.draft_only IS NOT TRUE"); + let sql = sqlb.sql().map_err(|_e| { + tracing::error!("failed to build sql: {}", _e); + Error::internal_error("failed to build sql", None) + })?; + let mut tx = user_db + .clone() + .begin(authed) + .await + .map_err(|_e| Error::internal_error("failed to begin transaction", None))?; + let item = sqlx::query_as::<_, ItemSchema>(&sql) + .fetch_one(&mut *tx) + .await + .map_err(|_e| { + tracing::error!("failed to fetch item schema: {}", _e); + Error::internal_error("failed to fetch item schema", None) + })?; + tx.commit() + .await + .map_err(|_e| Error::internal_error("failed to commit transaction", None))?; + Ok(item.schema) + } + + /// Reverses the transformation of a path. + /// + /// This function takes a transformed path and reverses the transformation applied by `transform_path`. + /// It checks if the path starts with "h" (indicating a Hub script) and removes the prefix if present. + /// It then determines the type of the item (script or flow) based on the prefix. + /// This is used in call_tool to get the original path, and the type of the item. + /// + /// # Parameters + /// - `transformed_path`: The transformed path to reverse. + /// + /// # Returns + /// - `Result<(&str, String, bool), String>`: A tuple containing the original path, the type of the item, and a boolean indicating if it's a Hub script. + /// - `Err(String)`: If the path is invalid. + fn reverse_transform(transformed_path: &str) -> Result<(&str, String, bool), String> { + let is_hub = transformed_path.starts_with("h"); + let transformed_path = if is_hub { + transformed_path[1..].to_string() + } else { + transformed_path.to_string() + }; + let type_str = if transformed_path.starts_with("s-") { + "script" + } else if transformed_path.starts_with("f-") { + "flow" + } else { + return Err(format!( + "Invalid prefix in transformed path: {}", + transformed_path + )); + }; + + let mangled_path = &transformed_path[2..]; + + // Check if this path was previously transformed with special underscore handling + let is_special_path = mangled_path.starts_with("f_"); + + let original_path = if is_hub { + let parts = mangled_path.split("-").collect::>(); + parts[0].to_string() + } else if is_special_path { + const TEMP_PLACEHOLDER: &str = "@@UNDERSCORE@@"; + let path_with_placeholder = mangled_path.replace("__", TEMP_PLACEHOLDER); + let path_with_slashes = path_with_placeholder.replace('_', "/"); + path_with_slashes.replace(TEMP_PLACEHOLDER, "_") + } else { + mangled_path.replacen('_', "/", 2) + }; + + Ok((type_str, original_path, is_hub)) + } + + async fn inner_get_resources_types( + user_db: &UserDB, + authed: &ApiAuthed, + workspace_id: &str, + ) -> Result, Error> { + let mut sqlb = SqlBuilder::select_from("resource_type as o"); + sqlb.fields(&["o.name", "o.description"]); + sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)); + let sql = sqlb.sql().map_err(|_e| { + tracing::error!("failed to build sql: {}", _e); + Error::internal_error("failed to build sql", None) + })?; + let mut tx = user_db + .clone() + .begin(authed) + .await + .map_err(|_e| Error::internal_error("failed to begin transaction", None))?; + let rows = sqlx::query_as::<_, ResourceType>(&sql) + .fetch_all(&mut *tx) + .await + .map_err(|_e| { + tracing::error!("Failed to fetch resource types: {}", _e); + Error::internal_error("failed to fetch resource types", None) + })?; + tx.commit() + .await + .map_err(|_e| Error::internal_error("failed to commit transaction", None))?; + Ok(rows) + } + + async fn inner_get_resources( + user_db: &UserDB, + authed: &ApiAuthed, + workspace_id: &str, + resource_type: &str, + ) -> Result, Error> { + let mut sqlb = SqlBuilder::select_from("resource as o"); + sqlb.fields(&["o.path", "o.description", "o.resource_type"]); + sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)); + sqlb.and_where("o.resource_type = ?".bind(&resource_type)); + let sql = sqlb.sql().map_err(|_e| { + tracing::error!("failed to build sql: {}", _e); + Error::internal_error("failed to build sql", None) + })?; + let mut tx = user_db + .clone() + .begin(authed) + .await + .map_err(|_e| Error::internal_error("failed to begin transaction", None))?; + let rows = sqlx::query_as::<_, ResourceInfo>(&sql) + .fetch_all(&mut *tx) + .await + .map_err(|_e| { + tracing::error!("Failed to fetch resources: {}", _e); + Error::internal_error("failed to fetch resources", None) + })?; + tx.commit() + .await + .map_err(|_e| Error::internal_error("failed to commit transaction", None))?; + + Ok(rows) + } + + async fn inner_get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Send + Unpin>( + user_db: &UserDB, + authed: &ApiAuthed, + workspace_id: &str, + scope_type: &str, + item_type: &str, + ) -> Result, Error> { + let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type)); + sqlb.fields(&["o.path", "o.summary", "o.description", "o.schema"]); + if scope_type == "favorites" { + sqlb.join("favorite") + .on("favorite.favorite_kind = ? AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?".bind(&item_type) + .bind(&authed.username)); + } + sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)) + .and_where("o.archived = false") + .and_where("o.draft_only IS NOT TRUE") + .order_by( + if item_type == "flow" { + "o.edited_at" + } else { + "o.created_at" + }, + false, + ) + .limit(100); + let sql = sqlb.sql().map_err(|_e| { + tracing::error!("failed to build sql: {}", _e); + Error::internal_error("failed to build sql", None) + })?; + let mut tx = user_db + .clone() + .begin(authed) + .await + .map_err(|_e| Error::internal_error("failed to begin transaction", None))?; + let rows = sqlx::query_as::<_, T>(&sql) + .fetch_all(&mut *tx) + .await + .map_err(|_e| { + tracing::error!("Failed to fetch {}: {}", item_type, _e); + Error::internal_error(format!("failed to fetch {}", item_type), None) + })?; + tx.commit() + .await + .map_err(|_e| Error::internal_error("failed to commit transaction", None))?; + Ok(rows) + } + + async fn inner_get_scripts_from_hub( + db: &DB, + scope_integrations: Option<&str>, + ) -> Result, Error> { + let query_params = Some(vec![ + ("limit", "100".to_string()), + ("with_schema", "true".to_string()), + ("apps", scope_integrations.unwrap_or("").to_string()), + ]); + let url = format!("{}/scripts/top", *HUB_BASE_URL.read().await); + let (_status_code, _headers, response) = + query_elems_from_hub(&HTTP_CLIENT, &url, query_params, &db) + .await + .map_err(|e| { + tracing::error!("Failed to get items from hub: {}", e); + Error::internal_error(format!("Failed to get items from hub: {}", e), None) + })?; + let body_bytes = to_bytes(response, usize::MAX).await.map_err(|e| { + tracing::error!("Failed to read response body: {}", e); + Error::internal_error(format!("Failed to read response body: {}", e), None) + })?; + let body_str = String::from_utf8(body_bytes.to_vec()).map_err(|e| { + tracing::error!("Failed to decode response body: {}", e); + Error::internal_error(format!("Failed to decode response body: {}", e), None) + })?; + let hub_response: HubResponse = serde_json::from_str(&body_str).map_err(|e| { + tracing::error!("Failed to parse hub response: {}", e); + Error::internal_error(format!("Failed to parse hub response: {}", e), None) + })?; + + Ok(hub_response.asks) + } + + /// Transforms a value if it's an object. + /// + /// This function takes a key and a value, and a schema object. + /// If the value is a string that starts with "$res:", it returns the value as is. + /// Otherwise, it checks if the key is defined in the schema and if it's an object type. + /// If it is, it transforms the value to a string. This is because some clients do not support object types. + /// # Parameters + /// - `key`: The key of the value to transform. + /// - `value`: The value to transform. + /// - `schema_obj`: The schema object. + /// + /// # Returns + /// - `Value`: The transformed value. + fn transform_value_if_object( + key: &str, + value: &Value, + schema_obj: &Option, + ) -> Value { + if value.is_string() && value.as_str().unwrap().starts_with("$res:") { + return value.clone(); + } + + let schema_obj = match schema_obj { + Some(s) => s, + None => return value.clone(), + }; + + // Check if property is defined in schema and is an object type + let is_obj_type = match schema_obj.properties.get(key) { + Some(property) => { + let prop_type = property.get("type").and_then(|t| t.as_str()); + prop_type == Some("object") + } + None => false, + }; + + // If it's an object type and we received a string, try to parse it + if is_obj_type && value.is_string() { + if let Some(str_val) = value.as_str() { + if let Ok(obj_val) = serde_json::from_str::(str_val) { + return obj_val; + } + } + } + + value.clone() + } + + /// Reverses the transformation of a key. + /// + /// This function takes a transformed key and a schema object. + /// It then reverses the transformation applied by `apply_key_transformation`. This can be subject to collisions, but it's unlikely and is ok for our use case. + /// # Parameters + /// - `transformed_key`: The transformed key to reverse. + /// - `schema_obj`: The schema object. + /// + /// # Returns + /// - `String`: The original key. + fn reverse_transform_key(transformed_key: &str, schema_obj: &Option) -> String { + let schema_obj = match schema_obj { + Some(s) => s, + None => { + // No schema available, return the key as is (best guess) + return transformed_key.to_string(); + } + }; + + for original_key_in_schema in schema_obj.properties.keys() { + // Apply the SAME forward transformation to the schema key + let potential_transformed_key = + Runner::apply_key_transformation(original_key_in_schema); + + // If it matches the key we received, we found the likely original + if potential_transformed_key == transformed_key { + return original_key_in_schema.clone(); + } + } + + transformed_key.to_string() + } + + /// Applies a key transformation to a key. + /// + /// This function takes a key and replaces spaces with underscores. + /// It also removes any characters that are not alphanumeric or underscores. + /// This is used when listing, because we can't have names with spaces or special characters in the schema properties. + /// # Parameters + /// - `key`: The key to transform. + /// + /// # Returns + /// - `String`: The transformed key. + fn apply_key_transformation(key: &str) -> String { + key.replace(' ', "_") + .chars() + .filter(|c| c.is_alphanumeric() || *c == '_') + .collect::() + } + + /// Transforms the schema for resources. + /// + /// This function takes a schema and a database connection, and attempts to transform the schema for resources. + /// It replaces invalid characters in property keys with underscores and converts object properties to strings. + /// It also fetches resource type information and adds it to the description of resource properties. + /// + /// # Parameters + /// - `schema`: The schema to transform. + /// - `user_db`: The database connection. + /// - `authed`: The authenticated user. + /// - `w_id`: The workspace ID. + /// - `resources_cache`: A mutable reference to the resources cache. + /// - `resources_types`: A reference to the resource types. + /// + /// # Returns + /// - `Result`: The transformed schema. + /// - `Err(Error)`: If the transformation fails. + async fn transform_schema_for_resources( + schema: &SchemaType, + user_db: &UserDB, + authed: &ApiAuthed, + w_id: &str, + resources_cache: &mut HashMap>, + resources_types: &Vec, + ) -> Result { + let mut schema_obj: SchemaType = schema.clone(); + + // replace invalid char in property key with underscore + let replacements: Vec<(String, String, serde_json::Value)> = schema_obj + .properties + .iter() + .filter_map(|(key, value)| { + if key.chars().any(|c| !c.is_alphanumeric() && c != '_') { + let new_key = Runner::apply_key_transformation(key); + Some((key.clone(), new_key, value.clone())) + } else { + None + } + }) + .collect(); + + for (old_key, new_key, value) in replacements { + schema_obj.properties.remove(&old_key); + schema_obj.properties.insert(new_key, value); + } + + for (_key, prop_value) in schema_obj.properties.iter_mut() { + if let serde_json::Value::Object(prop_map) = prop_value { + // transform object properties to string because some client does not support object, might change in the future + if let Some(type_value) = prop_map.get("type") { + if let serde_json::Value::String(type_str) = type_value { + if type_str == "object" { + prop_map.insert( + "type".to_string(), + serde_json::Value::String("string".to_string()), + ); + } + } + } + // if property is a resource, fetch the resource type infos, and add each available resource to the description + if let Some(format_value) = prop_map.get("format") { + if let serde_json::Value::String(format_str) = format_value { + if format_str.starts_with("resource-") { + let resource_type_key = + format_str.split("-").last().unwrap_or_default().to_string(); + let resource_type = resources_types + .iter() + .find(|rt| rt.name == resource_type_key); + let resource_type_obj = resource_type.cloned().unwrap_or_else(|| { + tracing::info!("Resource type not found: {}", resource_type_key); + ResourceType { name: resource_type_key.clone(), description: None } + }); + + if !resources_cache.contains_key(&resource_type_key) { + let available_resources = Runner::inner_get_resources( + user_db, + authed, + &w_id, + &resource_type_key, + ) + .await; + + match available_resources { + Ok(cache_data) => { + resources_cache + .insert(resource_type_key.clone(), cache_data); + } + Err(e) => { + tracing::error!( + "Failed to fetch resource cache data: {}", + e + ); + continue; // Skip this property if fetching failed + } + } + } + + if let Some(resource_cache) = resources_cache.get(&resource_type_key) { + let resources_count = resource_cache.len(); + let description = format!( + "This is a resource named `{}` with the following description: `{}`.\nThe path of the resource should be used to specify the resource.\n{}", + resource_type_obj.name, + resource_type_obj.description.as_deref().unwrap_or("No description"), + if resources_count == 0 { + "This resource does not have any available instances, you should create one from your windmill workspace." + } else if resources_count > 1 { + "This resource has multiple available instances, you should precisely select the one you want to use." + } else { + "There is 1 resource available." + } + ); + prop_map.insert( + "type".to_string(), + serde_json::Value::String("string".to_string()), + ); + prop_map.insert( + "description".to_string(), + serde_json::Value::String(description), + ); + if resources_count > 0 { + let resources_description = resource_cache + .iter() + .map(|resource| { + format!( + "{}: $res:{}", + resource + .description + .as_deref() + .unwrap_or("No title"), + resource.path + ) + }) + .collect::>() + .join("\n"); + + prop_map.insert( + "description".to_string(), + serde_json::Value::String(format!( + "{}\nHere are the available resources, in the format title:path. Title can be empty. Path should be used to specify the resource:\n{}", + prop_map.get("description").unwrap_or(&serde_json::Value::String("No description".to_string())), + resources_description + )), + ); + } + } + } + } + } + } else { + tracing::warn!( + "Schema property value is not a JSON object: {:?}", + prop_value + ); + } + } + + Ok(schema_obj) + } + + /// Fetches the schema for a Hub script. + /// + /// This function takes a script path and a database connection, and attempts to fetch the schema for the script. + /// It strips the path to remove any leading slashes, and then attempts to retrieve the full script using `get_full_hub_script_by_path`. + /// If successful, it converts the schema string to a `Schema` object. + /// If the schema cannot be converted, it logs a warning and returns `None`. + /// + /// # Parameters + /// - `path`: The path of the script to fetch the schema for. + /// - `db`: The database connection. + /// + /// # Returns + /// - `Ok(Option)`: The schema if found, otherwise `None`. + /// - `Err(Error)`: If the request fails. + async fn get_hub_script_schema(path: &str, db: &DB) -> Result, Error> { + let strip_path = StripPath(path.to_string()); + let res = get_full_hub_script_by_path(strip_path, &HTTP_CLIENT, Some(db)) + .await + .map_err(|e| { + tracing::error!("Failed to get hub script: {}", e); + Error::internal_error(format!("Failed to get hub script: {}", e), None) + })?; + match serde_json::from_str::(res.schema.get()) { + Ok(schema) => Ok(Some(schema)), + Err(e) => { + tracing::warn!("Failed to convert schema: {}", e); + Ok(None) + } + } + } + + /// Creates a `Tool` from a `ToolableItem`. + /// + /// This function takes an item that implements the `ToolableItem` trait and converts it into an RMCP `Tool`. + /// It handles both workspace scripts/flows and Hub scripts differently, depending on the item type. + /// + /// # Parameters + /// - `item`: The item to convert to a `Tool`. + /// - `user_db`: The database connection. + /// - `authed`: The authenticated user. + /// - `workspace_id`: The workspace ID. + /// - `resources_cache`: A mutable reference to the resources cache. + /// - `resources_types`: A reference to the resource types. + /// + /// # Returns + /// - `Ok(Tool)`: The created `Tool`. + async fn create_tool_from_item( + item: &T, + user_db: &UserDB, + authed: &ApiAuthed, + workspace_id: &str, + resources_cache: &mut HashMap>, + resources_types: &Vec, + ) -> Result { + let is_hub = item.is_hub(); + let path = item.get_path_or_id(); + let item_type = item.item_type(); + let description = format!( + "This is a {} named `{}` with the following description: `{}`.{}", + item_type, + item.get_summary(), + item.get_description(), + if is_hub { + format!( + " It is a tool used for the following app: {}", + item.get_integration_type() + .unwrap_or("No integration type".to_string()) + ) + } else { + "".to_string() + } + ); + let schema_obj = Runner::transform_schema_for_resources( + &item.get_schema(), + user_db, + authed, + &workspace_id, + resources_cache, + &resources_types, + ) + .await?; + let input_schema_map = match serde_json::to_value(schema_obj) { + Ok(Value::Object(map)) => map, + Ok(_) => { + tracing::warn!("Schema object for tool '{}' did not serialize to a JSON object, using empty schema.", path); + serde_json::Map::new() + } + Err(e) => { + tracing::error!( + "Failed to serialize schema object for tool '{}': {}. Using empty schema.", + path, + e + ); + serde_json::Map::new() + } + }; + Ok(Tool { + name: Cow::Owned(path), + description: Some(Cow::Owned(description)), + input_schema: Arc::new(input_schema_map), + annotations: None, + }) + } +} + +impl ServerHandler for Runner { + /// Handles the `CallTool` request from the MCP client. + /// + /// This involves: + /// 1. Parsing arguments and extracting context (DB, Auth). + /// 2. Reversing the tool name (`request.name`) to get the original path and type using `reverse_transform`. + /// 3. Handling Hub scripts: If identified as a Hub script, searches the Hub for the actual script ID. + /// 4. Fetching the schema for the item (needed for argument transformation). + /// 5. Transforming incoming arguments: + /// - Reversing key transformations (e.g., `user_input` back to `user input`). + /// - Parsing stringified JSON objects back into JSON values based on schema type. + /// 6. Executing the corresponding script or flow using internal Windmill runners. + /// 7. Formatting the execution result into an RMCP `CallToolResult`. + /// + /// # Parameters + /// - `request`: The `CallToolRequestParam` containing the tool name and arguments. + /// - `context`: The `RequestContext` providing access to workspace ID, DB connections, auth info. + /// + /// # Returns + /// - `Ok(CallToolResult)`: On successful execution, containing the output. + /// - `Err(Error)`: If any step fails (parsing, DB access, execution, reversing transform, hub search). + async fn call_tool( + &self, + request: CallToolRequestParam, + context: RequestContext, + ) -> Result { + let parse_args = |args_opt: Option| -> Result { + args_opt.map(Value::Object).ok_or_else(|| { + Error::invalid_params( + "Missing arguments for tool", + Some(request.name.clone().into()), + ) + }) + }; + + let authed = context + .req_extensions + .get::() + .ok_or_else(|| Error::internal_error("ApiAuthed not found", None))?; + let db = context + .req_extensions + .get::() + .ok_or_else(|| Error::internal_error("DB not found", None))?; + let user_db = context + .req_extensions + .get::() + .ok_or_else(|| Error::internal_error("UserDB not found", None))?; + let args = parse_args(request.arguments)?; + + let (tool_type, path, is_hub) = + Runner::reverse_transform(&request.name).unwrap_or_default(); + + let item_schema = if is_hub { + Runner::get_hub_script_schema(&format!("hub/{}", path), db).await? + } else { + Runner::get_item_schema(&path, user_db, authed, &context.workspace_id, &tool_type) + .await? + }; + + let schema_obj = if let Some(ref s) = item_schema { + match serde_json::from_str::(s.0.get()) { + Ok(val) => Some(val), + Err(e) => { + tracing::warn!("Failed to parse schema: {}", e); + None + } + } + } else { + None + }; + + let push_args = if let Value::Object(map) = args.clone() { + let mut args_hash = HashMap::new(); + for (k, v) in map { + // need to transform back the key without invalid characters to the original key + let original_key = Runner::reverse_transform_key(&k, &schema_obj); + + // object properties are transformed to string because some client does not support object, might change in the future + let transformed_v = Runner::transform_value_if_object(&k, &v, &schema_obj); + args_hash.insert(original_key, to_raw_value(&transformed_v)); + } + windmill_queue::PushArgsOwned { extra: None, args: args_hash } + } else { + windmill_queue::PushArgsOwned::default() + }; + + let w_id = context.workspace_id.clone(); + let script_or_flow_path = if is_hub { + StripPath(format!("hub/{}", path)) + } else { + StripPath(path) + }; + let run_query = RunJobQuery::default(); + + let result = if tool_type == "script" { + run_wait_result_script_by_path_internal( + db.clone(), + run_query, + script_or_flow_path, + authed.clone(), + user_db.clone(), + w_id.clone(), + push_args, + ) + .await + } else { + run_wait_result_flow_by_path_internal( + db.clone(), + run_query, + script_or_flow_path, + authed.clone(), + user_db.clone(), + push_args, + w_id.clone(), + ) + .await + }; + + match result { + Ok(response) => { + let body_bytes = to_bytes(response.into_body(), usize::MAX) + .await + .map_err(|e| { + Error::internal_error(format!("Failed to read response body: {}", e), None) + })?; + let body_str = String::from_utf8(body_bytes.to_vec()).map_err(|e| { + Error::internal_error(format!("Failed to decode response body: {}", e), None) + })?; + Ok(CallToolResult::success(vec![Content::text(body_str)])) + } + Err(e) => Err(Error::internal_error( + format!("Failed to run script: {}", e), + None, + )), + } + } + + /// Fetches available tools (scripts, flows, hub scripts) based on the user's scope. + /// + /// - Determines scope (all, favorites, hub-specific) from auth token. + /// - Fetches relevant items (workspace scripts/flows, hub scripts) concurrently. + /// - Fetches resource type information needed for schema enrichment. + /// - Transforms each item into an RMCP `Tool` definition, including schema adjustments + /// (like resource description enrichment and object->string conversion). + /// + /// # Parameters + /// - `_request`: Optional pagination parameters (currently ignored). + /// - `_context`: The `RequestContext` providing workspace ID, DB, auth. + /// + /// # Returns + /// - `Ok(ListToolsResult)`: A list of `Tool` definitions. Pagination is not yet implemented. + /// - `Err(Error)`: If fetching data from DB or Hub fails. + async fn list_tools( + &self, + _request: Option, + mut _context: RequestContext, + ) -> Result { + let workspace_id = _context.workspace_id.clone(); + let db = _context + .req_extensions + .get::() + .ok_or_else(|| Error::internal_error("DB not found", None))?; + let user_db = _context + .req_extensions + .get::() + .ok_or_else(|| Error::internal_error("UserDB not found", None))?; + let authed = _context + .req_extensions + .get::() + .ok_or_else(|| Error::internal_error("ApiAuthed not found", None))?; + let owned_scope = authed.scopes.as_ref().and_then(|scopes| { + scopes + .iter() + .find(|scope| scope.starts_with("mcp:") && !scope.contains("hub")) + }); + let hub_scope = authed + .scopes + .as_ref() + .and_then(|scopes| scopes.iter().find(|scope| scope.starts_with("mcp:hub"))); + let scope_type = owned_scope.map_or("all", |scope| { + let parts = scope.split(":").collect::>(); + parts[1] + }); + let scope_integrations = hub_scope.and_then(|scope| { + let parts = scope.split(":").collect::>(); + if parts.len() == 3 { + Some(parts[2]) + } else { + None + } + }); + + let scripts_fn = Runner::inner_get_items::( + user_db, + authed, + &workspace_id, + scope_type, + "script", + ); + let flows_fn = + Runner::inner_get_items::(user_db, authed, &workspace_id, scope_type, "flow"); + let resources_types_fn = Runner::inner_get_resources_types(user_db, authed, &workspace_id); + let hub_scripts_fn = Runner::inner_get_scripts_from_hub(db, scope_integrations.as_deref()); + let (scripts, flows, resources_types, hub_scripts) = if scope_integrations.is_some() { + let (scripts, flows, resources_types, hub_scripts) = + try_join!(scripts_fn, flows_fn, resources_types_fn, hub_scripts_fn)?; + (scripts, flows, resources_types, hub_scripts) + } else { + let (scripts, flows, resources_types) = + try_join!(scripts_fn, flows_fn, resources_types_fn)?; + (scripts, flows, resources_types, vec![]) + }; + + let mut resources_cache: HashMap> = HashMap::new(); + let mut tools: Vec = Vec::new(); + + for script in scripts { + tools.push( + Runner::create_tool_from_item( + &script, + user_db, + authed, + &workspace_id, + &mut resources_cache, + &resources_types, + ) + .await?, + ); + } + + for flow in flows { + tools.push( + Runner::create_tool_from_item( + &flow, + user_db, + authed, + &workspace_id, + &mut resources_cache, + &resources_types, + ) + .await?, + ); + } + + for hub_script in hub_scripts { + tools.push( + Runner::create_tool_from_item( + &hub_script, + user_db, + authed, + &workspace_id, + &mut resources_cache, + &resources_types, + ) + .await?, + ); + } + + Ok(ListToolsResult { tools, next_cursor: None }) + } + + fn get_info(&self) -> ServerInfo { + ServerInfo { + protocol_version: Default::default(), + capabilities: ServerCapabilities::builder() + .enable_tools() + .enable_tool_list_changed() + .build(), + server_info: Implementation::from_build_env(), + instructions: Some("This server provides a list of scripts and flows the user can run on Windmill. Each flow and script is a tool callable with their respective arguments.".to_string()), + } + } + + async fn initialize( + &self, + _request: InitializeRequestParam, + _context: RequestContext, + ) -> Result { + Ok(self.get_info()) + } + + async fn list_resources( + &self, + _request: Option, + _context: RequestContext, + ) -> Result { + Ok(ListResourcesResult { resources: vec![], next_cursor: None }) + } + + async fn list_prompts( + &self, + _request: Option, + _context: RequestContext, + ) -> Result { + Ok(ListPromptsResult::default()) + } + + async fn list_resource_templates( + &self, + _request: Option, + _context: RequestContext, + ) -> Result { + Ok(ListResourceTemplatesResult::default()) + } +} + +pub fn setup_mcp_server(addr: SocketAddr, path: &str) -> anyhow::Result<(SseServer, Router)> { + let config = SseServerConfig { + bind: addr, + sse_path: "/sse".to_string(), + post_path: "/message".to_string(), + full_message_path: path.to_string(), + ct: CancellationToken::new(), + sse_keep_alive: None, + }; + + Ok(SseServer::new(config)) +} diff --git a/backend/windmill-api/src/mqtt_triggers.rs b/backend/windmill-api/src/mqtt_triggers.rs index 67c4c12d5b..a29648be22 100644 --- a/backend/windmill-api/src/mqtt_triggers.rs +++ b/backend/windmill-api/src/mqtt_triggers.rs @@ -3,6 +3,7 @@ use crate::{ db::{ApiAuthed, DB}, jobs::{run_flow_by_path_inner, run_script_by_path_inner, RunJobQuery}, resources::try_get_resource_from_db_as, + trigger_helpers::TriggerJobArgs, users::fetch_api_authed, }; use windmill_queue::TriggerKind; @@ -16,7 +17,7 @@ use axum::{ routing::{delete, get, post}, Router, }; -use base64::prelude::*; +use base64::{engine, prelude::*}; use bytes::Bytes; use http::StatusCode; use itertools::Itertools; @@ -51,8 +52,6 @@ use rand::seq::SliceRandom; use serde_json::value::RawValue; use sqlx::types::Json as SqlxJson; -use windmill_queue::PushArgsOwned; - pub fn workspaced_service() -> Router { Router::new() .route("/create", post(create_mqtt_trigger)) @@ -82,12 +81,20 @@ enum Error { } async fn run_job( - args: Option>>, - extra: Option>>, + payload: &[u8], + trigger_info: HashMap>, db: &DB, trigger: &MqttTrigger, ) -> anyhow::Result<()> { - let args = PushArgsOwned { args: args.unwrap_or_default(), extra }; + let args = MqttTrigger::build_job_args( + &trigger.script_path, + trigger.is_flow, + &trigger.workspace_id, + db, + payload, + trigger_info, + ) + .await?; let authed = fetch_api_authed( trigger.edited_by.clone(), @@ -111,7 +118,6 @@ async fn run_job( StripPath(trigger.script_path.to_owned()), run_query, args, - None, ) .await?; } else { @@ -123,7 +129,6 @@ async fn run_job( StripPath(trigger.script_path.to_owned()), run_query, args, - None, ) .await?; } @@ -1090,31 +1095,27 @@ impl EventLoop for V3EventLoop { } async fn handle_publish_packet(db: &DB, mqtt: &MqttConfig, payload: Bytes, publish: PublishData) { - let args = HashMap::from([("payload".to_string(), to_raw_value(&payload.as_ref()))]); - let extra = Some(HashMap::from([( - "wm_trigger".to_string(), - to_raw_value(&serde_json::json!({ - "kind": "mqtt", - "mqtt": { - "topic": publish.topic, - "retain": publish.retain, - "pkid": publish.pkid, - "qos": publish.qos, - "v5": publish.v5.map(|properties| { - serde_json::json!({ - "payload_format_indicator": properties.payload_format_indicator, - "topic_alias": properties.topic_alias, - "response_topic": properties.response_topic, - "correlation_data": properties.correlation_data.as_deref(), - "user_properties": properties.user_properties, - "subscription_identifiers": properties.subscription_identifiers, - "content_type": properties.content_type, - }) + let trigger_info = HashMap::from([ + ("topic".to_string(), to_raw_value(&publish.topic)), + ("retain".to_string(), to_raw_value(&publish.retain)), + ("pkid".to_string(), to_raw_value(&publish.pkid)), + ("qos".to_string(), to_raw_value(&publish.qos)), + ( + "v5".to_string(), + to_raw_value(&publish.v5.map(|properties| { + serde_json::json!({ + "payload_format_indicator": properties.payload_format_indicator, + "topic_alias": properties.topic_alias, + "response_topic": properties.response_topic, + "correlation_data": properties.correlation_data.as_deref(), + "user_properties": properties.user_properties, + "subscription_identifiers": properties.subscription_identifiers, + "content_type": properties.content_type, }) - } - })), - )])); - mqtt.handle(&db, Some(args), extra).await; + })), + ), + ]); + mqtt.handle(&db, payload.as_ref(), trigger_info).await; } async fn handle_event(db: &DB, mqtt: &MqttConfig, handler: H, mut event_loop: E) -> () @@ -1225,12 +1226,12 @@ impl MqttConfig { async fn handle( &self, db: &DB, - args: Option>>, - extra: Option>>, + payload: &[u8], + trigger_info: HashMap>, ) -> () { match self { - MqttConfig::Trigger(trigger) => trigger.handle(&db, args, extra).await, - MqttConfig::Capture(capture) => capture.handle(&db, args, extra).await, + MqttConfig::Trigger(trigger) => trigger.handle(&db, payload, trigger_info).await, + MqttConfig::Capture(capture) => capture.handle(&db, payload, trigger_info).await, } } } @@ -1406,10 +1407,10 @@ impl MqttTrigger { async fn handle( &self, db: &DB, - args: Option>>, - extra: Option>>, + payload: &[u8], + trigger_info: HashMap>, ) -> () { - if let Err(err) = run_job(args, extra, db, self).await { + if let Err(err) = run_job(payload, trigger_info, db, self).await { report_critical_error( format!("Failed to trigger job from mqtt {}: {:?}", self.path, err), db.clone(), @@ -1421,6 +1422,21 @@ impl MqttTrigger { } } +impl TriggerJobArgs<&[u8]> for MqttTrigger { + fn v1_payload_fn(payload: &[u8]) -> HashMap> { + HashMap::from([("payload".to_string(), to_raw_value(&payload))]) + } + + fn v2_payload_fn(payload: &[u8]) -> HashMap> { + let base64_payload = engine::general_purpose::STANDARD.encode(payload); + HashMap::from([("payload".to_string(), to_raw_value(&base64_payload))]) + } + + fn trigger_kind() -> TriggerKind { + TriggerKind::Mqtt + } +} + struct PublishData { topic: String, retain: bool, @@ -1675,19 +1691,19 @@ impl CaptureConfigForMqttTrigger { async fn handle( &self, db: &DB, - args: Option>>, - extra: Option>>, + payload: &[u8], + trigger_info: HashMap>, ) -> () { - let args = PushArgsOwned { args: args.unwrap_or_default(), extra: None }; - let extra = extra.as_ref().map(to_raw_value); + let (main_args, preprocessor_args) = + MqttTrigger::build_capture_payloads(payload, trigger_info); if let Err(err) = insert_capture_payload( db, &self.workspace_id, &self.path, self.is_flow, &TriggerKind::Mqtt, - args, - extra, + main_args, + preprocessor_args, &self.owner, ) .await diff --git a/backend/windmill-api/src/postgres_triggers/mod.rs b/backend/windmill-api/src/postgres_triggers/mod.rs index d041f1f508..0be19d6c4c 100644 --- a/backend/windmill-api/src/postgres_triggers/mod.rs +++ b/backend/windmill-api/src/postgres_triggers/mod.rs @@ -2,6 +2,7 @@ use crate::{ db::{ApiAuthed, DB}, jobs::{run_flow_by_path_inner, run_script_by_path_inner, RunJobQuery}, resources::try_get_resource_from_db_as, + trigger_helpers::TriggerJobArgs, users::fetch_api_authed, }; use chrono::Utc; @@ -30,7 +31,6 @@ use handler::{ Relations, }; use windmill_common::{db::UserDB, error::Error, utils::StripPath}; -use windmill_queue::PushArgsOwned; mod bool; mod converter; mod handler; @@ -250,12 +250,19 @@ pub fn workspaced_service() -> Router { } async fn run_job( - args: Option>>, - extra: Option>>, + payload: HashMap>, db: &DB, trigger: &PostgresTrigger, ) -> anyhow::Result<()> { - let args = PushArgsOwned { args: args.unwrap_or_default(), extra }; + let args = PostgresTrigger::build_job_args( + &trigger.script_path, + trigger.is_flow, + &trigger.workspace_id, + db, + payload, + HashMap::new(), + ) + .await?; let authed = fetch_api_authed( trigger.edited_by.clone(), @@ -279,7 +286,6 @@ async fn run_job( StripPath(trigger.script_path.to_owned()), run_query, args, - None, ) .await?; } else { @@ -291,7 +297,6 @@ async fn run_job( StripPath(trigger.script_path.to_owned()), run_query, args, - None, ) .await?; } diff --git a/backend/windmill-api/src/postgres_triggers/trigger.rs b/backend/windmill-api/src/postgres_triggers/trigger.rs index 746dca99eb..f340666977 100644 --- a/backend/windmill-api/src/postgres_triggers/trigger.rs +++ b/backend/windmill-api/src/postgres_triggers/trigger.rs @@ -12,6 +12,7 @@ use crate::{ run_job, }, resources::try_get_resource_from_db_as, + trigger_helpers::TriggerJobArgs, users::fetch_api_authed, }; use windmill_queue::TriggerKind; @@ -31,7 +32,6 @@ use sqlx::types::Json as SqlxJson; use windmill_common::{ db::UserDB, error, utils::report_critical_error, worker::to_raw_value, INSTANCE_NAME, }; -use windmill_queue::PushArgsOwned; use super::{ drop_logical_replication_slot_query, drop_publication_query, get_database_connection, @@ -369,13 +369,8 @@ impl PostgresTrigger { .await } - async fn handle( - &self, - db: &DB, - args: Option>>, - extra: Option>>, - ) -> () { - if let Err(err) = run_job(args, extra, db, self).await { + async fn handle(&self, db: &DB, payload: HashMap>) -> () { + if let Err(err) = run_job(payload, db, self).await { report_critical_error( format!( "Failed to trigger job from postgres {}: {:?}", @@ -390,6 +385,20 @@ impl PostgresTrigger { } } +impl TriggerJobArgs>> for PostgresTrigger { + fn v1_payload_fn(payload: HashMap>) -> HashMap> { + payload + } + + fn v2_payload_fn(payload: HashMap>) -> HashMap> { + payload + } + + fn trigger_kind() -> TriggerKind { + TriggerKind::Postgres + } +} + struct PgInfo<'a> { postgres_resource_path: &'a str, publication_name: &'a str, @@ -467,7 +476,6 @@ impl PostgresConfig { let client = PostgresSimpleClient::new(&database).await?; - let publication = client .execute_query(&format!( "SELECT pubname FROM pg_publication WHERE pubname = {}", @@ -508,15 +516,10 @@ impl PostgresConfig { } } - async fn handle( - &self, - db: &DB, - args: Option>>, - extra: Option>>, - ) -> () { + async fn handle(&self, db: &DB, payload: HashMap>) -> () { match self { - PostgresConfig::Trigger(trigger) => trigger.handle(&db, args, extra).await, - PostgresConfig::Capture(capture) => capture.handle(&db, args, extra).await, + PostgresConfig::Trigger(trigger) => trigger.handle(&db, payload).await, + PostgresConfig::Capture(capture) => capture.handle(&db, payload).await, } } @@ -606,7 +609,7 @@ async fn listen_to_transactions( } }; - + let message = match message { Ok(message) => message, Err(err) => { @@ -678,13 +681,9 @@ async fn listen_to_transactions( ("old_row".to_string(), to_raw_value(&old_row)), ("row".to_string(), to_raw_value(&row)), ]); - let extra = Some(HashMap::from([( - "wm_trigger".to_string(), - to_raw_value(&serde_json::json!({"kind": "postgres", })), - )])); - - - let _ = pg.handle(&db, Some(database_info), extra).await; + + + let _ = pg.handle(&db, database_info).await; } Some((o_id, old_row, row, transaction_type)) => { let relation = match relations.get_relation(o_id) { @@ -694,7 +693,7 @@ async fn listen_to_transactions( continue; } }; - + if let Err(err) = old_row { tracing::error!( transaction_type = ?transaction_type, @@ -707,7 +706,7 @@ async fn listen_to_transactions( relation.name, ); } - + if let Err(err) = row { tracing::error!( transaction_type = ?transaction_type, @@ -720,7 +719,7 @@ async fn listen_to_transactions( relation.name, ); } - + } _ => {} } @@ -919,22 +918,17 @@ impl CaptureConfigForPostgresTrigger { } } - async fn handle( - &self, - db: &DB, - args: Option>>, - extra: Option>>, - ) -> () { - let args = PushArgsOwned { args: args.unwrap_or_default(), extra: None }; - let extra = extra.as_ref().map(to_raw_value); + async fn handle(&self, db: &DB, payload: HashMap>) -> () { + let main_args = PostgresTrigger::build_job_args_v2(false, payload.clone(), HashMap::new()); + let preprocessor_args = PostgresTrigger::build_job_args_v2(true, payload, HashMap::new()); if let Err(err) = insert_capture_payload( db, &self.workspace_id, &self.path, self.is_flow, &TriggerKind::Postgres, - args, - extra, + main_args, + preprocessor_args, &self.owner, ) .await diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 8e707a15d2..080c07cc91 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -40,6 +40,7 @@ use std::{ }; use windmill_audit::audit_ee::audit_log; use windmill_audit::ActionKind; +use windmill_worker::process_relative_imports; use windmill_common::error::to_anyhow; @@ -264,9 +265,12 @@ async fn list_scripts( if lq.show_archived.unwrap_or(false) { sqlb.and_where_eq( - "o.created_at", - "(select max(created_at) from script where o.path = path - AND workspace_id = ?)" + "o.ctid", + "(SELECT ctid FROM script + WHERE path = o.path + AND workspace_id = ? + ORDER BY created_at DESC + LIMIT 1)" .bind(&w_id), ); sqlb.and_where_eq("archived", true); @@ -649,8 +653,13 @@ async fn create_script_internal<'c>( ) { Some(String::new()) } else { - ns.lock - .and_then(|e| if e.is_empty() { None } else { Some(e) }) + ns.lock.as_ref().and_then(|e| { + if e.is_empty() { + None + } else { + Some(e.to_string()) + } + }) }; let needs_lock_gen = lock.is_none() && codebase.is_none(); @@ -676,12 +685,32 @@ async fn create_script_internal<'c>( let (no_main_func, has_preprocessor) = match lang { ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Deno | ScriptLang::Nativets => { - let args = windmill_parser_ts::parse_deno_signature(&ns.content, true, true, None)?; - (args.no_main_func, args.has_preprocessor) + let args = windmill_parser_ts::parse_deno_signature(&ns.content, true, true, None); + match args { + Ok(args) => (args.no_main_func, args.has_preprocessor), + Err(e) => { + tracing::warn!( + "Error parsing deno signature when deploying script {}: {:?}", + ns.path, + e + ); + (None, None) + } + } } ScriptLang::Python3 => { - let args = windmill_parser_py::parse_python_signature(&ns.content, None, true)?; - (args.no_main_func, args.has_preprocessor) + let args = windmill_parser_py::parse_python_signature(&ns.content, None, true); + match args { + Ok(args) => (args.no_main_func, args.has_preprocessor), + Err(e) => { + tracing::warn!( + "Error parsing python signature when deploying script {}: {:?}", + ns.path, + e + ); + (None, None) + } + } } _ => (ns.no_main_func, ns.has_preprocessor), }; @@ -894,6 +923,40 @@ async fn create_script_internal<'c>( .await?; Ok((hash, new_tx)) } else { + let db2 = db.clone(); + let w_id2 = w_id.clone(); + let authed2 = authed.clone(); + let permissioned_as2 = permissioned_as.clone(); + let script_path2 = script_path.clone(); + let parent_path = p_path_opt.clone(); + let lock = ns.lock.clone(); + let deployment_message = ns.deployment_message.clone(); + let content = ns.content.clone(); + let language = ns.language.clone(); + tokio::spawn(async move { + // wait for 10 seconds to make sure the script is deployed and that the CLI sync that pushed it (f one) is complete + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + if let Err(e) = process_relative_imports( + &db2, + None, + None, + &w_id2, + &script_path2, + parent_path, + deployment_message, + &content, + &Some(language), + &authed2.email, + &authed2.username, + &permissioned_as2, + lock, + ) + .await + { + tracing::error!(%e, "error processing relative imports"); + } + }); + handle_deployment_metadata( &authed.email, &authed.username, @@ -949,7 +1012,7 @@ async fn get_script_by_path( AND favorite.usr = $3 WHERE s.path = $1 AND s.workspace_id = $2 - AND s.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2)", + ORDER BY s.created_at DESC LIMIT 1", ) .bind(path) .bind(w_id) @@ -958,9 +1021,7 @@ async fn get_script_by_path( .await? } else { sqlx::query_as::<_, ScriptWithStarred>( - "SELECT *, NULL as starred FROM script WHERE path = $1 AND workspace_id = $2 \ - AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \ - workspace_id = $2)", + "SELECT *, NULL as starred FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", ) .bind(path) .bind(w_id) @@ -1000,9 +1061,8 @@ async fn get_script_by_path_w_draft( let script_o = sqlx::query_as::<_, ScriptWDraft>( "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email FROM script LEFT JOIN draft ON script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script' - WHERE script.path = $1 AND script.workspace_id = $2 \ - AND script.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \ - workspace_id = $2)", + WHERE script.path = $1 AND script.workspace_id = $2 + ORDER BY script.created_at DESC LIMIT 1", ) .bind(path) .bind(w_id) @@ -1024,7 +1084,7 @@ async fn get_script_history( "SELECT s.hash as hash, dm.deployment_msg as deployment_msg FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash WHERE s.workspace_id = $1 AND s.path = $2 - ORDER by created_at DESC", + ORDER by s.created_at DESC", w_id, path.to_path(), ) @@ -1052,7 +1112,7 @@ async fn get_latest_version( "SELECT s.hash as hash, dm.deployment_msg as deployment_msg FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash WHERE s.workspace_id = $1 AND s.path = $2 - ORDER by created_at DESC", + ORDER by s.created_at DESC LIMIT 1", w_id, path.to_path(), ) @@ -1148,7 +1208,15 @@ async fn toggle_workspace_error_handler( match error_handler_maybe { Some(_) => { sqlx::query_scalar!( - "UPDATE script SET ws_error_handler_muted = $3 WHERE workspace_id = $2 AND path = $1 AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2)", + "UPDATE script + SET ws_error_handler_muted = $3 + WHERE ctid = ( + SELECT ctid FROM script + WHERE path = $1 AND workspace_id = $2 + ORDER BY created_at DESC + LIMIT 1 + ) +", path.to_path(), w_id, req.muted, @@ -1169,6 +1237,7 @@ async fn toggle_workspace_error_handler( async fn get_tokened_raw_script_by_path( Extension(user_db): Extension, + Extension(db): Extension, Path((w_id, token, path)): Path<(String, String, StripPath)>, Extension(cache): Extension>, ) -> Result { @@ -1176,7 +1245,13 @@ async fn get_tokened_raw_script_by_path( .get_authed(Some(w_id.clone()), &token) .await .ok_or_else(|| Error::NotAuthorized("Invalid token".to_string()))?; - return raw_script_by_path(authed, Extension(user_db), Path((w_id, path))).await; + return raw_script_by_path( + authed, + Extension(user_db), + Extension(db), + Path((w_id, path)), + ) + .await; } async fn get_empty_ts_script_by_path() -> String { @@ -1186,22 +1261,25 @@ async fn get_empty_ts_script_by_path() -> String { async fn raw_script_by_path( authed: ApiAuthed, Extension(user_db): Extension, + Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, ) -> Result { - raw_script_by_path_internal(path, user_db, authed, w_id, false).await + raw_script_by_path_internal(path, user_db, db, authed, w_id, false).await } async fn raw_script_by_path_unpinned( authed: ApiAuthed, Extension(user_db): Extension, + Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, ) -> Result { - raw_script_by_path_internal(path, user_db, authed, w_id, true).await + raw_script_by_path_internal(path, user_db, db, authed, w_id, true).await } async fn raw_script_by_path_internal( path: StripPath, user_db: UserDB, + db: DB, authed: ApiAuthed, w_id: String, unpin: bool, @@ -1227,10 +1305,7 @@ async fn raw_script_by_path_internal( let mut tx = user_db.begin(&authed).await?; let content_o = sqlx::query_scalar!( - "SELECT content FROM script WHERE path = $1 AND workspace_id = $2 \ - AND - created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND archived = false AND \ - workspace_id = $2)", + "SELECT content FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1", path, w_id ) @@ -1238,6 +1313,22 @@ async fn raw_script_by_path_internal( .await?; tx.commit().await?; + if content_o.is_none() { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1)", + path, + w_id + ) + .fetch_one(&db) + .await?; + if exists.unwrap_or(false) { + return Err(Error::NotFound(format!( + "Script {path} not visible to {} but exists", + authed.username + ))); + } + } + let content = not_found_if_none(content_o, "Script", path)?; if unpin { @@ -1254,8 +1345,7 @@ async fn exists_script_by_path( let path = path.to_path(); let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 AND - created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2))", + "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1)", path, w_id ) @@ -1372,9 +1462,7 @@ pub async fn require_is_writer(authed: &ApiAuthed, path: &str, w_id: &str, db: D path, w_id, db, - "SELECT extra_perms FROM script WHERE path = $1 AND workspace_id = $2 \ - AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \ - workspace_id = $2)", + "SELECT extra_perms FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", "script", ) .await; diff --git a/backend/windmill-api/src/tracing_init.rs b/backend/windmill-api/src/tracing_init.rs index 9f34e93592..c5c841b2dc 100644 --- a/backend/windmill-api/src/tracing_init.rs +++ b/backend/windmill-api/src/tracing_init.rs @@ -33,6 +33,8 @@ impl OnResponse for MyOnResponse { let status = response.status().as_u16(); if response.status().is_success() || response.status().is_redirection() { tracing::info!(latency = latency, status = status, "response") + } else if response.status().as_u16() == 404 { + tracing::warn!(latency = latency, status = status, "response") } else { tracing::error!(latency = latency, status = status, "response") } diff --git a/backend/windmill-api/src/trigger_helpers.rs b/backend/windmill-api/src/trigger_helpers.rs new file mode 100644 index 0000000000..69a5a014a4 --- /dev/null +++ b/backend/windmill-api/src/trigger_helpers.rs @@ -0,0 +1,332 @@ +use quick_cache::sync::Cache; +use serde::Deserialize; +use serde_json::value::RawValue; +use std::collections::HashMap; +use windmill_common::{ + error::Result, + get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, + scripts::{ScriptHash, ScriptLang}, + worker::to_raw_value, + FlowVersionInfo, +}; +use windmill_queue::{PushArgsOwned, TriggerKind}; + +use crate::db::DB; + +type RunnableFormatCacheKey = (String, i64, TriggerKind); + +lazy_static::lazy_static! { + pub static ref RUNNABLE_FORMAT_VERSION_CACHE: Cache = Cache::new(1000); +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash, Copy)] +pub struct RunnableFormat { + pub version: RunnableFormatVersion, + pub has_preprocessor: bool, +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash, Copy)] +pub enum RunnableFormatVersion { + V1, + V2, +} + +struct ScriptInfo { + has_preprocessor: Option, + language: ScriptLang, + content: String, + schema: Option>, +} + +struct FlowInfo { + has_preprocessor: Option, + is_v1_preprocessor: Option, + schema: Option>, +} + +#[derive(Debug, Deserialize)] +struct PropertyDefinition { + r#type: Option, +} + +#[derive(Debug, Deserialize)] +struct PartialSchema { + properties: Option>, +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub enum RunnableId { + FlowPath(String), + ScriptId(ScriptId), +} + +impl RunnableId { + pub fn from_script_hash(hash: ScriptHash) -> Self { + Self::ScriptId(ScriptId::ScriptHash(hash)) + } + + pub fn from_script_path(path: &str) -> Self { + Self::ScriptId(ScriptId::ScriptPath(path.to_string())) + } + + pub fn from_flow_path(path: &str) -> Self { + Self::FlowPath(path.to_string()) + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub enum ScriptId { + ScriptPath(String), + ScriptHash(ScriptHash), +} + +impl ScriptId { + async fn get_script_hash(self, workspace_id: &str, db: &DB) -> Result { + let hash = match self { + ScriptId::ScriptPath(path) => { + let info = get_latest_deployed_hash_for_path(db, workspace_id, &path).await?; + info.hash + } + ScriptId::ScriptHash(hash) => hash.0, + }; + + Ok(hash) + } +} + +async fn get_script_info( + db: &DB, + workspace_id: &str, + hash: i64, +) -> std::result::Result { + sqlx::query_as!(ScriptInfo, "SELECT has_preprocessor, language as \"language: _\", content, schema as \"schema: _\" FROM script WHERE workspace_id = $1 AND hash = $2", workspace_id, hash) + .fetch_one(db) + .await +} + +fn runnable_format_from_schema( + trigger_kind: &TriggerKind, + has_preprocessor: bool, + schema: Option>, +) -> RunnableFormat { + match trigger_kind { + TriggerKind::Mqtt + if schema.as_ref().is_some_and(|schema| { + schema.properties.as_ref().is_some_and(|properties| { + properties.iter().any(|(key, def)| { + key == "payload" && def.r#type.as_ref().is_some_and(|t| t == "array") + }) + }) + }) => + { + RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor } + } + TriggerKind::Kafka | TriggerKind::Nats + if schema.as_ref().is_some_and(|schema| { + schema + .properties + .as_ref() + .is_some_and(|properties| properties.keys().any(|key| key == "msg")) + }) => + { + RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor } + } + _ => RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor }, + } +} +pub async fn get_runnable_format( + runnable_id: RunnableId, + workspace_id: &str, + db: &DB, + trigger_kind: &TriggerKind, +) -> Result { + match runnable_id { + RunnableId::FlowPath(path) => { + let FlowVersionInfo { version, .. } = + get_latest_flow_version_info_for_path(db, workspace_id, &path, true).await?; + + let key = (workspace_id.to_string(), version, trigger_kind.clone()); + + let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); + + if let Some(runnable_format) = runnable_format { + tracing::debug!("Using cached runnable format for flow {path}"); + return Ok(runnable_format); + } + + let flow_info = sqlx::query_as!( + FlowInfo, + "SELECT + value->'preprocessor_module' IS NOT NULL as has_preprocessor, + value->'preprocessor_module'->'value'->'input_transforms'->'wm_trigger' IS NOT NULL as is_v1_preprocessor, + schema as \"schema: _\" + FROM flow + WHERE workspace_id = $1 + AND path = $2", + workspace_id, + path + ) + .fetch_one(db) + .await?; + + let has_preprocessor = flow_info.has_preprocessor.unwrap_or(false); + let is_v1_preprocessor = flow_info.is_v1_preprocessor.unwrap_or(false); + + let runnable_format = if has_preprocessor && is_v1_preprocessor { + RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: true } + } else { + runnable_format_from_schema(trigger_kind, has_preprocessor, flow_info.schema) + }; + + RUNNABLE_FORMAT_VERSION_CACHE.insert(key, runnable_format); + + Ok(runnable_format) + } + RunnableId::ScriptId(script_id) => { + let hash = script_id.get_script_hash(workspace_id, db).await?; + let key = (workspace_id.to_string(), hash, trigger_kind.clone()); + let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); + + if let Some(runnable_format) = runnable_format { + tracing::debug!("Using cached runnable format for script {hash}"); + return Ok(runnable_format); + } + + let script_info = get_script_info(db, workspace_id, hash).await?; + + let has_preprocessor = script_info.has_preprocessor.unwrap_or(false); + + let runnable_format = if has_preprocessor { + let args = match script_info.language { + ScriptLang::Bun + | ScriptLang::Bunnative + | ScriptLang::Deno + | ScriptLang::Nativets => { + let args = windmill_parser_ts::parse_deno_signature( + &script_info.content, + true, + false, + Some("preprocessor".to_string()), + )?; + Some(args.args) + } + ScriptLang::Python3 => { + let args = windmill_parser_py::parse_python_signature( + &script_info.content, + Some("preprocessor".to_string()), + false, + )?; + Some(args.args) + } + _ => None, + }; + + if args.is_some_and(|args| args.iter().any(|arg| arg.name == "wm_trigger")) { + RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: true } + } else { + runnable_format_from_schema(trigger_kind, has_preprocessor, script_info.schema) + } + } else { + runnable_format_from_schema(trigger_kind, has_preprocessor, script_info.schema) + }; + + RUNNABLE_FORMAT_VERSION_CACHE.insert(key, runnable_format); + + Ok(runnable_format) + } + } +} + +#[allow(dead_code)] +pub trait TriggerJobArgs { + fn v1_payload_fn(payload: T) -> HashMap>; + fn v2_payload_fn(payload: T) -> HashMap> { + Self::v1_payload_fn(payload) + } + fn trigger_kind() -> TriggerKind; + + fn build_job_args_v2( + has_preprocessor: bool, + payload: T, + info: HashMap>, + ) -> PushArgsOwned { + let trigger_kind = Self::trigger_kind(); + let mut args = Self::v2_payload_fn(payload); + if has_preprocessor { + args.insert("kind".to_string(), to_raw_value(&trigger_kind.to_key())); + args.extend(info); + let args = HashMap::from([("event".to_string(), to_raw_value(&args))]); + PushArgsOwned { args, extra: None } + } else { + PushArgsOwned { args, extra: None } + } + } + + fn build_job_args_v1( + has_preprocessor: bool, + payload: T, + info: HashMap>, + ) -> PushArgsOwned { + let trigger_kind = Self::trigger_kind(); + let trigger_key = trigger_kind.to_key(); + let args = Self::v1_payload_fn(payload); + let extra = if has_preprocessor { + Some(HashMap::from([( + "wm_trigger".to_string(), + to_raw_value(&serde_json::json!({ + "kind": trigger_key, + trigger_key: info + })), + )])) + } else { + None + }; + + PushArgsOwned { args, extra } + } + + async fn build_job_args( + runnable_path: &str, + is_flow: bool, + w_id: &str, + db: &DB, + payload: T, + info: HashMap>, + ) -> Result { + let runnable_id = if is_flow { + RunnableId::from_flow_path(runnable_path) + } else { + RunnableId::from_script_path(runnable_path) + }; + Self::build_job_args_from_runnable_id(runnable_id, w_id, db, payload, info).await + } + + async fn build_job_args_from_runnable_id( + runnable_id: RunnableId, + w_id: &str, + db: &DB, + payload: T, + info: HashMap>, + ) -> Result { + let runnable_format = + get_runnable_format(runnable_id, w_id, db, &Self::trigger_kind()).await?; + + match runnable_format { + RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor } => { + Ok(Self::build_job_args_v1(has_preprocessor, payload, info)) + } + RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor } => { + Ok(Self::build_job_args_v2(has_preprocessor, payload, info)) + } + } + } + + fn build_capture_payloads( + payload: T, + info: HashMap>, + ) -> (PushArgsOwned, PushArgsOwned) { + let main_args = Self::build_job_args_v2(false, payload.clone(), info.clone()); + let preprocessor_args = Self::build_job_args_v2(true, payload, info); + (main_args, preprocessor_args) + } +} diff --git a/backend/windmill-api/src/websocket_triggers.rs b/backend/windmill-api/src/websocket_triggers.rs index 9be3829d83..c23daae33f 100644 --- a/backend/windmill-api/src/websocket_triggers.rs +++ b/backend/windmill-api/src/websocket_triggers.rs @@ -38,6 +38,7 @@ use crate::{ jobs::{ run_flow_by_path_inner, run_script_by_path_inner, run_wait_result_internal, RunJobQuery, }, + trigger_helpers::TriggerJobArgs, users::fetch_api_authed, }; @@ -608,7 +609,6 @@ async fn wait_runnable_result( StripPath(path.clone()), RunJobQuery::default(), args, - None, ) .await?; @@ -635,7 +635,6 @@ async fn wait_runnable_result( StripPath(path.clone()), RunJobQuery::default(), args, - None, ) .await?; @@ -890,10 +889,11 @@ impl WebsocketTrigger { async fn handle( &self, db: &DB, - args: PushArgsOwned, + msg: &str, + trigger_info: HashMap>, return_message_channels: Option, ) -> () { - if let Err(err) = run_job(db, self, args, return_message_channels).await { + if let Err(err) = run_job(db, self, &msg, trigger_info, return_message_channels).await { report_critical_error( format!( "Failed to trigger job from WebSocket {}: {:?}", @@ -919,6 +919,16 @@ impl WebsocketTrigger { } } +impl TriggerJobArgs<&str> for WebsocketTrigger { + fn v1_payload_fn(payload: &str) -> HashMap> { + HashMap::from([("msg".to_string(), to_raw_value(&payload))]) + } + + fn trigger_kind() -> TriggerKind { + TriggerKind::Websocket + } +} + #[derive(Deserialize)] struct CaptureConfigForWebsocket { trigger_config: SqlxJson, @@ -985,15 +995,18 @@ impl CaptureConfigForWebsocket { Some(()) } - async fn handle(&self, db: &DB, args: PushArgsOwned) -> () { + async fn handle(&self, db: &DB, msg: &str, trigger_info: HashMap>) -> () { + let (main_args, preprocessor_args) = + WebsocketTrigger::build_capture_payloads(&msg, trigger_info); + if let Err(err) = insert_capture_payload( db, &self.workspace_id, &self.path, self.is_flow, &TriggerKind::Websocket, - PushArgsOwned { args: args.args, extra: None }, - args.extra.as_ref().map(to_raw_value), + main_args, + preprocessor_args, &self.owner, ) .await @@ -1259,20 +1272,15 @@ async fn listen_to_websocket( } } if should_handle { - - let args = HashMap::from([("msg".to_string(), to_raw_value(&text))]); - let extra = Some(HashMap::from([( - "wm_trigger".to_string(), - to_raw_value(&serde_json::json!({"kind": "websocket", "websocket": { "url": url }})), - )])); - - let args = PushArgsOwned { args, extra }; + let trigger_info = HashMap::from([ + ("url".to_string(), to_raw_value(&url)), + ]); match &ws { WebsocketEnum::Trigger(ws_trigger) => { - ws_trigger.handle(&db, args, return_message_channels.clone()).await; + ws_trigger.handle(&db, &text, trigger_info, return_message_channels.clone()).await; }, WebsocketEnum::Capture(capture) => { - capture.handle(&db, args).await; + capture.handle(&db, &text, trigger_info).await; }, } } @@ -1311,9 +1319,20 @@ async fn listen_to_websocket( async fn run_job( db: &DB, trigger: &WebsocketTrigger, - args: PushArgsOwned, + msg: &str, + trigger_info: HashMap>, return_message_channels: Option, ) -> anyhow::Result<()> { + let args = WebsocketTrigger::build_job_args( + &trigger.script_path, + trigger.is_flow, + &trigger.workspace_id, + db, + msg, + trigger_info, + ) + .await?; + let authed = fetch_api_authed( trigger.edited_by.clone(), trigger.email.clone(), @@ -1374,7 +1393,6 @@ async fn run_job( runnable_path, run_query, args, - None, ) .await?; } else { @@ -1386,7 +1404,6 @@ async fn run_job( runnable_path, run_query, args, - None, ) .await?; } diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index b799313933..fea688c78a 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -692,6 +692,7 @@ pub(crate) async fn tarball_workspace( workspace_id, delivery_type AS "delivery_type: _", delivery_config AS "delivery_config: _", + subscription_mode AS "subscription_mode: _", path, script_path, is_flow, diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index a0278a4c2e..d3f7ed4c00 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -12,7 +12,7 @@ tantivy = [] prometheus = ["dep:prometheus"] loki = ["dep:tracing-loki"] benchmark = [] -parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts"] +parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts", "dep:datafusion"] aws_auth = ["dep:aws-sdk-sts", "dep:aws-config"] otel = ["dep:opentelemetry-semantic-conventions", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk", "dep:opentelemetry", "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing", "dep:tonic"] @@ -44,6 +44,9 @@ tracing = { workspace = true } axum = { workspace = true } hyper = { workspace = true } tokio = { workspace = true } +tokio-stream.workspace = true +tokio-util.workspace = true +datafusion = { workspace = true, optional = true} reqwest = { workspace = true } tracing-subscriber = { workspace = true } lazy_static.workspace = true @@ -67,6 +70,7 @@ async-stream.workspace = true const_format.workspace = true crc.workspace = true windmill-macros.workspace = true +windmill-parser-sql.workspace = true jsonwebtoken.workspace = true backon.workspace = true @@ -76,6 +80,8 @@ quick_cache.workspace = true pin-project-lite.workspace = true futures.workspace = true tempfile.workspace = true +systemstat.workspace = true +size.workspace = true opentelemetry-semantic-conventions = { workspace = true, optional = true } opentelemetry-otlp = { workspace = true, optional = true } diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index 9d1ee74675..5823fccf7c 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -317,7 +317,7 @@ pub mod aws { use crate::error::to_anyhow; use super::*; - use crate::utils::empty_string_as_none; + use crate::utils::empty_as_none; use aws_config::{BehaviorVersion, Region}; use aws_sdk_sts::{ config::Credentials as AwsCredentials, @@ -363,7 +363,7 @@ pub mod aws { #[derive(Debug, Deserialize)] pub struct CredentialsAuth { - #[serde(deserialize_with = "empty_string_as_none")] + #[serde(deserialize_with = "empty_as_none")] pub region: Option, #[serde(rename = "awsAccessKeyId")] pub aws_access_key_id: String, @@ -374,7 +374,7 @@ pub mod aws { #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "snake_case")] pub struct OidcAuth { - #[serde(deserialize_with = "empty_string_as_none")] + #[serde(deserialize_with = "empty_as_none")] pub region: Option, #[serde(rename = "roleArn")] pub role_arn: String, diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index aac1457f56..03d8ec7f03 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -570,15 +570,18 @@ pub mod script { let fut = CACHE.get_or_insert_async(hash, async move { match conn { Connection::Sql(db) => fetch_script_from_db(&db, hash, loc).await, - Connection::Http(_) => Err(error::Error::InternalErr(format!( - "Cannot fetch script in HTTP mode" - ))), + Connection::Http(client) => { + let r = client + .get::(&format!("/api/agent_workers/script/{}", hash.0)) + .await?; + Ok(r.into()) + } } }); fut.map_ok(|ScriptFull { data, meta }| (data, meta)) } - async fn fetch_script_from_db( + pub async fn fetch_script_from_db( db: &DB, hash: ScriptHash, loc: &'static Location<'_>, diff --git a/backend/windmill-common/src/ee.rs b/backend/windmill-common/src/ee.rs index 2a820475f7..7acb430f1b 100644 --- a/backend/windmill-common/src/ee.rs +++ b/backend/windmill-common/src/ee.rs @@ -98,3 +98,9 @@ pub async fn worker_groups_alerts(_db: &DB) {} #[cfg(feature = "enterprise")] pub async fn jobs_waiting_alerts(_db: &DB) {} + +#[cfg(feature = "enterprise")] +pub async fn low_disk_alerts(_db: &DB, _server_mode: bool, _worker_mode: bool, _workers: Vec) { + // Implementation is not open source +} + diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index feced2b6d5..a6d43a5fc4 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -66,6 +66,8 @@ pub enum Error { DatabaseMigration(#[from] MigrateError), #[error("Non-zero exit status for {0}: {1}")] ExitStatus(String, i32), + #[error("ExecutionRawError: {0}")] + ExecutionRawError(Box), #[error("Error: {error:#} @{location:#}")] Anyhow { error: anyhow::Error, location: String }, #[error("Error: {0:#?}")] diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 1f6c265e7d..443461af64 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -22,7 +22,8 @@ use crate::{ error::Error, more_serde::{default_empty_string, default_id, default_null, default_true, is_default}, scripts::{Schema, ScriptHash, ScriptLang}, - worker::{to_raw_value, Connection}, DB, + worker::{to_raw_value, Connection}, + DB, }; #[derive(Serialize, Deserialize, sqlx::FromRow)] @@ -135,10 +136,11 @@ pub struct FlowValue { pub concurrency_key: Option, } -#[derive(Deserialize, Serialize, Debug, Clone)] +#[derive(Default, Deserialize, Serialize, Debug, Clone)] pub struct StopAfterIf { pub expr: String, pub skip_if_stopped: bool, + pub error_message: Option, } #[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 109a0f6056..61184b627c 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -37,6 +37,7 @@ pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url"; pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels"; pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui"; +pub const CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING: &str = "critical_alerts_on_db_oversize"; pub const DEV_INSTANCE_SETTING: &str = "dev_instance"; pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 6cad6579fb..83752b125c 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -5,7 +5,7 @@ use futures_core::Stream; use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; -use sqlx::{types::Json, Pool, Postgres, Transaction}; +use sqlx::{types::Json, Pool, Postgres}; use tokio::io::AsyncReadExt; use uuid::Uuid; @@ -17,10 +17,11 @@ use crate::{ error::{self, to_anyhow, Error}, flow_status::{FlowStatus, RestartedFrom}, flows::{FlowNodeId, FlowValue, Retry}, - get_latest_deployed_hash_for_path, + get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, scripts::{ScriptHash, ScriptLang}, users::username_to_permissioned_as, worker::{to_raw_value, TMP_DIR}, + FlowVersionInfo, ScriptHashInfo, }; #[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone)] @@ -331,6 +332,7 @@ pub enum JobPayload { path: String, dedicated_worker: Option, apply_preprocessor: bool, + version: i64, }, RestartedFlow { completed_job_id: Uuid, @@ -385,9 +387,9 @@ pub struct OnBehalfOf { pub permissioned_as: String, } -pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgres>>( +pub async fn script_path_to_payload<'e, A: sqlx::Acquire<'e, Database = Postgres> + Send>( script_path: &str, - db: E, + db: A, w_id: &str, skip_preprocessor: Option, ) -> error::Result<( @@ -407,10 +409,10 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre None, ) } else { - let ( - script_hash, + let ScriptHashInfo { + hash, tag, - custom_concurrency_key, + concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, @@ -418,11 +420,12 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre dedicated_worker, priority, delete_after_use, - script_timeout, + timeout, has_preprocessor, on_behalf_of_email, created_by, - ) = get_latest_deployed_hash_for_path(db, w_id, script_path).await?; + .. + } = get_latest_deployed_hash_for_path(db, w_id, script_path).await?; let on_behalf_of = if let Some(email) = on_behalf_of_email { Some(OnBehalfOf { @@ -435,9 +438,9 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre ( JobPayload::ScriptHash { - hash: script_hash, + hash: ScriptHash(hash), path: script_path.to_owned(), - custom_concurrency_key, + custom_concurrency_key: concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl: cache_ttl, @@ -449,7 +452,7 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre }, tag, delete_after_use, - script_timeout, + timeout, on_behalf_of, ) }; @@ -462,52 +465,6 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre )) } -pub async fn script_hash_to_tag_and_limits<'c>( - script_hash: &ScriptHash, - db: &mut Transaction<'c, Postgres>, - w_id: &String, -) -> error::Result<( - Option, - Option, - Option, - Option, - Option, - ScriptLang, - Option, - Option, - Option, - Option, - Option, - String, -)> { - let script = sqlx::query!( - "select tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, on_behalf_of_email, created_by from script where hash = $1 AND workspace_id = $2", - script_hash.0, - w_id - ) - .fetch_one(&mut **db) - .await - .map_err(|e| { - Error::internal_err(format!( - "querying getting tag for hash {script_hash}: {e:#}" - )) - })?; - Ok(( - script.tag, - script.concurrency_key, - script.concurrent_limit, - script.concurrency_time_window_s, - script.cache_ttl, - script.language, - script.dedicated_worker, - script.priority, - script.delete_after_use, - script.timeout, - script.on_behalf_of_email, - script.created_by, - )) -} - pub async fn get_payload_tag_from_prefixed_path( path: &str, db: &DB, @@ -517,18 +474,10 @@ pub async fn get_payload_tag_from_prefixed_path( script_path_to_payload(path.strip_prefix("script/").unwrap(), db, w_id, Some(true)).await? } else if path.starts_with("flow/") { let path = path.strip_prefix("flow/").unwrap().to_string(); - let r = sqlx::query!( - "SELECT tag, dedicated_worker from flow WHERE path = $1 and workspace_id = $2", - &path, - &w_id, - ) - .fetch_optional(db) - .await?; - let (tag, dedicated_worker) = r - .map(|x| (x.tag, x.dedicated_worker)) - .unwrap_or_else(|| (None, None)); + let FlowVersionInfo { dedicated_worker, tag, version, .. } = + get_latest_flow_version_info_for_path(db, w_id, &path, true).await?; ( - JobPayload::Flow { path, dedicated_worker, apply_preprocessor: false }, + JobPayload::Flow { path, dedicated_worker, apply_preprocessor: false, version }, tag, None, None, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 0d2c746d3e..e1c2f4aa15 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -6,7 +6,9 @@ * LICENSE-AGPL for a copy of the license. */ +use quick_cache::sync::Cache; use std::{ + future::Future, net::SocketAddr, str::FromStr, sync::{ @@ -115,6 +117,7 @@ lazy_static::lazy_static! { pub static ref CRITICAL_ERROR_CHANNELS: Arc>> = Arc::new(RwLock::new(vec![])); + pub static ref CRITICAL_ALERTS_ON_DB_OVERSIZE: Arc>> = Arc::new(RwLock::new(None)); pub static ref JOB_RETENTION_SECS: Arc> = Arc::new(RwLock::new(0)); @@ -122,8 +125,15 @@ lazy_static::lazy_static! { pub static ref INSTANCE_NAME: String = rd_string(5); + pub static ref DEPLOYED_SCRIPT_HASH_CACHE: Cache<(String, String), ExpiringLatestVersionId> = Cache::new(1000); + pub static ref FLOW_VERSION_CACHE: Cache<(String, String), ExpiringLatestVersionId> = Cache::new(1000); + pub static ref DEPLOYED_SCRIPT_INFO_CACHE: Cache<(String, i64), ScriptHashInfo> = Cache::new(1000); + pub static ref FLOW_INFO_CACHE: Cache<(String, i64), FlowVersionInfo> = Cache::new(1000); + } +const LATEST_VERSION_ID_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(60); + pub async fn shutdown_signal( tx: KillpillSender, mut rx: tokio::sync::broadcast::Receiver<()>, @@ -335,54 +345,207 @@ type Tag = String; pub type DB = Pool; -pub async fn get_latest_deployed_hash_for_path<'e, E: sqlx::Executor<'e, Database = Postgres>>( +#[derive(Clone)] +pub struct ExpiringLatestVersionId { + id: i64, + expires_at: std::time::Instant, +} + +#[derive(Clone)] +pub struct ScriptHashInfo { + pub path: String, + pub hash: i64, + pub tag: Option, + pub concurrency_key: Option, + pub concurrent_limit: Option, + pub concurrency_time_window_s: Option, + pub cache_ttl: Option, + pub language: ScriptLang, + pub dedicated_worker: Option, + pub priority: Option, + pub delete_after_use: Option, + pub timeout: Option, + pub has_preprocessor: Option, + pub on_behalf_of_email: Option, + pub created_by: String, +} + +pub fn get_latest_deployed_hash_for_path< + 'a, + 'e, + E: sqlx::Acquire<'e, Database = Postgres> + Send + 'a, +>( + db: E, + w_id: &'a str, + script_path: &'a str, +) -> impl Future> + Send + 'a { + async move { + let mut conn = db.acquire().await?; + let cache_key = (w_id.to_string(), script_path.to_string()); + + let hash = match DEPLOYED_SCRIPT_HASH_CACHE.get(&cache_key) { + Some(cached_hash) if cached_hash.expires_at > std::time::Instant::now() => { + tracing::debug!( + "Using cached script hash {} for {script_path}", + cached_hash.id + ); + cached_hash.id + } + _ => { + tracing::debug!("Fetching script hash for {script_path}"); + let hash = sqlx::query_scalar!( + "select hash from script where path = $1 AND workspace_id = $2 AND deleted = false AND lock IS not NULL AND lock_error_logs IS NULL ORDER BY created_at DESC LIMIT 1", + script_path, + w_id + ) + .fetch_optional(&mut *conn) + .await?; + + let hash = utils::not_found_if_none(hash, "script", script_path)?; + + DEPLOYED_SCRIPT_HASH_CACHE.insert( + cache_key, + ExpiringLatestVersionId { + id: hash, + expires_at: std::time::Instant::now() + LATEST_VERSION_ID_CACHE_TTL, + }, + ); + + hash + } + }; + + get_script_info_for_hash(&mut *conn, w_id, hash).await + } +} + +pub async fn get_script_info_for_hash<'e, E: sqlx::PgExecutor<'e>>( db: E, w_id: &str, - script_path: &str, -) -> error::Result<( - scripts::ScriptHash, - Option, - Option, - Option, - Option, - Option, - ScriptLang, - Option, - Option, - Option, - Option, - Option, - Option, - String, -)> { - let r_o = sqlx::query!( - "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by from script where path = $1 AND workspace_id = $2 AND - created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND - deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)", - script_path, - w_id - ) - .fetch_optional(db) - .await?; + hash: i64, +) -> error::Result { + let key = (w_id.to_string(), hash); - let script = utils::not_found_if_none(r_o, "deployed script", script_path)?; + match DEPLOYED_SCRIPT_INFO_CACHE.get(&key) { + Some(info) => { + tracing::debug!("Using cached deployed script info for {hash}"); + Ok(info) + } + _ => { + tracing::debug!("Fetching deployed script info for {hash}"); + let info = sqlx::query_as!( + ScriptHashInfo, + "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by, path from script where hash = $1 AND workspace_id = $2", + hash, + w_id + ) + .fetch_optional(db) + .await?; - Ok(( - scripts::ScriptHash(script.hash), - script.tag, - script.concurrency_key, - script.concurrent_limit, - script.concurrency_time_window_s, - script.cache_ttl, - script.language, - script.dedicated_worker, - script.priority, - script.delete_after_use, - script.timeout, - script.has_preprocessor, - script.on_behalf_of_email, - script.created_by, - )) + let info = utils::not_found_if_none(info, "script", &hash.to_string())?; + + DEPLOYED_SCRIPT_INFO_CACHE.insert(key, info.clone()); + + Ok(info) + } + } +} + +#[derive(Clone)] +pub struct FlowVersionInfo { + pub version: i64, + pub tag: Option, + pub early_return: Option, + pub has_preprocessor: Option, + pub on_behalf_of_email: Option, + pub edited_by: String, + pub dedicated_worker: Option, +} + +pub fn get_latest_flow_version_info_for_path< + 'a, + 'e, + A: sqlx::Acquire<'e, Database = Postgres> + Send + 'a, +>( + db: A, + w_id: &'a str, + path: &'a str, + use_cache: bool, +) -> impl Future> + Send + 'a { + // as instructed in the docstring of sqlx::Acquire + async move { + let mut conn = db.acquire().await?; + + let cache_key = (w_id.to_string(), path.to_string()); + let cached_version = if use_cache { + FLOW_VERSION_CACHE.get(&cache_key) + } else { + None + }; + + let version = match cached_version { + Some(cached_version) if cached_version.expires_at > std::time::Instant::now() => { + tracing::debug!("Using cached flow version {} for {path}", cached_version.id); + cached_version.id + } + _ => { + tracing::debug!("Fetching flow version for {path}"); + let version = sqlx::query_scalar!( + "SELECT flow_version.id from flow + INNER JOIN flow_version + ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] + WHERE flow.path = $1 and flow.workspace_id = $2", + path, + w_id + ) + .fetch_optional(&mut *conn) + .await?; + + let version = utils::not_found_if_none(version, "flow", path)?; + + FLOW_VERSION_CACHE.insert( + cache_key, + ExpiringLatestVersionId { + id: version, + expires_at: std::time::Instant::now() + LATEST_VERSION_ID_CACHE_TTL, + }, + ); + + version + } + }; + + let key = (w_id.to_string(), version); + + match FLOW_INFO_CACHE.get(&key) { + Some(info) => { + tracing::debug!("Using cached flow version info for {version} ({path})"); + Ok(info) + } + _ => { + tracing::debug!("Fetching flow version info for {version} ({path})"); + let info = sqlx::query_as!( + FlowVersionInfo, + "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by, flow_version.id AS version + FROM flow + INNER JOIN flow_version + ON flow_version.id = $3 + WHERE flow.path = $1 and flow.workspace_id = $2", + path, + w_id, + version + ) + .fetch_optional(&mut *conn) + .await?; + + let info = utils::not_found_if_none(info, "flow", path)?; + + FLOW_INFO_CACHE.insert(key, info.clone()); + + Ok(info) + } + } + } } pub async fn get_latest_hash_for_path<'c>( diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 495f45912e..d49e64865e 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -16,10 +16,35 @@ use object_store::{aws::AmazonS3Builder, ClientOptions}; use reqwest::header::HeaderMap; use serde::{Deserialize, Serialize}; #[cfg(feature = "parquet")] -use std::sync::Arc; +use std::sync::{Arc, Mutex}; #[cfg(feature = "parquet")] use tokio::sync::RwLock; +#[cfg(feature = "parquet")] +use crate::error::to_anyhow; +#[cfg(feature = "parquet")] +use crate::utils::rd_string; +#[cfg(feature = "parquet")] +use bytes::Bytes; +#[cfg(feature = "parquet")] +use datafusion::arrow::array::{RecordBatch, RecordBatchWriter}; +#[cfg(feature = "parquet")] +use datafusion::arrow::error::ArrowError; +#[cfg(feature = "parquet")] +use datafusion::arrow::json::writer::JsonArray; +#[cfg(feature = "parquet")] +use datafusion::arrow::{csv, json}; +#[cfg(feature = "parquet")] +use datafusion::parquet::arrow::ArrowWriter; +#[cfg(feature = "parquet")] +use futures::TryStreamExt; +#[cfg(feature = "parquet")] +use std::io::Write; +#[cfg(feature = "parquet")] +use tokio::task; +#[cfg(feature = "parquet")] +use windmill_parser_sql::S3ModeFormat; + #[cfg(feature = "parquet")] lazy_static::lazy_static! { @@ -480,3 +505,184 @@ pub fn bundle(w_id: &str, hash: &str) -> String { pub fn raw_app(w_id: &str, version: &i64) -> String { format!("/home/rfiszel/raw_app/{}/{}", w_id, version) } + +// Originally used a Arc> +// But cannot call .close() on it because it moves the value and the object is not Sized +#[cfg(feature = "parquet")] +enum RecordBatchWriterEnum { + Parquet(ArrowWriter), + Csv(csv::Writer), + Json(json::Writer), +} + +#[cfg(feature = "parquet")] +impl RecordBatchWriter for RecordBatchWriterEnum { + fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> { + match self { + RecordBatchWriterEnum::Parquet(w) => w.write(batch).map_err(|e| e.into()), + RecordBatchWriterEnum::Csv(w) => w.write(batch), + RecordBatchWriterEnum::Json(w) => w.write(batch), + } + } + + fn close(self) -> Result<(), ArrowError> { + match self { + RecordBatchWriterEnum::Parquet(w) => w.close().map_err(|e| e.into()).map(drop), + RecordBatchWriterEnum::Csv(w) => w.close(), + RecordBatchWriterEnum::Json(w) => w.close(), + } + } +} + +#[cfg(feature = "parquet")] +struct ChannelWriter { + sender: tokio::sync::mpsc::Sender>, +} + +#[cfg(feature = "parquet")] +impl Write for ChannelWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let data: Bytes = buf.to_vec().into(); + self.sender.blocking_send(Ok(data)).map_err(|e| { + std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + format!("Channel send error: {}", e), + ) + })?; + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[cfg(not(feature = "parquet"))] +pub async fn convert_json_line_stream>( + mut _stream: impl futures::TryStreamExt> + Unpin, + _output_format: windmill_parser_sql::S3ModeFormat, +) -> anyhow::Result>> { + Ok(async_stream::stream! { + yield Err(anyhow::anyhow!("Parquet feature is not enabled. Cannot convert JSON line stream.")); + }) +} + +#[cfg(feature = "parquet")] +pub async fn convert_json_line_stream>( + mut stream: impl TryStreamExt> + Unpin, + output_format: S3ModeFormat, +) -> anyhow::Result>> { + const MAX_MPSC_SIZE: usize = 1000; + + use datafusion::{execution::context::SessionContext, prelude::NdJsonReadOptions}; + use futures::StreamExt; + use std::path::PathBuf; + use tokio::io::AsyncWriteExt; + + let mut path = PathBuf::from(std::env::temp_dir()); + path.push(format!("{}.json", rd_string(8))); + let path_str = path + .to_str() + .ok_or_else(|| anyhow::anyhow!("Invalid path"))?; + + // Write the stream to a temporary file + let mut file: tokio::fs::File = tokio::fs::File::create(&path).await.map_err(to_anyhow)?; + + while let Some(chunk) = stream.next().await { + match chunk { + Ok(chunk) => { + // Convert the chunk to bytes and write it to the file + let b: bytes::Bytes = serde_json::to_string(&chunk)?.into(); + file.write_all(&b).await?; + file.write_all(b"\n").await?; + } + Err(e) => { + tokio::fs::remove_file(&path).await?; + return Err(e.into()); + } + } + } + + file.flush().await?; + file.sync_all().await?; + drop(file); + + let ctx = SessionContext::new(); + ctx.register_json( + "my_table", + path_str, + NdJsonReadOptions { ..Default::default() }, + ) + .await + .map_err(to_anyhow)?; + + let df = ctx.sql("SELECT * FROM my_table").await.map_err(to_anyhow)?; + let schema = df.schema().clone().into(); + let mut datafusion_stream = df.execute_stream().await.map_err(to_anyhow)?; + + let (tx, rx) = tokio::sync::mpsc::channel(MAX_MPSC_SIZE); + let writer: Arc>> = + Arc::new(Mutex::new(Some(match output_format { + S3ModeFormat::Parquet => RecordBatchWriterEnum::Parquet( + ArrowWriter::try_new(ChannelWriter { sender: tx.clone() }, Arc::new(schema), None) + .map_err(to_anyhow)?, + ), + + S3ModeFormat::Csv => { + RecordBatchWriterEnum::Csv(csv::Writer::new(ChannelWriter { sender: tx.clone() })) + } + S3ModeFormat::Json => { + RecordBatchWriterEnum::Json(json::Writer::<_, JsonArray>::new(ChannelWriter { + sender: tx.clone(), + })) + } + }))); + + // This spawn is so that the data is sent in the background. Else the function would deadlock + // when hitting the mpsc channel limit + task::spawn(async move { + while let Some(batch_result) = datafusion_stream.next().await { + let batch: RecordBatch = match batch_result { + Ok(batch) => batch, + Err(e) => { + tracing::error!("Error in datafusion stream: {:?}", &e); + match tx.send(Err(e.into())).await { + Ok(_) => {} + Err(e) => tracing::error!("Failed to write error to channel: {:?}", &e), + } + break; + } + }; + let writer = writer.clone(); + // Writer calls blocking_send which would crash if called from the async context + let write_result = task::spawn_blocking(move || { + // SAFETY: We await so the code is actually sequential, lock unwrap cannot panic + // Second unwrap is ok because we initialized the option with Some + writer.lock().unwrap().as_mut().unwrap().write(&batch) + }) + .await; + match write_result { + Ok(Ok(_)) => {} + Ok(Err(e)) => { + tracing::error!("Error writing batch: {:?}", &e); + match tx.send(Err(e.into())).await { + Ok(_) => {} + Err(e) => tracing::error!("Failed to write error to channel: {:?}", &e), + } + } + Err(e) => tracing::error!("Error in blocking task: {:?}", &e), + }; + } + task::spawn_blocking(move || { + writer.lock().unwrap().take().unwrap().close()?; + drop(writer); + Ok::<_, anyhow::Error>(()) + }) + .await??; + drop(ctx); + tokio::fs::remove_file(&path).await?; + Ok::<_, anyhow::Error>(()) + }); + + Ok(tokio_stream::wrappers::ReceiverStream::new(rx)) +} diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 82491a52a4..8b67284c4e 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -265,7 +265,7 @@ pub struct ScriptHistoryUpdate { pub deployment_msg: Option, } -#[derive(Serialize, Deserialize, Debug, sqlx::Type)] +#[derive(Serialize, Deserialize, Debug, sqlx::Type, Clone)] #[sqlx(transparent)] #[serde(transparent)] pub struct Schema(pub sqlx::types::Json>); diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 62c193031a..981b71eccf 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -97,8 +97,10 @@ lazy_static::lazy_static! { search_addon = true; println!("Binary is in 'standalone' mode with search enabled"); Mode::Standalone - } - else { + } else if &x == "mcp" { + println!("Binary is in 'mcp' mode"); + Mode::MCP + } else { if &x != "standalone" { eprintln!("mode not recognized, defaulting to standalone: {x}"); } else { @@ -346,6 +348,7 @@ pub enum Mode { Server, Standalone, Indexer, + MCP, } impl std::fmt::Display for Mode { @@ -356,6 +359,7 @@ impl std::fmt::Display for Mode { Mode::Server => write!(f, "server"), Mode::Standalone => write!(f, "standalone"), Mode::Indexer => write!(f, "indexer"), + Mode::MCP => write!(f, "mcp"), } } } @@ -467,13 +471,28 @@ pub async fn report_recovered_critical_error( } } -pub fn empty_string_as_none<'de, D>( - deserializer: D, -) -> std::result::Result, D::Error> +pub trait IsEmpty { + fn is_empty(&self) -> bool; +} + +impl IsEmpty for String { + fn is_empty(&self) -> bool { + self.is_empty() + } +} + +impl IsEmpty for Vec { + fn is_empty(&self) -> bool { + self.is_empty() + } +} + +pub fn empty_as_none<'de, D, T>(deserializer: D) -> std::result::Result, D::Error> where D: Deserializer<'de>, + T: Deserialize<'de> + IsEmpty, { - let option = as serde::Deserialize>::deserialize(deserializer)?; + let option = as serde::Deserialize>::deserialize(deserializer)?; Ok(option.filter(|s| !s.is_empty())) } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index cdffe6f6a1..ef4cfa0f46 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -381,12 +381,8 @@ fn normalize_path(path: &Path) -> PathBuf { } ret } -pub fn write_file_at_user_defined_location( - job_dir: &str, - user_defined_path: &str, - content: &str, - mode: Option, -) -> error::Result { + +pub fn is_allowed_file_location(job_dir: &str, user_defined_path: &str) -> error::Result { let job_dir = Path::new(job_dir); let user_path = PathBuf::from(user_defined_path); @@ -405,6 +401,17 @@ pub fn write_file_at_user_defined_location( .into()); } + Ok(normalized_full_path) +} + +pub fn write_file_at_user_defined_location( + job_dir: &str, + user_defined_path: &str, + content: &str, + mode: Option, +) -> error::Result { + let normalized_full_path = is_allowed_file_location(job_dir, user_defined_path)?; + let full_path = normalized_full_path.as_path(); if let Some(parent_dir) = full_path.parent() { std::fs::create_dir_all(parent_dir)?; @@ -1388,18 +1395,66 @@ pub async fn load_worker_config( tracing::debug!("Custom tags priority set: {:?}", priority_tags_sorted); let env_vars_static = config.env_vars_static.unwrap_or_default().clone(); - let resolved_env_vars: HashMap = env_vars_static - .keys() - .map(|x| x.to_string()) - .chain(config.env_vars_allowlist.unwrap_or_default()) - .chain( - std::env::var("WHITELIST_ENVS") - .ok() - .map(|x| x.split(',').map(|x| x.to_string()).collect_vec()) - .unwrap_or_default() - .into_iter(), - ) - .sorted() + let resolved_env_vars: HashMap = load_env_vars( + config + .env_vars_allowlist + .unwrap_or_default() + .into_iter() + .chain(load_whitelist_env_vars_from_env()) + .chain(env_vars_static.keys().map(|x| x.to_string())), + &env_vars_static, + ); + + Ok(WorkerConfig { + worker_tags, + priority_tags_sorted, + dedicated_worker, + init_bash: config + .init_bash + .or_else(|| load_init_bash_from_env()) + .and_then(|x| if x.is_empty() { None } else { Some(x) }), + cache_clear: config.cache_clear, + pip_local_dependencies: config + .pip_local_dependencies + .or_else(|| load_pip_local_dependencies_from_env()), + additional_python_paths: config + .additional_python_paths + .or_else(|| load_additional_python_paths_from_env()), + env_vars: resolved_env_vars, + }) +} + +pub fn load_init_bash_from_env() -> Option { + std::env::var("INIT_SCRIPT") + .ok() + .and_then(|x| if x.is_empty() { None } else { Some(x) }) +} + +pub fn load_pip_local_dependencies_from_env() -> Option> { + std::env::var("PIP_LOCAL_DEPENDENCIES") + .ok() + .map(|x| x.split(',').map(|x| x.to_string()).collect_vec()) +} + +pub fn load_additional_python_paths_from_env() -> Option> { + std::env::var("ADDITIONAL_PYTHON_PATHS") + .ok() + .map(|x| x.split(':').map(|x| x.to_string()).collect_vec()) +} + +pub fn load_whitelist_env_vars_from_env() -> std::vec::IntoIter { + std::env::var("WHITELIST_ENVS") + .ok() + .map(|x| x.split(',').map(|x| x.to_string()).collect_vec()) + .unwrap_or_default() + .into_iter() +} + +pub fn load_env_vars( + iter: impl Iterator, + env_vars_static: &HashMap, +) -> HashMap { + iter.sorted() .unique() .map(|envvar_name| { ( @@ -1412,34 +1467,7 @@ pub async fn load_worker_config( }), ) }) - .collect(); - - Ok(WorkerConfig { - worker_tags, - priority_tags_sorted, - dedicated_worker, - init_bash: config - .init_bash - .or_else(|| std::env::var("INIT_SCRIPT").ok()) - .and_then(|x| if x.is_empty() { None } else { Some(x) }), - cache_clear: config.cache_clear, - pip_local_dependencies: config.pip_local_dependencies.or_else(|| { - let pip_local_dependencies = std::env::var("PIP_LOCAL_DEPENDENCIES") - .ok() - .map(|x| x.split(',').map(|x| x.to_string()).collect()); - if pip_local_dependencies == Some(vec!["".to_string()]) { - None - } else { - pip_local_dependencies - } - }), - additional_python_paths: config.additional_python_paths.or_else(|| { - std::env::var("ADDITIONAL_PYTHON_PATHS") - .ok() - .map(|x| x.split(':').map(|x| x.to_string()).collect()) - }), - env_vars: resolved_env_vars, - }) + .collect() } #[derive(Clone, PartialEq, Debug)] diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index c92933ef92..74e002f545 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -164,7 +164,7 @@ pub async fn cancel_single_job<'c>( let username = username.to_string(); let w_id = w_id.to_string(); let db = db.clone(); - tracing::info!("cancelling job {:?}", db); + tracing::info!("cancelling job {:?}", job_running.id); let job_running = job_running.clone(); tokio::task::spawn(async move { let reason: String = reason @@ -1943,7 +1943,7 @@ async fn handle_successful_schedule<'a, 'c, T: Serialize + Send + Sync>( Ok(()) } -#[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone)] +#[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash)] #[sqlx(type_name = "TRIGGER_KIND", rename_all = "lowercase")] #[serde(rename_all = "lowercase")] pub enum TriggerKind { @@ -1959,6 +1959,24 @@ pub enum TriggerKind { Gcp } + +impl TriggerKind { + pub fn to_key(&self) -> String { + match self { + TriggerKind::Webhook => "webhook".to_string(), + TriggerKind::Http => "http".to_string(), + TriggerKind::Websocket => "websocket".to_string(), + TriggerKind::Kafka => "kafka".to_string(), + TriggerKind::Email => "email".to_string(), + TriggerKind::Nats => "nats".to_string(), + TriggerKind::Mqtt => "mqtt".to_string(), + TriggerKind::Sqs => "sqs".to_string(), + TriggerKind::Postgres => "postgres".to_string(), + TriggerKind::Gcp => "gcp".to_string(), + } + } +} + #[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone)] #[sqlx(type_name = "JOB_TRIGGER_KIND", rename_all = "lowercase")] #[serde(rename_all = "lowercase")] @@ -2569,6 +2587,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>( if query.is_empty() { tracing::warn!("No suspended pull queries available"); + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; return Ok((None, false)); } @@ -2590,6 +2609,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>( if queries.is_empty() { tracing::warn!("No pull queries available"); + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; return Ok((None, false)); } @@ -2651,13 +2671,38 @@ fn interpolate_args(x: String, args: &PushArgs, workspace_id: &str) -> String { let mut interpolated = workspaced.clone(); for cap in RE_ARG_TAG.captures_iter(&workspaced) { let arg_name = cap.get(1).unwrap().as_str(); - let arg_value = args - .args - .get(arg_name) - .or(args.extra.as_ref().and_then(|x| x.get(arg_name))) - .map(|x| x.get()) - .unwrap_or_default() - .trim_matches('"'); + let arg_value = if arg_name.contains('.') { + let parts: Vec<&str> = arg_name.split('.').collect(); + let root = parts[0]; + let mut value = args + .args + .get(root) + .or(args.extra.as_ref().and_then(|x| x.get(root))) + .map(|x| x.get()) + .unwrap_or_default().to_string(); + + for part in parts.iter().skip(1) { + if let Ok(obj) = serde_json::from_str::(&value) { + value = obj.get(part) + .and_then(|v| Some(v.to_string())) + .unwrap_or_default() + .as_str().to_string(); + } else { + value = "".to_string(); // Invalid JSON or missing field + break; + } + } + value.trim_matches('"').to_string() + } else { + args.args + .get(arg_name) + .or(args.extra.as_ref().and_then(|x| x.get(arg_name))) + .map(|x| x.get()) + .unwrap_or_default() + .trim_matches('"') + .to_string() + }; + tracing::error!("arg_value: {}", arg_value); interpolated = interpolated.replace(format!("$args[{}]", arg_name).as_str(), &arg_value); } @@ -3225,7 +3270,7 @@ pub fn empty_result() -> Box { // } lazy_static::lazy_static! { - pub static ref RE_ARG_TAG: Regex = Regex::new(r#"\$args\[(\w+)\]"#).unwrap(); + pub static ref RE_ARG_TAG: Regex = Regex::new(r#"\$args\[((?:\w+\.)*\w+)\]"#).unwrap(); } // #[instrument(level = "trace", skip_all)] @@ -3234,7 +3279,7 @@ pub async fn push<'c, 'd>( mut tx: PushIsolationLevel<'c>, workspace_id: &str, job_payload: JobPayload, - mut args: PushArgs<'d>, + args: PushArgs<'d>, user: &str, mut email: &str, mut permissioned_as: String, @@ -3450,17 +3495,10 @@ pub async fn push<'c, 'd>( priority, apply_preprocessor, } => { - let extra = args.extra.get_or_insert_with(HashMap::new); if apply_preprocessor { preprocessed = Some(false); - extra.entry("wm_trigger".to_string()).or_insert_with(|| { - to_raw_value(&serde_json::json!({ - "kind": "webhook", - })) - }); - } else { - extra.remove("wm_trigger"); - } + } + ( Some(hash.0), Some(path), @@ -3826,19 +3864,8 @@ pub async fn push<'c, 'd>( priority, ) } - JobPayload::Flow { path, dedicated_worker, apply_preprocessor } => { + JobPayload::Flow { path, dedicated_worker, apply_preprocessor, version } => { let mut ntx = tx.into_tx().await?; - // Fetch the latest version of the flow. - let version = sqlx::query_scalar!( - "SELECT flow.versions[array_upper(flow.versions, 1)] AS \"version!: i64\" - FROM flow WHERE path = $1 AND workspace_id = $2", - &path, - &workspace_id - ) - .fetch_optional(&mut *ntx) - .await? - .ok_or_else(|| Error::internal_err(format!("not found flow at path {:?}", path)))?; - // Do not use the lite version unless all workers are updated. let data = if *DISABLE_FLOW_SCRIPT || (!*MIN_VERSION_IS_AT_LEAST_1_432.read().await && !*CLOUD_HOSTED) @@ -3862,17 +3889,10 @@ pub async fn push<'c, 'd>( let concurrency_time_window_s = value.concurrency_time_window_s; let concurrent_limit = value.concurrent_limit; - let extra = args.extra.get_or_insert_with(HashMap::new); if !apply_preprocessor { value.preprocessor_module = None; - extra.remove("wm_trigger"); } else { preprocessed = Some(false); - extra.entry("wm_trigger".to_string()).or_insert_with(|| { - to_raw_value(&serde_json::json!({ - "kind": "webhook", - })) - }); } // this is a new flow being pushed, status is set to `value`. diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index af1fca6f5d..bb7d5df906 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -15,8 +15,10 @@ use std::str::FromStr; use windmill_common::db::Authed; use windmill_common::ee::LICENSE_KEY_VALID; use windmill_common::flows::Retry; +use windmill_common::get_latest_flow_version_info_for_path; use windmill_common::jobs::JobPayload; use windmill_common::schedule::schedule_to_user; +use windmill_common::FlowVersionInfo; use windmill_common::DB; use windmill_common::{ error::{self, Result}, @@ -114,21 +116,21 @@ pub async fn push_scheduled_job<'c>( } let (payload, tag, timeout, on_behalf_of_email, created_by) = if schedule.is_flow { - let r = sqlx::query!( - "SELECT tag, dedicated_worker, on_behalf_of_email, edited_by from flow WHERE path = $1 and workspace_id = $2", - &schedule.script_path, + let FlowVersionInfo { + version, tag, dedicated_worker, on_behalf_of_email, edited_by, .. + } = get_latest_flow_version_info_for_path( + &mut *tx, &schedule.workspace_id, + &schedule.script_path, + false, ) - .fetch_optional(&mut *tx) .await?; - let (tag, dedicated_worker, on_behalf_of_email, edited_by) = r - .map(|x| (x.tag, x.dedicated_worker, x.on_behalf_of_email, x.edited_by)) - .unwrap_or_else(|| (None, None, None, "".to_string())); ( JobPayload::Flow { path: schedule.script_path.clone(), dedicated_worker, apply_preprocessor: false, + version, }, tag, None, diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index e4de400d84..63295208a2 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -90,6 +90,7 @@ deno_tls = { workspace = true, optional = true } deno_permissions = { workspace = true, optional = true } deno_io = { workspace = true, optional = true } deno_error = { workspace = true, optional = true } +async-stream.workspace = true postgres-native-tls.workspace = true native-tls.workspace = true diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 7e0d1bd09c..1e47edc67e 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -5,19 +5,22 @@ use std::{collections::HashMap, os::unix::fs::PermissionsExt, path::PathBuf, pro use std::{collections::HashMap, path::PathBuf, process::Stdio}; use anyhow::anyhow; +use futures::future::try_join_all; use itertools::Itertools; +use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use tokio::process::Command; use uuid::Uuid; use windmill_common::{ error, worker::{ - to_raw_value, write_file, write_file_at_user_defined_location, Connection, WORKER_CONFIG, + is_allowed_file_location, to_raw_value, write_file, write_file_at_user_defined_location, + Connection, WORKER_CONFIG, }, }; use windmill_queue::MiniPulledJob; -use windmill_parser_yaml::{AnsibleRequirements, ResourceOrVariablePath}; +use windmill_parser_yaml::{AnsibleRequirements, GitRepo, ResourceOrVariablePath}; use windmill_queue::{append_logs, CanceledBy}; use crate::{ @@ -28,8 +31,8 @@ use crate::{ }, handle_child::handle_child, python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion}, - AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, - PY_INSTALL_DIR, TZ_ENV, + AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, + PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, }; lazy_static::lazy_static! { @@ -41,7 +44,294 @@ lazy_static::lazy_static! { } const NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT: &str = include_str!("../nsjail/run.ansible.config.proto"); +const WINDMILL_ANSIBLE_PASSWORD_FILENAME: &str = ".windmill.ansible_vault_password_file"; +async fn clone_repo( + repo: &GitRepo, + job_dir: &str, + job_id: &Uuid, + worker_name: &str, + conn: &Connection, + mem_peak: &mut i32, + canceled_by: &mut Option, + w_id: &str, + occupancy_metrics: &mut OccupancyMetrics, + git_ssh_cmd: &str, +) -> error::Result { + let target_path = is_allowed_file_location(job_dir, &repo.target_path)?; + + let mut clone_cmd = Command::new(GIT_PATH.as_str()); + clone_cmd + .current_dir(job_dir) + .env_clear() + .envs(PROXY_ENVS.clone()) + .env("PATH", PATH_ENV.as_str()) + .env("TZ", TZ_ENV.as_str()) + .env("GIT_SSH_COMMAND", git_ssh_cmd) + .args(["clone", "--quiet"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if let Some(branch) = &repo.branch { + clone_cmd.args(["--branch", branch]); + } + clone_cmd.arg(&repo.url); + clone_cmd.arg(&target_path); + + let clone_cmd_child = start_child_process(clone_cmd, GIT_PATH.as_str()).await?; + handle_child( + job_id, + conn, + mem_peak, + canceled_by, + clone_cmd_child, + false, + worker_name, + w_id, + "git clone", + None, + false, + &mut Some(occupancy_metrics), + None, + ) + .await?; + + // Checkout specific commit if provided + if let Some(commit) = &repo.commit { + let mut checkout_cmd = Command::new(GIT_PATH.as_str()); + checkout_cmd + .current_dir(job_dir) + .env_clear() + .envs(PROXY_ENVS.clone()) + .env("PATH", PATH_ENV.as_str()) + .env("TZ", TZ_ENV.as_str()) + .env("GIT_SSH_COMMAND", git_ssh_cmd) + .arg("-C") + .arg(&target_path) + .args(["checkout", "--quiet", commit]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let checkout_cmd_child = start_child_process(checkout_cmd, GIT_PATH.as_str()).await?; + handle_child( + job_id, + conn, + mem_peak, + canceled_by, + checkout_cmd_child, + false, + worker_name, + w_id, + "git checkout", + None, + false, + &mut Some(occupancy_metrics), + None, + ) + .await?; + } + + let mut rev_parse_cmd = Command::new(GIT_PATH.as_str()); + + let commit_hash_output = rev_parse_cmd + .current_dir(job_dir) + .env_clear() + .envs(PROXY_ENVS.clone()) + .env("PATH", PATH_ENV.as_str()) + .env("TZ", TZ_ENV.as_str()) + .env("GIT_SSH_COMMAND", git_ssh_cmd) + .arg("-C") + .arg(&target_path) + .args(["rev-parse", "HEAD"]) + .stderr(Stdio::piped()) + .output() + .await?; + + if !commit_hash_output.status.success() { + let stderr = String::from_utf8(commit_hash_output.stderr)?; + return Err(anyhow!("Error getting git repo commit hash: {stderr}").into()); + } + + let commit_hash = String::from_utf8(commit_hash_output.stdout)? + .trim() + .to_string(); + + Ok(commit_hash) +} + +pub fn create_empty_dir(path: &PathBuf) -> std::io::Result<()> { + if path.exists() { + if path.is_dir() { + let mut entries = std::fs::read_dir(&path)?; + if entries.next().is_some() { + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!( + "Directory '{}' already exists and is not empty", + path.display() + ), + )); + } + Ok(()) + } else { + Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!("Path '{}' exists and is not a directory", path.display()), + )) + } + } else { + std::fs::create_dir_all(path) + } +} + +async fn clone_repo_without_history( + repo: &GitRepo, + full_commit: &str, + job_dir: &str, + job_id: &Uuid, + worker_name: &str, + conn: &Connection, + mem_peak: &mut i32, + canceled_by: &mut Option, + w_id: &str, + occupancy_metrics: &mut OccupancyMetrics, + git_ssh_cmd: &str, +) -> error::Result<()> { + let target_path = is_allowed_file_location(job_dir, &repo.target_path)?; + + create_empty_dir(&target_path)?; + + let mut init_cmd = Command::new(GIT_PATH.as_str()); + init_cmd + .current_dir(job_dir) + .env_clear() + .envs(PROXY_ENVS.clone()) + .env("PATH", PATH_ENV.as_str()) + .env("TZ", TZ_ENV.as_str()) + .arg("-C") + .arg(&target_path) + .args(["init", "--quiet"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if let Some(branch) = &repo.branch { + init_cmd.args(["--initial-branch", branch]); + } + + let init_cmd_child = start_child_process(init_cmd, GIT_PATH.as_str()).await?; + handle_child( + job_id, + conn, + mem_peak, + canceled_by, + init_cmd_child, + false, + worker_name, + w_id, + "git init", + None, + false, + &mut Some(occupancy_metrics), + None, + ) + .await?; + + let mut add_remote_cmd = Command::new(GIT_PATH.as_str()); + add_remote_cmd + .current_dir(job_dir) + .env_clear() + .envs(PROXY_ENVS.clone()) + .env("PATH", PATH_ENV.as_str()) + .env("TZ", TZ_ENV.as_str()) + .env("GIT_SSH_COMMAND", git_ssh_cmd) + .arg("-C") + .arg(&target_path) + .args(vec!["remote", "add", "origin", &repo.url]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let add_remote_cmd_child = start_child_process(add_remote_cmd, GIT_PATH.as_str()).await?; + handle_child( + job_id, + conn, + mem_peak, + canceled_by, + add_remote_cmd_child, + false, + worker_name, + w_id, + "git add remote", + None, + false, + &mut Some(occupancy_metrics), + None, + ) + .await?; + + let mut fetch_cmd = Command::new(GIT_PATH.as_str()); + fetch_cmd + .current_dir(job_dir) + .env_clear() + .envs(PROXY_ENVS.clone()) + .env("PATH", PATH_ENV.as_str()) + .env("TZ", TZ_ENV.as_str()) + .env("GIT_SSH_COMMAND", git_ssh_cmd) + .arg("-C") + .arg(&target_path) + .args(vec!["fetch", "--depth=1", "--quiet", "origin", full_commit]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let fetch_cmd_child = start_child_process(fetch_cmd, GIT_PATH.as_str()).await?; + handle_child( + job_id, + conn, + mem_peak, + canceled_by, + fetch_cmd_child, + false, + worker_name, + w_id, + "git fetch", + None, + false, + &mut Some(occupancy_metrics), + None, + ) + .await?; + + let mut checkout_cmd = Command::new(GIT_PATH.as_str()); + checkout_cmd + .current_dir(job_dir) + .env_clear() + .envs(PROXY_ENVS.clone()) + .env("PATH", PATH_ENV.as_str()) + .env("TZ", TZ_ENV.as_str()) + .env("GIT_SSH_COMMAND", git_ssh_cmd) + .arg("-C") + .arg(&target_path) + .args(["checkout", "--quiet", "FETCH_HEAD"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let checkout_cmd_child = start_child_process(checkout_cmd, GIT_PATH.as_str()).await?; + handle_child( + job_id, + conn, + mem_peak, + canceled_by, + checkout_cmd_child, + false, + worker_name, + w_id, + "git checkout", + None, + false, + &mut Some(occupancy_metrics), + None, + ) + .await?; + + Ok(()) +} async fn handle_ansible_python_deps( job_dir: &str, requirements_o: Option<&String>, @@ -118,7 +408,7 @@ async fn handle_ansible_python_deps( Ok(additional_python_paths) } -async fn install_galaxy_collections( +pub async fn install_galaxy_collections( collections_yml: &str, job_dir: &str, job_id: &Uuid, @@ -128,6 +418,7 @@ async fn install_galaxy_collections( canceled_by: &mut Option, conn: &Connection, occupancy_metrics: &mut OccupancyMetrics, + git_ssh_cmd: &str, ) -> anyhow::Result<()> { write_file(job_dir, "requirements.yml", collections_yml)?; @@ -138,15 +429,52 @@ async fn install_galaxy_collections( conn, ) .await; - let mut galaxy_command = Command::new(ANSIBLE_GALAXY_PATH.as_str()); - galaxy_command + + let mut galaxy_roles_cmd = Command::new(ANSIBLE_GALAXY_PATH.as_str()); + galaxy_roles_cmd .current_dir(job_dir) .env_clear() .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) - // .env("BASE_INTERNAL_URL", base_internal_url) - // .env("HOME", HOME_ENV.as_str()) + .env("GIT_SSH_COMMAND", git_ssh_cmd) + .args(vec![ + "role", + "install", + "-r", + "requirements.yml", + "-p", + "./roles", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let child = start_child_process(galaxy_roles_cmd, ANSIBLE_GALAXY_PATH.as_str()).await?; + handle_child( + job_id, + conn, + mem_peak, + canceled_by, + child, + !*DISABLE_NSJAIL, + worker_name, + w_id, + "ansible-galaxy role install", + None, + false, + &mut Some(occupancy_metrics), + None, + ) + .await?; + + let mut galaxy_collections_cmd = Command::new(ANSIBLE_GALAXY_PATH.as_str()); + galaxy_collections_cmd + .current_dir(job_dir) + .env_clear() + .envs(PROXY_ENVS.clone()) + .env("PATH", PATH_ENV.as_str()) + .env("TZ", TZ_ENV.as_str()) + .env("GIT_SSH_COMMAND", git_ssh_cmd) .args(vec![ "collection", "install", @@ -158,7 +486,7 @@ async fn install_galaxy_collections( .stdout(Stdio::piped()) .stderr(Stdio::piped()); - let child = start_child_process(galaxy_command, ANSIBLE_GALAXY_PATH.as_str()).await?; + let child = start_child_process(galaxy_collections_cmd, ANSIBLE_GALAXY_PATH.as_str()).await?; handle_child( job_id, conn, @@ -168,7 +496,7 @@ async fn install_galaxy_collections( !*DISABLE_NSJAIL, worker_name, w_id, - "ansible galaxy install", + "ansible-galaxy collection install", None, false, &mut Some(occupancy_metrics), @@ -179,6 +507,258 @@ async fn install_galaxy_collections( Ok(()) } +#[derive(Serialize, Deserialize)] +pub struct AnsibleDependencyLocks { + pub python_lockfile: String, + pub git_repos: HashMap, // URL to full commit hash + pub collections_and_roles: String, + pub collections_and_roles_logs: String, + // pub collection_versions: HashMap, // + // pub role_versions: HashMap, +} + +pub async fn get_collection_locks( + job_dir: &str, +) -> anyhow::Result<(HashMap, String)> { + let mut ansible_cmd = Command::new(ANSIBLE_GALAXY_PATH.as_str()); + + ansible_cmd + .current_dir(job_dir) + .args(["collection", "list", "--format", "json", "-p", "./"]); + + let output = ansible_cmd.output().await?; + + let mut ret = HashMap::new(); + let mut logs = String::new(); + + if !output.status.success() { + let stderr = String::from_utf8(output.stderr)?; + return Err(anyhow!( + "Error getting ansible collection versions: {stderr}" + )); + } + + let stdout = String::from_utf8(output.stdout)?; + + let val: serde_json::Value = serde_json::from_str(&stdout)?; + + let Some(own_collections) = val.get(format!("{}/ansible_collections", job_dir)) else { + return Ok((ret, logs)); + }; + + let collections = own_collections.as_object().ok_or(anyhow!( + "Expected an object (map) for the `ansible-galaxy collection list` command output and got {}", + own_collections + ))?; + + for (c_name, c) in collections.iter() { + if let Some(v) = c.get("version").and_then(|v| v.as_str()) { + // TODO: Check if version is not something like `(undefined)` + ret.insert(c_name.clone(), v.to_string()); + } else { + logs.push_str(&format!("Failed to get version for collection `{}`. Expected an object with a string in the `version` field but received {}\n", c_name, c)); + } + } + + Ok((ret, logs)) +} + +pub async fn get_role_locks(job_dir: &str) -> anyhow::Result<(HashMap, String)> { + let mut ansible_cmd = Command::new(ANSIBLE_GALAXY_PATH.as_str()); + + ansible_cmd + .current_dir(job_dir) + .args(["role", "list", "-p", "./roles"]); + + let output = ansible_cmd.output().await?; + let mut ret = HashMap::new(); + let mut logs = String::new(); + + if !output.status.success() { + let stderr = String::from_utf8(output.stderr)?; + logs.push_str(&format!("Error getting ansible role versions: {stderr}")); + return Ok((ret, logs)); + } + + let stdout = String::from_utf8(output.stdout)?; + + let mut lines = stdout.lines(); + + while let Some(line) = lines.next() { + if line == format!("# {}/roles", job_dir) { + break; + } + } + + for line in lines { + let line = line.strip_prefix("-").unwrap_or(line); + let mut cols = line.split(","); + + if let Some(name) = cols.next().map(|n| n.trim()) { + if let Some(version) = cols.next().map(|v| v.trim()) { + // TODO: Check if version is not something like `(undefined)` + ret.insert(name.to_string(), version.to_string()); + } else { + logs.push_str(&format!("Failed to get version for role `{}`.", name)); + } + } + } + + Ok((ret, logs)) +} + +pub async fn get_git_repo_full_head_commit_hash( + repo: &GitRepo, + git_ssh_cmd: &str, +) -> anyhow::Result { + let mut git_cmd = Command::new(GIT_PATH.as_str()); + + git_cmd + .env("GIT_SSH_COMMAND", git_ssh_cmd) + .args(["ls-remote", &repo.url, "HEAD"]); + + let output = git_cmd.stderr(Stdio::piped()).output().await?; + + if !output.status.success() { + let stderr = String::from_utf8(output.stderr)?; + return Err(anyhow!("Error getting git repo commit hash: {stderr}")); + } + + let stdout = String::from_utf8(output.stdout)?; + + let lines: Vec<&str> = stdout.lines().collect(); + + if lines.len() != 1 { + return Err(anyhow!("Unexpected output format for git ls-remote",)); + } + + Ok(lines + .first() + .ok_or(anyhow!( + "The HEAD commit hash was not found for repo `{}`", + &repo.url + ))? + .split_whitespace() + .next() + .map(|s| s.to_string()) + .ok_or(anyhow!("Unexpected output format for git ls-remote"))?) +} + +pub async fn get_git_repos_lock( + repos: &Vec, + job_dir: &str, + job_id: &Uuid, + worker_name: &str, + conn: &Connection, + mem_peak: &mut i32, + canceled_by: &mut Option, + w_id: &str, + occupancy_metrics: &mut OccupancyMetrics, + git_ssh_cmd: &str, +) -> anyhow::Result> { + let mut ret = HashMap::new(); + + for repo in repos { + if repo.commit.is_some() { + ret.insert( + repo.url.to_string(), + clone_repo( + repo, + job_dir, + job_id, + worker_name, + conn, + mem_peak, + canceled_by, + w_id, + occupancy_metrics, + git_ssh_cmd, + ) + .await?, + ); + } else { + ret.insert( + repo.url.to_string(), + get_git_repo_full_head_commit_hash(repo, git_ssh_cmd).await?, + ); + } + } + + Ok(ret) +} + +pub fn create_ansible_cfg( + reqs: Option<&AnsibleRequirements>, + job_dir: &str, + vault_password_file_exists: bool, +) -> error::Result<()> { + let mut passwords_cfg = String::new(); + if vault_password_file_exists { + passwords_cfg.push_str(&format!( + "vault_password_file = {WINDMILL_ANSIBLE_PASSWORD_FILENAME}\n" + )); + } + if let Some(vault_ids) = reqs.as_ref().map(|r| &r.vault_id) { + if !vault_ids.is_empty() { + let password_files = vault_ids.join(","); + + passwords_cfg.push_str(&format!("vault_identity_list = {password_files}\n")); + } + } + let ansible_cfg_content = format!( + r#" +[defaults] +collections_path = ./ +roles_path = ./roles +home={job_dir}/.ansible +local_tmp={job_dir}/.ansible/tmp +remote_tmp={job_dir}/.ansible/tmp +{passwords_cfg} +"# + ); + + write_file(job_dir, "ansible.cfg", &ansible_cfg_content)?; + + Ok(()) +} + +pub async fn get_git_ssh_cmd( + reqs: &AnsibleRequirements, + job_dir: &str, + client: &AuthedClient, +) -> error::Result { + let ssh_id_files = try_join_all(reqs.git_ssh_identity.iter().enumerate().map( + async |(i, var_path)| -> error::Result { + let id_file_name = format!(".ssh_id_priv_{}", i); + let loc = is_allowed_file_location(job_dir, &id_file_name)?; + + let mut content = client.get_variable_value(var_path).await.map_err(|e| { + error::Error::NotFound(format!( + "Variable {var_path} not found for git ssh identity: {e:#}" + )) + })?; + content.push_str("\n"); + + let file = write_file(job_dir, &id_file_name, &content)?; + + #[cfg(unix)] + { + let perm = std::os::unix::fs::PermissionsExt::from_mode(0o600); + file.set_permissions(perm)?; + } + + Ok(format!( + " -i '{}'", + loc.to_string_lossy().replace('\'', r"'\''") + )) + }, + )) + .await?; + + let git_ssh_cmd = format!("ssh -o StrictHostKeyChecking=no{}", ssh_id_files.join("")); + Ok(git_ssh_cmd) +} + pub async fn handle_ansible_job( requirements_o: Option<&String>, job_dir: &str, @@ -202,13 +782,30 @@ pub async fn handle_ansible_job( "ansible", )?; + let req_lockfiles: Option = if let Some(s) = requirements_o { + if let Ok(lockfile) = serde_json::from_str(s) { + Some(lockfile) + } else { + append_logs( + &job.id, + &job.workspace_id, + format!("WARN: lockfile could not be parsed: `{s}`"), + conn, + ) + .await; + None + } + } else { + None + }; + let (logs, reqs, playbook) = windmill_parser_yaml::parse_ansible_reqs(inner_content)?; append_logs(&job.id, &job.workspace_id, logs, conn).await; write_file(job_dir, "main.yml", &playbook)?; let additional_python_paths = handle_ansible_python_deps( job_dir, - requirements_o, + req_lockfiles.as_ref().map(|r| &r.python_lockfile), reqs.as_ref(), &job.workspace_id, &job.id, @@ -221,6 +818,11 @@ pub async fn handle_ansible_job( ) .await?; + let git_ssh_cmd = &match &reqs { + Some(r) => get_git_ssh_cmd(r, job_dir, client).await?, + None => "ssh".to_string(), + }; + let interpolated_args; if let Some(args) = &job.args { let mut args = args.0.clone(); @@ -268,23 +870,93 @@ pub async fn handle_ansible_job( .unwrap_or_else(|| vec![]); let mut nsjail_extra_mounts = vec![]; - if let Some(r) = reqs { - if let Some(db) = conn.as_sql() { - nsjail_extra_mounts = create_file_resources( + if let Some(r) = reqs.as_ref() { + nsjail_extra_mounts = create_file_resources( + &job.id, + &job.workspace_id, + job_dir, + interpolated_args.as_ref(), + &r, + &client, + conn, + ) + .await?; + + for repo in &r.git_repos { + append_logs( &job.id, &job.workspace_id, - job_dir, - interpolated_args.as_ref(), - &r, - &client, - db, + format!("\nCloning {}...\n", &repo.url), + conn, ) - .await?; + .await; + if let Some(full_commit_hash) = req_lockfiles + .as_ref() + .and_then(|r| r.git_repos.get(&repo.url)) + { + clone_repo_without_history( + repo, + full_commit_hash, + job_dir, + &job.id, + worker_name, + conn, + mem_peak, + canceled_by, + &job.workspace_id, + occupancy_metrics, + git_ssh_cmd, + ) + .await + .map_err(|e| anyhow!("Failed to clone git repo `{}`: {e}", repo.url))?; + } else { + if req_lockfiles.is_some() { + append_logs( + &job.id, + &job.workspace_id, + format!("Warning: `{}` is using latest commit because the lockfile didn't store a commit hash for this repo. Updates to the repo could break the deployed playbook.\n", &repo.url), + conn, + ) + .await; + } + clone_repo( + repo, + job_dir, + &job.id, + worker_name, + conn, + mem_peak, + canceled_by, + &job.workspace_id, + occupancy_metrics, + git_ssh_cmd, + ) + .await + .map_err(|e| anyhow!("Failed to clone git repo `{}`: {e}", repo.url))?; + } + + append_logs( + &job.id, + &job.workspace_id, + format!("Cloned {} into {}\n", &repo.url, &repo.target_path), + conn, + ) + .await; } - if let Some(collections) = r.collections { + if let Some(collections) = r.roles_and_collections.as_ref() { + let empty = String::new(); + let (lockfile, logs) = req_lockfiles + .as_ref() + .map(|r| (&r.collections_and_roles, &r.collections_and_roles_logs)) + .unwrap_or((collections, &empty)); + + if !logs.is_empty() { + append_logs(&job.id, &job.workspace_id, logs, conn).await; + } + install_galaxy_collections( - collections.as_str(), + lockfile, job_dir, &job.id, worker_name, @@ -293,10 +965,12 @@ pub async fn handle_ansible_job( canceled_by, conn, occupancy_metrics, + git_ssh_cmd, ) .await?; } } + append_logs( &job.id, &job.workspace_id, @@ -304,17 +978,17 @@ pub async fn handle_ansible_job( conn, ) .await; - let ansible_cfg_content = format!( - r#" -[defaults] -collections_path = ./ -roles_path = ./roles -home={job_dir}/.ansible -local_tmp={job_dir}/.ansible/tmp -remote_tmp={job_dir}/.ansible/tmp -"# - ); - write_file(job_dir, "ansible.cfg", &ansible_cfg_content)?; + + let vault_password_file_exists = match reqs.as_ref().and_then(|x| x.vault_password.as_ref()) { + Some(var_path) => { + let password = client.get_variable_value(&var_path).await?; + write_file(job_dir, WINDMILL_ANSIBLE_PASSWORD_FILENAME, &password)?; + true + } + None => false, + }; + + create_ansible_cfg(reqs.as_ref(), job_dir, vault_password_file_exists)?; let mut reserved_variables = get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; @@ -498,7 +1172,7 @@ async fn create_file_resources( args: Option<&HashMap>>, r: &AnsibleRequirements, client: &crate::AuthedClient, - db: &sqlx::Pool, + conn: &Connection, ) -> error::Result> { let mut logs = String::new(); let mut nsjail_mounts: Vec = vec![]; @@ -568,7 +1242,7 @@ async fn create_file_resources( file_res.target_path, file_res.resource_path )); } - append_logs(job_id, w_id, logs, &Connection::Sql(db.clone())).await; + append_logs(job_id, w_id, logs, conn).await; Ok(nsjail_mounts) } diff --git a/backend/windmill-worker/src/bigquery_executor.rs b/backend/windmill-worker/src/bigquery_executor.rs index 9cc9fd6c90..7a6c0ff92d 100644 --- a/backend/windmill-worker/src/bigquery_executor.rs +++ b/backend/windmill-worker/src/bigquery_executor.rs @@ -1,20 +1,24 @@ use std::collections::HashMap; use futures::future::BoxFuture; -use futures::{FutureExt, TryFutureExt}; +use futures::{FutureExt, StreamExt}; use reqwest::Client; use serde_json::{json, value::RawValue, Value}; use windmill_common::error::to_anyhow; +use windmill_common::s3_helpers::convert_json_line_stream; use windmill_common::worker::Connection; use windmill_common::{error::Error, worker::to_raw_value}; use windmill_parser_sql::{ - parse_bigquery_sig, parse_db_resource, parse_sql_blocks, parse_sql_statement_named_params, + parse_bigquery_sig, parse_db_resource, parse_s3_mode, parse_sql_blocks, + parse_sql_statement_named_params, }; use windmill_queue::CanceledBy; use serde::Deserialize; -use crate::common::{build_http_client, OccupancyMetrics}; +use crate::common::{ + build_http_client, s3_mode_args_to_worker_data, OccupancyMetrics, S3ModeWorkerData, +}; use crate::handle_child::run_future_with_polling_update_job_poller; use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args; use crate::{ @@ -31,6 +35,16 @@ struct BigqueryResponse { totalRows: Option, schema: Option, jobComplete: bool, + pageToken: Option, + jobReference: Option, +} + +#[allow(non_snake_case)] +#[derive(Deserialize, Clone)] +struct BigQueryResponseJobReference { + jobId: String, + projectId: String, + location: Option, } #[derive(Deserialize)] @@ -74,6 +88,7 @@ fn do_bigquery_inner<'a>( column_order: Option<&'a mut Option>>, skip_collect: bool, http_client: &'a Client, + s3: Option, ) -> windmill_common::error::Result>>> { let param_names = parse_sql_statement_named_params(query, '@'); @@ -120,69 +135,80 @@ fn do_bigquery_inner<'a>( e.to_string() )) })?; + let rows = handle_bigquery_response(&result, &s3, column_order).await?; - if !result.jobComplete { - return Err(Error::ExecutionErr( - "BigQuery API did not answer query in time".to_string(), - )); + if let Some(s3) = s3 { + let cloned_s3 = s3.clone(); + let cloned_http_client = http_client.clone(); + let cloned_token = token.to_string(); + let rows_stream = async_stream::stream! { + for row in rows.iter() { + yield Ok::<_, windmill_common::error::Error>(row.clone()); + } + let mut next_page_token = result.pageToken; + let Some(job_reference) = result.jobReference.clone() else { + return; + }; + while let Some(ref next_page_token_value) = next_page_token { + let response2 = cloned_http_client + .get( + format!("https://bigquery.googleapis.com/bigquery/v2/projects/{}/queries/{}", job_reference.projectId, job_reference.jobId), + ) + .bearer_auth(cloned_token.as_str()) + .query(&[ + ("pageToken", next_page_token_value.as_str()), + ("maxResults", "10000"), + ("timeoutMs", timeout_ms.to_string().as_str()), + ("location", job_reference.location.as_ref().unwrap_or(&"US".to_string()).as_str()), + ]) + .send() + .await + .map_err(|e| { + Error::ExecutionErr(format!("Could not send query to BigQuery API: {}", e)) + })?; + + if let Err(e) = response2.error_for_status_ref() { + match response2.json::().await { + Ok(bq_err) => { + yield Err(Error::ExecutionErr(format!( + "Error from BigQuery API: {}", + bq_err.error.message + ))) + .map_err(to_anyhow)?; + return; + }, + Err(_) => { + yield Err(Error::ExecutionErr(format!( + "Error from BigQuery API could not be parsed: {}", + e.to_string() + ))) + .map_err(to_anyhow)?; + return; + }, + } + } + + let result2 = response2.json::().await.map_err(|e| { + Error::ExecutionErr(format!( + "BigQuery API response could not be parsed: {}", + e.to_string() + )) + })?; + let rows = handle_bigquery_response(&result2, &Some(cloned_s3.clone()), None).await?; + for row in rows.into_iter() { + yield Ok::<_, windmill_common::error::Error>(row); + } + next_page_token = result2.pageToken; + } + }; + + let stream = + convert_json_line_stream(rows_stream.boxed(), s3.format).await?; + s3.upload(stream.boxed()).await?; + + return Ok(to_raw_value(&s3.object_key)); } - if result.rows.is_none() || result.rows.as_ref().unwrap().len() == 0 { - return Ok(serde_json::from_str("[]").unwrap()); - } - - if result.schema.is_none() { - return Err(Error::ExecutionErr( - "Incomplete response from BigQuery API".to_string(), - )); - } - - if result - .totalRows - .unwrap_or(json!("")) - .as_str() - .unwrap_or("") - .parse::() - .unwrap_or(0) - > 10000 - { - return Err(Error::ExecutionErr( - "More than 10000 rows were requested, use LIMIT 10000 to limit the number of rows".to_string(), - )); - } - - if let Some(column_order) = column_order { - *column_order = Some( - result - .schema - .as_ref() - .unwrap() - .fields - .iter() - .map(|x| x.name.clone()) - .collect::>(), - ); - } - - let rows = result - .rows - .unwrap() - .iter() - .map(|row| { - let mut row_map = serde_json::Map::new(); - row.f - .iter() - .zip(result.schema.as_ref().unwrap().fields.iter()) - .for_each(|(field, schema)| { - row_map.insert( - schema.name.clone(), - parse_val(&field.v, &schema.r#type, &schema), - ); - }); - Value::from(row_map) - }) - .collect::>(); - Ok(to_raw_value(&rows)) } } @@ -204,6 +230,79 @@ fn do_bigquery_inner<'a>( Ok(result_f.boxed()) } +async fn handle_bigquery_response<'a>( + result: &BigqueryResponse, + s3: &Option, + column_order: Option<&'a mut Option>>, +) -> windmill_common::error::Result> { + if !result.jobComplete { + return Err(Error::ExecutionErr( + "BigQuery API did not answer query in time".to_string(), + )); + } + + if result.rows.is_none() || result.rows.as_ref().unwrap().len() == 0 { + return Ok(serde_json::from_str("[]").unwrap()); + } + + if result.schema.is_none() { + return Err(Error::ExecutionErr( + "Incomplete response from BigQuery API".to_string(), + )); + } + + if s3.is_none() + && result + .totalRows + .as_ref() + .unwrap_or(&json!("")) + .as_str() + .unwrap_or("") + .parse::() + .unwrap_or(0) + > 10000 + { + return Err(Error::ExecutionErr( + "More than 10000 rows were requested, use LIMIT 10000 to limit the number of rows" + .to_string(), + )); + } + + if let Some(column_order) = column_order { + *column_order = Some( + result + .schema + .as_ref() + .unwrap() + .fields + .iter() + .map(|x| x.name.clone()) + .collect::>(), + ); + } + + let rows = result + .rows + .as_ref() + .unwrap() + .iter() + .map(|row| { + let mut row_map = serde_json::Map::new(); + row.f + .iter() + .zip(result.schema.as_ref().unwrap().fields.iter()) + .for_each(|(field, schema)| { + row_map.insert( + schema.name.clone(), + parse_val(&field.v, &schema.r#type, &schema), + ); + }); + Value::from(row_map) + }) + .collect::>(); + Ok(rows) +} + use windmill_queue::MiniPulledJob; pub async fn do_bigquery( @@ -220,6 +319,7 @@ pub async fn do_bigquery( let bigquery_args = build_args_values(job, client, conn).await?; let inline_db_res_path = parse_db_resource(&query); + let s3 = parse_s3_mode(&query)?.map(|s3| s3_mode_args_to_worker_data(s3, client.clone(), job)); let db_arg = if let Some(inline_db_res_path) = inline_db_res_path { Some( @@ -332,6 +432,7 @@ pub async fn do_bigquery( None, annotations.return_last_result && i < queries.len() - 1, &http_client, + s3.clone(), ) }) .collect::>>()?; @@ -361,6 +462,7 @@ pub async fn do_bigquery( Some(column_order), false, &http_client, + s3, )? }; @@ -370,7 +472,7 @@ pub async fn do_bigquery( conn, mem_peak, canceled_by, - result_f.map_err(to_anyhow), + result_f, worker_name, &job.workspace_id, &mut Some(occupancy_metrics), diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 91284c9d41..6a6ad120e7 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -42,7 +42,7 @@ use windmill_common::{ error::{self, Result}, get_latest_hash_for_path, scripts::ScriptLang, - worker::{exists_in_cache, save_cache, write_file, Connection, DISABLE_BUNDLING}, + worker::{exists_in_cache, save_cache, to_raw_value, write_file, Connection, DISABLE_BUNDLING}, DB, }; @@ -111,7 +111,7 @@ pub async fn gen_bun_lockfile( let mut empty_deps = false; - if let Some(raw_deps) = raw_deps { + if let Some(raw_deps) = raw_deps.as_ref() { gen_bunfig(job_dir).await?; write_file(job_dir, "package.json", raw_deps.as_str())?; } else { @@ -201,10 +201,21 @@ pub async fn gen_bun_lockfile( } if export_pkg { - let mut content = "".to_string(); + let mut content; { let mut file = File::open(format!("{job_dir}/package.json")).await?; - file.read_to_string(&mut content).await?; + let mut buf = String::default(); + file.read_to_string(&mut buf).await?; + if raw_deps.is_some() { + let mut json_map: HashMap> = serde_json::from_str(&buf)?; + json_map.insert( + "generatedFromPackageJson".to_string(), + to_raw_value(&"true".to_string()), + ); + content = serde_json::to_string_pretty(&json_map)?; + } else { + content = buf; + } } if !npm_mode { #[cfg(any(target_os = "linux", target_os = "macos"))] diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 2699b89d02..086cee7e39 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -31,6 +31,7 @@ use windmill_common::{ }; use anyhow::{anyhow, bail, Result}; +use windmill_parser_sql::{s3_mode_extension, S3ModeArgs, S3ModeFormat}; use windmill_queue::MiniPulledJob; use std::ops::AsyncFn; @@ -1579,3 +1580,55 @@ pub async fn par_install_language_dependencies<'a>( } Ok(()) } + +#[derive(Clone)] +pub struct S3ModeWorkerData { + pub client: AuthedClient, + pub object_key: String, + pub format: S3ModeFormat, + pub storage: Option, + pub workspace_id: String, +} + +impl S3ModeWorkerData { + pub async fn upload(&self, stream: S) -> error::Result + where + S: futures::stream::TryStream + Send + 'static, + S::Error: Into>, + bytes::Bytes: From, + { + self.client + .upload_s3_file( + self.workspace_id.as_str(), + self.object_key.clone(), + self.storage.clone(), + stream, + ) + .await + } +} + +pub fn s3_mode_args_to_worker_data( + s3: S3ModeArgs, + client: AuthedClient, + job: &MiniPulledJob, +) -> S3ModeWorkerData { + S3ModeWorkerData { + client, + storage: s3.storage, + format: s3.format, + object_key: format!( + "{}/{}.{}", + s3.prefix.unwrap_or_else(|| format!( + "wmill_datalake/{}", + job.runnable_path + .as_ref() + .map(|s| s.as_str()) + .unwrap_or("unknown_script") + )), + job.id, + s3_mode_extension(s3.format) + ), + workspace_id: job.workspace_id.clone(), + } +} diff --git a/backend/windmill-worker/src/graphql_executor.rs b/backend/windmill-worker/src/graphql_executor.rs index 5da8b2d193..1fbc547c30 100644 --- a/backend/windmill-worker/src/graphql_executor.rs +++ b/backend/windmill-worker/src/graphql_executor.rs @@ -1,6 +1,5 @@ use std::collections::HashMap; -use anyhow::anyhow; use futures::{stream, TryStreamExt}; use serde_json::{json, value::RawValue}; use sqlx::types::Json; @@ -134,11 +133,13 @@ pub async fn do_graphql( .map_err(|e| Error::ExecutionErr(e.to_string()))?; if let Some(errors) = result.errors { - return Err(anyhow!(errors - .into_iter() - .map(|x| x.message) - .collect::>() - .join("\n"),)); + return Err(Error::ExecutionErr( + errors + .into_iter() + .map(|x| x.message) + .collect::>() + .join("\n"), + )); } // And then check that we got back the same string we sent over. diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 713046207f..6cf4272a58 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -526,7 +526,7 @@ pub async fn run_future_with_polling_update_job_poller( get_mem: S, ) -> error::Result where - Fut: Future>, + Fut: Future>, S: stream::Stream + Unpin, { let (tx, rx) = broadcast::channel::<()>(3); diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index 33a9b57690..5c9af6760b 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -789,7 +789,7 @@ pub async fn eval_fetch_timeout( w_id: &str, load_client: bool, occupation_metrics: &mut OccupancyMetrics, -) -> anyhow::Result> { +) -> windmill_common::error::Result> { use windmill_queue::append_logs; let (sender, mut receiver) = oneshot::channel::(); @@ -933,7 +933,7 @@ pub async fn eval_fetch_timeout( let r = runtime.block_on(future)?; // tracing::info!("total: {:?}", instant.elapsed()); - r + r as windmill_common::error::Result> }); let res = run_future_with_polling_update_job_poller( @@ -942,7 +942,7 @@ pub async fn eval_fetch_timeout( conn, mem_peak, canceled_by, - async { result_f.await? }, + async { result_f.await.map_err(windmill_common::error::to_anyhow)? }, worker_name, w_id, &mut Some(occupation_metrics), @@ -1004,22 +1004,26 @@ async fn eval_fetch( script_entrypoint_override: Option, load_client: bool, job_id: &Uuid, -) -> anyhow::Result> { +) -> windmill_common::error::Result> { if load_client { if let Some(env_code) = env_code.as_ref() { let _ = js_runtime .load_side_es_module_from_code( - &deno_core::resolve_url("file:///windmill.ts")?, + &deno_core::resolve_url("file:///windmill.ts").map_err(error::to_anyhow)?, format!("{env_code}\n{}", WINDMILL_CLIENT.to_string()), ) - .await?; + .await + .map_err(error::to_anyhow)?; } } use anyhow::Context; + use deno_core::error::CoreError; + use windmill_common::{error, worker::to_raw_value}; + let source = format!("{}\n{expr}", env_code.unwrap_or_default()); let _ = js_runtime .load_side_es_module_from_code( - &deno_core::resolve_url("file:///eval.ts")?, - format!("{}\n{expr}", env_code.unwrap_or_default()), + &deno_core::resolve_url("file:///eval.ts").map_err(error::to_anyhow)?, + source.to_string(), ) .await .map_err(|e| { @@ -1052,15 +1056,50 @@ import("file:///eval.ts").then((module) => module.{main_override}(...args)).then .map_err(|e| { write_error_expr(expr, &job_id); e - }) - .context("native script event loop")?; + }); - let scope = &mut js_runtime.handle_scope(); - let local = v8::Local::new(scope, global); - // Deserialize a `v8` object into a Rust type using `serde_v8`, - // in this case deserialize to a JSON `Value`. - let r = serde_v8::from_v8::>(scope, local)?; - Ok(unsafe_raw(r.unwrap_or_else(|| "null".to_string()))) + match global { + Ok(global) => { + let scope = &mut js_runtime.handle_scope(); + let local = v8::Local::new(scope, global); + // Deserialize a `v8` object into a Rust type using `serde_v8`, + // in this case deserialize to a JSON `Value`. + let r = serde_v8::from_v8::>(scope, local).map_err(error::to_anyhow)?; + Ok(unsafe_raw(r.unwrap_or_else(|| "null".to_string()))) + } + Err(CoreError::Js(e)) => { + let stack_head = e.frames.first().and_then(|f| { + if f.file_name.as_ref().is_some_and(|x| x == "file:///eval.ts") { + Some(format!( + "{}\n", + source + .lines() + .nth((f.line_number.unwrap_or(1)) as usize - 1) + .unwrap_or("") + .to_string() + )) + } else { + None + } + }); + let stack_s = format!( + "{}{}", + stack_head.unwrap_or("".to_string()), + e.stack.unwrap_or("".to_string()) + ); + let stack = if stack_s.is_empty() { + None + } else { + Some(stack_s) + }; + Err(Error::ExecutionRawError(to_raw_value(&serde_json::json!({ + "message": e.message, + "stack": stack, + "name": e.name, + })))) + } + Err(e) => Err(Error::ExecutionErr(e.print_with_cause())), + } } #[cfg(feature = "deno_core")] diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 2b0a5d6f4a..64a9cf2c88 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -49,6 +49,8 @@ mod worker_flow; mod worker_lockfiles; mod worker_utils; +pub use worker_lockfiles::process_relative_imports; + pub use worker::*; pub use result_processor::handle_job_error; diff --git a/backend/windmill-worker/src/mssql_executor.rs b/backend/windmill-worker/src/mssql_executor.rs index d827e57095..4c727b786e 100644 --- a/backend/windmill-worker/src/mssql_executor.rs +++ b/backend/windmill-worker/src/mssql_executor.rs @@ -1,5 +1,6 @@ use base64::{engine::general_purpose, Engine as _}; use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc}; +use futures::StreamExt; use regex::Regex; use serde::Deserialize; use serde_json::value::RawValue; @@ -8,16 +9,17 @@ use tiberius::{AuthMethod, Client, ColumnData, Config, FromSqlOwned, Query, Row, use tokio::net::TcpStream; use tokio_util::compat::TokioAsyncWriteCompatExt; use uuid::Uuid; +use windmill_common::s3_helpers::convert_json_line_stream; use windmill_common::{ error::{self, to_anyhow, Error}, - utils::empty_string_as_none, + utils::empty_as_none, worker::{to_raw_value, Connection}, }; -use windmill_parser_sql::{parse_db_resource, parse_mssql_sig}; +use windmill_parser_sql::{parse_db_resource, parse_mssql_sig, parse_s3_mode}; use windmill_queue::MiniPulledJob; use windmill_queue::{append_logs, CanceledBy}; -use crate::common::{build_args_values, OccupancyMetrics}; +use crate::common::{build_args_values, s3_mode_args_to_worker_data, OccupancyMetrics}; use crate::handle_child::run_future_with_polling_update_job_poller; use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args; use crate::AuthedClient; @@ -35,13 +37,13 @@ struct MssqlDatabase { #[serde(default, deserialize_with = "deserialize_aad_token")] aad_token: Option, trust_cert: Option, - #[serde(default, deserialize_with = "empty_string_as_none")] + #[serde(default, deserialize_with = "empty_as_none")] ca_cert: Option, } #[derive(Debug, Deserialize)] struct AadToken { - #[serde(default, deserialize_with = "empty_string_as_none")] + #[serde(default, deserialize_with = "empty_as_none")] token: Option, } @@ -63,6 +65,7 @@ pub async fn do_mssql( let mssql_args = build_args_values(job, client, conn).await?; let inline_db_res_path = parse_db_resource(&query); + let s3 = parse_s3_mode(&query)?.map(|s3| s3_mode_args_to_worker_data(s3, client.clone(), job)); let db_arg = if let Some(inline_db_res_path) = inline_db_res_path { Some( @@ -197,27 +200,42 @@ pub async fn do_mssql( // A response to a query is a stream of data, that must be // polled to the end before querying again. Using streams allows // fetching data in an asynchronous manner, if needed. - let stream = prepared_query.query(&mut client).await.map_err(to_anyhow)?; - let results = stream.into_results().await.map_err(to_anyhow)?; - let len = results.len(); - let mut json_results = vec![]; - for (i, statement_result) in results.into_iter().enumerate() { - if annotations.return_last_result && i < len - 1 { - continue; - } - let mut json_rows = vec![]; - for row in statement_result { - let row = row_to_json(row)?; - json_rows.push(row); - } - json_results.push(json_rows); - } + if let Some(s3) = s3 { + let rows_stream = async_stream::stream! { + let mut stream = prepared_query.query(&mut client).await.map_err(to_anyhow)?.into_row_stream().map(|row| { + row_to_json(row.map_err(to_anyhow)?).map_err(to_anyhow) + }); + while let Some(row) = stream.next().await { + yield row; + } + }; - if annotations.return_last_result && json_results.len() > 0 { - Ok(to_raw_value(&json_results.pop().unwrap())) + let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?; + s3.upload(stream.boxed()).await?; + + Ok(serde_json::value::to_raw_value(&s3.object_key)?) } else { - Ok(to_raw_value(&json_results)) + let stream = prepared_query.query(&mut client).await.map_err(to_anyhow)?; + let results = stream.into_results().await.map_err(to_anyhow)?; + let len = results.len(); + let mut json_results = vec![]; + for (i, statement_result) in results.into_iter().enumerate() { + if annotations.return_last_result && i < len - 1 { + continue; + } + let mut json_rows = vec![]; + for row in statement_result { + let row = row_to_json(row)?; + json_rows.push(row); + } + json_results.push(json_rows); + } + if annotations.return_last_result && json_results.len() > 0 { + Ok(to_raw_value(&json_results.pop().unwrap())) + } else { + Ok(to_raw_value(&json_results)) + } } }; diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index a1309395a7..3bfe3afb49 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -1,27 +1,31 @@ use std::{collections::HashMap, sync::Arc}; +use anyhow::anyhow; use base64::Engine; -use futures::{future::BoxFuture, FutureExt}; +use futures::{future::BoxFuture, FutureExt, StreamExt}; use itertools::Itertools; use mysql_async::{ consts::ColumnType, prelude::*, FromValueError, OptsBuilder, Params, Row, SslOpts, }; +use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; use serde_json::{json, value::RawValue, Value}; +use std::str::FromStr; use tokio::sync::Mutex; use windmill_common::{ error::{to_anyhow, Error}, + s3_helpers::convert_json_line_stream, worker::{to_raw_value, Connection}, }; use windmill_parser_sql::{ - parse_db_resource, parse_mysql_sig, parse_sql_blocks, parse_sql_statement_named_params, - RE_ARG_MYSQL_NAMED, + parse_db_resource, parse_mysql_sig, parse_s3_mode, parse_sql_blocks, + parse_sql_statement_named_params, RE_ARG_MYSQL_NAMED, }; use windmill_queue::CanceledBy; use windmill_queue::MiniPulledJob; use crate::{ - common::{build_args_values, OccupancyMetrics}, + common::{build_args_values, s3_mode_args_to_worker_data, OccupancyMetrics, S3ModeWorkerData}, handle_child::run_future_with_polling_update_job_poller, sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args, AuthedClient, @@ -37,13 +41,14 @@ struct MysqlDatabase { ssl: Option, } -pub fn do_mysql_inner<'a>( +fn do_mysql_inner<'a>( query: &'a str, all_statement_values: &Params, conn: Arc>, column_order: Option<&'a mut Option>>, skip_collect: bool, -) -> windmill_common::error::Result>>> { + s3: Option, +) -> windmill_common::error::Result>>> { let param_names = parse_sql_statement_named_params(query, ':') .into_iter() .map(|x| x.into_bytes()) @@ -69,6 +74,38 @@ pub fn do_mysql_inner<'a>( .map_err(to_anyhow)?; Ok(to_raw_value(&Value::Array(vec![]))) + } else if let Some(ref s3) = s3 { + let query = query.to_string(); + let rows_stream = async_stream::stream! { + let mut conn = conn.lock().await; + let mut result = match conn.exec_iter(query, statement_values).await.map_err(to_anyhow) { + Ok(result) => result, + Err(e) => { + yield Err(anyhow!("Error executing query: {:?}", e)); + return; + } + }; + loop { + let row = result.next().await; + match row { + Ok(Some(row)) => { + yield Ok(convert_row_to_value(row)); + } + Ok(None) => { + break; + } + Err(e) => { + yield Err(anyhow!("Error fetching row: {:?}", e)); + return; + } + } + } + }; + + let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?; + s3.upload(stream.boxed()).await?; + + Ok(serde_json::value::to_raw_value(&s3.object_key)?) } else { let rows: Vec = conn .lock() @@ -116,6 +153,7 @@ pub async fn do_mysql( let job_args = build_args_values(job, client, conn).await?; let inline_db_res_path = parse_db_resource(&query); + let s3 = parse_s3_mode(&query)?.map(|s3| s3_mode_args_to_worker_data(s3, client.clone(), job)); let db_arg = if let Some(inline_db_res_path) = inline_db_res_path { Some( @@ -250,6 +288,7 @@ pub async fn do_mysql( conn_a.clone(), None, annotations.return_last_result && i < queries.len() - 1, + s3.clone(), ) }) .collect::>>()?; @@ -275,6 +314,7 @@ pub async fn do_mysql( conn_a.clone(), Some(column_order), false, + s3, )? }; @@ -303,26 +343,38 @@ pub async fn do_mysql( return Ok(raw_result); } +// 2023-12-01T16:18:00.000Z +static DATE_REGEX_TZ: Lazy = Lazy::new(|| { + regex::Regex::new(r"(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})\.(\d+)Z").unwrap() +}); +// 2025-04-21 10:08:00 +static DATE_REGEX: Lazy = + Lazy::new(|| regex::Regex::new(r"(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})").unwrap()); + fn string_date_to_mysql_date(s: &str) -> mysql_async::Value { - // 2023-12-01T16:18:00.000Z - let re = regex::Regex::new(r"(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})\.(\d+)Z").unwrap(); - let caps = re.captures(s); + let caps = DATE_REGEX_TZ.captures(s).or_else(|| DATE_REGEX.captures(s)); if let Some(caps) = caps { mysql_async::Value::Date( - caps.get(1).unwrap().as_str().parse().unwrap_or_default(), - caps.get(2).unwrap().as_str().parse().unwrap_or_default(), - caps.get(3).unwrap().as_str().parse().unwrap_or_default(), - caps.get(4).unwrap().as_str().parse().unwrap_or_default(), - caps.get(5).unwrap().as_str().parse().unwrap_or_default(), - caps.get(6).unwrap().as_str().parse().unwrap_or_default(), - caps.get(7).unwrap().as_str().parse().unwrap_or_default(), + get_capture_by_index(&caps, 1), + get_capture_by_index(&caps, 2), + get_capture_by_index(&caps, 3), + get_capture_by_index(&caps, 4), + get_capture_by_index(&caps, 5), + get_capture_by_index(&caps, 6), + get_capture_by_index(&caps, 7), ) } else { mysql_async::Value::Date(0, 0, 0, 0, 0, 0, 0) } } +fn get_capture_by_index(caps: ®ex::Captures, n: usize) -> T { + caps.get(n) + .and_then(|s| s.as_str().parse::().ok()) + .unwrap_or_default() +} + fn convert_row_to_value(row: Row) -> serde_json::Value { let mut map = serde_json::Map::new(); diff --git a/backend/windmill-worker/src/oracledb_executor.rs b/backend/windmill-worker/src/oracledb_executor.rs index 7c63e02c81..50ea9a2ec3 100644 --- a/backend/windmill-worker/src/oracledb_executor.rs +++ b/backend/windmill-worker/src/oracledb_executor.rs @@ -43,7 +43,7 @@ pub fn do_oracledb_inner<'a>( conn: Arc>, column_order: Option<&'a mut Option>>, skip_collect: bool, -) -> windmill_common::error::Result>>> { +) -> windmill_common::error::Result>>> { let qw = query.trim_end_matches(';').to_string(); let result_f = async move { diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 79002e4d1f..e12ca71626 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -8,7 +8,7 @@ use anyhow::Context; use base64::{engine, Engine as _}; use chrono::Utc; use futures::future::BoxFuture; -use futures::{FutureExt, TryStreamExt}; +use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt}; use itertools::Itertools; use native_tls::{Certificate, TlsConnector}; use postgres_native_tls::MakeTlsConnector; @@ -27,14 +27,18 @@ use tokio_postgres::{ use uuid::Uuid; use windmill_common::error::to_anyhow; use windmill_common::error::{self, Error}; +use windmill_common::s3_helpers::convert_json_line_stream; use windmill_common::worker::{to_raw_value, Connection, CLOUD_HOSTED}; use windmill_parser::{Arg, Typ}; use windmill_parser_sql::{ - parse_db_resource, parse_pg_statement_arg_indices, parse_pgsql_sig, parse_sql_blocks, + parse_db_resource, parse_pg_statement_arg_indices, parse_pgsql_sig, parse_s3_mode, + parse_sql_blocks, }; use windmill_queue::{CanceledBy, MiniPulledJob}; -use crate::common::{build_args_values, sizeof_val, OccupancyMetrics}; +use crate::common::{ + build_args_values, s3_mode_args_to_worker_data, sizeof_val, OccupancyMetrics, S3ModeWorkerData, +}; use crate::handle_child::run_future_with_polling_update_job_poller; use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args; use crate::{AuthedClient, MAX_RESULT_SIZE}; @@ -68,7 +72,8 @@ fn do_postgresql_inner<'a>( column_order: Option<&'a mut Option>>, siz: &'a AtomicUsize, skip_collect: bool, -) -> error::Result>>> { + s3: Option, +) -> error::Result>>> { let mut query_params = vec![]; let arg_indices = parse_pg_statement_arg_indices(&query); @@ -106,6 +111,20 @@ fn do_postgresql_inner<'a>( .execute_raw(&query, query_params) .await .map_err(to_anyhow)?; + } else if let Some(ref s3) = s3 { + let rows_stream = client + .query_raw(&query, query_params) + .map_err(to_anyhow) + .await? + .map_err(to_anyhow) + .map(|row_result| { + row_result.and_then(|row| postgres_row_to_json_value(row).map_err(to_anyhow)) + }); + + let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?; + s3.upload(stream.boxed()).await?; + + return Ok(serde_json::value::to_raw_value(&s3.object_key)?); } else { let rows = client .query_raw(&query, query_params) @@ -136,17 +155,17 @@ fn do_postgresql_inner<'a>( if *CLOUD_HOSTED { let siz = siz.load(Ordering::Relaxed); if siz > MAX_RESULT_SIZE * 4 { - return Err(anyhow::anyhow!( + return Err(Error::ExecutionErr(format!( "Query result too large for cloud (size = {} > {})", siz, - MAX_RESULT_SIZE & 4 - )); + MAX_RESULT_SIZE & 4, + ))); } } if let Ok(v) = r { res.push(v); } else { - return Err(to_anyhow(r.err().unwrap())); + return Err(to_anyhow(r.err().unwrap()).into()); } } } @@ -172,6 +191,8 @@ pub async fn do_postgresql( let inline_db_res_path = parse_db_resource(&query); + let s3 = parse_s3_mode(&query)?.map(|s3| s3_mode_args_to_worker_data(s3, client.clone(), job)); + let db_arg = if let Some(inline_db_res_path) = inline_db_res_path { Some( client @@ -321,6 +342,7 @@ pub async fn do_postgresql( None, &size, annotations.return_last_result && i < queries.len() - 1, + s3.clone(), ) }) .collect::>>()?; @@ -347,6 +369,7 @@ pub async fn do_postgresql( Some(column_order), &size, false, + s3, )? }; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index c642abae49..a776bbe7c7 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -77,6 +77,7 @@ use crate::{ start_child_process, OccupancyMetrics, }, handle_child::handle_child, + worker_lockfiles::LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT, worker_utils::ping_job_status, AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, INSTANCE_PYTHON_VERSION, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR, @@ -2303,8 +2304,16 @@ fn split_requirements(requirements: &str) -> Vec<&str> { /// Check requirements/lockfile to figure out python version assigned to it. fn get_pyv_from_requirements_lines(requirements_lines: &[&str]) -> PyVersion { // If script is deployed we can try to parse first line to get assigned version + + let index = if requirements_lines.get(0).map_or(false, |line| { + line.starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT) + }) { + 1 + } else { + 0 + }; if let Some(v) = requirements_lines - .get(0) + .get(index) .and_then(|line| PyVersion::parse_version(*line)) { // We have valid assigned version, we use it diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index e1b2a763f6..0bde6fcdc6 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -373,6 +373,7 @@ pub async fn process_result( } } } + Error::ExecutionRawError(e) => to_raw_value(&e), err @ _ => to_raw_value(&SerializedError { message: format!("execution error:\n{err:#}",), name: "ExecutionErr".to_string(), diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index a4aba420c0..84d2d0776d 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -2,22 +2,28 @@ use base64::{engine, Engine as _}; use chrono::Datelike; use core::fmt::Write; use futures::future::BoxFuture; -use futures::{FutureExt, TryFutureExt}; +use futures::{FutureExt, StreamExt, TryStreamExt}; use jsonwebtoken::{encode, Algorithm, EncodingKey, Header}; use reqwest::{Client, Response}; use serde_json::{json, value::RawValue, Value}; use sha2::{Digest, Sha256}; use std::collections::HashMap; use windmill_common::error::to_anyhow; +use windmill_common::s3_helpers::convert_json_line_stream; use windmill_common::worker::Connection; use windmill_common::{error::Error, worker::to_raw_value}; -use windmill_parser_sql::{parse_db_resource, parse_snowflake_sig, parse_sql_blocks}; +use windmill_parser_sql::{ + parse_db_resource, parse_s3_mode, parse_snowflake_sig, parse_sql_blocks, +}; use windmill_queue::{CanceledBy, MiniPulledJob, HTTP_CLIENT}; use serde::{Deserialize, Serialize}; -use crate::common::{build_http_client, resolve_job_timeout, OccupancyMetrics}; +use crate::common::{ + build_http_client, resolve_job_timeout, s3_mode_args_to_worker_data, OccupancyMetrics, + S3ModeWorkerData, +}; use crate::handle_child::run_future_with_polling_update_job_poller; use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args; use crate::{common::build_args_values, AuthedClient}; @@ -124,6 +130,7 @@ fn do_snowflake_inner<'a>( column_order: Option<&'a mut Option>>, skip_collect: bool, http_client: &'a Client, + s3: Option, ) -> windmill_common::error::Result>>> { let sig = parse_snowflake_sig(&query) .map_err(|x| Error::ExecutionErr(x.to_string()))? @@ -175,7 +182,7 @@ fn do_snowflake_inner<'a>( .parse_snowflake_response::() .await?; - if response.resultSetMetaData.numRows > 10000 { + if s3.is_none() && response.resultSetMetaData.numRows > 10000 { return Err(Error::ExecutionErr( "More than 10000 rows were requested, use LIMIT 10000 to limit the number of rows" .to_string(), @@ -192,54 +199,72 @@ fn do_snowflake_inner<'a>( ); } - let mut rows = response.data; + // Clones are because, in s3 mode, reqwest::Body::wrap_stream requires the stream to be + // 'static even though it doesn't make sense to be in our case since the request is + // awaited and the stream is fully read before the function returns. + // Turns out it is a real pain to trick the compiler, even using unsafe + let cloned_account_identifier: String = account_identifier.to_string(); + let cloned_token = token.to_string(); - if response.resultSetMetaData.partitionInfo.len() > 1 { - for idx in 1..response.resultSetMetaData.partitionInfo.len() { - let url = format!( - "https://{}.snowflakecomputing.com/api/v2/statements/{}", - account_identifier.to_uppercase(), - response.statementHandle - ); - let mut request = HTTP_CLIENT - .get(url) - .bearer_auth(token) - .query(&[("partition", idx.to_string())]); - - if token_is_keypair { - request = - request.header("X-Snowflake-Authorization-Token-Type", "KEYPAIR_JWT"); - } - - let response = request - .send() - .await - .parse_snowflake_response::() - .await?; - - rows.extend(response.data); + let rows_stream = async_stream::stream! { + for row in response.data { + yield Ok::, windmill_common::error::Error>(row); } + + if response.resultSetMetaData.partitionInfo.len() > 1 { + for idx in 1..response.resultSetMetaData.partitionInfo.len() { + let url = format!( + "https://{}.snowflakecomputing.com/api/v2/statements/{}", + cloned_account_identifier.to_uppercase(), + response.statementHandle + ); + let mut request = HTTP_CLIENT + .get(url) + .bearer_auth(cloned_token.as_str()) + .query(&[("partition", idx.to_string())]); + + if token_is_keypair { + request = + request.header("X-Snowflake-Authorization-Token-Type", "KEYPAIR_JWT"); + } + + let response = request + .send() + .await + .parse_snowflake_response::() + .await?; + + for row in response.data { + yield Ok(row); + } + } + } + }; + + let rows_stream = rows_stream.map_ok(move |row| { + let mut row_map = serde_json::Map::new(); + row.iter() + .zip(response.resultSetMetaData.rowType.iter()) + .for_each(|(val, row_type)| { + row_map.insert(row_type.name.clone(), parse_val(&val, &row_type.r#type)); + }); + row_map + }); + + if let Some(s3) = s3 { + let rows_stream = + rows_stream.map(|r| serde_json::value::to_value(&r?).map_err(to_anyhow)); + let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?; + s3.upload(stream.boxed()).await?; + Ok(to_raw_value(&s3.object_key)) + } else { + let rows = rows_stream + .collect::>() + .await + .into_iter() + .collect::, _>>()?; + Ok(to_raw_value(&rows)) } - - let rows = to_raw_value( - &rows - .iter() - .map(|row| { - let mut row_map = serde_json::Map::new(); - row.iter() - .zip(response.resultSetMetaData.rowType.iter()) - .for_each(|(val, row_type)| { - row_map.insert( - row_type.name.clone(), - parse_val(&val, &row_type.r#type), - ); - }); - row_map - }) - .collect::>(), - ); - - Ok(rows) } }; @@ -260,6 +285,7 @@ pub async fn do_snowflake( let snowflake_args = build_args_values(job, client, conn).await?; let inline_db_res_path = parse_db_resource(&query); + let s3 = parse_s3_mode(&query)?.map(|s3| s3_mode_args_to_worker_data(s3, client.clone(), job)); let db_arg = if let Some(inline_db_res_path) = inline_db_res_path { Some( @@ -391,6 +417,7 @@ pub async fn do_snowflake( None, annotations.return_last_result && i < queries.len() - 1, &http_client, + s3.clone(), ) }) .collect::>>()?; @@ -420,6 +447,7 @@ pub async fn do_snowflake( Some(column_order), false, &http_client, + s3.clone(), )? }; let r = run_future_with_polling_update_job_poller( @@ -428,7 +456,7 @@ pub async fn do_snowflake( conn, mem_peak, canceled_by, - result_f.map_err(to_anyhow), + result_f, worker_name, &job.workspace_id, &mut Some(occupancy_metrics), diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index fed83e07e0..8590603c2f 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -39,7 +39,7 @@ use windmill_common::METRICS_DEBUG_ENABLED; #[cfg(feature = "prometheus")] use windmill_common::METRICS_ENABLED; -use reqwest::Response; +use reqwest::{Body, Response}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use sqlx::types::Json; use std::{ @@ -306,6 +306,7 @@ lazy_static::lazy_static! { pub static ref NSJAIL_PATH: String = std::env::var("NSJAIL_PATH").unwrap_or_else(|_| "nsjail".to_string()); pub static ref PATH_ENV: String = std::env::var("PATH").unwrap_or_else(|_| String::new()); pub static ref HOME_ENV: String = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()); + pub static ref GIT_PATH: String = std::env::var("GIT_PATH").unwrap_or_else(|_| "/usr/bin/git".to_string()); pub static ref NODE_PATH: Option = std::env::var("NODE_PATH").ok(); @@ -519,6 +520,46 @@ impl AuthedClient { _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), } } + + pub async fn upload_s3_file( + &self, + workspace_id: &str, + object_key: String, + storage: Option, + body: S, + ) -> error::Result + where + S: futures::stream::TryStream + Send + 'static, + S::Error: Into>, + bytes::Bytes: From, + { + let mut query = vec![("file_key", object_key)]; + if let Some(storage) = storage { + query.push(("storage", storage)); + } + self.force_client + .as_ref() + .unwrap_or(&HTTP_CLIENT) + .post(format!( + "{}/api/w/{}/job_helpers/upload_s3_file", + self.base_internal_url, workspace_id + )) + .query(&query) + .header( + reqwest::header::ACCEPT, + reqwest::header::HeaderValue::from_static("application/json"), + ) + .header( + reqwest::header::AUTHORIZATION, + reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token)) + .map_err(|e| error::Error::BadConfig(e.to_string()))?, + ) + .body(Body::wrap_stream(body)) + .send() + .await + .context(format!("Sent upload_s3_file request",)) + .map_err(error::Error::from) + } } #[derive(Clone)] @@ -1268,7 +1309,7 @@ pub async fn run_worker( if job.is_err() && !same_worker_job.recoverable { tracing::error!( worker = %worker_name, hostname = %hostname, - "failed to fetch same_worker job on a non recoverable job, exiting" + "failed to fetch same_worker job on a non recoverable job, exiting: {job:?}", ); job_completed_tx .kill() @@ -1821,26 +1862,14 @@ async fn queue_init_bash_maybe<'c>( same_worker_tx: SameWorkerSender, worker_name: &str, ) -> anyhow::Result { - let uuid_content = match conn { - Connection::Sql(db) => { - if let Some(content) = WORKER_CONFIG.read().await.init_bash.clone() { - Some(( - push_init_job(db, content.clone(), worker_name).await?, - content, - )) - } else { - None - } - } - Connection::Http(client) => { - let init_script = std::env::var("INIT_SCRIPT"); - if init_script.is_ok() { - let content = init_script.unwrap(); - Some((queue_init_job(client, &content).await?, content)) - } else { - None - } - } + let uuid_content = if let Some(content) = WORKER_CONFIG.read().await.init_bash.clone() { + let uuid = match conn { + Connection::Sql(db) => push_init_job(db, content.clone(), worker_name).await?, + Connection::Http(client) => queue_init_job(client, &content).await?, + }; + Some((uuid, content)) + } else { + None }; if let Some((uuid, content)) = uuid_content { same_worker_tx diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 63cf414c53..3a88d50f9f 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -26,7 +26,6 @@ use sqlx::types::Json; use sqlx::{FromRow, Postgres, Transaction}; use tracing::instrument; use uuid::Uuid; -use windmill_common::add_time; use windmill_common::auth::JobPerms; #[cfg(feature = "benchmark")] use windmill_common::bench::BenchmarkIter; @@ -35,15 +34,18 @@ use windmill_common::db::Authed; use windmill_common::flow_status::{ ApprovalConditions, FlowStatusModuleWParent, Iterator as FlowIterator, JobResult, }; -use windmill_common::flows::{add_virtual_items_if_necessary, Branch, FlowNodeId}; +use windmill_common::flows::{add_virtual_items_if_necessary, Branch, FlowNodeId, StopAfterIf}; use windmill_common::jobs::{ - script_hash_to_tag_and_limits, script_path_to_payload, JobKind, JobPayload, OnBehalfOf, - RawCode, ENTRYPOINT_OVERRIDE, + script_path_to_payload, JobKind, JobPayload, OnBehalfOf, RawCode, ENTRYPOINT_OVERRIDE, }; use windmill_common::scripts::ScriptHash; use windmill_common::users::username_to_permissioned_as; use windmill_common::utils::WarnAfterExt; use windmill_common::worker::to_raw_value; +use windmill_common::{ + add_time, get_latest_flow_version_info_for_path, get_script_info_for_hash, FlowVersionInfo, + ScriptHashInfo, +}; use windmill_common::{ error::{self, to_anyhow, Error}, flow_status::{ @@ -85,7 +87,6 @@ pub async fn update_flow_status_after_job_completion( ) -> error::Result>> { // this is manual tailrecursion because async_recursion blows up the stack potentially_crash_for_testing(); - let mut rec = RecUpdateFlowStatusAfterJobCompletion { flow, job_id_for_status: job_id_for_status.clone(), @@ -184,6 +185,30 @@ struct RecoveryObject { recover: Option, } +fn get_stop_after_if_data( + stop_early: bool, + stop_after_if: Option<&StopAfterIf>, +) -> (bool, Option) { + if let Some(stop_after_if) = stop_after_if { + let err_msg = stop_early + .then(|| { + let err_msg = stop_after_if.error_message.as_ref().and_then(|message| { + let err_start_msg = "Flow early stop"; + let s = if message.is_empty() { + format!("{}: {}", err_start_msg, &stop_after_if.expr) + } else { + format!("{}: {}", err_start_msg, message) + }; + Some(s) + }); + err_msg + }) + .flatten(); + return (stop_after_if.skip_if_stopped, err_msg); + } + return (false, None); +} + // #[instrument(level = "trace", skip_all)] pub async fn update_flow_status_after_job_completion_internal( db: &DB, @@ -208,6 +233,7 @@ pub async fn update_flow_status_after_job_completion_internal( flow_job, flow_data, stop_early, + stop_early_err_msg, skip_if_stop_early, nresult, is_failure_step, @@ -216,34 +242,34 @@ pub async fn update_flow_status_after_job_completion_internal( // tracing::debug!("UPDATE FLOW STATUS: {flow:?} {success} {result:?} {w_id} {depth}"); let (job_kind, script_hash, old_status, raw_flow) = sqlx::query!( - "SELECT - kind AS \"job_kind!: JobKind\", - runnable_id AS \"script_hash: ScriptHash\", - flow_status AS \"flow_status!: Json>\", - raw_flow AS \"raw_flow: Json>\" - FROM v2_job INNER JOIN v2_job_status ON v2_job.id = v2_job_status.id WHERE v2_job.id = $1 AND v2_job.workspace_id = $2 LIMIT 1", - flow, - w_id - ) - .fetch_one(db) - .await - .map_err(|e| { - Error::internal_err(format!( - "fetching flow status {flow} while reporting {success} {result:?}: {e:#}" - )) - }) - .and_then(|record| { - Ok(( - record.job_kind, - record.script_hash, - serde_json::from_str::(record.flow_status.0.get()).map_err(|e| { - Error::internal_err(format!( - "requiring current module to be parsable as FlowStatus: {e:?}" - )) - })?, - record.raw_flow, - )) - })?; + "SELECT + kind AS \"job_kind!: JobKind\", + runnable_id AS \"script_hash: ScriptHash\", + flow_status AS \"flow_status!: Json>\", + raw_flow AS \"raw_flow: Json>\" + FROM v2_job INNER JOIN v2_job_status ON v2_job.id = v2_job_status.id WHERE v2_job.id = $1 AND v2_job.workspace_id = $2 LIMIT 1", + flow, + w_id + ) + .fetch_one(db) + .await + .map_err(|e| { + Error::internal_err(format!( + "fetching flow status {flow} while reporting {success} {result:?}: {e:#}" + )) + }) + .and_then(|record| { + Ok(( + record.job_kind, + record.script_hash, + serde_json::from_str::(record.flow_status.0.get()).map_err(|e| { + Error::internal_err(format!( + "requiring current module to be parsable as FlowStatus: {e:?}" + )) + })?, + record.raw_flow, + )) + })?; let flow_data = cache::job::fetch_flow(db, job_kind, script_hash) .or_else(|_| cache::job::fetch_preview_flow(db, &flow, raw_flow)) @@ -299,7 +325,7 @@ pub async fn update_flow_status_after_job_completion_internal( let is_failure_step = old_status.step >= old_status.modules.len() as i32 && old_status.modules.len() > 0; - let (mut stop_early, mut skip_if_stop_early, continue_on_error) = + let (mut stop_early, mut stop_early_err_msg, mut skip_if_stop_early, continue_on_error) = if let Some(se) = stop_early_override { //do not stop early if module is a flow step let step = match module_step { @@ -315,7 +341,6 @@ pub async fn update_flow_status_after_job_completion_internal( } current_module - .as_ref() .map(|module| { serde_json::from_str::(module.value.get()) .map(|v| v.r#type == "flow") @@ -327,19 +352,19 @@ pub async fn update_flow_status_after_job_completion_internal( }; if is_flow { - (false, false, false) + (false, None, false, false) } else { - (true, se, false) + (true, None, se, false) } } else if is_failure_step || matches!(module_step, Step::PreprocessorStep) { - (false, false, false) - } else if let Some(current_module) = current_module.as_ref() { + (false, None, false, false) + } else if let Some(current_module) = current_module { let stop_early = success && !is_branch_all - && if let Some(ref expr) = current_module + && if let Some(expr) = current_module .stop_after_if .as_ref() - .map(|x| x.expr.clone()) + .map(|x| x.expr.as_str()) { let all_iters = match &module_status { @@ -352,9 +377,9 @@ pub async fn update_flow_status_after_job_completion_internal( }; let args = sqlx::query_scalar!( "SELECT - args AS \"args: Json>>\" - FROM v2_job - WHERE id = $1", + args AS \"args: Json>>\" + FROM v2_job + WHERE id = $1", flow ) .fetch_one(db) @@ -376,59 +401,53 @@ pub async fn update_flow_status_after_job_completion_internal( } else { false }; + let (skip_if_stopped, stop_early_err_msg) = + get_stop_after_if_data(stop_early, current_module.stop_after_if.as_ref()); ( stop_early, - current_module - .stop_after_if - .as_ref() - .map(|x| x.skip_if_stopped) - .unwrap_or(false), + stop_early_err_msg.filter(|_| !(is_loop || is_branch_all)), + skip_if_stopped, current_module.continue_on_error.unwrap_or(false), ) } else { - (false, false, false) + (false, None, false, false) }; - let skip_branch_failure = match module_status { + let skip_seq_branch_failure = match module_status { FlowStatusModule::InProgress { branchall: Some(BranchAllStatus { branch, .. }), - parallel, + parallel: false, .. - } => compute_skip_branchall_failure( - job_id_for_status, - *branch, - *parallel, - db, - current_module, - ) - .await? - .unwrap_or(false), + } => { + compute_skip_branchall_failure(branch.to_owned(), false, current_module, &None) + .await? + } _ => false, }; if matches!(module_step, Step::PreprocessorStep) { sqlx::query!( "WITH job_result AS ( - SELECT result - FROM v2_job_completed - WHERE id = $1 - ) - UPDATE v2_job - SET args = COALESCE( - CASE - WHEN job_result.result IS NULL THEN NULL - WHEN jsonb_typeof(job_result.result) = 'object' - THEN job_result.result - WHEN jsonb_typeof(job_result.result) = 'null' - THEN NULL - ELSE jsonb_build_object('value', job_result.result) - END, - '{}'::jsonb - ), - preprocessed = TRUE - FROM job_result - WHERE v2_job.id = $2; - ", + SELECT result + FROM v2_job_completed + WHERE id = $1 + ) + UPDATE v2_job + SET args = COALESCE( + CASE + WHEN job_result.result IS NULL THEN NULL + WHEN jsonb_typeof(job_result.result) = 'object' + THEN job_result.result + WHEN jsonb_typeof(job_result.result) = 'null' + THEN NULL + ELSE jsonb_build_object('value', job_result.result) + END, + '{}'::jsonb + ), + preprocessed = TRUE + FROM job_result + WHERE v2_job.id = $2; + ", job_id_for_status, flow ) @@ -463,46 +482,46 @@ pub async fn update_flow_status_after_job_completion_internal( }; let nindex = if let Some(position) = position { - sqlx::query_scalar!( - "UPDATE v2_job_status SET - flow_status = JSONB_SET( - JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), - ARRAY['modules', $1::TEXT, 'iterator', 'index'], - ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb - ) - WHERE id = $2 - RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", - old_status.step, - flow, - position as i32, - json!(success) - ) - } else { - sqlx::query_scalar!( - "UPDATE v2_job_status SET - flow_status = JSONB_SET( - flow_status, - ARRAY['modules', $1::TEXT, 'iterator', 'index'], - ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb - ) - WHERE id = $2 - RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", - old_status.step, - flow - ) - } - .fetch_one(&mut *tx) - .await.map_err(|e| { - Error::internal_err(format!( - "error while fetching iterator index: {e:#}" - )) - })? - .ok_or_else(|| Error::internal_err(format!("requiring an index in InProgress")))?; + sqlx::query_scalar!( + "UPDATE v2_job_status SET + flow_status = JSONB_SET( + JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), + ARRAY['modules', $1::TEXT, 'iterator', 'index'], + ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb + ) + WHERE id = $2 + RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", + old_status.step, + flow, + position as i32, + json!(success) + ) + } else { + sqlx::query_scalar!( + "UPDATE v2_job_status SET + flow_status = JSONB_SET( + flow_status, + ARRAY['modules', $1::TEXT, 'iterator', 'index'], + ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb + ) + WHERE id = $2 + RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", + old_status.step, + flow + ) + } + .fetch_one(&mut *tx) + .await.map_err(|e| { + Error::internal_err(format!( + "error while fetching iterator index: {e:#}" + )) + })? + .ok_or_else(|| Error::internal_err(format!("requiring an index in InProgress")))?; tracing::info!( - "parallel iteration {job_id_for_status} of flow {flow} update nindex: {nindex} len: {len}", - nindex = nindex, - len = itered.len() - ); + "parallel iteration {job_id_for_status} of flow {flow} update nindex: {nindex} len: {len}", + nindex = nindex, + len = itered.len() + ); (nindex, itered.len() as i32) } (_, Some(BranchAllStatus { len, .. })) => { @@ -513,42 +532,42 @@ pub async fn update_flow_status_after_job_completion_internal( }; let nindex = if let Some(position) = position { - sqlx::query_scalar!( - "UPDATE v2_job_status SET - flow_status = JSONB_SET( - JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), - ARRAY['modules', $1::TEXT, 'branchall', 'branch'], - ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb - ) - WHERE id = $2 - RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", - old_status.step, - flow, - position as i32, - json!(success) - ) - } else { - sqlx::query_scalar!( - "UPDATE v2_job_status SET - flow_status = JSONB_SET( - flow_status, - ARRAY['modules', $1::TEXT, 'branchall', 'branch'], - ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb - ) - WHERE id = $2 - RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", - old_status.step, - flow - ) - } - .fetch_one(&mut *tx) - .await - .map_err(|e| { - Error::internal_err(format!( - "error while fetching branchall index: {e:#}" - )) - })? - .ok_or_else(|| Error::internal_err(format!("requiring an index in InProgress")))?; + sqlx::query_scalar!( + "UPDATE v2_job_status SET + flow_status = JSONB_SET( + JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), + ARRAY['modules', $1::TEXT, 'branchall', 'branch'], + ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb + ) + WHERE id = $2 + RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", + old_status.step, + flow, + position as i32, + json!(success) + ) + } else { + sqlx::query_scalar!( + "UPDATE v2_job_status SET + flow_status = JSONB_SET( + flow_status, + ARRAY['modules', $1::TEXT, 'branchall', 'branch'], + ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb + ) + WHERE id = $2 + RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", + old_status.step, + flow + ) + } + .fetch_one(&mut *tx) + .await + .map_err(|e| { + Error::internal_err(format!( + "error while fetching branchall index: {e:#}" + )) + })? + .ok_or_else(|| Error::internal_err(format!("requiring an index in InProgress")))?; (nindex, *len as i32) } _ => Err(Error::internal_err(format!( @@ -571,50 +590,50 @@ pub async fn update_flow_status_after_job_completion_internal( } let new_status = if skip_loop_failures - || sqlx::query_scalar!( - "SELECT status = 'success' OR status = 'skipped' AS \"success!\" FROM v2_job_completed WHERE id = ANY($1)", - jobs.as_slice() - ) - .fetch_all(&mut *tx) - .await - .map_err(|e| { - Error::internal_err(format!( - "error while fetching sucess from completed_jobs: {e:#}" - )) - })? - .into_iter() - .all(|x| x) - { - success = true; - FlowStatusModule::Success { - id: module_status.id(), - job: job_id_for_status.clone(), - flow_jobs: Some(jobs.clone()), - flow_jobs_success: flow_jobs_success.clone(), - branch_chosen: None, - approvers: vec![], - failed_retries: vec![], - skipped: false, - } - } else { - success = false; - FlowStatusModule::Failure { - id: module_status.id(), - job: job_id_for_status.clone(), - flow_jobs: Some(jobs.clone()), - flow_jobs_success: flow_jobs_success.clone(), - branch_chosen: None, - failed_retries: vec![], - } - }; + || sqlx::query_scalar!( + "SELECT status = 'success' OR status = 'skipped' AS \"success!\" FROM v2_job_completed WHERE id = ANY($1)", + jobs.as_slice() + ) + .fetch_all(&mut *tx) + .await + .map_err(|e| { + Error::internal_err(format!( + "error while fetching sucess from completed_jobs: {e:#}" + )) + })? + .into_iter() + .all(|x| x) + { + success = true; + FlowStatusModule::Success { + id: module_status.id(), + job: job_id_for_status.clone(), + flow_jobs: Some(jobs.clone()), + flow_jobs_success: flow_jobs_success.clone(), + branch_chosen: None, + approvers: vec![], + failed_retries: vec![], + skipped: false, + } + } else { + success = false; + FlowStatusModule::Failure { + id: module_status.id(), + job: job_id_for_status.clone(), + flow_jobs: Some(jobs.clone()), + flow_jobs_success: flow_jobs_success.clone(), + branch_chosen: None, + failed_retries: vec![], + } + }; let r = sqlx::query_scalar!( - "DELETE FROM parallel_monitor_lock WHERE parent_flow_id = $1 RETURNING last_ping", - flow, - ).fetch_optional(db).await.map_err(|e| { - Error::internal_err(format!( - "error while deleting parallel_monitor_lock: {e:#}" - )) - })?; + "DELETE FROM parallel_monitor_lock WHERE parent_flow_id = $1 RETURNING last_ping", + flow, + ).fetch_optional(db).await.map_err(|e| { + Error::internal_err(format!( + "error while deleting parallel_monitor_lock: {e:#}" + )) + })?; if r.is_some() { tracing::info!( @@ -633,10 +652,10 @@ pub async fn update_flow_status_after_job_completion_internal( if parallelism.is_some() { sqlx::query!( "UPDATE v2_job_queue q SET suspend = 0 - FROM v2_job j, v2_job_status f - WHERE parent_job = $1 - AND f.id = j.id AND q.id = j.id - AND suspend = $2 AND (f.flow_status->'step')::int = 0", + FROM v2_job j, v2_job_status f + WHERE parent_job = $1 + AND f.id = j.id AND q.id = j.id + AND suspend = $2 AND (f.flow_status->'step')::int = 0", flow, nindex ) @@ -650,12 +669,12 @@ pub async fn update_flow_status_after_job_completion_internal( } let r = sqlx::query_scalar!( - "DELETE FROM parallel_monitor_lock WHERE parent_flow_id = $1 and job_id = $2 RETURNING last_ping", - flow, - job_id_for_status - ).fetch_optional(db).await.map_err(|e| { - Error::internal_err(format!("error while removing parallel_monitor_lock: {e:#}")) - })?; + "DELETE FROM parallel_monitor_lock WHERE parent_flow_id = $1 and job_id = $2 RETURNING last_ping", + flow, + job_id_for_status + ).fetch_optional(db).await.map_err(|e| { + Error::internal_err(format!("error while removing parallel_monitor_lock: {e:#}")) + })?; if r.is_some() { tracing::info!( "parallel flow has removed lock on its parent, last ping was {:?}", @@ -696,7 +715,7 @@ pub async fn update_flow_status_after_job_completion_internal( flow_jobs_success, flow_jobs, .. - } if branch.to_owned() < len - 1 && (success || skip_branch_failure) => { + } if branch.to_owned() < len - 1 && (success || skip_seq_branch_failure) => { if let Some(jobs) = flow_jobs { set_success_in_flow_job_success( flow_jobs_success, @@ -736,7 +755,9 @@ pub async fn update_flow_status_after_job_completion_internal( } } } - if success || (flow_jobs.is_some() && (skip_loop_failures || skip_branch_failure)) { + if success + || (flow_jobs.is_some() && (skip_loop_failures || skip_seq_branch_failure)) + { let is_skipped = if current_module.as_ref().is_some_and(|m| m.skip_if.is_some()) { sqlx::query_scalar!( @@ -793,11 +814,22 @@ pub async fn update_flow_status_after_job_completion_internal( } }; + let skip_parallel_branchall_failure = match (module_status, new_status.as_ref()) { + ( + FlowStatusModule::InProgress { branchall: Some(_), parallel: true, .. }, + Some(FlowStatusModule::Success { flow_jobs_success, .. }), + ) => compute_skip_branchall_failure(0, true, current_module, flow_jobs_success).await?, + ( + FlowStatusModule::InProgress { branchall: Some(_), parallel: true, .. }, + Some(FlowStatusModule::Failure { flow_jobs_success, .. }), + ) => compute_skip_branchall_failure(0, true, current_module, flow_jobs_success).await?, + _ => false, + }; let step_counter = if inc_step_counter { sqlx::query!( "UPDATE v2_job_status - SET flow_status = JSONB_SET(flow_status, ARRAY['step'], $1) - WHERE id = $2", + SET flow_status = JSONB_SET(flow_status, ARRAY['step'], $1) + WHERE id = $2", json!(old_status.step + 1), flow ) @@ -826,20 +858,20 @@ pub async fn update_flow_status_after_job_completion_internal( if let Some(new_status) = new_status.as_ref() { if is_failure_step { let parent_module = sqlx::query_scalar!( - "SELECT flow_status->'failure_module'->>'parent_module' FROM v2_job_status WHERE id = $1", - flow - ) - .fetch_one(&mut *tx) - .await.map_err(|e| { - Error::internal_err(format!( - "error while fetching failure module: {e:#}" - )) - })?; + "SELECT flow_status->'failure_module'->>'parent_module' FROM v2_job_status WHERE id = $1", + flow + ) + .fetch_one(&mut *tx) + .await.map_err(|e| { + Error::internal_err(format!( + "error while fetching failure module: {e:#}" + )) + })?; sqlx::query!( "UPDATE v2_job_status - SET flow_status = JSONB_SET(flow_status, ARRAY['failure_module'], $1) - WHERE id = $2", + SET flow_status = JSONB_SET(flow_status, ARRAY['failure_module'], $1) + WHERE id = $2", json!(FlowStatusModuleWParent { parent_module, module_status: new_status.clone() @@ -856,8 +888,8 @@ pub async fn update_flow_status_after_job_completion_internal( } else if matches!(module_step, Step::PreprocessorStep) { sqlx::query!( "UPDATE v2_job_status - SET flow_status = JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1) - WHERE id = $2", + SET flow_status = JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1) + WHERE id = $2", json!(new_status), flow ) @@ -871,8 +903,8 @@ pub async fn update_flow_status_after_job_completion_internal( } else { sqlx::query!( "UPDATE v2_job_status - SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2) - WHERE id = $3", + SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2) + WHERE id = $3", old_status.step.to_string(), json!(new_status), flow @@ -885,40 +917,44 @@ pub async fn update_flow_status_after_job_completion_internal( if let Some(job_result) = new_status.job_result() { sqlx::query!( - "UPDATE v2_job_status - SET flow_leaf_jobs = JSONB_SET(coalesce(flow_leaf_jobs, '{}'::jsonb), ARRAY[$1::TEXT], $2) - WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $3), $3) = id", - new_status.id(), - json!(job_result), - flow - ) - .execute(&mut *tx) - .await.map_err(|e| { - Error::internal_err(format!( - "error while setting leaf jobs: {e:#}" - )) - })?; + "UPDATE v2_job_status + SET flow_leaf_jobs = JSONB_SET(coalesce(flow_leaf_jobs, '{}'::jsonb), ARRAY[$1::TEXT], $2) + WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $3), $3) = id", + new_status.id(), + json!(job_result), + flow + ) + .execute(&mut *tx) + .await.map_err(|e| { + Error::internal_err(format!( + "error while setting leaf jobs: {e:#}" + )) + })?; } } } - let nresult = match &new_status { - Some(FlowStatusModule::Success { flow_jobs: Some(jobs), .. }) - | Some(FlowStatusModule::Failure { flow_jobs: Some(jobs), .. }) => { - Arc::new(retrieve_flow_jobs_results(db, w_id, jobs).await?) + let mut nresult = if let Some(stop_early_err_msg) = stop_early_err_msg.as_ref() { + Arc::new(to_raw_value(stop_early_err_msg)) + } else { + match &new_status { + Some(FlowStatusModule::Success { flow_jobs: Some(jobs), .. }) + | Some(FlowStatusModule::Failure { flow_jobs: Some(jobs), .. }) => { + Arc::new(retrieve_flow_jobs_results(db, w_id, jobs).await?) + } + _ => result.clone(), } - _ => result.clone(), }; match &new_status { Some(FlowStatusModule::Success { .. }) if is_loop || is_branch_all => { - if let Some(ref expr) = current_module + if let Some(stop_after_all_iters_if) = current_module .as_ref() - .and_then(|m| m.stop_after_all_iters_if.as_ref().map(|x| x.expr.clone())) + .and_then(|m| m.stop_after_all_iters_if.as_ref()) { let args = sqlx::query_scalar!( "SELECT args AS \"args: Json>>\" - FROM v2_job WHERE id = $1", + FROM v2_job WHERE id = $1", flow ) .fetch_one(db) @@ -928,7 +964,7 @@ pub async fn update_flow_status_after_job_completion_internal( })?; let should_stop = compute_bool_from_expr( - &expr, + &stop_after_all_iters_if.expr, Marc::new(args.unwrap_or_default().0), nresult.clone(), None, @@ -940,15 +976,14 @@ pub async fn update_flow_status_after_job_completion_internal( .await?; if should_stop { - stop_early = should_stop; - skip_if_stop_early = current_module - .as_ref() - .and_then(|m| { - m.stop_after_all_iters_if - .as_ref() - .map(|x| x.skip_if_stopped) - }) - .unwrap_or(false); + stop_early = true; + let (skip_if_stopped, err_msg_internal) = + get_stop_after_if_data(should_stop, Some(stop_after_all_iters_if)); + skip_if_stop_early = skip_if_stopped; + if err_msg_internal.is_some() { + stop_early_err_msg = err_msg_internal; + nresult = Arc::new(to_raw_value(&stop_early_err_msg)); + } } } } @@ -960,8 +995,8 @@ pub async fn update_flow_status_after_job_completion_internal( { sqlx::query!( "UPDATE v2_job_status - SET flow_status = flow_status - 'retry' - WHERE id = $1", + SET flow_status = flow_status - 'retry' + WHERE id = $1", flow ) .execute(&mut *tx) @@ -985,7 +1020,12 @@ pub async fn update_flow_status_after_job_completion_internal( _ if flow_job.is_canceled() => false, true => !is_last_step, false if unrecoverable => false, - false if skip_branch_failure || skip_loop_failures || continue_on_error => { + false + if skip_seq_branch_failure + || skip_parallel_branchall_failure + || skip_loop_failures + || continue_on_error => + { !is_last_step } false @@ -1029,6 +1069,7 @@ pub async fn update_flow_status_after_job_completion_internal( flow_job, flow_data, stop_early, + stop_early_err_msg, skip_if_stop_early, nresult, is_failure_step, @@ -1056,9 +1097,9 @@ pub async fn update_flow_status_after_job_completion_internal( // run the cleanup step only when the root job is complete if !_cleanup_module.flow_jobs_to_clean.is_empty() { tracing::debug!( - "Cleaning up jobs arguments, result and logs as they were marked as delete_after_use {:?}", - _cleanup_module.flow_jobs_to_clean - ); + "Cleaning up jobs arguments, result and logs as they were marked as delete_after_use {:?}", + _cleanup_module.flow_jobs_to_clean + ); sqlx::query!( "UPDATE v2_job SET args = '{}'::jsonb WHERE id = ANY($1)", &_cleanup_module.flow_jobs_to_clean, @@ -1107,14 +1148,15 @@ pub async fn update_flow_status_after_job_completion_internal( } let success = success && (!is_failure_step || result_has_recover_true(nresult.clone())) - && !skip_error_handler; + && !skip_error_handler + && stop_early_err_msg.is_none(); add_time!(bench, "flow status update 1"); if success { add_completed_job( db, &flow_job, - success, + true, stop_early && skip_if_stop_early, Json(&nresult), None, @@ -1128,7 +1170,7 @@ pub async fn update_flow_status_after_job_completion_internal( add_completed_job( db, &flow_job, - success, + false, stop_early && skip_if_stop_early, Json( &serde_json::from_str::(nresult.get()).unwrap_or_else( @@ -1227,12 +1269,12 @@ async fn set_success_in_flow_job_success<'c>( if let Some(position) = position { sqlx::query!( "UPDATE v2_job_status SET - flow_status = JSONB_SET( - flow_status, - ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], - $4 - ) - WHERE id = $2", + flow_status = JSONB_SET( + flow_status, + ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], + $4 + ) + WHERE id = $2", old_status.step as i32, flow, position as i32, @@ -1255,8 +1297,8 @@ async fn retrieve_flow_jobs_results( ) -> error::Result> { let results = sqlx::query!( "SELECT result, id - FROM v2_job_completed - WHERE id = ANY($1) AND workspace_id = $2", + FROM v2_job_completed + WHERE id = ANY($1) AND workspace_id = $2", job_uuids.as_slice(), w_id ) @@ -1274,47 +1316,48 @@ async fn retrieve_flow_jobs_results( .ok_or_else(|| Error::internal_err(format!("missing job result for {}", j))) }) .collect::, _>>()?; - tracing::debug!("Retrieved results for flow jobs {:?}", results); Ok(to_raw_value(&results)) } async fn compute_skip_branchall_failure<'c>( - job: &Uuid, branch: usize, parallel: bool, - db: &DB, flow_module: Option<&FlowModule>, -) -> Result, Error> { - let branch = if parallel { - sqlx::query_scalar!("SELECT runnable_path FROM v2_job WHERE id = $1", job) - .fetch_one(db) - .await - .map_err(|e| { - Error::internal_err(format!("error during retrieval of branchall index: {e:#}")) - })? - .map(|p| { - BRANCHALL_INDEX_RE - .captures(&p) - .map(|x| x.get(1).unwrap().as_str().parse::().ok()) - .flatten() - .ok_or(Error::internal_err(format!( - "could not parse branchall index from path: {p}" - ))) - }) - .ok_or_else(|| { - Error::internal_err(format!("no branchall script path found for job {job}")) - })?? - } else { - branch as i32 - }; - Ok(flow_module + successes: &Option>>, +) -> windmill_common::error::Result { + let branches = flow_module .and_then(|x| x.get_branches_skip_failures().ok()) - .and_then(|x| { + .map(|x| { x.branches - .get(branch as usize) - .map(|x| x.skip_failure.unwrap_or(false)) - })) + .iter() + .map(|b| b.skip_failure.unwrap_or(false)) + .collect::>() + }); + if parallel { + if let Some(successes) = successes { + for (i, success) in successes.iter().enumerate() { + if branches + .as_ref() + .and_then(|x| x.get(i)) + .unwrap_or(&false) + .to_owned() + { + continue; + } + if !(success.unwrap_or(false)) { + return Ok(false); + } + } + Ok(true) + } else { + Ok(false) + } + } else { + Ok(branches + .and_then(|x| x.get(branch as usize).map(|b| b.to_owned())) + .unwrap_or(false)) + } } // async fn retrieve_cleanup_module<'c>(flow_uuid: Uuid, db: &DB) -> Result { @@ -1644,24 +1687,24 @@ async fn push_next_flow_job( .await?; if no_flow_overlap { let overlapping = sqlx::query_scalar!( - // Query plan: - // - use of the `ix_v2_job_root_by_path` index; hence the `parent_job IS NULL` - // clause. - // - select from `v2_job` first, then join with `v2_job_queue` to avoid a full - // table scan on `running = true`. - "SELECT id - FROM v2_job j JOIN v2_job_queue USING (id) - WHERE j.workspace_id = $2 AND trigger_kind = 'schedule' AND trigger = $1 AND runnable_path = $4 - AND parent_job IS NULL - AND j.id != $3 - AND running = true", - schedule_path.as_ref().unwrap(), - flow_job.workspace_id.as_str(), - flow_job.id, - flow_job.runnable_path() - ) - .fetch_all(db) - .await?; + // Query plan: + // - use of the `ix_v2_job_root_by_path` index; hence the `parent_job IS NULL` + // clause. + // - select from `v2_job` first, then join with `v2_job_queue` to avoid a full + // table scan on `running = true`. + "SELECT id + FROM v2_job j JOIN v2_job_queue USING (id) + WHERE j.workspace_id = $2 AND trigger_kind = 'schedule' AND trigger = $1 AND runnable_path = $4 + AND parent_job IS NULL + AND j.id != $3 + AND running = true", + schedule_path.as_ref().unwrap(), + flow_job.workspace_id.as_str(), + flow_job.id, + flow_job.runnable_path() + ) + .fetch_all(db) + .await?; if overlapping.len() > 0 { let overlapping_str = overlapping .iter() @@ -1669,24 +1712,24 @@ async fn push_next_flow_job( .collect::>() .join(", "); job_completed_tx - .send(SendResult::UpdateFlow { - flow: flow_job.id, - success: true, - result: serde_json::from_str( - &format!("\"not allowed to overlap with {overlapping_str}, scheduling next iteration\""), - ) - .unwrap(), - stop_early_override: Some(true), - w_id: flow_job.workspace_id.clone(), - worker_dir: worker_dir.to_string(), - token: client.token.clone(), - }) - .await - .map_err(|e| { - Error::internal_err(format!( - "error sending update flow message to job completed channel: {e:#}" - )) - })?; + .send(SendResult::UpdateFlow { + flow: flow_job.id, + success: true, + result: serde_json::from_str( + &format!("\"not allowed to overlap with {overlapping_str}, scheduling next iteration\""), + ) + .unwrap(), + stop_early_override: Some(true), + w_id: flow_job.workspace_id.clone(), + worker_dir: worker_dir.to_string(), + token: client.token.clone(), + }) + .await + .map_err(|e| { + Error::internal_err(format!( + "error sending update flow message to job completed channel: {e:#}" + )) + })?; return Ok(None); } @@ -1778,13 +1821,13 @@ async fn push_next_flow_job( .context("lock flow in queue")?; let resumes = sqlx::query_as::<_, ResumeRow>( - "SELECT value, approver, resume_id, approved FROM resume_job WHERE job = $1 ORDER BY created_at ASC", - ) - .bind(last) - .fetch_all(&mut *tx) - .await? - .into_iter() - .collect::>(); + "SELECT value, approver, resume_id, approved FROM resume_job WHERE job = $1 ORDER BY created_at ASC", + ) + .bind(last) + .fetch_all(&mut *tx) + .await? + .into_iter() + .collect::>(); resume_messages.extend(resumes.iter().map(|r| to_raw_value(&r.value))); approvers.extend(resumes.iter().map(|r| { @@ -1814,22 +1857,22 @@ async fn push_next_flow_job( .insert("previous_result".to_string(), arc_last_job_result.clone()); let eval_result = serde_json::from_str::>( - eval_timeout( - expr.to_string(), - context, - Some(arc_flow_job_args.clone()), - None, - None, - None - ) - .await - .map_err(|e| { - Error::ExecutionErr(format!( - "Error during isolated evaluation of expression `{expr}`:\n{e:#}" - )) - })? - .get(), - ); + eval_timeout( + expr.to_string(), + context, + Some(arc_flow_job_args.clone()), + None, + None, + None + ) + .await + .map_err(|e| { + Error::ExecutionErr(format!( + "Error during isolated evaluation of expression `{expr}`:\n{e:#}" + )) + })? + .get(), + ); if eval_result.is_ok() { user_groups_required = eval_result.ok().unwrap_or(Vec::new()) } else { @@ -1848,8 +1891,8 @@ async fn push_next_flow_job( }; sqlx::query!( "UPDATE v2_job_status - SET flow_status = JSONB_SET(flow_status, ARRAY['approval_conditions'], $1) - WHERE id = $2", + SET flow_status = JSONB_SET(flow_status, ARRAY['approval_conditions'], $1) + WHERE id = $2", json!(approval_conditions), flow_job.id ) @@ -1887,42 +1930,42 @@ async fn push_next_flow_job( resume_messages.push(to_raw_value(&js)); audit_log( - &mut *tx, - &audit_author, - "jobs.suspend_resume", - ActionKind::Update, - &flow_job.workspace_id, - Some(&serde_json::json!({"approved": false, "job_id": flow_job.id, "details": "Suspend timed out without approval but can continue".to_string()}).to_string()), - None, - ) - .await?; + &mut *tx, + &audit_author, + "jobs.suspend_resume", + ActionKind::Update, + &flow_job.workspace_id, + Some(&serde_json::json!({"approved": false, "job_id": flow_job.id, "details": "Suspend timed out without approval but can continue".to_string()}).to_string()), + None, + ) + .await?; } sqlx::query!( - "UPDATE v2_job_status - SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'approvers'], $2) - WHERE id = $3", - (status.step - 1).to_string(), - json!(resumes - .into_iter() - .map(|r| Approval { - resume_id: r.resume_id as u16, - approver: r - .approver.clone() - .unwrap_or_else(|| "unknown".to_string()) - }) - .collect::>() - ), - flow_job.id - ) - .execute(&mut *tx) - .await?; + "UPDATE v2_job_status + SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'approvers'], $2) + WHERE id = $3", + (status.step - 1).to_string(), + json!(resumes + .into_iter() + .map(|r| Approval { + resume_id: r.resume_id as u16, + approver: r + .approver.clone() + .unwrap_or_else(|| "unknown".to_string()) + }) + .collect::>() + ), + flow_job.id + ) + .execute(&mut *tx) + .await?; // Remove the approval conditions from the flow status sqlx::query!( "UPDATE v2_job_status - SET flow_status = flow_status - 'approval_conditions' - WHERE id = $1", + SET flow_status = flow_status - 'approval_conditions' + WHERE id = $1", flow_job.id ) .execute(&mut *tx) @@ -1939,14 +1982,14 @@ async fn push_next_flow_job( { sqlx::query!( "WITH suspend AS ( - UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + $3 - WHERE id = $4 - RETURNING id - ) UPDATE v2_job_status SET flow_status = JSONB_SET( - flow_status, - ARRAY['modules', flow_status->>'step'::TEXT], - $1 - ) WHERE id = (SELECT id FROM suspend)", + UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + $3 + WHERE id = $4 + RETURNING id + ) UPDATE v2_job_status SET flow_status = JSONB_SET( + flow_status, + ARRAY['modules', flow_status->>'step'::TEXT], + $1 + ) WHERE id = (SELECT id FROM suspend)", json!(FlowStatusModule::WaitingForEvents { id: status_module.id(), count: required_events, @@ -1963,7 +2006,7 @@ async fn push_next_flow_job( sqlx::query!( "UPDATE v2_job_runtime SET ping = NULL - WHERE id = $1", + WHERE id = $1", flow_job.id, ) .execute(&mut *tx) @@ -1976,15 +2019,15 @@ async fn push_next_flow_job( } else { if is_disapproved.is_none() { audit_log( - &mut *tx, - &audit_author, - "jobs.suspend_resume", - ActionKind::Update, - &flow_job.workspace_id, - Some(&serde_json::json!({"approved": false, "job_id": flow_job.id, "details": "Suspend timed out without approval and is cancelled".to_string()}).to_string()), - None, - ) - .await?; + &mut *tx, + &audit_author, + "jobs.suspend_resume", + ActionKind::Update, + &flow_job.workspace_id, + Some(&serde_json::json!({"approved": false, "job_id": flow_job.id, "details": "Suspend timed out without approval and is cancelled".to_string()}).to_string()), + None, + ) + .await?; } tx.commit().await?; @@ -2086,22 +2129,22 @@ async fn push_next_flow_job( context.insert("previous_result".to_string(), arc_last_job_result.clone()); serde_json::from_str( - eval_timeout( - expr.to_string(), - context, - Some(arc_flow_job_args.clone()), - None, - None, - None, - ) - .await - .map_err(|e| { - Error::ExecutionErr(format!( - "Error during isolated evaluation of expression `{expr}`:\n{e:#}" - )) - })? - .get(), - ) + eval_timeout( + expr.to_string(), + context, + Some(arc_flow_job_args.clone()), + None, + None, + None, + ) + .await + .map_err(|e| { + Error::ExecutionErr(format!( + "Error during isolated evaluation of expression `{expr}`:\n{e:#}" + )) + })? + .get(), + ) } }; match json_value.and_then(|x| serde_json::from_str::(x.get())) { @@ -2150,18 +2193,18 @@ async fn push_next_flow_job( scheduled_for_o = Some(from_now(retry_in)); status.retry.failed_jobs.push(job.clone()); sqlx::query!( - "UPDATE v2_job_status - SET flow_status = JSONB_SET(JSONB_SET(flow_status, ARRAY['retry'], $1), ARRAY['modules', $3::TEXT, 'failed_retries'], $4) - WHERE id = $2", - json!(RetryStatus { fail_count, ..status.retry.clone() }), - flow_job.id, - status.step.to_string(), - json!(status.retry.failed_jobs) - ) - .execute(db) - .warn_after_seconds(2) - .await - .context("update flow retry")?; + "UPDATE v2_job_status + SET flow_status = JSONB_SET(JSONB_SET(flow_status, ARRAY['retry'], $1), ARRAY['modules', $3::TEXT, 'failed_retries'], $4) + WHERE id = $2", + json!(RetryStatus { fail_count, ..status.retry.clone() }), + flow_job.id, + status.step.to_string(), + json!(status.retry.failed_jobs) + ) + .execute(db) + .warn_after_seconds(2) + .await + .context("update flow retry")?; status_module = FlowStatusModule::WaitingForPriorSteps { id: status_module.id() }; // we get the args from the last failed job @@ -2188,8 +2231,8 @@ async fn push_next_flow_job( if module.retry.as_ref().is_some_and(|x| x.has_attempts()) { sqlx::query!( "UPDATE v2_job_status - SET flow_status = JSONB_SET(flow_status, ARRAY['retry'], $1) - WHERE id = $2", + SET flow_status = JSONB_SET(flow_status, ARRAY['retry'], $1) + WHERE id = $2", json!(RetryStatus { fail_count: 0, failed_jobs: vec![] }), flow_job.id ) @@ -2247,7 +2290,7 @@ async fn push_next_flow_job( } else if let Some(id) = get_args_from_id { let args = sqlx::query_scalar!( "SELECT args AS \"args: Json>>\" - FROM v2_job WHERE id = $1 AND workspace_id = $2", + FROM v2_job WHERE id = $1 AND workspace_id = $2", id, &flow_job.workspace_id ) @@ -2335,9 +2378,9 @@ async fn push_next_flow_job( NextFlowTransform::EmptyInnerFlows { branch_chosen } => { let raw_status = sqlx::query_scalar!( "UPDATE v2_job_status - SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2) - WHERE id = $3 - RETURNING flow_status AS \"flow_status: Json>\"", + SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2) + WHERE id = $3 + RETURNING flow_status AS \"flow_status: Json>\"", status.step.to_string(), json!(FlowStatusModule::Success { id: status_module.id(), @@ -2559,14 +2602,14 @@ async fn push_next_flow_job( .or_else(|| Some(flow_job.id)) { sqlx::query_as!( - JobPerms, - "SELECT email, username, is_admin, is_operator, groups, folders FROM job_perms WHERE job_id = $1 AND workspace_id = $2", - root_job, - flow_job.workspace_id, - ) - .fetch_optional(&mut *tx) - .await? - .map(|x| x.into()) + JobPerms, + "SELECT email, username, is_admin, is_operator, groups, folders FROM job_perms WHERE job_id = $1 AND workspace_id = $2", + root_job, + flow_job.workspace_id, + ) + .fetch_optional(&mut *tx) + .await? + .map(|x| x.into()) } else { None } @@ -2637,10 +2680,10 @@ async fn push_next_flow_job( if i as u16 >= p { sqlx::query!( "UPDATE v2_job_queue SET - suspend = $1, - suspend_until = now() + interval '14 day', - running = true - WHERE id = $2", + suspend = $1, + suspend_until = now() + interval '14 day', + running = true + WHERE id = $2", (i as u16 - p + 1) as i32, uuid, ) @@ -2657,14 +2700,14 @@ async fn push_next_flow_job( })?; sqlx::query!( - "UPDATE v2_job_status - SET flow_status = JSONB_SET(flow_status, ARRAY['cleanup_module', 'flow_jobs_to_clean'], COALESCE(flow_status->'cleanup_module'->'flow_jobs_to_clean', '[]'::jsonb) || $1) - WHERE id = $2", - uuid_singleton_json, - root_job.unwrap_or(flow_job.id) - ) - .execute(&mut *inner_tx) - .await?; + "UPDATE v2_job_status + SET flow_status = JSONB_SET(flow_status, ARRAY['cleanup_module', 'flow_jobs_to_clean'], COALESCE(flow_status->'cleanup_module'->'flow_jobs_to_clean', '[]'::jsonb) || $1) + WHERE id = $2", + uuid_singleton_json, + root_job.unwrap_or(flow_job.id) + ) + .execute(&mut *inner_tx) + .await?; } tx = inner_tx; @@ -2683,7 +2726,7 @@ async fn push_next_flow_job( for uuid in &uuids { sqlx::query!( "INSERT INTO parallel_monitor_lock (parent_flow_id, job_id) - VALUES ($1, $2)", + VALUES ($1, $2)", flow_job.id, uuid ) @@ -2787,12 +2830,12 @@ async fn push_next_flow_job( Step::FailureStep => { sqlx::query!( "UPDATE v2_job_status SET - flow_status = JSONB_SET( - JSONB_SET(flow_status, ARRAY['failure_module'], $1), - ARRAY['step'], - $2 - ) - WHERE id = $3", + flow_status = JSONB_SET( + JSONB_SET(flow_status, ARRAY['failure_module'], $1), + ARRAY['step'], + $2 + ) + WHERE id = $3", json!(FlowStatusModuleWParent { parent_module: Some(current_id.clone()), module_status: new_status @@ -2806,12 +2849,12 @@ async fn push_next_flow_job( Step::PreprocessorStep => { sqlx::query!( "UPDATE v2_job_status SET - flow_status = JSONB_SET( - JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1), - ARRAY['step'], - $2 - ) - WHERE id = $3", + flow_status = JSONB_SET( + JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1), + ARRAY['step'], + $2 + ) + WHERE id = $3", json!(new_status), json!(-1), flow_job.id @@ -2822,12 +2865,12 @@ async fn push_next_flow_job( Step::Step(i) => { sqlx::query!( "UPDATE v2_job_status SET - flow_status = JSONB_SET( - JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), - ARRAY['step'], - $3 - ) - WHERE id = $4", + flow_status = JSONB_SET( + JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), + ARRAY['step'], + $3 + ) + WHERE id = $4", i as i32, json!(new_status), json!(i), @@ -2850,8 +2893,8 @@ async fn push_next_flow_job( if continue_on_same_worker { if !is_one_uuid { return Err(Error::BadRequest( - "Cannot continue on same worker with multiple jobs, parallel cannot be used in conjunction with same_worker".to_string(), - )); + "Cannot continue on same worker with multiple jobs, parallel cannot be used in conjunction with same_worker".to_string(), + )); } } tx.commit().warn_after_seconds(3).await?; @@ -3702,11 +3745,11 @@ async fn next_forloop_status( itered.clone() }; let (index, next) = index - .checked_add(1) - .and_then(|i| itered_new.get(i).map(|next| (i, next))) - .with_context(|| { - format!("Could not find iteration number {index} restarting inside the for-loop flow. It's possible the itered-array has changed and this value isn't available anymore.") - })?; + .checked_add(1) + .and_then(|i| itered_new.get(i).map(|next| (i, next))) + .with_context(|| { + format!("Could not find iteration number {index} restarting inside the for-loop flow. It's possible the itered-array has changed and this value isn't available anymore.") + })?; ForLoopStatus::NextIteration(ForloopNextIteration { index, @@ -3828,20 +3871,15 @@ async fn flow_to_payload( w_id: &str, db: &DB, ) -> Result { - let record = sqlx::query!( - "SELECT on_behalf_of_email, edited_by FROM flow WHERE path = $1 AND workspace_id = $2", - path, - w_id, - ) - .fetch_one(db) - .await - .map_err(|e| Error::NotFound(format!("fetching flow: {e:#}")))?; - let on_behalf_of = if let Some(email) = record.on_behalf_of_email { - Some(OnBehalfOf { email, permissioned_as: username_to_permissioned_as(&record.edited_by) }) + let FlowVersionInfo { version, on_behalf_of_email, edited_by, .. } = + get_latest_flow_version_info_for_path(db, w_id, &path, true).await?; + let on_behalf_of = if let Some(email) = on_behalf_of_email { + Some(OnBehalfOf { email, permissioned_as: username_to_permissioned_as(&edited_by) }) } else { None }; - let payload = JobPayload::Flow { path, dedicated_worker: None, apply_preprocessor: false }; + let payload = + JobPayload::Flow { path, dedicated_worker: None, apply_preprocessor: false, version }; Ok(JobPayloadWithTag { payload, tag: None, delete_after_use, timeout: None, on_behalf_of }) } @@ -3871,9 +3909,9 @@ async fn script_to_payload( } else { let hash = script_hash.unwrap(); let mut tx: sqlx::Transaction<'_, sqlx::Postgres> = db.begin().await?; - let ( + let ScriptHashInfo { tag, - custom_concurrency_key, + concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, @@ -3881,10 +3919,11 @@ async fn script_to_payload( dedicated_worker, priority, delete_after_use, - script_timeout, + timeout, on_behalf_of_email, created_by, - ) = script_hash_to_tag_and_limits(&hash, &mut tx, &flow_job.workspace_id).await?; + .. + } = get_script_info_for_hash(&mut *tx, &flow_job.workspace_id, hash.0).await?; let on_behalf_of = if let Some(email) = on_behalf_of_email { Some(OnBehalfOf { email, permissioned_as: username_to_permissioned_as(&created_by) }) } else { @@ -3894,7 +3933,7 @@ async fn script_to_payload( JobPayload::ScriptHash { hash, path: script_path, - custom_concurrency_key, + custom_concurrency_key: concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl: module.cache_ttl.map(|x| x as i32).ok_or(cache_ttl).ok(), @@ -3905,7 +3944,7 @@ async fn script_to_payload( }, tag_override.to_owned().or(tag), delete_after_use, - script_timeout, + timeout, on_behalf_of, ) }; @@ -4006,7 +4045,7 @@ async fn get_previous_job_result( Some(FlowStatusModule::Success { job, .. }) => Ok(Some( sqlx::query_scalar!( "SELECT result AS \"result!: Json>\" - FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", job, w_id ) diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index cab717a94f..7744ef341b 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -3,6 +3,8 @@ use std::collections::HashMap; use std::fs::{create_dir_all, remove_dir_all}; use std::path::{Component, Path, PathBuf}; +#[cfg(feature = "python")] +use crate::ansible_executor::{get_git_repos_lock, AnsibleDependencyLocks}; use async_recursion::async_recursion; use serde_json::value::RawValue; use serde_json::{json, Value}; @@ -18,6 +20,8 @@ use windmill_common::scripts::ScriptHash; #[cfg(feature = "python")] use windmill_common::worker::PythonAnnotations; use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, Connection}; +#[cfg(feature = "python")] +use windmill_parser_yaml::AnsibleRequirements; use windmill_common::{ apps::AppScriptId, @@ -61,17 +65,17 @@ pub async fn update_script_dependency_map( relative_imports: Vec, ) -> error::Result<()> { let importer_kind = "script"; + + let mut tx = db.begin().await?; + tx = clear_dependency_parent_path(parent_path, script_path, w_id, importer_kind, tx).await?; + + tx = clear_dependency_map_for_item(script_path, w_id, importer_kind, tx, &None).await?; + if !relative_imports.is_empty() { let mut logs = "".to_string(); logs.push_str("\n--- RELATIVE IMPORTS ---\n\n"); logs.push_str(&relative_imports.join("\n")); - let mut tx = db.begin().await?; - tx = - clear_dependency_parent_path(parent_path, script_path, w_id, importer_kind, tx).await?; - - tx = clear_dependency_map_for_item(script_path, w_id, importer_kind, tx, &None).await?; - tx = add_relative_imports_to_dependency_map( script_path, w_id, @@ -82,9 +86,10 @@ pub async fn update_script_dependency_map( None, ) .await?; - tx.commit().await?; append_logs(job_id, w_id, logs, &db.into()).await; } + tx.commit().await?; + Ok(()) } @@ -364,44 +369,22 @@ pub async fn handle_dependency_job( tracing::error!(%e, "error handling deployment metadata"); } - let relative_imports = - extract_relative_imports(&script_data.code, script_path, &job.script_lang); - if let Some(relative_imports) = relative_imports { - update_script_dependency_map( - &job.id, - db, - w_id, - &parent_path, - script_path, - relative_imports, - ) - .await?; - let already_visited = job - .args - .as_ref() - .map(|x| { - x.get("already_visited") - .map(|v| serde_json::from_str::>(v.get()).ok()) - .flatten() - }) - .flatten() - .unwrap_or_default(); - if let Err(e) = trigger_dependents_to_recompute_dependencies( - w_id, - script_path, - deployment_message, - parent_path, - &job.permissioned_as_email, - &job.created_by, - &job.permissioned_as, - db, - already_visited, - ) - .await - { - tracing::error!(%e, "error triggering dependents to recompute dependencies"); - } - } + process_relative_imports( + db, + Some(job.id), + job.args.as_ref(), + &job.workspace_id, + script_path, + parent_path, + deployment_message, + &script_data.code, + &job.script_lang, + &job.permissioned_as_email, + &job.created_by, + &job.permissioned_as, + None, + ) + .await?; Ok(to_raw_value_owned( json!({ "status": "Successful lock file generation", "lock": content }), @@ -439,6 +422,82 @@ fn remove_ansi_codes(s: &str) -> String { ANSI_REGEX.replace_all(s, "").to_string() } +pub async fn process_relative_imports( + db: &sqlx::Pool, + job_id: Option, + args: Option<&Json>>>, + w_id: &str, + script_path: &str, + parent_path: Option, + deployment_message: Option, + code: &str, + script_lang: &Option, + permissioned_as_email: &str, + created_by: &str, + permissioned_as: &str, + lock: Option, +) -> error::Result<()> { + let relative_imports = extract_relative_imports(&code, script_path, script_lang); + if let Some(relative_imports) = relative_imports { + if (script_lang.is_some_and(|v| v == ScriptLang::Bun) + && lock + .as_ref() + .is_some_and(|v| v.contains("generatedFromPackageJson"))) + || (script_lang.is_some_and(|v| v == ScriptLang::Python3) + && lock + .as_ref() + .is_some_and(|v| v.starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT))) + { + // if the lock file is generated from a package.json/requirements.txt, we need to clear the dependency map + // because we do not want to have dependencies be recomputed automatically. Empty relative imports passed + // to update_script_dependency_map will clear the dependency map. + update_script_dependency_map( + &job_id.unwrap_or_else(|| Uuid::nil()), + db, + w_id, + &parent_path, + script_path, + vec![], + ) + .await?; + } else { + update_script_dependency_map( + &job_id.unwrap_or_else(|| Uuid::nil()), + db, + w_id, + &parent_path, + script_path, + relative_imports, + ) + .await?; + } + let already_visited = args + .map(|x| { + x.get("already_visited") + .map(|v| serde_json::from_str::>(v.get()).ok()) + .flatten() + }) + .flatten() + .unwrap_or_default(); + if let Err(e) = trigger_dependents_to_recompute_dependencies( + w_id, + script_path, + deployment_message, + parent_path, + permissioned_as_email, + created_by, + permissioned_as, + db, + already_visited, + ) + .await + { + tracing::error!(%e, "error triggering dependents to recompute dependencies"); + } + } + Ok(()) +} + async fn trigger_dependents_to_recompute_dependencies( w_id: &str, script_path: &str, @@ -485,9 +544,9 @@ async fn trigger_dependents_to_recompute_dependencies( match r { Ok(r) => JobPayload::Dependencies { path: s.importer_path.clone(), - hash: r.0, - language: r.6, - dedicated_worker: r.7, + hash: ScriptHash(r.hash), + language: r.language, + dedicated_worker: r.dedicated_worker, }, Err(err) => { tracing::error!( @@ -649,13 +708,15 @@ pub async fn handle_flow_dependency_job( tx = clear_dependency_parent_path(&parent_path, &job_path, &job.workspace_id, "flow", tx) .await?; - sqlx::query!( + if !skip_flow_update { + sqlx::query!( "DELETE FROM workspace_runnable_dependencies WHERE flow_path = $1 AND workspace_id = $2", job_path, job.workspace_id ) - .execute(&mut *tx) - .await?; + .execute(&mut *tx) + .await?; + } let modified_ids; let errors; (flow.modules, tx, modified_ids, errors) = lock_modules( @@ -673,6 +734,7 @@ pub async fn handle_flow_dependency_job( token, &nodes_to_relock, occupancy_metrics, + skip_flow_update, ) .await?; if !errors.is_empty() { @@ -841,6 +903,7 @@ async fn lock_modules<'c>( token: &str, locks_to_reload: &Option>, occupancy_metrics: &mut OccupancyMetrics, + skip_flow_update: bool, // (modules to replace old seq (even unmmodified ones), new transaction, modified ids) ) ) -> Result<( Vec, @@ -892,6 +955,7 @@ async fn lock_modules<'c>( token, locks_to_reload, occupancy_metrics, + skip_flow_update, )) .await?; e.value = FlowModuleValue::ForloopFlow { @@ -926,6 +990,7 @@ async fn lock_modules<'c>( token, locks_to_reload, occupancy_metrics, + skip_flow_update, )) .await?; nmodified_ids.extend(inner_modified_ids); @@ -952,6 +1017,7 @@ async fn lock_modules<'c>( token, locks_to_reload, occupancy_metrics, + skip_flow_update, )) .await?; e.value = FlowModuleValue::WhileloopFlow { @@ -983,6 +1049,7 @@ async fn lock_modules<'c>( token, locks_to_reload, occupancy_metrics, + skip_flow_update, )) .await?; nmodified_ids.extend(inner_modified_ids); @@ -1007,6 +1074,7 @@ async fn lock_modules<'c>( token, locks_to_reload, occupancy_metrics, + skip_flow_update, )) .await?; errors.extend(ninner_errors); @@ -1017,7 +1085,9 @@ async fn lock_modules<'c>( } .into(); } - FlowModuleValue::Script { path, hash, .. } if !path.starts_with("hub/") => { + FlowModuleValue::Script { path, hash, .. } + if !path.starts_with("hub/") && !skip_flow_update => + { sqlx::query!( "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id) VALUES ($1, $2, $3, FALSE, $4) ON CONFLICT DO NOTHING", job_path, @@ -1028,7 +1098,7 @@ async fn lock_modules<'c>( .execute(&mut *tx) .await?; } - FlowModuleValue::Flow { path, .. } => { + FlowModuleValue::Flow { path, .. } if !skip_flow_update => { sqlx::query!( "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, workspace_id) VALUES ($1, $2, TRUE, $3) ON CONFLICT DO NOTHING", job_path, @@ -1888,6 +1958,126 @@ async fn python_dep( req } +#[cfg(feature = "python")] +async fn ansible_dep( + reqs: AnsibleRequirements, + job_id: &Uuid, + mem_peak: &mut i32, + canceled_by: &mut Option, + job_dir: &str, + db: &sqlx::Pool, + worker_name: &str, + w_id: &str, + worker_dir: &str, + occupancy_metrics: &mut OccupancyMetrics, + token: &str, + base_internal_url: &str, +) -> std::result::Result { + use windmill_parser_yaml::add_versions_to_requirements_yaml; + + use crate::{ + ansible_executor::{ + create_ansible_cfg, get_collection_locks, get_git_ssh_cmd, get_role_locks, + install_galaxy_collections, + }, + AuthedClient, + }; + + let python_lockfile = python_dep( + reqs.python_reqs.join("\n").to_string(), + job_id, + mem_peak, + canceled_by, + job_dir, + db, + worker_name, + w_id, + worker_dir, + &mut Some(occupancy_metrics), + None, + PythonAnnotations::default(), + ) + .await?; + + let conn = &Connection::Sql(db.clone()); + + let authed_client = AuthedClient { + base_internal_url: base_internal_url.to_string(), + token: token.to_string(), + workspace: w_id.to_string(), + force_client: None, + }; + + let git_ssh_cmd = get_git_ssh_cmd(&reqs, job_dir, &authed_client).await?; + + let git_repos = get_git_repos_lock( + &reqs.git_repos, + job_dir, + job_id, + worker_name, + conn, + mem_peak, + canceled_by, + w_id, + occupancy_metrics, + &git_ssh_cmd, + ) + .await?; + + let ansible_lockfile; + + create_ansible_cfg(Some(&reqs), job_dir, false)?; + + if let Some(collections) = reqs.roles_and_collections.as_ref() { + install_galaxy_collections( + collections, + job_dir, + job_id, + worker_name, + w_id, + mem_peak, + canceled_by, + conn, + occupancy_metrics, + &git_ssh_cmd, + ) + .await?; + + let (collection_versions, logs1) = get_collection_locks(job_dir).await?; + + let (role_versions, logs2) = if collections.contains("roles:") { + get_role_locks(job_dir).await? + } else { + (HashMap::new(), String::new()) + }; + + let (reqs_yaml, logs3) = + add_versions_to_requirements_yaml(&collections, &role_versions, &collection_versions)?; + + let logs = format!("\n{logs1}\n{logs2}\n{logs3}\n"); + + append_logs(job_id, w_id, &logs, conn).await; + + ansible_lockfile = AnsibleDependencyLocks { + python_lockfile, + git_repos, + collections_and_roles: reqs_yaml, + collections_and_roles_logs: logs, + }; + } else { + ansible_lockfile = AnsibleDependencyLocks { + python_lockfile, + git_repos, + collections_and_roles: String::new(), + collections_and_roles_logs: String::new(), + }; + } + + serde_json::to_string(&ansible_lockfile).map_err(|e| e.into()) +} + +pub const LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT: &str = "# from requirements.txt"; + async fn capture_dependency_job( job_id: &Uuid, job_language: &ScriptLang, @@ -1956,6 +2146,13 @@ async fn capture_dependency_job( anns, ) .await + .map(|res| { + if raw_deps { + format!("{}\n{}", LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT, res) + } else { + res + } + }) } } ScriptLang::Ansible => { @@ -1972,10 +2169,9 @@ async fn capture_dependency_job( )); } let (_logs, reqs, _) = windmill_parser_yaml::parse_ansible_reqs(job_raw_code)?; - let reqs = reqs.map(|r| r.python_reqs.join("\n")).unwrap_or_default(); - python_dep( - reqs, + ansible_dep( + reqs.unwrap_or_default(), job_id, mem_peak, canceled_by, @@ -1984,9 +2180,9 @@ async fn capture_dependency_job( worker_name, w_id, worker_dir, - &mut Some(occupancy_metrics), - None, - PythonAnnotations::default(), + occupancy_metrics, + token, + base_internal_url, ) .await } diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index ee61b90d51..a3b195de7a 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.483.1"; +export const VERSION = "v1.490.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/conf.ts b/cli/conf.ts index 14de9bcdda..e2a4c1cdbf 100644 --- a/cli/conf.ts +++ b/cli/conf.ts @@ -63,5 +63,5 @@ export async function mergeConfigWithConfigFile( opts: T ): Promise { const configFile = await readConfigFile(); - return Object.assign(configFile, opts); + return Object.assign(configFile ?? {}, opts); } diff --git a/cli/main.ts b/cli/main.ts index ab60532ce1..9be55b1c60 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -63,7 +63,7 @@ export { // } // }); -export const VERSION = "1.483.1"; +export const VERSION = "1.490.0"; const command = new Command() .name("wmill") diff --git a/cli/sync.ts b/cli/sync.ts index f31cf3f328..867b8c4d17 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -609,11 +609,16 @@ export async function* readDirRecursiveWithIgnore( while (stack.length > 0) { const e = stack.pop()!; + // console.log(e.path); yield e; for await (const e2 of e.c()) { - if (e2.path.startsWith(".git" + SEP)) { - continue; + if (e2.isDirectory) { + const dirName = e2.path.split(SEP).pop(); + if (dirName == "node_modules" || dirName?.startsWith(".")) { + continue; + } } + // console.log(e2.path); stack.push({ path: e2.path, ignored: e.ignored || ignore(e2.path, e2.isDirectory), diff --git a/docker/DockerfileFull b/docker/DockerfileFull index c36682c4e9..78eec753d1 100644 --- a/docker/DockerfileFull +++ b/docker/DockerfileFull @@ -1,8 +1,8 @@ FROM ghcr.io/windmill-labs/windmill:dev # Rust -COPY --from=rust:1.81.0 /usr/local/cargo /usr/local/cargo -COPY --from=rust:1.81.0 /usr/local/rustup /usr/local/rustup +COPY --from=rust:1.86.0 /usr/local/cargo /usr/local/cargo +COPY --from=rust:1.86.0 /usr/local/rustup /usr/local/rustup # Ansible RUN uv tool install ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -s -t "$UV_TOOL_BIN_DIR/" || true diff --git a/docker/DockerfileFullEe b/docker/DockerfileFullEe index 25cc47b2dd..138038559c 100644 --- a/docker/DockerfileFullEe +++ b/docker/DockerfileFullEe @@ -20,8 +20,8 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ FROM ghcr.io/windmill-labs/windmill-ee:dev # Rust -COPY --from=rust:1.81.0 /usr/local/cargo /usr/local/cargo -COPY --from=rust:1.81.0 /usr/local/rustup /usr/local/rustup +COPY --from=rust:1.86.0 /usr/local/cargo /usr/local/cargo +COPY --from=rust:1.86.0 /usr/local/rustup /usr/local/rustup # Ansible RUN uv tool install ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -s -t "$UV_TOOL_BIN_DIR/" || true diff --git a/examples/deploy/aws-eks-cloudformation/README.md b/examples/deploy/aws-eks-cloudformation/README.md new file mode 100644 index 0000000000..16c1a372eb --- /dev/null +++ b/examples/deploy/aws-eks-cloudformation/README.md @@ -0,0 +1,46 @@ +# windmill-cloudformation +Cloudformation Template for Windmill on AWS EKS + +## Overview + +This CloudFormation template automatically deploys Windmill on AWS EKS. The deployment includes: + +- An EKS cluster with configurable node types and sizes +- An RDS PostgreSQL database for Windmill data +- AWS Load Balancer Controller for handling ingress traffic +- Proper network configuration with VPC, subnets, and security groups +- A fully automated installation of Windmill via Helm + +## Parameters + +The template accepts various parameters to customize your deployment: + +- **NodeInstanceType**: EC2 instance type for EKS worker nodes (t3.small to r5.2xlarge) +- **NodeGroupSize**: Number of EKS worker nodes +- **RdsInstanceClass**: RDS instance class for the PostgreSQL database (db.t3.micro to db.r5.2xlarge) +- **DBPassword**: Password for the PostgreSQL database +- **WorkerReplicas**: Number of Windmill worker replicas +- **NativeWorkerReplicas**: Number of Windmill native worker replicas +- **Enterprise**: Enable Windmill [Enterprise features](https://www.windmill.dev/docs/misc/plans_details#upgrading-to-enterprise-edition) (requires license key) + +## Customization + +To modify the Helm chart configuration or update the template, refer to the official Windmill Helm chart repository: +[https://github.com/windmill-labs/windmill-helm-charts](https://github.com/windmill-labs/windmill-helm-charts) + +## Documentation + +For more information about Windmill's Helm chart deployment options, see: +[https://www.windmill.dev/docs/advanced/self_host#helm-chart](https://www.windmill.dev/docs/advanced/self_host#helm-chart) + +For detailed information about setting up RDS for Windmill on AWS: +[https://www.windmill.dev/docs/advanced/self_host/aws_ecs#create-a-rds-database](https://www.windmill.dev/docs/advanced/self_host/aws_ecs#create-a-rds-database) + +## Deployment + +1. Upload the CloudFormation template to your AWS account +2. Fill in the required parameters +3. Deploy the stack +4. Access Windmill using the URL provided in the Outputs section of the stack + +After deployment, you can access Windmill via the LoadBalancer URL shown in the CloudFormation stack outputs. diff --git a/examples/deploy/aws-eks-cloudformation/quicklaunch.yaml b/examples/deploy/aws-eks-cloudformation/quicklaunch.yaml new file mode 100644 index 0000000000..3dda495ebe --- /dev/null +++ b/examples/deploy/aws-eks-cloudformation/quicklaunch.yaml @@ -0,0 +1,688 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: Deploy Windmill on EKS with Helm + +Parameters: + NodeInstanceType: + Type: String + Default: t3.medium + AllowedValues: + - t3.small + - t3.medium + - t3.large + - t3.xlarge + - t3.2xlarge + - m5.large + - m5.xlarge + - m5.2xlarge + - m5.4xlarge + - c5.large + - c5.xlarge + - c5.2xlarge + - r5.large + - r5.xlarge + - r5.2xlarge + Description: EC2 instance type for the EKS worker nodes + NodeGroupSize: + Type: Number + Default: 2 + RdsInstanceClass: + Type: String + Default: db.t3.small + AllowedValues: + - db.t3.micro + - db.t3.small + - db.t3.medium + - db.t3.large + - db.t3.xlarge + - db.m5.large + - db.m5.xlarge + - db.m5.2xlarge + - db.r5.large + - db.r5.xlarge + - db.r5.2xlarge + Description: RDS instance class for the PostgreSQL database + DBPassword: + Type: String + NoEcho: true + WorkerReplicas: + Type: Number + Default: 2 + NativeWorkerReplicas: + Type: Number + Default: 1 + Enterprise: + Type: String + Default: false + AllowedValues: + - true + - false + Description: Enable Windmill Enterprise features (requires license key) + +Mappings: + RegionMap: + us-east-1: + AMI: ami-0cff7528ff583bf9a + us-east-2: + AMI: ami-0cd3c7f72edd5b06d + us-west-1: + AMI: ami-0d9858aa3c6322f73 + us-west-2: + AMI: ami-098e42ae54c764c35 + ca-central-1: + AMI: ami-00f881f027a6d74a0 + eu-west-1: + AMI: ami-04dd4500af104442f + eu-west-2: + AMI: ami-0eb260c4d5475b901 + eu-west-3: + AMI: ami-05e8e20cef0eaa9d0 + eu-central-1: + AMI: ami-0bad4a5e987bdebde + ap-northeast-1: + AMI: ami-0b7546e839d7ace12 + ap-northeast-2: + AMI: ami-0fd0765afb77bcca7 + ap-southeast-1: + AMI: ami-0c802847a7dd848c0 + ap-southeast-2: + AMI: ami-07620139298af599e + ap-south-1: + AMI: ami-0851b76e8b1bce90b + sa-east-1: + AMI: ami-054a31f1b3bf90920 + +Resources: + VPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: 10.0.0.0/16 + EnableDnsSupport: true + EnableDnsHostnames: true + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-vpc" + - Key: !Sub "kubernetes.io/cluster/${AWS::StackName}-cluster" + Value: shared + + InternetGateway: + Type: AWS::EC2::InternetGateway + + AttachGateway: + Type: AWS::EC2::VPCGatewayAttachment + Properties: + VpcId: !Ref VPC + InternetGatewayId: !Ref InternetGateway + + PublicSubnet1: + Type: AWS::EC2::Subnet + Properties: + VpcId: !Ref VPC + CidrBlock: 10.0.1.0/24 + AvailabilityZone: !Select [0, !GetAZs ""] + MapPublicIpOnLaunch: true + Tags: + - Key: kubernetes.io/role/elb + Value: "1" + - Key: !Sub "kubernetes.io/cluster/${AWS::StackName}-cluster" + Value: shared + - Key: Name + Value: !Sub ${AWS::StackName}-public-subnet-1 + + PublicSubnet2: + Type: AWS::EC2::Subnet + Properties: + VpcId: !Ref VPC + CidrBlock: 10.0.2.0/24 + AvailabilityZone: !Select [1, !GetAZs ""] + MapPublicIpOnLaunch: true + Tags: + - Key: kubernetes.io/role/elb + Value: "1" + - Key: !Sub "kubernetes.io/cluster/${AWS::StackName}-cluster" + Value: shared + - Key: Name + Value: !Sub ${AWS::StackName}-public-subnet-2 + + RouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: !Ref VPC + + PublicRoute: + Type: AWS::EC2::Route + DependsOn: AttachGateway + Properties: + RouteTableId: !Ref RouteTable + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: !Ref InternetGateway + + SubnetRouteTableAssociation1: + Type: AWS::EC2::SubnetRouteTableAssociation + DependsOn: PublicRoute + Properties: + SubnetId: !Ref PublicSubnet1 + RouteTableId: !Ref RouteTable + + SubnetRouteTableAssociation2: + Type: AWS::EC2::SubnetRouteTableAssociation + DependsOn: PublicRoute + Properties: + SubnetId: !Ref PublicSubnet2 + RouteTableId: !Ref RouteTable + + EKSClusterRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + - eks.amazonaws.com + - ec2.amazonaws.com + Action: + - sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/AmazonEKSClusterPolicy + - arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy + Policies: + - PolicyName: EKSAccess + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - eks:* + - ec2:DescribeInstances + - ec2:DescribeRouteTables + - ec2:DescribeSecurityGroups + - ec2:DescribeSubnets + - ec2:DescribeVpcs + - iam:GetRole + - iam:ListRoles + Resource: "*" + - Effect: Allow + Action: + - ssm:GetParameter + - ssm:PutParameter + Resource: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/${AWS::StackName}/*" + - PolicyName: KubernetesAccess + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - eks:DescribeCluster + - eks:ListClusters + - eks:AccessKubernetesApi + Resource: !Sub "arn:aws:eks:${AWS::Region}:${AWS::AccountId}:cluster/${AWS::StackName}-cluster" + + EKSNodeRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + - ec2.amazonaws.com + Action: + - sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy + - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly + - arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy + + EKSCluster: + Type: AWS::EKS::Cluster + Properties: + Name: !Sub "${AWS::StackName}-cluster" + RoleArn: !GetAtt EKSClusterRole.Arn + ResourcesVpcConfig: + SubnetIds: + - !Ref PublicSubnet1 + - !Ref PublicSubnet2 + EndpointPublicAccess: true + AccessConfig: + AuthenticationMode: API_AND_CONFIG_MAP + BootstrapClusterCreatorAdminPermissions: true + DependsOn: + - PublicRoute + - VPCCleanup + + EKSClusterAccess: + Type: AWS::EKS::AccessEntry + Properties: + ClusterName: !Ref EKSCluster + PrincipalArn: !GetAtt EKSClusterRole.Arn + Type: STANDARD + Username: admin + AccessPolicies: + - PolicyArn: arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy + AccessScope: + Type: cluster + + EKSNodeGroup: + Type: AWS::EKS::Nodegroup + DependsOn: + - WindmillDB + - EKSCluster + Properties: + ClusterName: !Ref EKSCluster + NodeRole: !GetAtt EKSNodeRole.Arn + Subnets: + - !Ref PublicSubnet1 + - !Ref PublicSubnet2 + ScalingConfig: + MinSize: 1 + DesiredSize: !Ref NodeGroupSize + MaxSize: 4 + InstanceTypes: + - !Ref NodeInstanceType + + WindmillDBSubnetGroup: + Type: AWS::RDS::DBSubnetGroup + Properties: + DBSubnetGroupDescription: Subnet group for RDS instance + SubnetIds: + - !Ref PublicSubnet1 + - !Ref PublicSubnet2 + + WindmillDBSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Allow PostgreSQL access from EKS nodes + VpcId: !Ref VPC + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 5432 + ToPort: 5432 + CidrIp: 10.0.0.0/16 + + WindmillDB: + Type: AWS::RDS::DBInstance + Properties: + DBInstanceIdentifier: !Sub "${AWS::StackName}-db" + AllocatedStorage: 20 + DBInstanceClass: !Ref RdsInstanceClass + Engine: postgres + EngineVersion: 17.2 + MasterUsername: postgres + MasterUserPassword: !Ref DBPassword + DBName: windmill + PubliclyAccessible: false + DBSubnetGroupName: !Ref WindmillDBSubnetGroup + VPCSecurityGroups: + - !Ref WindmillDBSecurityGroup + DependsOn: + - WindmillDBSubnetGroup + + WindmillInstallerInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Roles: + - !Ref EKSClusterRole + + WindmillInstallerSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Security group for Windmill installer instance + VpcId: !Ref VPC + SecurityGroupEgress: + - IpProtocol: -1 + FromPort: -1 + ToPort: -1 + CidrIp: 0.0.0.0/0 + + WindmillInstaller: + Type: AWS::EC2::Instance + CreationPolicy: + ResourceSignal: + Timeout: PT30M # Gives 30 minutes for the installation to complete + DependsOn: + - EKSNodeGroup + - WindmillDB + Properties: + ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] + InstanceType: t3.micro + IamInstanceProfile: !Ref WindmillInstallerInstanceProfile + SubnetId: !Ref PublicSubnet1 + SecurityGroupIds: + - !Ref WindmillInstallerSecurityGroup + UserData: + Fn::Base64: !Sub | + #!/bin/bash + set -e # Exit on any error + + # Install required tools + yum update -y + yum install -y aws-cli jq postgresql15 aws-cfn-bootstrap + + # Set up logging directory with correct permissions + mkdir -p /var/log/windmill-installer + touch /var/log/windmill-installer/install.log + + # Create installation directory + mkdir -p /opt/windmill-installer + + # Install kubectl + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x kubectl + mv kubectl /usr/local/bin/ + + # Install helm + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + chmod +x get_helm.sh + ./get_helm.sh + + # Create the installation script + cat << 'EOF' > /opt/windmill-installer/install.sh + #!/bin/bash + set -e + + # Configure kubectl + aws sts get-caller-identity > /dev/null + export AWS_SDK_LOAD_CONFIG=1 + export KUBECONFIG=/root/.kube/config + aws eks update-kubeconfig --name ${AWS::StackName}-cluster --region ${AWS::Region} + + # Add debugging for each kubectl attempt + echo "HOME: $HOME" + echo "KUBECONFIG: $KUBECONFIG" + echo "User: $(whoami)" + echo "AWS Identity: $(aws sts get-caller-identity)" + echo "Trying kubectl command..." + kubectl get nodes || echo "Command failed with status $?" + + echo "Waiting for EKS nodes to be ready..." + while true; do + # Force credential refresh on each attempt + aws sts get-caller-identity > /dev/null + aws eks update-kubeconfig --name ${AWS::StackName}-cluster --region ${AWS::Region} + + if kubectl get nodes &>/dev/null; then + READY_NODES=$(kubectl get nodes -o json | jq -r '.items[] | select(.status.conditions[] | select(.type=="Ready" and .status=="True")) | .metadata.name' | wc -l) + DESIRED_NODES=${NodeGroupSize} + if [ "$READY_NODES" -eq "$DESIRED_NODES" ]; then + echo "All nodes are ready" + break + fi + echo "Found $READY_NODES ready nodes out of $DESIRED_NODES desired nodes" + else + echo "Waiting for cluster access..." + fi + sleep 30 + done + + echo "Installing AWS Load Balancer Controller..." + # Install AWS Load Balancer Controller + helm repo add eks https://aws.github.io/eks-charts + helm repo update + + helm install aws-load-balancer-controller eks/aws-load-balancer-controller \ + -n kube-system \ + --set clusterName=${AWS::StackName}-cluster \ + --set region=${AWS::Region} \ + --set vpcId=${VPC} + + echo "Waiting for AWS Load Balancer Controller to be ready..." + kubectl wait --namespace kube-system \ + --for=condition=ready pod \ + --selector=app.kubernetes.io/name=aws-load-balancer-controller \ + --timeout=300s + + echo "Waiting for RDS to be available..." + while true; do + if pg_isready -h ${WindmillDB.Endpoint.Address} -p 5432 -U postgres 2>/dev/null; then + echo "Database is ready" + break + fi + echo "Database not ready yet..." + sleep 30 + done + + echo "Creating namespace and installing Windmill..." + kubectl create namespace windmill + + # Add helm repo and install Windmill + helm repo add windmill https://windmill-labs.github.io/windmill-helm-charts + helm repo update + + helm install ${AWS::StackName} windmill/windmill \ + --namespace windmill \ + --set windmill.databaseUrl="postgres://postgres:${DBPassword}@${WindmillDB.Endpoint.Address}/windmill?sslmode=require" \ + --set windmill.baseDomain=windmill.local \ + --set windmill.baseProtocol=http \ + --set windmill.appReplicas=${WorkerReplicas} \ + --set windmill.lspReplicas=2 \ + --set windmill.workerGroups[0].name=default \ + --set windmill.workerGroups[0].mode=worker \ + --set windmill.workerGroups[0].replicas=${WorkerReplicas} \ + --set windmill.workerGroups[1].name=native \ + --set windmill.workerGroups[1].mode=worker \ + --set windmill.workerGroups[1].replicas=${NativeWorkerReplicas} \ + --set windmill.app.service.spec.type=LoadBalancer \ + --set windmill.app.service.spec.sessionAffinity=None \ + --set windmill.app.service.port=8000 \ + --set windmill.app.service.ports[0].port=8000 \ + --set windmill.app.service.ports[0].targetPort=8000 \ + --set windmill.app.service.ports[0].protocol=TCP \ + --set postgresql.enabled=false \ + --set enterprise.enabled=${Enterprise} + + # Change service to LoadBalancer + echo "Changing service to LoadBalancer" + kubectl patch service windmill-app -n windmill -p '{"spec":{"type":"LoadBalancer","sessionAffinity":"None"}}' + kubectl patch service windmill-app -n windmill -p '{"spec":{"ports":[{"name":"api","port":8000,"targetPort":8000,"protocol":"TCP"},{"name":"http","port":80,"targetPort":8000,"protocol":"TCP"}]}}' + + # Wait for LoadBalancer to get an address + echo "Waiting for LoadBalancer address..." + while true; do + LB_HOSTNAME=$(kubectl get svc -n windmill windmill-app -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null) + if [ ! -z "$LB_HOSTNAME" ]; then + break + fi + echo "Waiting for LoadBalancer hostname..." + sleep 30 + done + + # Store the LoadBalancer hostname in SSM + aws ssm put-parameter \ + --region ${AWS::Region} \ + --name "/${AWS::StackName}/loadbalancer-hostname" \ + --value "$LB_HOSTNAME" \ + --type "String" \ + --overwrite + + # Signal CloudFormation that installation is complete + echo "Signal CloudFormation that installation is complete" + /opt/aws/bin/cfn-signal -e $? \ + --stack ${AWS::StackName} \ + --resource WindmillInstaller \ + --region ${AWS::Region} + + # Self-terminate this instance + echo "Self-terminating instance" + aws ec2 terminate-instances --instance-ids $(curl -s http://169.254.169.254/latest/meta-data/instance-id) --region ${AWS::Region} + EOF + + # Set permissions and run the installation script + chmod +x /opt/windmill-installer/install.sh + + # Run the installation script directly (not as ec2-user) + cd /opt/windmill-installer && ./install.sh > /var/log/windmill-installer/install.log 2>&1 + + LoadBalancerHostnameLookup: + Type: Custom::SSMParameterLookup + DependsOn: WindmillInstaller + Properties: + ServiceToken: !GetAtt LookupSSMParameterFunction.Arn + ParameterName: !Sub "/${AWS::StackName}/loadbalancer-hostname" + + LookupSSMParameterFunction: + Type: AWS::Lambda::Function + Properties: + Handler: index.handler + Role: !GetAtt LambdaExecutionRole.Arn + Timeout: 300 + Runtime: nodejs18.x + Code: + ZipFile: !Sub | + const { SSMClient, GetParameterCommand } = require('@aws-sdk/client-ssm'); + const response = require('cfn-response'); + + exports.handler = async (event, context) => { + if (event.RequestType === 'Delete') { + return response.send(event, context, response.SUCCESS); + } + + try { + const ssmClient = new SSMClient(); + + // Loop to check for the parameter until it's not "pending" + let tries = 0; + let paramValue = "pending"; + + while (paramValue === "pending" && tries < 20) { + const params = { + Name: event.ResourceProperties.ParameterName, + WithDecryption: false + }; + + const result = await ssmClient.send(new GetParameterCommand(params)); + paramValue = result.Parameter.Value; + + if (paramValue === "pending") { + await new Promise(resolve => setTimeout(resolve, 15000)); // wait 15 seconds + tries++; + } + } + + return response.send(event, context, response.SUCCESS, { + HostnameValue: paramValue + }); + } catch (error) { + console.error(error); + return response.send(event, context, response.FAILED, { error: error.message }); + } + }; + + LambdaExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + Policies: + - PolicyName: SSMParameterAccess + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - ssm:GetParameter + Resource: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/${AWS::StackName}/*" + + VPCCleanupFunction: + Type: AWS::Lambda::Function + Properties: + Handler: index.handler + Role: !GetAtt VPCCleanupRole.Arn + Timeout: 300 + Runtime: nodejs18.x + Code: + ZipFile: | + const { ElasticLoadBalancingClient, DescribeLoadBalancersCommand, + DeleteLoadBalancerCommand } = require('@aws-sdk/client-elastic-load-balancing'); + const response = require('cfn-response'); + + exports.handler = async (event, context) => { + if (event.RequestType !== 'Delete') { + return response.send(event, context, response.SUCCESS); + } + + try { + const elb = new ElasticLoadBalancingClient(); + const vpcId = event.ResourceProperties.VpcId; + + // Find and delete Classic Load Balancers in the VPC + const lbResponse = await elb.send(new DescribeLoadBalancersCommand({})); + let deleted = false; + + for (const lb of lbResponse.LoadBalancerDescriptions || []) { + if (lb.VPCId === vpcId) { + console.log(`Deleting Classic Load Balancer: ${lb.LoadBalancerName}`); + await elb.send(new DeleteLoadBalancerCommand({ + LoadBalancerName: lb.LoadBalancerName + })); + deleted = true; + } + } + + if (deleted) { + // Wait for deletion to complete + console.log('Waiting 30 seconds for load balancer deletion to complete...'); + await new Promise(r => setTimeout(r, 30000)); + } + + return response.send(event, context, response.SUCCESS); + } catch (error) { + console.error('Error deleting load balancers:', error); + return response.send(event, context, response.FAILED, {error: error.message}); + } + }; + + VPCCleanupRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + Policies: + - PolicyName: VPCCleanupPolicy + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - ec2:DescribeAddresses + - ec2:DisassociateAddress + - ec2:DescribeNetworkInterfaces + - elasticloadbalancing:DescribeLoadBalancers + - elasticloadbalancing:DeleteLoadBalancer + - elasticloadbalancingv2:DescribeLoadBalancers + - elasticloadbalancingv2:DeleteLoadBalancer + Resource: "*" + + VPCCleanup: + Type: Custom::VPCCleanup + Properties: + ServiceToken: !GetAtt VPCCleanupFunction.Arn + VpcId: !Ref VPC + +Outputs: + ClusterName: + Description: EKS cluster name + Value: !Sub "${AWS::StackName}-cluster" + + DatabaseEndpoint: + Description: RDS instance endpoint + Value: !GetAtt WindmillDB.Endpoint.Address + + LoadBalancerHostname: + Description: Windmill LoadBalancer hostname + Value: !Sub "http://${LoadBalancerHostnameLookup.HostnameValue}" diff --git a/flake.nix b/flake.nix index f821279959..447f498531 100644 --- a/flake.nix +++ b/flake.nix @@ -17,6 +17,7 @@ extensions = [ "rust-src" # for rust-analyzer "rust-analyzer" + "rustfmt" ]; }; buildInputs = with pkgs; [ @@ -94,6 +95,7 @@ oracle-instantclient # LSP/Local dev svelte-language-server + ansible taplo ]); packages = [ @@ -158,6 +160,7 @@ ]; inherit PKG_CONFIG_PATH RUSTY_V8_ARCHIVE; + GIT_PATH = "${pkgs.git}/bin/git"; NODE_ENV = "development"; NODE_OPTIONS = "--max-old-space-size=16384"; DATABASE_URL = "postgres://postgres:changeme@127.0.0.1:5432/"; @@ -172,13 +175,16 @@ JAVA_PATH = "${pkgs.jdk21}/bin/java"; JAVAC_PATH = "${pkgs.jdk21}/bin/javac"; COURSIER_PATH = "${coursier}/coursier"; - # for related places search: ADD_NEW_LANG + # for related places search: ADD_NEW_LANG FLOCK_PATH = "${pkgs.flock}/bin/flock"; CARGO_PATH = "${rust}/bin/cargo"; DOTNET_PATH = "${pkgs.dotnet-sdk_9}/bin/dotnet"; DOTNET_ROOT = "${pkgs.dotnet-sdk_9}/share/dotnet"; ORACLE_LIB_DIR = "${pkgs.oracle-instantclient.lib}/lib"; + ANSIBLE_PLAYBOOK_PATH = "${pkgs.ansible}/bin/ansible-playbook"; + ANSIBLE_GALAXY_PATH = "${pkgs.ansible}/bin/ansible-galaxy"; RUST_LOG = "debug"; + SQLX_OFFLINE = "true"; }; packages.default = self.packages.${system}.windmill; packages.windmill-client = pkgs.buildNpmPackage { diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 2cea18aab6..1fa3ad37d4 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.483.1", + "version": "1.490.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.483.1", + "version": "1.490.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -75,7 +75,7 @@ "windmill-parser-wasm-py": "^1.477.1", "windmill-parser-wasm-regex": "^1.481.0", "windmill-parser-wasm-rust": "^1.429.0", - "windmill-parser-wasm-ts": "^1.438.2", + "windmill-parser-wasm-ts": "^1.486.1", "windmill-parser-wasm-yaml": "^1.429.0", "windmill-sql-datatype-parser-wasm": "^1.318.0", "y-monaco": "^0.1.4", @@ -2309,10 +2309,6 @@ "ag-grid-community": "31.3.4" } }, - "node_modules/ag-grid-enterprise/node_modules/ag-charts-community": { - "version": "9.3.2", - "license": "MIT" - }, "node_modules/agentkeepalive": { "version": "4.6.0", "license": "MIT", @@ -4127,8 +4123,9 @@ } }, "node_modules/esrap": { - "version": "1.4.5", - "license": "MIT", + "version": "1.4.6", + "resolved": "https://registry.npmjs.org/esrap/-/esrap-1.4.6.tgz", + "integrity": "sha512-F/D2mADJ9SHY3IwksD4DAXjTt7qt7GWUf3/8RhCNWmC/67tyb55dpimHmy7EplakFaflV0R/PC+fdSPqrRHAQw==", "dependencies": { "@jridgewell/sourcemap-codec": "^1.4.15" } @@ -9716,8 +9713,9 @@ } }, "node_modules/svelte": { - "version": "5.22.6", - "license": "MIT", + "version": "5.28.2", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.28.2.tgz", + "integrity": "sha512-FbWBxgWOpQfhKvoGJv/TFwzqb4EhJbwCD17dB0tEpQiw1XyUEKZJtgm4nA4xq3LLsMo7hu5UY/BOFmroAxKTMg==", "dependencies": { "@ampproject/remapping": "^2.3.0", "@jridgewell/sourcemap-codec": "^1.5.0", @@ -9728,7 +9726,7 @@ "axobject-query": "^4.1.0", "clsx": "^2.1.1", "esm-env": "^1.2.1", - "esrap": "^1.4.3", + "esrap": "^1.4.6", "is-reference": "^3.0.3", "locate-character": "^3.0.0", "magic-string": "^0.30.11", @@ -10915,7 +10913,9 @@ "version": "1.429.0" }, "node_modules/windmill-parser-wasm-ts": { - "version": "1.438.2" + "version": "1.486.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.486.1.tgz", + "integrity": "sha512-nv7nPpZA5O0Zsve6W2Sm3mxCSxMqg2rfyYHNqozIpBaGhbe8SpgnbyrSaXKsGKSc/bpwpANHE7nVCp23pQUK6Q==" }, "node_modules/windmill-parser-wasm-yaml": { "version": "1.429.0" @@ -11222,8 +11222,6 @@ }, "node_modules/zod": { "version": "3.24.2", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.2.tgz", - "integrity": "sha512-lY7CDW43ECgW9u1TcT3IoXHflywfVqDYze4waEz812jR/bZ8FHDsl7pFQoSZTz5N+2NqRXs8GBwnAwo3ZNxqhQ==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/frontend/package.json b/frontend/package.json index 9ba7dd084d..fdfeda2e4c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.483.1", + "version": "1.490.0", "scripts": { "dev": "vite dev", "build": "vite build", @@ -144,7 +144,7 @@ "windmill-parser-wasm-py": "^1.477.1", "windmill-parser-wasm-regex": "^1.481.0", "windmill-parser-wasm-rust": "^1.429.0", - "windmill-parser-wasm-ts": "^1.438.2", + "windmill-parser-wasm-ts": "^1.486.1", "windmill-parser-wasm-yaml": "^1.429.0", "windmill-sql-datatype-parser-wasm": "^1.318.0", "y-monaco": "^0.1.4", diff --git a/frontend/src/lib/assets/app.css b/frontend/src/lib/assets/app.css index 6a386c559d..6e0623940d 100644 --- a/frontend/src/lib/assets/app.css +++ b/frontend/src/lib/assets/app.css @@ -4,10 +4,21 @@ @tailwind utilities; @layer base { - *, - ::before, - ::after { - @apply dark:border-gray-600; + + /* Light mode: default border color */ + .border, .border-t, .border-r, .border-b, .border-l, + .border-x, .border-y, + .divide-x > :not([hidden]) ~ :not([hidden]), + .divide-y > :not([hidden]) ~ :not([hidden]) { + border-color: #e5e7eb; /* gray-200 */ + } + + /* Dark mode: change border color */ + .dark .border, .dark .border-t, .dark .border-r, .dark .border-b, .dark .border-l, + .dark .border-x, .dark .border-y, + .dark .divide-x > :not([hidden]) ~ :not([hidden]), + .dark .divide-y > :not([hidden]) ~ :not([hidden]) { + border-color: #4b5563; /* gray-700 */ } /* Chrome, Edge, and Safari */ @@ -82,6 +93,22 @@ } } +.driver-popover-title { + @apply leading-6 !text-primary !text-base; +} + +.driver-popover-description { + @apply !text-secondary !text-sm; +} + +.driver-popover { + @apply p-6 !bg-surface !max-w-2xl; +} + +.panel-item { + @apply border dark:border-gray-600 border-gray-200 flex gap-1 truncate font-normal justify-between w-full items-center py-1 px-2 rounded-sm duration-200; +} + .splitpanes--vertical > .splitpanes__pane { transition: none !important; } @@ -165,3 +192,4 @@ svelte-virtual-list-contents > * + * { rgba(0, 0, 192, 0.6) 20px ); } + diff --git a/frontend/src/lib/components/ApiConnectForm.svelte b/frontend/src/lib/components/ApiConnectForm.svelte index 9009899aec..53a6bd278e 100644 --- a/frontend/src/lib/components/ApiConnectForm.svelte +++ b/frontend/src/lib/components/ApiConnectForm.svelte @@ -153,9 +153,7 @@ viewJsonSchema = false try { schema = resourceTypeInfo.schema as any - schema.order = schema.order ?? Object.keys(schema.properties).sort() - notFound = false } catch (e) { notFound = true diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index dbbf7e9e89..180e0e57e9 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -126,15 +126,18 @@ oneOf.length >= 2 && (!oneOfSelected || !oneOf.some((o) => o.title === oneOfSelected) || !value) ) { - if (value && value['label'] && oneOf.some((o) => o.title === value['label'])) { + const tagKey = oneOf.find((o) => Object.keys(o.properties ?? {}).includes('kind')) + ? 'kind' + : 'label' + if (value && value[tagKey] && oneOf.some((o) => o.title === value[tagKey])) { const existingValue = JSON.parse(JSON.stringify(value)) - oneOfSelected = value['label'] + oneOfSelected = value[tagKey] await tick() value = existingValue } else { - const label = oneOf[0]['title'] - oneOfSelected = label - value = { ...(typeof value === 'object' ? (value ?? {}) : {}), label } + const variantTitle = oneOf[0]['title'] + oneOfSelected = variantTitle + value = { ...(typeof value === 'object' ? (value ?? {}) : {}), [tagKey]: variantTitle } } } } @@ -145,8 +148,11 @@ function onOneOfChange() { const label = value?.['label'] + const kind = value?.['kind'] if (label && oneOf && oneOf.some((o) => o.title == label) && oneOfSelected != label) { oneOfSelected = label + } else if (kind && oneOf && oneOf.some((o) => o.title == kind) && oneOfSelected != kind) { + oneOfSelected = kind } } @@ -820,7 +826,10 @@ for (const key of prevValueKeys) { toKeep[key] = value[key] } - value = { ...toKeep, label: detail } + const tagKey = oneOf.find((o) => Object.keys(o.properties ?? {}).includes('kind')) + ? 'kind' + : 'label' + value = { ...toKeep, [tagKey]: detail } }} let:item > @@ -849,7 +858,7 @@ }} bind:args={value} dndType={`nested-${title}`} - hiddenArgs={['label']} + hiddenArgs={['label', 'kind']} on:reorder={(e) => { if (oneOf && oneOf[objIdx]) { const keys = e.detail @@ -868,7 +877,7 @@ {onlyMaskPassword} {disablePortal} {disabled} - hiddenArgs={['label']} + hiddenArgs={['label', 'kind']} schema={{ properties: obj.properties, order: obj.order, diff --git a/frontend/src/lib/components/BoundedInputNumber b/frontend/src/lib/components/BoundedInputNumber deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frontend/src/lib/components/ChannelSelector.svelte b/frontend/src/lib/components/ChannelSelector.svelte new file mode 100644 index 0000000000..f7db63ddc2 --- /dev/null +++ b/frontend/src/lib/components/ChannelSelector.svelte @@ -0,0 +1,86 @@ + + +
+
+
+ - {#if !isFetching} - {#if teams.length === 0} - - {:else} - - {#each teams as team} - - {/each} - {/if} - {:else} - - {/if} - -
-
- -
+
+
+ {#if platform === 'teams'} + + + {#if $enterpriseLicense} + sendUserToast('Failed to load teams: ' + e.detail.message, true)} + /> + {/if} + {:else} + {/if} - {:else} - - {/if} - Not connected + Not connected +
{/if} diff --git a/frontend/src/lib/components/DBManager.svelte b/frontend/src/lib/components/DBManager.svelte new file mode 100644 index 0000000000..cfca62c71b --- /dev/null +++ b/frontend/src/lib/components/DBManager.svelte @@ -0,0 +1,206 @@ + + + + +
+ {#if dbSupportsSchemas} + + {/if} + +
+
+ {#each filteredTableKeys as tableKey} + + {/each} +
+ + + + {#if tableKey} + {#await getColDefs(tableKey) then colDefs} + {#if colDefs && colDefs?.length} + {@const dbTableOps = dbTableOpsFactory({ colDefs, tableKey })} + + {/if} + {/await} + {/if} + + + + (askingForConfirmation = undefined)} + on:confirmed={askingForConfirmation?.onConfirm ?? (() => {})} +/> + +{#if dbTableEditorProps} + (dbTableEditorState = { open: false })} + > + (dbTableEditorState = { open: false })} + title="Create a new table" + > + { + await dbTableEditorProps.onConfirm(values) + dbTableEditorState = { open: false } + }} + /> + + +{/if} diff --git a/frontend/src/lib/components/DBManagerDrawerButton.svelte b/frontend/src/lib/components/DBManagerDrawerButton.svelte new file mode 100644 index 0000000000..1120f4211f --- /dev/null +++ b/frontend/src/lib/components/DBManagerDrawerButton.svelte @@ -0,0 +1,272 @@ + + + { + if (e.key === 'Escape') { + if (replResultData) { + replResultData = undefined + } + } + }} +/> + +{#if shouldDisplayError} + + Schema could not be loaded. Please check the permissions of the resource. + +{:else} + + + { + if (replResultData) { + replResultData = undefined + } else { + isDrawerOpen = false + } + }} + CloseIcon={replResultData ? ArrowLeft : undefined} + noPadding + > + {#if dbSchema && $workspaceStore} + + + + +
{ + // Only proceed if the click is directly on this div and not on the child elements + if (e.target === e.currentTarget) { + replResultData = undefined + } + }} + > + {#if replResultData} + {#key replResultData} + + {/key} + {/if} +
+ + dbTableOpsWithPreviewScripts({ + colDefs, + tableKey, + resourcePath, + resourceType, + workspace: $workspaceStore + })} + dbTableActionsFactory={[ + dbDeleteTableActionWithPreviewScript({ + resourcePath, + resourceType, + workspace: $workspaceStore + }) + ]} + {refresh} + dbTableEditorPropsFactory={({ selectedSchemaKey }) => ({ + resourceType, + previewSql: (values) => + makeCreateTableQuery(values, resourceType, selectedSchemaKey), + async onConfirm(values) { + await runPreviewJobAndPollResult({ + workspace: $workspaceStore, + requestBody: { + args: { database: '$res:' + resourcePath }, + content: makeCreateTableQuery(values, resourceType, selectedSchemaKey), + language: getLanguageByResourceType(resourceType) + } + }) + refresh() + } + })} + /> + + + { + replResultData = data + }} + placeholderTableName={sortArray( + Object.keys( + dbSchema?.schema[ + 'public' in dbSchema?.schema + ? 'public' + : 'dbo' in dbSchema?.schema + ? 'dbo' + : Object.keys(dbSchema?.schema)?.[0] + ] + ) + )?.[0]} + /> + + + {:else} + + + + + + {/if} + + + + - - - - - - {#if dbSchema.lang !== 'graphql' && (dbSchema.schema?.public || dbSchema.schema?.PUBLIC || dbSchema.schema?.dbo)} - - - - - {/if} - {#if dbSchema.lang === 'graphql'} - {#await import('$lib/components/GraphqlSchemaViewer.svelte')} - - {:then Module} - - {/await} - {:else} - - {/if} - - -{:else if shouldDisplayError} - - Schema could not be loaded. Please check the permissions of the resource. - + + + +{/if} +{#if dbSchema.lang === 'graphql'} + {#await import('$lib/components/GraphqlSchemaViewer.svelte')} + + {:then Module} + + {/await} +{:else} + {/if} diff --git a/frontend/src/lib/components/DBTable.svelte b/frontend/src/lib/components/DBTable.svelte new file mode 100644 index 0000000000..337f66b0fd --- /dev/null +++ b/frontend/src/lib/components/DBTable.svelte @@ -0,0 +1,206 @@ + + + + +
+
+ + {#if dbTableOps.onInsert} + { + if (!$workspaceStore) return + dbTableOps.onInsert?.({ values }).then((result) => { + refresh?.() + sendUserToast('Row inserted') + }) + }} + /> + {/if} +
+
+
+ +
{ + if ((e.ctrlKey || e.metaKey) && e.key === 'c') { + const selectedCell = api?.getFocusedCell() + if (selectedCell) { + const rowIndex = selectedCell.rowIndex + const colId = selectedCell.column?.getId() + const rowNode = api?.getDisplayedRowAtIndex(rowIndex) + const selectedValue = rowNode?.data?.[colId] + navigator.clipboard.writeText(selectedValue) + sendUserToast('Copied cell value to clipboard', false) + } + } + }} + >
+
+ +
+
+ + Download +
+ {#if rowCount} + {firstRow}{'->'}{lastRow + 1} of {rowCount} rows + {:else} + {firstRow}{'->'}{lastRow + 1} + {/if} +
+
+
diff --git a/frontend/src/lib/components/DBTableEditor.svelte b/frontend/src/lib/components/DBTableEditor.svelte new file mode 100644 index 0000000000..8381b60c45 --- /dev/null +++ b/frontend/src/lib/components/DBTableEditor.svelte @@ -0,0 +1,435 @@ + + + + + + +
+
+ + +
+ + + + + + Name + Type + Primary + + + + {#each values.columns as column, i} + + + + + + + + {#snippet trigger()} + + {/snippet} + {#snippet content()} + {#if datatypeHasLength(column.datatype)} + + {/if} + + {#if !column.primaryKey} + + {/if} + {/snippet} + + + + + + +
+
+ + + + + + Table + Columns + + + + {#each values.foreignKeys as foreignKey, foreignKeyIndex} + {@const fkErrors = errors?.foreignKeys?.[foreignKeyIndex]} + + + (column.sourceColumn = e.detail.value)} + items={values.columns.map((c) => c.name)} + clearable={false} + /> +
+ +
+ + + + + + + + ON UPDATE + + {/snippet} + + {/if} +
+
+ {/each} + +
+ + {/each} + + + + + + + +
+
+ + + + (askingForConfirmation = undefined)} + on:confirmed={askingForConfirmation?.onConfirm ?? (() => {})} +> + {#if askingForConfirmation?.codeContent} +
+ + {askingForConfirmation.codeContent} + + +
+ {/if} + diff --git a/frontend/src/lib/components/DateInput.svelte b/frontend/src/lib/components/DateInput.svelte index dec411f6cc..c39e6149e3 100644 --- a/frontend/src/lib/components/DateInput.svelte +++ b/frontend/src/lib/components/DateInput.svelte @@ -10,12 +10,13 @@ export let dateFormat: string | undefined = 'dd-MM-yyyy' export let disabled: boolean = false + const defaultDateFormat = 'dd-MM-yyyy' + const defaultHtmlDateFormat = 'yyyy-MM-dd' + let date: string | undefined = computeDate(value) const dispatch = createEventDispatcher() - const defaultDateFormat = 'dd-MM-yyyy' - const defaultHtmlDateFormat = 'yyyy-MM-dd' function computeDate(value: string | undefined) { if (dateFormat === undefined) { dateFormat = defaultDateFormat diff --git a/frontend/src/lib/components/DefaultTagsInner.svelte b/frontend/src/lib/components/DefaultTagsInner.svelte index bc2fc14fba..5e4e824787 100644 --- a/frontend/src/lib/components/DefaultTagsInner.svelte +++ b/frontend/src/lib/components/DefaultTagsInner.svelte @@ -103,7 +103,9 @@ }} disabled={!$enterpriseLicense || !$superadmin} > - Save {#if !$superadmin} superadmin only {/if} + Save {#if !$superadmin} + superadmin only + {/if} {#if workspaceConnectedToTeams} -
-
- +
+
+
+ +
+

Teams Channel

-

Teams Channel

-
- -
-
- + +
+ (handlerExtraArgs['channel'] = e.detail.channel_id)} + selectedChannel={handlerExtraArgs['channel'] + ? (teams_channels.find((ch) => ch.channel_id === handlerExtraArgs['channel']) as any) + : undefined} + /> +
+ +

- This workspace is connected to Team: {teams_team_name} + This workspace is connected to Team: {teams_team_name}

Each workspace can only be connected to one Microsoft Teams team. You can configure it under sendTeamsMessage(handlerExtraArgs['channel'] ?? '')} size="xs">Send test message {#if connectionTestJob.in_progress} - + {:else if connectionTestJob.is_success} {:else} diff --git a/frontend/src/lib/components/FlowBuilderTutorials.svelte b/frontend/src/lib/components/FlowBuilderTutorials.svelte index 3f483a4ff9..f482081157 100644 --- a/frontend/src/lib/components/FlowBuilderTutorials.svelte +++ b/frontend/src/lib/components/FlowBuilderTutorials.svelte @@ -81,17 +81,3 @@ You need to create a new flow before starting the tutorial.
- - diff --git a/frontend/src/lib/components/FlowHistoryJobPicker.svelte b/frontend/src/lib/components/FlowHistoryJobPicker.svelte index 5a9a661a74..293a4af24c 100644 --- a/frontend/src/lib/components/FlowHistoryJobPicker.svelte +++ b/frontend/src/lib/components/FlowHistoryJobPicker.svelte @@ -8,6 +8,7 @@ export let path: string export let selected: string | undefined = undefined + export let selectInitial: boolean = false const dispatch = createEventDispatcher() async function loadInitial() { @@ -19,7 +20,9 @@ perPage: 1 }) if (jobs.length > 0) { - dispatch('select', { jobId: jobs[0].id, initial: true }) + if (selectInitial) { + dispatch('select', { jobId: jobs[0].id, initial: true }) + } } else { dispatch('nohistory') } diff --git a/frontend/src/lib/components/FlowMetadata.svelte b/frontend/src/lib/components/FlowMetadata.svelte index 3ab5d3d416..ef090c9afb 100644 --- a/frontend/src/lib/components/FlowMetadata.svelte +++ b/frontend/src/lib/components/FlowMetadata.svelte @@ -3,7 +3,7 @@ import { base } from '$lib/base' import JobStatus from '$lib/components/JobStatus.svelte' import { displayDate, truncateRev } from '$lib/utils' - import ScheduleEditor from './ScheduleEditor.svelte' + import ScheduleEditor from '$lib/components/triggers/schedules/ScheduleEditor.svelte' import TimeAgo from './TimeAgo.svelte' import { workspaceStore } from '$lib/stores' import Tooltip from './Tooltip.svelte' diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index 448e6e89b4..23efc93695 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -19,6 +19,8 @@ import JsonInputs from './JsonInputs.svelte' import FlowHistoryJobPicker from './FlowHistoryJobPicker.svelte' import { NEVER_TESTED_THIS_FAR } from './flows/models' + import { writable, type Writable } from 'svelte/store' + import type { DurationStatus, GraphModuleState } from './graph' export let previewMode: 'upTo' | 'whole' export let open: boolean @@ -26,12 +28,21 @@ export let jobId: string | undefined = undefined export let job: Job | undefined = undefined - let selectedJobStep: string | undefined = undefined - let branchOrIterationN: number = 0 - let restartBranchNames: [number, string][] = [] + export let initial: boolean = false - let selectedJobStepIsTopLevel: boolean | undefined = undefined - let selectedJobStepType: 'single' | 'forloop' | 'branchall' = 'single' + export let selectedJobStep: string | undefined = undefined + export let selectedJobStepIsTopLevel: boolean | undefined = undefined + export let selectedJobStepType: 'single' | 'forloop' | 'branchall' = 'single' + export let rightColumnSelect: 'timeline' | 'node_status' | 'node_definition' | 'user_states' = + 'timeline' + + export let branchOrIterationN: number = 0 + export let scrollTop: number = 0 + + export let localModuleStates: Writable> = writable({}) + export let localDurationStatuses: Writable> = writable({}) + + let restartBranchNames: [number, string][] = [] let isRunning: boolean = false let jobProgressReset: () => void @@ -59,7 +70,6 @@ const dispatch = createEventDispatcher() let renderCount: number = 0 - let initial: boolean = false let schemaFormWithArgPicker: SchemaFormWithArgPicker | undefined = undefined let currentJobId: string | undefined = undefined @@ -210,6 +220,19 @@ } }) } + + let scrollableDiv: HTMLDivElement | undefined = undefined + function handleScroll() { + scrollTop = scrollableDiv?.scrollTop ?? 0 + } + + $: scrollableDiv && onScrollableDivChange() + + function onScrollableDivChange() { + if (scrollTop != 0 && scrollableDiv) { + scrollableDiv.scrollTop = scrollTop + } + } @@ -375,7 +398,11 @@
-
+
handleScroll()} + >
{ loadIndividualStepsStates() }} @@ -484,6 +512,9 @@
{/if} { $executionCount = $executionCount + 1 }} - on:jobsLoaded={({ detail }) => { - job = detail + on:jobsLoaded={() => { if (initial) { console.log('loading initial steps after initial job loaded') loadIndividualStepsStates() } }} bind:selectedJobStep + bind:rightColumnSelect /> {:else}
Flow status will be displayed here
diff --git a/frontend/src/lib/components/FlowStatusViewer.svelte b/frontend/src/lib/components/FlowStatusViewer.svelte index b6eb933af7..ff58e71c90 100644 --- a/frontend/src/lib/components/FlowStatusViewer.svelte +++ b/frontend/src/lib/components/FlowStatusViewer.svelte @@ -3,7 +3,7 @@ import FlowStatusViewerInner from './FlowStatusViewerInner.svelte' import type { FlowState } from './flows/flowState' import { createEventDispatcher, setContext } from 'svelte' - import type { FlowStatusViewerContext } from './graph' + import type { DurationStatus, FlowStatusViewerContext, GraphModuleState } from './graph' import { isOwner as loadIsOwner } from '$lib/utils' import { userStore, workspaceStore } from '$lib/stores' import type { Job } from '$lib/gen' @@ -19,9 +19,13 @@ export let hideNodeDefinition = false export let hideJobId = false export let hideDownloadLogs = false - + export let rightColumnSelect: 'timeline' | 'node_status' | 'node_definition' | 'user_states' = + 'timeline' export let isOwner = false export let wideResults = false + export let localModuleStates: Writable> = writable({}) + export let localDurationStatuses: Writable> = writable({}) + export let job: Job | undefined = undefined let lastJobId: string = jobId @@ -67,14 +71,18 @@ } dispatch('jobsLoaded', job) }} - globalDurationStatuses={[]} globalModuleStates={[]} + globalDurationStatuses={[]} + bind:localModuleStates + bind:localDurationStatuses bind:selectedNode={selectedJobStep} on:start on:done + bind:job {initialJob} {jobId} {workspaceId} {isOwner} {wideResults} + bind:rightColumnSelect /> diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 4e64417e5a..d25bf607da 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -13,7 +13,7 @@ import FlowPreviewStatus from './preview/FlowPreviewStatus.svelte' import { createEventDispatcher, getContext, tick } from 'svelte' import { onDestroy } from 'svelte' - import { Badge, Button, Tab } from './common' + import { Badge, Button, Skeleton, Tab } from './common' import DisplayResult from './DisplayResult.svelte' import Tabs from './common/tabs/Tabs.svelte' import { type DurationStatus, type FlowStatusViewerContext, type GraphModuleState } from './graph' @@ -81,7 +81,11 @@ export let isForloopSelected = false export let parentRecursiveRefresh: Record Promise> = {} export let job: Job | undefined = undefined + export let rightColumnSelect: 'timeline' | 'node_status' | 'node_definition' | 'user_states' = + 'timeline' + export let localModuleStates: Writable> = writable({}) + export let localDurationStatuses: Writable> = writable({}) let recursiveRefresh: Record Promise> = {} let jobResults: any[] = @@ -90,12 +94,12 @@ let retry_selected = '' let timeout: NodeJS.Timeout | undefined = undefined - let localModuleStates: Writable> = writable({}) - let localDurationStatuses: Writable> = writable({}) let expandedSubflows: Record = {} $: flowJobIds?.moduleId && onFlowModuleId() + let selectedId: Writable = writable(selectedNode) + function onFlowModuleId() { if (globalRefreshes) { let modId = flowJobIds?.moduleId @@ -491,6 +495,7 @@ async function updateJobId() { if (jobId !== job?.id) { + console.log('updating job id', globalDurationStatuses.length) $localModuleStates = {} flowTimeline?.reset() timeout && clearTimeout(timeout) @@ -793,8 +798,6 @@ let flowTimeline: FlowTimeline - let rightColumnSelect: 'timeline' | 'node_status' | 'node_definition' | 'user_states' = 'timeline' - function loadPreviousIters(lenToAdd: number) { let r = $localDurationStatuses[flowJobIds?.moduleId ?? ''] if (r.iteration_from) { @@ -1244,6 +1247,7 @@
+ {/if} diff --git a/frontend/src/lib/components/apps/editor/AppJobsDrawer.svelte b/frontend/src/lib/components/apps/editor/AppJobsDrawer.svelte index 99ab1b743e..aef77f3f51 100644 --- a/frontend/src/lib/components/apps/editor/AppJobsDrawer.svelte +++ b/frontend/src/lib/components/apps/editor/AppJobsDrawer.svelte @@ -241,7 +241,7 @@ {/if} {:else} -
+
{#if job?.id} diff --git a/frontend/src/lib/components/apps/editor/SettingsPanel.svelte b/frontend/src/lib/components/apps/editor/SettingsPanel.svelte index ae4143dd34..b2e0a4b52e 100644 --- a/frontend/src/lib/components/apps/editor/SettingsPanel.svelte +++ b/frontend/src/lib/components/apps/editor/SettingsPanel.svelte @@ -293,7 +293,7 @@ tooltip="This event is triggered when the script runs successfully." items={Object.keys($runnableComponents).filter((_id) => _id !== id)} bind:value={ - () => hiddenInlineScript.script.recomputeIds, + () => hiddenInlineScript.script.recomputeIds ?? [], (v) => { if ($app.hiddenInlineScripts[hiddenInlineScript.index]) { $app.hiddenInlineScripts[hiddenInlineScript.index].recomputeIds = v diff --git a/frontend/src/lib/components/apps/editor/component/ComponentInner.svelte b/frontend/src/lib/components/apps/editor/component/ComponentInner.svelte index acbe2d5b83..6f36d786cb 100644 --- a/frontend/src/lib/components/apps/editor/component/ComponentInner.svelte +++ b/frontend/src/lib/components/apps/editor/component/ComponentInner.svelte @@ -73,6 +73,7 @@ import AppRecomputeAll from '../../components/display/AppRecomputeAll.svelte' import AppUserResource from '../../components/inputs/AppUserResource.svelte' import type { AppComponent } from './components' + import { Button } from '$lib/components/common' export let component: AppComponent export let render: boolean @@ -82,700 +83,722 @@ export let initializing: boolean | undefined = undefined -{#if component.type === 'displaycomponent'} - -{:else if component.type === 'logcomponent'} - -{:else if component.type === 'jobidlogcomponent'} - -{:else if component.type === 'flowstatuscomponent'} - -{:else if component.type === 'jobidflowstatuscomponent'} - -{:else if component.type === 'barchartcomponent'} - -{:else if component.type === 'timeseriescomponent'} - -{:else if component.type === 'htmlcomponent'} - -{:else if component.type === 'customcomponent'} - -{:else if component.type === 'mardowncomponent'} - -{:else if component.type === 'vegalitecomponent'} - -{:else if component.type === 'plotlycomponent'} - -{:else if component.type === 'plotlycomponentv2'} - -{:else if component.type === 'scatterchartcomponent'} - -{:else if component.type === 'piechartcomponent'} - -{:else if component.type === 'agchartscomponent'} - -{:else if component.type === 'agchartscomponentee'} - -{:else if component.type === 'tablecomponent'} - -{:else if component.type === 'dbexplorercomponent'} - -{:else if component.type === 'aggridcomponent'} - -{:else if component.type === 'aggridcomponentee'} - -{:else if component.type === 'aggridinfinitecomponent'} - -{:else if component.type === 'aggridinfinitecomponentee'} - -{:else if component.type === 'textcomponent'} - -{:else if component.type === 'codeinputcomponent'} - -{:else if component.type === 'buttoncomponent'} - -{:else if component.type === 'downloadcomponent'} - -{:else if component.type === 'selectcomponent' || component.type === 'resourceselectcomponent'} - -{:else if component.type === 'userresourcecomponent'} - -{:else if component.type === 'multiselectcomponent'} - -{:else if component.type === 'multiselectcomponentv2'} - -{:else if component.type === 'formcomponent'} - -{:else if component.type === 'formbuttoncomponent'} - -{:else if component.type === 'checkboxcomponent'} - -{:else if component.type === 'textinputcomponent'} - -{:else if component.type === 'quillcomponent'} - -{:else if component.type === 'textareainputcomponent'} - -{:else if component.type === 'emailinputcomponent'} - -{:else if component.type === 'passwordinputcomponent'} - -{:else if component.type === 'dateinputcomponent'} - -{:else if component.type === 'timeinputcomponent'} - -{:else if component.type === 'datetimeinputcomponent'} - -{:else if component.type === 'numberinputcomponent'} - -{:else if component.type === 'currencycomponent'} - -{:else if component.type === 'slidercomponent'} - -{:else if component.type === 'dateslidercomponent'} - -{:else if component.type === 'horizontaldividercomponent'} - -{:else if component.type === 'verticaldividercomponent'} - -{:else if component.type === 'rangecomponent'} - -{:else if component.type === 'tabscomponent' && component.tabs} - -{:else if component.type === 'steppercomponent' && component.tabs} - -{:else if component.type === 'conditionalwrapper' && component.conditions} - -{:else if component.type === 'containercomponent'} - -{:else if component.type === 'listcomponent'} - -{:else if component.type === 'verticalsplitpanescomponent'} - -{:else if component.type === 'horizontalsplitpanescomponent'} - -{:else if component.type === 'iconcomponent'} - -{:else if component.type === 'fileinputcomponent'} - -{:else if component.type === 's3fileinputcomponent'} - -{:else if component.type === 'imagecomponent'} - -{:else if component.type === 'drawercomponent'} - -{:else if component.type === 'mapcomponent'} - -{:else if component.type === 'pdfcomponent'} - -{:else if component.type === 'modalcomponent'} - -{:else if component.type === 'schemaformcomponent'} - -{:else if component.type === 'selecttabcomponent'} - -{:else if component.type === 'selectstepcomponent'} - -{:else if component.type === 'chartjscomponent'} - -{:else if component.type === 'chartjscomponentv2'} - -{:else if component.type === 'carousellistcomponent'} - -{:else if component.type === 'accordionlistcomponent'} - -{:else if component.type === 'statcomponent'} - -{:else if component.type === 'menucomponent'} - -{:else if component.type === 'decisiontreecomponent' && component.nodes} - -{:else if component.type === 'alertcomponent'} - -{:else if component.type === 'navbarcomponent'} - -{:else if component.type === 'dateselectcomponent'} - -{:else if component.type === 'jobiddisplaycomponent'} - -{:else if component.type === 'recomputeallcomponent'} - -{/if} + + {#if component.type === 'displaycomponent'} + + {:else if component.type === 'logcomponent'} + + {:else if component.type === 'jobidlogcomponent'} + + {:else if component.type === 'flowstatuscomponent'} + + {:else if component.type === 'jobidflowstatuscomponent'} + + {:else if component.type === 'barchartcomponent'} + + {:else if component.type === 'timeseriescomponent'} + + {:else if component.type === 'htmlcomponent'} + + {:else if component.type === 'customcomponent'} + + {:else if component.type === 'mardowncomponent'} + + {:else if component.type === 'vegalitecomponent'} + + {:else if component.type === 'plotlycomponent'} + + {:else if component.type === 'plotlycomponentv2'} + + {:else if component.type === 'scatterchartcomponent'} + + {:else if component.type === 'piechartcomponent'} + + {:else if component.type === 'agchartscomponent'} + + {:else if component.type === 'agchartscomponentee'} + + {:else if component.type === 'tablecomponent'} + + {:else if component.type === 'dbexplorercomponent'} + + {:else if component.type === 'aggridcomponent'} + + {:else if component.type === 'aggridcomponentee'} + + {:else if component.type === 'aggridinfinitecomponent'} + + {:else if component.type === 'aggridinfinitecomponentee'} + + {:else if component.type === 'textcomponent'} + + {:else if component.type === 'codeinputcomponent'} + + {:else if component.type === 'buttoncomponent'} + + {:else if component.type === 'downloadcomponent'} + + {:else if component.type === 'selectcomponent' || component.type === 'resourceselectcomponent'} + + {:else if component.type === 'userresourcecomponent'} + + {:else if component.type === 'multiselectcomponent'} + + {:else if component.type === 'multiselectcomponentv2'} + + {:else if component.type === 'formcomponent'} + + {:else if component.type === 'formbuttoncomponent'} + + {:else if component.type === 'checkboxcomponent'} + + {:else if component.type === 'textinputcomponent'} + + {:else if component.type === 'quillcomponent'} + + {:else if component.type === 'textareainputcomponent'} + + {:else if component.type === 'emailinputcomponent'} + + {:else if component.type === 'passwordinputcomponent'} + + {:else if component.type === 'dateinputcomponent'} + + {:else if component.type === 'timeinputcomponent'} + + {:else if component.type === 'datetimeinputcomponent'} + + {:else if component.type === 'numberinputcomponent'} + + {:else if component.type === 'currencycomponent'} + + {:else if component.type === 'slidercomponent'} + + {:else if component.type === 'dateslidercomponent'} + + {:else if component.type === 'horizontaldividercomponent'} + + {:else if component.type === 'verticaldividercomponent'} + + {:else if component.type === 'rangecomponent'} + + {:else if component.type === 'tabscomponent' && component.tabs} + + {:else if component.type === 'steppercomponent' && component.tabs} + + {:else if component.type === 'conditionalwrapper' && component.conditions} + + {:else if component.type === 'containercomponent'} + + {:else if component.type === 'listcomponent'} + + {:else if component.type === 'verticalsplitpanescomponent'} + + {:else if component.type === 'horizontalsplitpanescomponent'} + + {:else if component.type === 'iconcomponent'} + + {:else if component.type === 'fileinputcomponent'} + + {:else if component.type === 's3fileinputcomponent'} + + {:else if component.type === 'imagecomponent'} + + {:else if component.type === 'drawercomponent'} + + {:else if component.type === 'mapcomponent'} + + {:else if component.type === 'pdfcomponent'} + + {:else if component.type === 'modalcomponent'} + + {:else if component.type === 'schemaformcomponent'} + + {:else if component.type === 'selecttabcomponent'} + + {:else if component.type === 'selectstepcomponent'} + + {:else if component.type === 'chartjscomponent'} + + {:else if component.type === 'chartjscomponentv2'} + + {:else if component.type === 'carousellistcomponent'} + + {:else if component.type === 'accordionlistcomponent'} + + {:else if component.type === 'statcomponent'} + + {:else if component.type === 'menucomponent'} + + {:else if component.type === 'decisiontreecomponent' && component.nodes} + + {:else if component.type === 'alertcomponent'} + + {:else if component.type === 'navbarcomponent'} + + {:else if component.type === 'dateselectcomponent'} + + {:else if component.type === 'jobiddisplaycomponent'} + + {:else if component.type === 'recomputeallcomponent'} + + {/if} + {#snippet failed(error, reset)} +
+

Rendering of component failed

+ 
{error}
+
+ +
+
+ {/snippet} + diff --git a/frontend/src/lib/components/apps/editor/component/components.ts b/frontend/src/lib/components/apps/editor/component/components.ts index a0b99e4805..2ff364f44a 100644 --- a/frontend/src/lib/components/apps/editor/component/components.ts +++ b/frontend/src/lib/components/apps/editor/component/components.ts @@ -91,21 +91,51 @@ export type CustomComponentConfig = { reactVersion?: string } } -export type TextComponent = BaseComponent<'textcomponent'> -export type TextInputComponent = BaseComponent<'textinputcomponent'> -export type QuillComponent = BaseComponent<'quillcomponent'> -export type CodeInputComponent = BaseComponent<'codeinputcomponent'> -export type TextareaInputComponent = BaseComponent<'textareainputcomponent'> -export type PasswordInputComponent = BaseComponent<'passwordinputcomponent'> -export type EmailInputComponent = BaseComponent<'emailinputcomponent'> -export type DateInputComponent = BaseComponent<'dateinputcomponent'> -export type TimeInputComponent = BaseComponent<'timeinputcomponent'> -export type DateTimeInputComponent = BaseComponent<'datetimeinputcomponent'> -export type NumberInputComponent = BaseComponent<'numberinputcomponent'> -export type CurrencyComponent = BaseComponent<'currencycomponent'> -export type SliderComponent = BaseComponent<'slidercomponent'> -export type DateSliderComponent = BaseComponent<'dateslidercomponent'> -export type RangeComponent = BaseComponent<'rangecomponent'> +export type TextComponent = BaseComponent<'textcomponent'> & { + onChange?: string[] +} +export type TextInputComponent = BaseComponent<'textinputcomponent'> & { + onChange?: string[] +} +export type QuillComponent = BaseComponent<'quillcomponent'> & { + onChange?: string[] +} +export type CodeInputComponent = BaseComponent<'codeinputcomponent'> & { + onChange?: string[] +} +export type TextareaInputComponent = BaseComponent<'textareainputcomponent'> & { + onChange?: string[] +} +export type PasswordInputComponent = BaseComponent<'passwordinputcomponent'> & { + onChange?: string[] +} +export type EmailInputComponent = BaseComponent<'emailinputcomponent'> & { + onChange?: string[] +} +export type DateInputComponent = BaseComponent<'dateinputcomponent'> & { + onChange?: string[] +} +export type TimeInputComponent = BaseComponent<'timeinputcomponent'> & { + onChange?: string[] +} +export type DateTimeInputComponent = BaseComponent<'datetimeinputcomponent'> & { + onChange?: string[] +} +export type NumberInputComponent = BaseComponent<'numberinputcomponent'> & { + onChange?: string[] +} +export type CurrencyComponent = BaseComponent<'currencycomponent'> & { + onChange?: string[] +} +export type SliderComponent = BaseComponent<'slidercomponent'> & { + onChange?: string[] +} +export type DateSliderComponent = BaseComponent<'dateslidercomponent'> & { + onChange?: string[] +} +export type RangeComponent = BaseComponent<'rangecomponent'> & { + onChange?: string[] +} export type HtmlComponent = BaseComponent<'htmlcomponent'> export type CustomComponent = BaseComponent<'customcomponent'> & { customComponent: CustomComponentConfig @@ -293,7 +323,9 @@ export type NavBarComponent = BaseComponent<'navbarcomponent'> & { navbarItems: NavbarItem[] } -export type DateSelectComponent = BaseComponent<'dateselectcomponent'> +export type DateSelectComponent = BaseComponent<'dateselectcomponent'> & { + onChange?: string[] +} export type RecomputeAllComponent = BaseComponent<'recomputeallcomponent'> @@ -2769,6 +2801,11 @@ See date-fns format for more information. By default, it is 'yyyy-MM-dd' documentationLink: 'https://date-fns.org/v2.30.0/docs/format', placeholder: 'yyyy-MM-dd' + }, + disabled: { + type: 'static', + value: false, + fieldType: 'boolean' } } } diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptsPanelList.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptsPanelList.svelte index 4ba20786dc..195558b9db 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptsPanelList.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptsPanelList.svelte @@ -250,9 +250,3 @@ - - diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/DecisionTreeGraphNode.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/DecisionTreeGraphNode.svelte index a72d54356b..f066647fa0 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/DecisionTreeGraphNode.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/DecisionTreeGraphNode.svelte @@ -70,7 +70,7 @@ { - if (flowModule?.value.type == 'script') { - flowModule.value.hash = undefined + if (flowModuleValue.type == 'script') { + dispatch('setHash', undefined) } }}>hash @@ -87,8 +88,8 @@ size="xs" btnClasses="text-tertiary inline-flex gap-1 items-center" on:click={() => { - if (flowModule?.value.type == 'script') { - flowModule.value.hash = latestHash + if (flowModuleValue.type == 'script') { + dispatch('setHash', latestHash) } }}>hash @@ -102,10 +103,10 @@ >
{/if} - - {:else if flowModule?.value.type === 'flow'} + + {:else if flowModuleValue.type === 'flow'} flow - + {/if}
diff --git a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte index 139223cb1c..2cd9f2d5e0 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte @@ -102,6 +102,7 @@ $: lastDeployedCode = onModulesChange(savedModule, flowModule) function onModulesChange(savedModule: FlowModule | undefined, flowModule: FlowModule) { + // console.log('onModulesChange', savedModule, flowModule) return savedModule?.value?.type === 'rawscript' && flowModule.value.type === 'rawscript' && savedModule.value.content !== flowModule.value.content @@ -145,7 +146,10 @@ } } let inputTransformSchemaForm: InputTransformSchemaForm | undefined = undefined + + let reloadError: string | undefined = undefined async function reload(flowModule: FlowModule) { + reloadError = undefined try { const { input_transforms, schema } = await loadSchemaFromModule(flowModule) validCode = true @@ -167,12 +171,16 @@ flowModule.value.type == 'script' || flowModule.value.type == 'flow' ) { - flowModule.value.input_transforms = input_transforms + if (!deepEqual(flowModule.value.input_transforms, input_transforms)) { + flowModule.value.input_transforms = input_transforms + } } } if (flowModule.value.type == 'rawscript' && flowModule.value.lock != undefined) { - flowModule.value.lock = undefined + if (flowModule.value.lock != undefined) { + flowModule.value.lock = undefined + } } await tick() if (!deepEqual(schema, $flowStateStore[flowModule.id]?.schema)) { @@ -184,6 +192,7 @@ } } catch (e) { validCode = false + reloadError = e?.message } } @@ -278,17 +287,31 @@ {#if flowModule.value}
{ forceReload++ reload(flowModule) }} {noEditor} - bind:flowModule + on:setHash={(e) => { + if (flowModule.value.type == 'script') { + flowModule.value.hash = e.detail + } + }} + bind:summary={flowModule.summary} > { + console.log('tagChange', e.detail) + if (flowModule.value.type == 'script') { + flowModule.value.tag_override = e.detail + } else if (flowModule.value.type == 'rawscript') { + flowModule.value.tag = e.detail + } + }} on:toggleSuspend={() => selectAdvanced('suspend')} on:toggleSleep={() => selectAdvanced('sleep')} on:toggleMock={() => selectAdvanced('mock')} @@ -333,7 +356,7 @@
{#if flowModule.value.type === 'rawscript' && !noEditor} -
+
{ + const content = event.detail if (flowModule.value.type === 'rawscript') { - flowModule.value.content = event.detail + if (flowModule.value.content !== content) { + flowModule.value.content = content + } + await reload(flowModule) } - await reload(flowModule) }} formatAction={() => { reload(flowModule) @@ -455,6 +481,12 @@ error={failureModule} noPadding > + {#if reloadError} +
+ {/if}
@@ -98,12 +100,35 @@ : undefined : result} {#if !parentLoopId && !isLoop} - + + { + if (flowModule.stop_after_if) { + flowModule.stop_after_if.error_message = event.detail === false ? undefined : '' + } + }} + options={{ + right: 'Raise an error message if stopped', + rightTooltip: + 'If enabled and the stop condition is met, an error message will be raised. A custom message can be provided; otherwise, a default message will be used.' + }} + /> +
+ {/if} + {#if raise_error_message_stop_after_if} + {/if} Stop condition expression @@ -132,13 +157,20 @@
{:else} {#if !parentLoopId && !isLoop} - +
+ + +
{/if} Stop condition expression @@ -170,7 +202,8 @@ } else { flowModule.stop_after_all_iters_if = { expr: 'result == undefined', - skip_if_stopped: false + skip_if_stopped: false, + error_message: undefined } } }} @@ -180,18 +213,41 @@ />
{#if flowModule.stop_after_all_iters_if} {#if !parentLoopId} - + + { + if (flowModule.stop_after_all_iters_if) { + flowModule.stop_after_all_iters_if.error_message = event.detail === false ? undefined : '' + } + }} + options={{ + right: 'Raise an error message if stopped', + rightTooltip: + 'If enabled and the stop condition is met, an error message will be raised. A custom message can be provided; otherwise, a default message will be used.' + }} + /> +
+ {/if} + {#if raise_error_message_stop_after_all_if} + {/if} Stop condition expression @@ -218,13 +274,22 @@
{:else} {#if !parentLoopId} - +
+ + +
{/if} Stop condition expression diff --git a/frontend/src/lib/components/flows/content/FlowModuleHeader.svelte b/frontend/src/lib/components/flows/content/FlowModuleHeader.svelte index 56a01e96c8..c19c5b7ab6 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleHeader.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleHeader.svelte @@ -29,13 +29,11 @@ const dispatch = createEventDispatcher() let customUi: undefined | FlowBuilderWhitelabelCustomUi = getContext('customUi') - - $: moduleRetry = module.retry?.constant || module.retry?.exponential
{#if module.value.type === 'script' || module.value.type === 'rawscript' || module.value.type == 'flow'} - {#if moduleRetry} + {#if module.retry?.constant || module.retry?.exponential} {/if} {#if customUi?.tagEdit != false} - + dispatch('tagChange', e.detail)} + /> {/if} {#if customUi?.scriptFork != false}
{#if module.value.type === 'rawscript'} - + dispatch('tagChange', e.detail)} + /> +
+ {:else} + + {/if} +
+{/if} {#if keys.length > 0} {#if !fullyCollapsed} @@ -132,10 +197,10 @@ : - {#if getTypeAsString(json[key]) === 'object'} + {#if getTypeAsString(jsonFiltered[key]) === 'object'} { - selectProp(key, json[key], true) + selectProp(key, jsonFiltered[key], true) }} - title={JSON.stringify(json[key])} + title={JSON.stringify(jsonFiltered[key])} > - {#if json[key] === NEVER_TESTED_THIS_FAR} + {#if jsonFiltered[key] === NEVER_TESTED_THIS_FAR} Test the flow to see a value - {:else if json[key] == undefined} + {:else if jsonFiltered[key] == undefined} undefined - {:else if json[key] == null} + {:else if jsonFiltered[key] == null} null - {:else if typeof json[key] == 'string'} - "{truncate(json[key], 200)}" - {:else if typeof json[key] == 'number' && Number.isInteger(json[key]) && !Number.isSafeInteger(json[key])} + {:else if typeof jsonFiltered[key] == 'string'} + "{truncate(jsonFiltered[key], 200)}" + {:else if typeof jsonFiltered[key] == 'number' && Number.isInteger(jsonFiltered[key]) && !Number.isSafeInteger(jsonFiltered[key])} - {truncate(JSON.stringify(json[key]), 200)} + {truncate(JSON.stringify(jsonFiltered[key]), 200)} @@ -181,7 +246,7 @@ {:else} - {truncate(JSON.stringify(json[key]), 200)} + {truncate(JSON.stringify(jsonFiltered[key]), 200)} {/if} @@ -198,20 +263,20 @@ {#if level == 0 && topBrackets}
{closeBracket} - {#if getTypeAsString(json) === 's3object'} + {#if getTypeAsString(jsonFiltered) === 's3object'} download @@ -240,7 +305,7 @@ {/if} {:else if topBrackets} {openBracket}{closeBracket} -{:else if json == undefined} +{:else if jsonFiltered == undefined} undefined {:else} No items diff --git a/frontend/src/lib/components/propertyPicker/PropPicker.svelte b/frontend/src/lib/components/propertyPicker/PropPicker.svelte index 8faf206f95..8c4da6dc22 100644 --- a/frontend/src/lib/components/propertyPicker/PropPicker.svelte +++ b/frontend/src/lib/components/propertyPicker/PropPicker.svelte @@ -197,7 +197,7 @@
-
+
@@ -768,7 +768,7 @@ bind:value={deploymentMsg} />
-
+
Path
-
+
{#if appPath == ''} Save this app once before you can publish it @@ -856,10 +856,10 @@ -
+

Public URL

-
+
Custom path is an enterprise only feature. -
+
{:else if !($userStore?.is_admin || $userStore?.is_super_admin)} Custom path can only be set by workspace admins -
+
{/if} { diff --git a/frontend/src/lib/components/raw_apps/RawAppInlineScriptEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppInlineScriptEditor.svelte index 7754e965e3..448406c008 100644 --- a/frontend/src/lib/components/raw_apps/RawAppInlineScriptEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppInlineScriptEditor.svelte @@ -109,7 +109,7 @@
+ >
{/if}
diff --git a/frontend/src/lib/components/raw_apps/RawAppInlineScriptPanelList.svelte b/frontend/src/lib/components/raw_apps/RawAppInlineScriptPanelList.svelte index 8bece2ea3a..7d7558b17c 100644 --- a/frontend/src/lib/components/raw_apps/RawAppInlineScriptPanelList.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppInlineScriptPanelList.svelte @@ -82,9 +82,3 @@
- - diff --git a/frontend/src/lib/components/raw_apps/RawAppPreview.svelte b/frontend/src/lib/components/raw_apps/RawAppPreview.svelte index 4957a520ba..083fc41a8a 100644 --- a/frontend/src/lib/components/raw_apps/RawAppPreview.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppPreview.svelte @@ -20,4 +20,4 @@ title="raw-app" srcDoc={htmlContent(workspace, version, { ctx: user, workspace })} class="w-full h-full min-h-screen bg-white border-none" -/> +> diff --git a/frontend/src/lib/components/runs/RunRow.svelte b/frontend/src/lib/components/runs/RunRow.svelte index 32c2e16f3a..e2bb280c91 100644 --- a/frontend/src/lib/components/runs/RunRow.svelte +++ b/frontend/src/lib/components/runs/RunRow.svelte @@ -12,7 +12,7 @@ isJobSelectable } from '$lib/utils' import { Badge, Button } from '../common' - import ScheduleEditor from '../ScheduleEditor.svelte' + import ScheduleEditor from '$lib/components/triggers/schedules/ScheduleEditor.svelte' import BarsStaggered from '$lib/components/icons/BarsStaggered.svelte' import { diff --git a/frontend/src/lib/components/schema/FlowPropertyEditor.svelte b/frontend/src/lib/components/schema/FlowPropertyEditor.svelte index e975324a42..1ef4606ab1 100644 --- a/frontend/src/lib/components/schema/FlowPropertyEditor.svelte +++ b/frontend/src/lib/components/schema/FlowPropertyEditor.svelte @@ -53,7 +53,7 @@ return oneOf.map((v) => ({ ...v, properties: Object.fromEntries( - Object.entries(v.properties ?? {}).filter(([k, v]) => k !== 'label') + Object.entries(v.properties ?? {}).filter(([k, v]) => k !== 'label' && k !== 'kind') ) })) } @@ -122,12 +122,17 @@ properties = structuredClone(changedSchema.properties) order = structuredClone(changedSchema.order) requiredProperty = structuredClone(changedSchema.required) + + const tagKey = oneOf?.find((o) => Object.keys(o.properties ?? {}).includes('kind')) + ? 'kind' + : 'label' + oneOf = changedSchema.oneOf?.map((v) => { return { ...v, properties: { ...(v.properties ?? {}), - label: { + [tagKey]: { type: 'string', enum: [v.title ?? ''] } diff --git a/frontend/src/lib/components/schema/PropertyEditor.svelte b/frontend/src/lib/components/schema/PropertyEditor.svelte index 9071a64841..6fa69f64dd 100644 --- a/frontend/src/lib/components/schema/PropertyEditor.svelte +++ b/frontend/src/lib/components/schema/PropertyEditor.svelte @@ -55,7 +55,9 @@ oneOfSchemas = oneOf.map((obj) => { return { properties: obj.properties - ? Object.fromEntries(Object.entries(obj.properties).filter(([k, v]) => k !== 'label')) + ? Object.fromEntries( + Object.entries(obj.properties).filter(([k, v]) => k !== 'label' && k !== 'kind') + ) : {}, order: obj.order } diff --git a/frontend/src/lib/components/settings/AIUserSettings.svelte b/frontend/src/lib/components/settings/AIUserSettings.svelte new file mode 100644 index 0000000000..ee32c6c071 --- /dev/null +++ b/frontend/src/lib/components/settings/AIUserSettings.svelte @@ -0,0 +1,64 @@ + + +
+

AI user settings

+ +
+ { + updateSetting(codeCompletionSessionEnabled, e.detail, 'codeCompletionSessionEnabled') + }} + checked={$codeCompletionSessionEnabled} + options={{ + right: 'Code completion', + rightTooltip: 'AI completion in the code editors' + }} + /> + { + updateSetting(metadataCompletionEnabled, e.detail, 'metadataCompletionEnabled') + }} + checked={$metadataCompletionEnabled} + options={{ + right: 'Metadata completion', + rightTooltip: 'AI completion for summaries and descriptions' + }} + /> + { + updateSetting(stepInputCompletionEnabled, e.detail, 'stepInputCompletionEnabled') + }} + checked={$stepInputCompletionEnabled} + options={{ + right: 'Flow step input completion', + rightTooltip: 'AI completion for flow step inputs' + }} + /> +
+
diff --git a/frontend/src/lib/components/settings/TokensTable.svelte b/frontend/src/lib/components/settings/TokensTable.svelte new file mode 100644 index 0000000000..9cc4f30ecb --- /dev/null +++ b/frontend/src/lib/components/settings/TokensTable.svelte @@ -0,0 +1,415 @@ + + +
+

Tokens

+
+ +
+
+
+ Authenticate to the Windmill API with access tokens. +
+ +
+ {#if newToken} +
+
+ Added token: +
+
+ Make sure to copy your personal access token now. You won't be able to see it again! +
+
+ {/if} + + {#if newMcpToken} +
+

New MCP URL:

+ +

+ Make sure to copy this URL now. You won't be able to see it again! +

+
+ {/if} + + {#if displayCreateToken} +
+

Add a new token

+ + {#if showMcpMode} +
+ { + mcpCreationMode = e.detail + if (e.detail) { + newTokenLabel = 'MCP token' + newTokenExpiration = undefined + newTokenWorkspace = $workspaceStore + } else { + newTokenLabel = undefined + newTokenExpiration = undefined + newTokenWorkspace = defaultNewTokenWorkspace + } + }} + checked={mcpCreationMode} + options={{ + right: 'Generate MCP URL', + rightTooltip: + 'Generate a new MCP URL to make your scripts and flows available as tools through your LLM clients.', + rightDocumentationLink: 'https://www.windmill.dev/docs/core_concepts/mcp' + }} + size="xs" + /> +
+ {/if} + + {#if scopes != undefined} +
+ Scope + {#each scopes as scope} + + {/each} +
+ {/if} + +
+ {#if mcpCreationMode} +
+ Scope + + + + +
+ +
+ Hub scripts (optional) + {#if loadingApps} +
Loading...
+ {:else if errorFetchApps} +
Error fetching apps
+ {:else} + + {/if} +
+ +
+ Workspace + +
+ {/if} + +
+ Label (optional) + +
+ +
+ Expires In (optional) + +
+
+ +
+ + +
+
+ {/if} +
+ +
+ + + prefix + label + expiration + scopes + + + + {#if tokens && tokens.length > 0} + {#each tokens as { token_prefix, expiration, label, scopes }} + + {token_prefix}**** + {label ?? ''} + {displayDate(expiration ?? '')} + {scopes?.join(', ') ?? ''} + + + + + {/each} + {:else if tokens && tokens.length === 0} + + There are no tokens yet + + {:else} + Loading... + {/if} + + +
+ {#if tokens?.length == 100} + + {/if} + {#if tokenPage > 1} + + {/if} +
+
diff --git a/frontend/src/lib/components/settings/UserInfoSettings.svelte b/frontend/src/lib/components/settings/UserInfoSettings.svelte new file mode 100644 index 0000000000..e7a46a30e6 --- /dev/null +++ b/frontend/src/lib/components/settings/UserInfoSettings.svelte @@ -0,0 +1,72 @@ + + +
+

User info

+
+ {#if passwordError} +
{passwordError}
+ {/if} +
+
+ + {#if login_type == 'password'} + + {:else if login_type == 'github'} + Authenticated through Github OAuth2. Cannot set a password. + {/if} +
+
+
+
diff --git a/frontend/src/lib/components/triggers/CaptureTable.svelte b/frontend/src/lib/components/triggers/CaptureTable.svelte index 71df4c2991..cc784e33b6 100644 --- a/frontend/src/lib/components/triggers/CaptureTable.svelte +++ b/frontend/src/lib/components/triggers/CaptureTable.svelte @@ -83,31 +83,41 @@ let capturesWithPayload: CaptureWithPayload[] = captures.map((capture) => { let newCapture: CaptureWithPayload = { ...capture } - if (capture.payload === 'WINDMILL_TOO_BIG') { + + const isLarge = + capture.main_args === 'WINDMILL_TOO_BIG' || + capture.preprocessor_args === 'WINDMILL_TOO_BIG' + if (isLarge) { newCapture = { ...capture, + payloadData: 'Too big to display here, select to view', getFullCapture: () => CaptureService.getCapture({ workspace: $workspaceStore!, id: capture.id }) } + return newCapture } - const trigger_extra = isObject(capture.trigger_extra) ? capture.trigger_extra : {} - newCapture.payloadData = - kind === 'preprocessor' - ? capture.payload === 'WINDMILL_TOO_BIG' - ? { - payload: capture.payload, - ...trigger_extra - } - : typeof capture.payload === 'object' + const preprocessor_args = isObject(capture.preprocessor_args) + ? capture.preprocessor_args + : {} + + if ('wm_trigger' in preprocessor_args) { + // v1 + newCapture.payloadData = + kind === 'preprocessor' + ? typeof capture.main_args === 'object' ? { - ...capture.payload, - ...trigger_extra + ...capture.main_args, + ...preprocessor_args } - : trigger_extra - : capture.payload + : preprocessor_args + : capture.main_args + } else { + // v2 + newCapture.payloadData = kind === 'preprocessor' ? preprocessor_args : capture.main_args + } return newCapture }) @@ -138,13 +148,23 @@ let payloadData: any = {} if (capture.getFullCapture) { const fullCapture = await capture.getFullCapture() - payloadData = - testKind === 'preprocessor' - ? { - ...(typeof fullCapture.payload === 'object' ? fullCapture.payload : {}), - ...(typeof fullCapture.trigger_extra === 'object' ? fullCapture.trigger_extra : {}) - } - : fullCapture.payload + const preprocessor_args = isObject(fullCapture.preprocessor_args) + ? fullCapture.preprocessor_args + : {} + if ('wm_trigger' in preprocessor_args) { + // v1 + payloadData = + testKind === 'preprocessor' + ? { + ...(typeof fullCapture.main_args === 'object' ? fullCapture.main_args : {}), + ...preprocessor_args + } + : fullCapture.main_args + } else { + // v2 + payloadData = + testKind === 'preprocessor' ? fullCapture.preprocessor_args : fullCapture.main_args + } } else { payloadData = structuredClone(capture.payloadData) } diff --git a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorConfigSection.svelte b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorConfigSection.svelte index d359085122..816dffd90f 100644 --- a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorConfigSection.svelte +++ b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorConfigSection.svelte @@ -50,7 +50,6 @@ const DEFAULT_PUSH_CONFIG: PushConfig = { audience: '', authenticate: false, - base_endpoint } async function loadAllPubSubTopicsFromProject() { @@ -87,7 +86,7 @@ function getBaseUrl(captureInfo: CaptureInfo | undefined) { if (captureInfo) { - return `${location.origin}${base}/api/w/${$workspaceStore}/capture_u/gcp/${ + return `${window.location.origin}${base}/api/w/${$workspaceStore}/capture_u/gcp/${ captureInfo.isFlow ? 'flow' : 'script' }` } else { @@ -271,19 +270,6 @@
- - -

Enable Google Cloud authentication for push delivery using a verified token. + {#if delivery_config.authenticate} + + + + {/if}

{/if}
diff --git a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte index 45a74ec93d..61a8684234 100644 --- a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte @@ -103,6 +103,7 @@ delivery_type = s.delivery_type subscription_id = s.subscription_id delivery_config = s.delivery_config + subscription_mode = s.subscription_mode is_flow = s.is_flow path = s.path enabled = s.enabled @@ -116,12 +117,12 @@ async function updateTrigger(): Promise { try { is_loading = true + const base_endpoint = `${window.location.origin}${base}` if (delivery_type === 'push') { if (!delivery_config) { sendUserToast('Must set route path when delivery type is push', true) return } - delivery_config.base_endpoint = `${window.location.origin}${base}` } else { delivery_config = undefined } @@ -131,13 +132,11 @@ path: initialPath, requestBody: { gcp_resource_path, - subscription_mode: { - subscription_mode, - subscription_id, - delivery_type, - delivery_config, - base_endpoint: `${window.location.origin}${base}` - }, + subscription_mode, + subscription_id, + delivery_type, + delivery_config, + base_endpoint, topic_id, path, script_path, @@ -151,13 +150,11 @@ workspace: $workspaceStore!, requestBody: { gcp_resource_path, - subscription_mode: { - subscription_mode, - subscription_id, - delivery_type, - delivery_config: delivery_config, - base_endpoint: `${window.location.origin}${base}` - }, + subscription_mode, + subscription_id, + delivery_type, + delivery_config, + base_endpoint, topic_id, path, script_path, @@ -266,7 +263,7 @@ btnClasses="ml-4 mt-2" color="dark" size="xs" - href={itemKind === 'flow' ? '/flows/add?hub=68' : '/scripts/add?hub=hub%2F14251'} + href={itemKind === 'flow' ? '/flows/add?hub=68' : '/scripts/add?hub=hub%2F11446'} target="_blank">Create from template {/if} @@ -281,7 +278,7 @@ bind:delivery_config bind:topic_id bind:subscription_mode - bind:path={path} + bind:path cloud_subscription_id={subscription_id} create_update_subscription_id={subscription_id} {can_write} diff --git a/frontend/src/lib/components/triggers/http/RouteBodyTransformerOption.svelte b/frontend/src/lib/components/triggers/http/RouteBodyTransformerOption.svelte index 62d54a5a35..d9f129851f 100644 --- a/frontend/src/lib/components/triggers/http/RouteBodyTransformerOption.svelte +++ b/frontend/src/lib/components/triggers/http/RouteBodyTransformerOption.svelte @@ -30,8 +30,10 @@ Wraps the payload in an object under the 'body' key, useful for handling unknown payloads. + Wraps the payload in an object under the 'body' key, useful for handling unknown payloads. + Note that this will have no effect when using a preprocessor. + Create from template {/if} diff --git a/frontend/src/lib/components/triggers/mqtt/MqttEditorConfigSection.svelte b/frontend/src/lib/components/triggers/mqtt/MqttEditorConfigSection.svelte index 0e1ad9c039..b2c11571ae 100644 --- a/frontend/src/lib/components/triggers/mqtt/MqttEditorConfigSection.svelte +++ b/frontend/src/lib/components/triggers/mqtt/MqttEditorConfigSection.svelte @@ -144,7 +144,7 @@
Create from template {/if} diff --git a/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte index 72b05a6a9e..6d3cab175f 100644 --- a/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte @@ -66,8 +66,8 @@ args.nats_resource_path = nDefaultValues?.nats_resource_path ?? '' args.subjects = nDefaultValues?.subjects ?? [''] args.use_jetstream = nDefaultValues?.use_jetstream ?? false - args.stream_name = args.use_jetstream ? nDefaultValues?.stream_name ?? '' : undefined - args.consumer_name = args.use_jetstream ? nDefaultValues?.consumer_name ?? '' : undefined + args.stream_name = args.use_jetstream ? (nDefaultValues?.stream_name ?? '') : undefined + args.consumer_name = args.use_jetstream ? (nDefaultValues?.consumer_name ?? '') : undefined initialScriptPath = '' fixedScriptPath = fixedScriptPath_ ?? '' script_path = fixedScriptPath @@ -242,7 +242,7 @@ btnClasses="ml-4 mt-2" color="dark" size="xs" - href={itemKind === 'flow' ? '/flows/add?hub=66' : '/scripts/add?hub=hub%2F11634'} + href={itemKind === 'flow' ? '/flows/add?hub=66' : '/scripts/add?hub=hub%2F19663'} target="_blank">Create from template {/if} diff --git a/frontend/src/lib/components/triggers/postgres/PostgresEditorConfigSection.svelte b/frontend/src/lib/components/triggers/postgres/PostgresEditorConfigSection.svelte index 9ed430bcee..00018eca3c 100644 --- a/frontend/src/lib/components/triggers/postgres/PostgresEditorConfigSection.svelte +++ b/frontend/src/lib/components/triggers/postgres/PostgresEditorConfigSection.svelte @@ -37,7 +37,7 @@ function updateValidity(publication: PublicationData) { isValid = !emptyString(postgres_resource_path) && - (!publication.table_to_track || publication.table_to_track.length === 0) + (!publication.table_to_track || publication.table_to_track.length !== 0) } $: updateValidity(publication) diff --git a/frontend/src/lib/components/triggers/scheduled/utils.ts b/frontend/src/lib/components/triggers/scheduled/utils.ts new file mode 100644 index 0000000000..2b7ab5930c --- /dev/null +++ b/frontend/src/lib/components/triggers/scheduled/utils.ts @@ -0,0 +1,34 @@ +import { JobService, ScheduleService } from "$lib/gen" +import { goto } from "$lib/navigation" +import { sendUserToast } from "$lib/utils" + +export async function runScheduleNow( + path: string, + schedulePath: string, + isFlow: boolean, + workspace_id: string +): Promise { + try { + const runByPath = isFlow ? JobService.runFlowByPath : JobService.runScriptByPath + const args = ( + await ScheduleService.getSchedule({ + workspace: workspace_id, + path: schedulePath + }) + ).args + const run = await runByPath({ + path, + requestBody: args ?? {}, + workspace: workspace_id + }) + + sendUserToast(`Schedule ${path} will run now`, false, [ + { + label: 'Go to the run page', + callback: () => goto('/run/' + run + '?workspace=' + workspace_id) + } + ]) + } catch (err) { + sendUserToast(`Cannot run schedule now: ${err.body}`, true) + } +} \ No newline at end of file diff --git a/frontend/src/lib/components/ScheduleEditor.svelte b/frontend/src/lib/components/triggers/schedules/ScheduleEditor.svelte similarity index 70% rename from frontend/src/lib/components/ScheduleEditor.svelte rename to frontend/src/lib/components/triggers/schedules/ScheduleEditor.svelte index 6e50584c65..f70e3a48cb 100644 --- a/frontend/src/lib/components/ScheduleEditor.svelte +++ b/frontend/src/lib/components/triggers/schedules/ScheduleEditor.svelte @@ -9,10 +9,14 @@ drawer?.openEdit(ePath, isFlow) } - export async function openNew(is_flow: boolean, initial_script_path?: string) { + export async function openNew( + is_flow: boolean, + initial_script_path?: string, + schedule_path?: string + ) { open = true await tick() - drawer?.openNew(is_flow, initial_script_path) + drawer?.openNew(is_flow, initial_script_path, schedule_path) } let drawer: ScheduleEditorInner diff --git a/frontend/src/lib/components/ScheduleEditorInner.svelte b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte similarity index 95% rename from frontend/src/lib/components/ScheduleEditorInner.svelte rename to frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte index 2bec0d2673..69fe6c77d4 100644 --- a/frontend/src/lib/components/ScheduleEditorInner.svelte +++ b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte @@ -17,7 +17,8 @@ ScriptService, type Flow, SettingService, - type Retry + type Retry, + type Schedule } from '$lib/gen' import { enterpriseLicense, userStore, workspaceStore } from '$lib/stores' import { canWrite, emptyString, formatCron, sendUserToast, cronV1toV2 } from '$lib/utils' @@ -25,11 +26,12 @@ import { createEventDispatcher } from 'svelte' import Section from '$lib/components/Section.svelte' import { List, Loader2, Save, AlertTriangle } from 'lucide-svelte' - import FlowRetries from './flows/content/FlowRetries.svelte' - import WorkerTagPicker from './WorkerTagPicker.svelte' - import Label from './Label.svelte' - import DateTimeInput from './DateTimeInput.svelte' import autosize from '$lib/autosize' + import DateTimeInput from '$lib/components/DateTimeInput.svelte' + import FlowRetries from '$lib/components/flows/content/FlowRetries.svelte' + import Label from '$lib/components/Label.svelte' + import WorkerTagPicker from '$lib/components/WorkerTagPicker.svelte' + import { runScheduleNow } from '../scheduled/utils' let optionTabSelected: 'error_handler' | 'recovery_handler' | 'success_handler' | 'retries' = 'error_handler' @@ -65,6 +67,7 @@ let failedTimes = 1 let failedExact = false let recoveredTimes = 1 + let duplicate = false let retry: Retry | undefined = undefined let script_path = '' @@ -94,16 +97,58 @@ } } - export async function openNew(nis_flow: boolean, initial_script_path?: string) { - drawerLoading = true - try { - drawer?.openDrawer() - args = {} - runnable = undefined - is_flow = nis_flow - schedule = '0 0 12 * *' - paused_until = undefined - showPauseUntil = false + async function setScheduleHandler(s?: Schedule) { + if (s) { + if (s.on_failure) { + let splitted = s.on_failure.split('/') + errorHandleritemKind = splitted[0] as 'flow' | 'script' + errorHandlerPath = splitted.slice(1)?.join('/') + errorHandlerCustomInitialPath = errorHandlerPath + failedTimes = s.on_failure_times ?? 1 + failedExact = s.on_failure_exact ?? false + errorHandlerExtraArgs = s.on_failure_extra_args ?? {} + errorHandlerSelected = getHandlerType('error', errorHandlerPath) + } else { + errorHandlerPath = undefined + errorHandleritemKind = 'script' + errorHandlerCustomInitialPath = undefined + errorHandlerExtraArgs = {} + failedExact = false + failedTimes = 1 + errorHandlerSelected = 'slack' + } + if (s.on_recovery) { + let splitted = s.on_recovery.split('/') + recoveryHandlerItemKind = splitted[0] as 'flow' | 'script' + recoveryHandlerPath = splitted.slice(1)?.join('/') + recoveryHandlerCustomInitialPath = recoveryHandlerPath + recoveredTimes = s.on_recovery_times ?? 1 + recoveryHandlerExtraArgs = s.on_recovery_extra_args ?? {} + recoveryHandlerSelected = getHandlerType('recovery', recoveryHandlerPath) + } else { + recoveryHandlerPath = undefined + recoveryHandlerItemKind = 'script' + recoveryHandlerCustomInitialPath = undefined + recoveredTimes = 1 + recoveryHandlerSelected = 'slack' + recoveryHandlerExtraArgs = {} + } + if (s.on_success) { + let splitted = s.on_success.split('/') + successHandlerItemKind = splitted[0] as 'flow' | 'script' + successHandlerPath = splitted.slice(1)?.join('/') + successHandlerCustomInitialPath = successHandlerPath + successHandlerExtraArgs = s.on_success_extra_args ?? {} + successHandlerSelected = getHandlerType('success', successHandlerPath) + } else { + successHandlerPath = undefined + successHandlerItemKind = 'script' + successHandlerCustomInitialPath = undefined + successHandlerSelected = 'slack' + successHandlerExtraArgs = {} + } + } + else { let defaultErrorHandlerMaybe = undefined let defaultRecoveryHandlerMaybe = undefined let defaultSuccessHandlerMaybe = undefined @@ -119,17 +164,6 @@ })) as any } - edit = false - itemKind = nis_flow ? 'flow' : 'script' - initialScriptPath = initial_script_path ?? '' - summary = '' - description = '' - no_flow_overlap = false - path = initialScriptPath - initialPath = initialScriptPath - script_path = initialScriptPath - await loadScript(script_path) - if (defaultErrorHandlerMaybe !== undefined && defaultErrorHandlerMaybe !== null) { wsErrorHandlerMuted = defaultErrorHandlerMaybe['wsErrorHandlerMuted'] let splitted = (defaultErrorHandlerMaybe['errorHandlerPath'] as string).split('/') @@ -181,7 +215,54 @@ successHandlerCustomInitialPath = undefined successHandlerSelected = 'slack' } - timezone = Intl.DateTimeFormat().resolvedOptions().timeZone + } + } + + export async function openNew( + nis_flow: boolean, + initial_script_path?: string, + schedule_path?: string + ) { + drawerLoading = true + try { + let s: Schedule | undefined + if (schedule_path) { + s = await ScheduleService.getSchedule({ + workspace: $workspaceStore!, + path: schedule_path + }) + duplicate = true + } + drawer?.openDrawer() + runnable = undefined + is_flow = s?.is_flow ?? nis_flow + edit = false + itemKind = is_flow ? 'flow' : 'script' + initialScriptPath = initial_script_path ?? '' + path = duplicate === true ? '' : initialScriptPath + + initialPath = path + cronVersion = s?.cron_version ?? 'v2' + initialCronVersion = cronVersion + isLatestCron = cronVersion == 'v2' + schedule = s?.schedule ?? '0 0 12 * *' + initialSchedule = schedule + timezone = s?.timezone ?? Intl.DateTimeFormat().resolvedOptions().timeZone + paused_until = s?.paused_until + showPauseUntil = paused_until !== undefined + summary = s?.summary ?? '' + description = s?.description ?? '' + script_path = s?.script_path ?? initialScriptPath + args = s?.args ?? {} + tag = s?.tag + + await loadScript(script_path) + + no_flow_overlap = s?.no_flow_overlap ?? false + wsErrorHandlerMuted = s?.ws_error_handler_muted ?? false + retry = s?.retry + + await setScheduleHandler(s) } finally { drawerLoading = false } @@ -322,6 +403,7 @@ workspace: $workspaceStore!, path: initialPath }) + is_flow = s.is_flow cronVersion = s.cron_version ?? 'v2' initialCronVersion = cronVersion isLatestCron = cronVersion == 'v2' @@ -334,63 +416,16 @@ summary = s.summary ?? '' description = s.description ?? '' script_path = s.script_path ?? '' - await loadScript(script_path) - - is_flow = s.is_flow - no_flow_overlap = s.no_flow_overlap ?? false - wsErrorHandlerMuted = s.ws_error_handler_muted ?? false - retry = s.retry - if (s.on_failure) { - let splitted = s.on_failure.split('/') - errorHandleritemKind = splitted[0] as 'flow' | 'script' - errorHandlerPath = splitted.slice(1)?.join('/') - errorHandlerCustomInitialPath = errorHandlerPath - failedTimes = s.on_failure_times ?? 1 - failedExact = s.on_failure_exact ?? false - errorHandlerExtraArgs = s.on_failure_extra_args ?? {} - errorHandlerSelected = getHandlerType('error', errorHandlerPath) - } else { - errorHandlerPath = undefined - errorHandleritemKind = 'script' - errorHandlerCustomInitialPath = undefined - errorHandlerExtraArgs = {} - failedExact = false - failedTimes = 1 - errorHandlerSelected = 'slack' - } - if (s.on_recovery) { - let splitted = s.on_recovery.split('/') - recoveryHandlerItemKind = splitted[0] as 'flow' | 'script' - recoveryHandlerPath = splitted.slice(1)?.join('/') - recoveryHandlerCustomInitialPath = recoveryHandlerPath - recoveredTimes = s.on_recovery_times ?? 1 - recoveryHandlerExtraArgs = s.on_recovery_extra_args ?? {} - recoveryHandlerSelected = getHandlerType('recovery', recoveryHandlerPath) - } else { - recoveryHandlerPath = undefined - recoveryHandlerItemKind = 'script' - recoveryHandlerCustomInitialPath = undefined - recoveredTimes = 1 - recoveryHandlerSelected = 'slack' - recoveryHandlerExtraArgs = {} - } - if (s.on_success) { - let splitted = s.on_success.split('/') - successHandlerItemKind = splitted[0] as 'flow' | 'script' - successHandlerPath = splitted.slice(1)?.join('/') - successHandlerCustomInitialPath = successHandlerPath - successHandlerExtraArgs = s.on_success_extra_args ?? {} - successHandlerSelected = getHandlerType('success', successHandlerPath) - } else { - successHandlerPath = undefined - successHandlerItemKind = 'script' - successHandlerCustomInitialPath = undefined - successHandlerSelected = 'slack' - successHandlerExtraArgs = {} - } args = s.args ?? {} can_write = canWrite(s.path, s.extra_perms, $userStore) tag = s.tag + + await loadScript(script_path) + + no_flow_overlap = s.no_flow_overlap ?? false + wsErrorHandlerMuted = s.ws_error_handler_muted ?? false + retry = s.retry + await setScheduleHandler(s) } catch (err) { sendUserToast(`Could not load schedule: ${err}`, true) } @@ -571,7 +606,7 @@ {#if !drawerLoading} {#if edit} -
+
+
{#if can_write}
@@ -749,7 +794,7 @@ Pick a script or flow to be triggered by the schedule

Create from template {/if} diff --git a/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte b/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte index 188750b0a9..bd8bb7d044 100644 --- a/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte +++ b/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte @@ -126,15 +126,15 @@ async function triggerJob() { cleanedRunnableArgs ?? {}, null, 2 - ).replaceAll('\n', '\n\t')});` + ).replaceAll('\n', '\n\t')});` } const endpoint = \`${url}\`; return await fetch(endpoint, { method: '${requestType === 'get_path' ? 'GET' : 'POST'}', headers: ${JSON.stringify(headers(), null, 2).replaceAll('\n', '\n\t\t')}${ - requestType === 'get_path' ? '' : `,\n\t\tbody` - } + requestType === 'get_path' ? '' : `,\n\t\tbody` + } }); }` } @@ -244,12 +244,12 @@ done` requestType === 'get_path' ? `&payload=${encodeURIComponent(btoa(JSON.stringify(cleanedRunnableArgs ?? {})))}` : '' - }` + }` : `${ requestType === 'get_path' ? `?payload=${encodeURIComponent(btoa(JSON.stringify(cleanedRunnableArgs ?? {})))}` : '' - }`) + }`) - {/if} diff --git a/frontend/src/lib/components/tutorials/Tutorial.svelte b/frontend/src/lib/components/tutorials/Tutorial.svelte index 5a2dd5014e..dfbdfdbcbe 100644 --- a/frontend/src/lib/components/tutorials/Tutorial.svelte +++ b/frontend/src/lib/components/tutorials/Tutorial.svelte @@ -1,11 +1,11 @@ - +{#if tutorial} + +{/if} diff --git a/frontend/src/lib/components/tutorials/TutorialInner.svelte b/frontend/src/lib/components/tutorials/TutorialInner.svelte new file mode 100644 index 0000000000..ce0784ba8e --- /dev/null +++ b/frontend/src/lib/components/tutorials/TutorialInner.svelte @@ -0,0 +1,3 @@ + diff --git a/frontend/src/lib/consts.ts b/frontend/src/lib/consts.ts index 5070c1bb1d..d319868795 100644 --- a/frontend/src/lib/consts.ts +++ b/frontend/src/lib/consts.ts @@ -1,3 +1,5 @@ +import type { DbType } from './components/apps/components/display/dbtable/utils' + export const DEFAULT_WEBHOOK_TYPE: 'async' | 'sync' = 'async' export const HOME_SHOW_HUB = true @@ -201,3 +203,11 @@ export const MSSQL_TYPES = [ 'decimal', 'bit' ] + +export const DB_TYPES: Record = { + bigquery: BIGQUERY_TYPES, + ms_sql_server: MSSQL_TYPES, + mysql: MYSQL_TYPES, + postgresql: POSTGRES_TYPES, + snowflake: SNOWFLAKE_TYPES +} diff --git a/frontend/src/lib/editorUtils.ts b/frontend/src/lib/editorUtils.ts index fb4f8782ee..f2679df653 100644 --- a/frontend/src/lib/editorUtils.ts +++ b/frontend/src/lib/editorUtils.ts @@ -146,7 +146,7 @@ export function extToLang(ext: string) { return 'nu' case 'java': return 'java' - // for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG default: return 'unknown' } diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index 9e90ebe74c..3d58eac5b7 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -631,15 +631,6 @@ export const TS_PREPROCESSOR_SCRIPT_INTRO = `/** * It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, Postgres, or email) * before passing it to \`main\`. This separates the trigger logic from the main logic and keeps the auto-generated runnable UI clean. * - * The preprocessor receives trigger metadata (\`wm_trigger\`) along with the main trigger arguments. - * The structure of \`wm_trigger\` and the main trigger arguments are specific to each trigger type: - * - Webhook/HTTP: \`(wm_trigger: { kind: 'http' | 'webhook', http?: { ... } }, body_key_1: any, body_key_2: any, ...)\` - * - Postgres: \`(wm_trigger: { kind: 'postgres' }, transaction_type: string, schema_name: string, table_name: string, old_row?: any, row: any)\` - * - WebSocket/Kafka/NATS/SQS: \`(wm_trigger: { kind: 'websocket' | 'kafka' | 'nats' | 'sqs', [kind]: { ... } }, msg: string)\` - * - MQTT: \`(wm_trigger: { kind: 'mqtt', [kind]: { ... } }, payload: Array)\` - * - GCP: \`(wm_trigger: { kind: 'gcp', [kind]: { ... } }, payload: string)\` - * - Email: \`(wm_trigger: { kind: 'email' }, raw_email: string, parsed_email: { ... })\` - * * The returned object defines the parameter values passed to \`main()\`. * e.g., { b: 1, a: 2 } → Calls \`main(2, 1)\`, assuming \`main\` is defined as \`main(a: number, b: number)\`. * Ensure that the parameter names in \`main\` match the keys in the returned object. @@ -652,15 +643,6 @@ export const TS_PREPROCESSOR_FLOW_INTRO = `/** * * It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, Postgres, or email) * before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. - * - * The preprocessor receives trigger metadata (\`wm_trigger\`) along with the main trigger arguments. - * The structure of \`wm_trigger\` and the main trigger arguments are specific to each trigger type: - * - Webhook/HTTP: \`(wm_trigger: { kind: 'http' | 'webhook', http?: { ... } }, body_key_1: any, body_key_2: any, ...)\` - * - Postgres: \`(wm_trigger: { kind: 'postgres' }, transaction_type: string, schema_name: string, table_name: string, old_row?: any, row: any)\` - * - WebSocket/Kafka/NATS/SQS: \`(wm_trigger: { kind: 'websocket' | 'kafka' | 'nats' | 'sqs', [kind]: { ... } }, msg: string)\` - * - MQTT: \`(wm_trigger: { kind: 'mqtt', [kind]: { ... } }, payload: Array)\` - * - GCP: \`(wm_trigger: { kind: 'gcp', [kind]: { ... } }, payload: string)\`* - * - Email: \`(wm_trigger: { kind: 'email' }, raw_email: string, parsed_email: { ... })\` * * The returned object determines the parameter values passed to the flow. * e.g., \`{ b: 1, a: 2 }\` → Calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. @@ -670,77 +652,92 @@ export const TS_PREPROCESSOR_FLOW_INTRO = `/** */\n` export const TS_PREPROCESSOR_MODULE_CODE = `export async function preprocessor( - /* - * Replace this comment with the parameters received from the trigger. - * Examples: \`bodyKey1\`, \`bodyKey2\` for Webhook/HTTP, \`msg\` for WebSocket, etc. - */ - - // The trigger metadata - wm_trigger: { - kind: 'http' | 'email' | 'webhook' | 'websocket' | 'kafka' | 'nats' | 'postgres' | 'sqs' | 'mqtt' | 'gcp', - http?: { - route: string // The route path, e.g. "/users/:id" - path: string // The actual path called, e.g. "/users/123" - method: string - params: Record // path parameters - query: Record // query parameters - headers: Record - }, - websocket?: { - url: string // The websocket url - }, - kafka?: { - brokers: string[] - topic: string - group_id: string - }, - nats?: { - servers: string[] - subject: string - headers?: Record - status?: number - description?: string - length: number - }, - sqs?: { - queue_url: string, - message_id?: string, - receipt_handle?: string, - attributes: Record, - message_attributes?: Record - }, - mqtt?: { - topic: string, - retain: boolean, - pkid: number, - qos: number, - v5?: { - payload_format_indicator?: number, - topic_alias?: number, - response_topic?: string, - correlation_data?: Array, - user_properties?: Array<[string, string]>, - subscription_identifiers?: Array, - content_type?: string + event: + | { + kind: "webhook"; + body: any, + raw_string: string | null, + query: Record; + headers: Record; + } + | { + kind: "http"; + body: any, + raw_string: string | null, + route: string; + path: string; + method: string; + params: Record; + query: Record; + headers: Record; + } + | { + kind: "email"; + parsed_email: any, + raw_email: string, + } + | { kind: "websocket"; msg: string; url: string } + | { + kind: "kafka"; + payload: string; + brokers: string[]; + topic: string; + group_id: string; + } + | { + kind: "nats"; + payload: string; + servers: string[]; + subject: string; + headers?: Record; + status?: number; + description?: string; + length: number; + } + | { + kind: "sqs"; + msg: string, + queue_url: string; + message_id?: string; + receipt_handle?: string; + attributes: Record; + message_attributes?: Record< + string, + { string_value?: string; data_type: string } + >; + } + | { + kind: "mqtt"; + payload: string, + topic: string; + retain: boolean; + pkid: number; + qos: number; + v5?: { + payload_format_indicator?: number; + topic_alias?: number; + response_topic?: string; + correlation_data?: Array; + user_properties?: Array<[string, string]>; + subscription_identifiers?: Array; + content_type?: string; + }; + } + | { + kind: "gcp"; + payload: string, + message_id: string; + subscription: string; + ordering_key?: string; + attributes?: Record; + delivery_type: "push" | "pull"; + headers?: Record; + publish_time?: string; } - }, - gcp?: { - message_id: string, - subscription: string, - ordering_key?: string, - attributes?: Record, - delivery_type: "push" | "pull", - headers?: Record, - publish_time?: string, - } - } ) { return { // return the args to be passed to the runnable - } + }; } ` @@ -778,15 +775,6 @@ export const PYTHON_PREPROCESSOR_SCRIPT_INTRO = `# Trigger preprocessor # It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, Postgres, or email) # before passing it to \`main\`. This separates the trigger logic from the main logic and keeps the auto-generated UI clean. # -# The preprocessor receives trigger metadata (\`wm_trigger\`) along with the main trigger arguments. -# The structure of \`wm_trigger\` and the main trigger arguments are specific to each trigger type: -# - Webhook/HTTP: \`(wm_trigger: { kind: 'http' | 'webhook', http?: { ... } }, body_key_1: any, body_key_2: any, ...)\` -# - Postgres: \`(wm_trigger: { kind: 'postgres' }, transaction_type: string, schema_name: string, table_name: string, old_row?: any, row: any)\` -# - WebSocket/Kafka/NATS/SQS: \`(wm_trigger: { kind: 'websocket' | 'kafka' | 'nats' | 'sqs', [kind]: { ... } }, msg: string)\` -# - MQTT: \`(wm_trigger: { kind: 'mqtt', [kind]: { ... } }, payload: Array)\` -# - GCP: \`(wm_trigger: { kind: 'gcp', [kind]: { ... } }, payload: string)\` -# - Email: \`(wm_trigger: { kind: 'email' }, raw_email: string, parsed_email: { ... })\` -# # The returned object defines the parameter values passed to \`main()\`. # e.g., { b: 1, a: 2 } → Calls \`main(2, 1)\`, assuming \`main\` is defined as \`main(a: int, b: int)\`. # Ensure that the parameter names in \`main\` match the keys in the returned object. @@ -797,15 +785,6 @@ export const PYTHON_PREPROCESSOR_FLOW_INTRO = `# Trigger preprocessor # # It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, Postgres, or email) # before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. -# -# The preprocessor receives the same data the flow would if no preprocessor was used, -# plus trigger metadata in the \`wm_trigger\` object: -# - Webhook/HTTP: \`(wm_trigger: { kind: 'http' | 'webhook', http?: { ... } }, body_key_1: any, body_key_2: any, ...)\` -# - Postgres: \`(wm_trigger: { kind: 'postgres' }, transaction_type: string, schema_name: string, table_name: string, old_row?:any, row: any)\` -# - WebSocket/Kafka/NATS/SQS: \`(wm_trigger: { kind: 'websocket' | 'kafka' | 'nats' | 'sqs', [kind]: { ... } }, msg: string)\` -# - MQTT: \`(wm_trigger: { kind: 'mqtt', [kind]: { ... } }, payload: Array)\` -# - GCP: \`(wm_trigger: { kind: 'gcp', [kind]: { ... } }, payload: string)\` -# - Email: \`(wm_trigger: { kind: 'email' }, raw_email: string, parsed_email: { ... })\` # # The returned object determines the parameter values passed to the flow. # e.g., \`{ b: 1, a: 2 }\` → Calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. @@ -813,85 +792,120 @@ export const PYTHON_PREPROCESSOR_FLOW_INTRO = `# Trigger preprocessor # # Learn more: https://www.windmill.dev/docs/core_concepts/preprocessors\n\n` -export const PYTHON_PREPROCESSOR_MODULE_CODE = `from typing import TypedDict, Literal -class Http(TypedDict): - route: str # The route path, e.g. "/users/:id" - path: str # The actual path called, e.g. "/users/123" +export const PYTHON_PREPROCESSOR_MODULE_CODE = `from typing import TypedDict, Literal, Optional, Union + + +class WebhookEvent(TypedDict): + kind: Literal["webhook"] + body: dict + raw_string: Optional[str] + query: dict[str, str] + headers: dict[str, str] + + +class HttpEvent(TypedDict): + kind: Literal["http"] + body: dict + raw_string: Optional[str] + route: str + path: str method: str params: dict[str, str] query: dict[str, str] headers: dict[str, str] -class Websocket(TypedDict): - url: str # The websocket url -class Kafka(TypedDict): - topic: str +class EmailEvent(TypedDict): + kind: Literal["email"] + parsed_email: dict + raw_email: str + + +class WebsocketEvent(TypedDict): + kind: Literal["websocket"] + msg: str + url: str + + +class KafkaEvent(TypedDict): + kind: Literal["kafka"] + payload: str brokers: list[str] + topic: str group_id: str -class Nats(TypedDict): + +class NatsEvent(TypedDict): + kind: Literal["nats"] + payload: str servers: list[str] subject: str - headers: dict[str, list[str]] | None - status: int | None - description: str | None + headers: Optional[dict[str, list[str]]] + status: Optional[int] + description: Optional[str] length: int + class MessageAttribute(TypedDict): - string_value: str | None + string_value: Optional[str] data_type: str -class Sqs(TypedDict): + +class SqsEvent(TypedDict): + kind: Literal["sqs"] + msg: str queue_url: str - message_id: str | None - receipt_handle: str | None + message_id: Optional[str] + receipt_handle: Optional[str] attributes: dict[str, str] - message_attributes: dict[str, MessageAttribute] | None + message_attributes: Optional[dict[str, MessageAttribute]] -class MqttV5Properties: - payload_format_indicator: int | None - topic_alias: int | None - response_topic: str | None - correlation_data: list[int] | None - user_properties: list[tuple[str, str]] | None - subscription_identifiers: list[int] | None - content_type: str | None -class Mqtt(TypedDict): +class MqttV5Properties(TypedDict, total=False): + payload_format_indicator: Optional[int] + topic_alias: Optional[int] + response_topic: Optional[str] + correlation_data: Optional[list[int]] + user_properties: Optional[list[tuple[str, str]]] + subscription_identifiers: Optional[list[int]] + content_type: Optional[str] + + +class MqttEvent(TypedDict): + kind: Literal["mqtt"] + payload: str topic: str retain: bool pkid: int qos: int - v5: MqttV5Properties | None + v5: Optional[MqttV5Properties] -class Gcp(TypedDict): + +class GcpEvent(TypedDict): + kind: Literal["gcp"] + payload: str message_id: str subscription: str - ordering_key: str | None - attributes: dict[str, str] | None + ordering_key: Optional[str] + attributes: Optional[dict[str, str]] delivery_type: Literal["push", "pull"] - headers: dict[str, str] | None - publish_time: str | None - - -class WmTrigger(TypedDict): - kind: Literal["http", "email", "webhook", "websocket", "kafka", "nats", "postgres", "sqs", "mqtt", "gcp"] - http: Http | None - websocket: Websocket | None - kafka: Kafka | None - nats: Nats | None - sqs: Sqs | None - mqtt: Mqtt | None - gcp: Gcp | None + headers: Optional[dict[str, str]] + publish_time: Optional[str] -def preprocessor( - # Replace this comment with the parameters received from the trigger. - # Examples: \`bodyKey1\`, \`bodyKey2\` for Webhook/HTTP, \`msg\` for WebSocket, etc. +Event = Union[ + WebhookEvent, + HttpEvent, + EmailEvent, + WebsocketEvent, + KafkaEvent, + NatsEvent, + SqsEvent, + MqttEvent, + GcpEvent, +] - # Trigger metadata - wm_trigger: WmTrigger, -): + +def preprocessor(event: Event): return { # return the args to be passed to the runnable } @@ -931,6 +945,12 @@ inventory: - resource_type: ansible_inventory # You can pin an inventory to this script by hardcoding the resource path: # resource: u/user/your_resource +# - name: hcloud.yml +# resource_type: dynamic_inventory + +options: + - verbosity: vvv + # File resources will be written in the relative \`target\` location before # running the playbook @@ -946,11 +966,15 @@ extra_vars: world_qualifier: type: string +# If using Ansible Vault: +# vault_password: u/user/ansible_vault_password + dependencies: galaxy: collections: - name: community.general - name: community.vmware + roles: python: - jmespath --- @@ -1027,7 +1051,7 @@ public class Main { } } ` -// for related places search: ADD_NEW_LANG +// for related places search: ADD_NEW_LANG export const INITIAL_CODE = { bun: { scriptInitCodeBlock: BUN_INIT_BLOCK, @@ -1114,8 +1138,8 @@ export const INITIAL_CODE = { }, java: { script: JAVA_INIT_CODE - }, - // for related places search: ADD_NEW_LANG + } + // for related places search: ADD_NEW_LANG } export function isInitialCode(content: string): boolean { @@ -1221,7 +1245,7 @@ export function initialCode( return INITIAL_CODE.nu.script } else if (language == 'java') { return INITIAL_CODE.java.script - // for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG } else if (language == 'bun' || language == 'bunnative') { if (kind == 'trigger') { return INITIAL_CODE.bun.trigger diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts index 20e48d235c..9a126bf080 100644 --- a/frontend/src/lib/stores.ts +++ b/frontend/src/lib/stores.ts @@ -28,6 +28,14 @@ export interface UserExt { folders_owners: string[] } +export interface UserWorkspace { + id: string + name: string + username: string + color: string | null + operator_settings?: OperatorSettings +} + const persistedWorkspace = BROWSER && getWorkspace() function getWorkspace(): string | undefined { @@ -60,31 +68,26 @@ export const superadmin = writable(undefined) export const devopsRole = writable(undefined) export const lspTokenStore = writable(undefined) export const hubBaseUrlStore = writable('https://hub.windmill.dev') -export const userWorkspaces: Readable< - Array<{ - id: string - name: string - username: string - color: string | null - operator_settings?: OperatorSettings - }> -> = derived([usersWorkspaceStore, superadmin], ([store, superadmin]) => { - const originalWorkspaces = store?.workspaces ?? [] - if (superadmin) { - return [ - ...originalWorkspaces.filter((x) => x.id != 'admins'), - { - id: 'admins', - name: 'Admins', - username: 'superadmin', - color: null, - operator_settings: null - } - ] - } else { - return originalWorkspaces +export const userWorkspaces: Readable> = derived( + [usersWorkspaceStore, superadmin], + ([store, superadmin]) => { + const originalWorkspaces = store?.workspaces ?? [] + if (superadmin) { + return [ + ...originalWorkspaces.filter((x) => x.id != 'admins'), + { + id: 'admins', + name: 'Admins', + username: 'superadmin', + color: null, + operator_settings: null + } + ] + } else { + return originalWorkspaces + } } -}) +) export const copilotInfo = writable<{ enabled: boolean codeCompletionModel?: AIProviderModel @@ -232,3 +235,16 @@ export const workspaceColor: Readable = derived( }) } ) + +export function getFlatTableNamesFromSchema(dbSchema: DBSchema | undefined): string[] { + const schema = dbSchema?.schema ?? {} + const tableNames: string[] = [] + + for (const schemaKey in schema) { + for (const tableKey in schema[schemaKey]) { + tableNames.push(`${schemaKey}.${tableKey}`) + } + } + + return tableNames +} diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index f5da07ee38..2609eba53d 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -9,7 +9,7 @@ import { deepEqual } from 'fast-equals' import YAML from 'yaml' -import type { UserExt } from './stores' +import { type UserExt } from './stores' import { sendUserToast } from './toast' import type { Job, Script } from './gen' import type { EnumType, SchemaProperty } from './common' @@ -17,6 +17,7 @@ import type { Schema } from './common' export { sendUserToast } import type { AnyMeltElement } from '@melt-ui/svelte' import type { RunsSelectionMode } from './components/runs/RunsBatchActionsDropdown.svelte' +import type { TriggerKind } from './components/triggers' export function isJobCancelable(j: Job): boolean { return j.type === 'QueuedJob' && !j.schedule_path && !j.canceled @@ -141,6 +142,17 @@ export function msToSec(ms: number | undefined, maximumFractionDigits?: number): }) } +export function removeTriggerKindIfUnused( + length: number, + triggerKind: TriggerKind, + usedTriggerKinds: string[] +) { + if (length === 0 && usedTriggerKinds.includes(triggerKind)) { + return usedTriggerKinds.filter((kind) => kind != triggerKind) + } + return usedTriggerKinds +} + export function msToReadableTime(ms: number | undefined): string { if (ms === undefined) return '?' @@ -680,6 +692,12 @@ export function sortObject(o: T & object): T { }, {}) as T } +export function sortArray(array: T[], compareFn?: (a: T, b: T) => number): T[] { + const arr = [...array] + arr.sort(compareFn) + return arr +} + export function generateRandomString(len: number = 24): string { let chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789' let result = '' @@ -1247,3 +1265,10 @@ export function getOS() { return 'Unknown OS' as const } + +import { type ClassValue, clsx } from 'clsx' +import { twMerge } from 'tailwind-merge' + +export function cn(...inputs: ClassValue[]) { + return twMerge(clsx(inputs)) +} diff --git a/frontend/src/lib/utils_deployable.ts b/frontend/src/lib/utils_deployable.ts index 355fe79071..2a81423b47 100644 --- a/frontend/src/lib/utils_deployable.ts +++ b/frontend/src/lib/utils_deployable.ts @@ -9,7 +9,7 @@ import { ScheduleService, SqsTriggerService, WebsocketTriggerService, - type NewGcpTrigger, + type GcpTriggerData, type WorkspaceDeployUISettings } from './gen' import type { TriggerKind } from './components/triggers' @@ -78,11 +78,9 @@ export async function existsTrigger( return await PostgresTriggerService.existsPostgresTrigger(data) } else if (triggerKind === 'sqs') { return await SqsTriggerService.existsSqsTrigger(data) - } - else if (triggerKind === 'gcp') { + } else if (triggerKind === 'gcp') { return await GcpTriggerService.existsGcpTrigger(data) - } - else if (triggerKind === 'websockets') { + } else if (triggerKind === 'websockets') { return await WebsocketTriggerService.existsWebsocketTrigger(data) } else if (triggerKind === 'nats') { return await NatsTriggerService.existsNatsTrigger(data) @@ -140,36 +138,31 @@ export async function getTriggersDeployData(kind: TriggerKind, path: string, wor createFn: NatsTriggerService.createNatsTrigger, updateFn: NatsTriggerService.updateNatsTrigger } - } - else if (kind === 'gcp') { + } else if (kind === 'gcp') { const gcpTrigger = await GcpTriggerService.getGcpTrigger({ workspace: workspace!, path: path }) gcpTrigger.subscription_id = '' + gcpTrigger.subscription_mode = 'create_update' + if (gcpTrigger.delivery_config) { gcpTrigger.delivery_config.audience = '' } - const data: NewGcpTrigger = { - subscription_mode: { - subscription_mode: 'create_update', - subscription_id: gcpTrigger.subscription_id, - delivery_type: gcpTrigger.delivery_type, - delivery_config: gcpTrigger.delivery_config, - base_endpoint: `${window.location.origin}${base}` - }, - ...gcpTrigger + const data: GcpTriggerData = { + ...gcpTrigger, + base_endpoint: + gcpTrigger.delivery_type === 'push' ? `${window.location.origin}${base}` : undefined } return { data, createFn: GcpTriggerService.createGcpTrigger, updateFn: GcpTriggerService.updateGcpTrigger - } - } - else if (kind === 'postgres') { + } + } else if (kind === 'postgres') { const postgresTrigger = await PostgresTriggerService.getPostgresTrigger({ workspace: workspace!, path: path @@ -321,26 +314,20 @@ export async function getTriggerValue(kind: TriggerKind, path: string, workspace replication_slot_name, publication_name } - } - else if (kind === 'gcp') { - const { - enabled, - script_path, - is_flow, - gcp_resource_path, - } = await GcpTriggerService.getGcpTrigger({ - workspace: workspace!, - path: path - }) + } else if (kind === 'gcp') { + const { enabled, script_path, is_flow, gcp_resource_path } = + await GcpTriggerService.getGcpTrigger({ + workspace: workspace!, + path: path + }) return { enabled, script_path, is_flow, - gcp_resource_path, + gcp_resource_path } - } - else if (kind === 'websockets') { + } else if (kind === 'websockets') { const { enabled, script_path, @@ -511,17 +498,14 @@ export async function getTriggerDependency(kind: TriggerKind, path: string, work }) result = retrieveKindsValues({ resource_path: postgres_resource_path, script_path, is_flow }) - } - else if (kind === 'gcp') { - const { gcp_resource_path, script_path, is_flow } = - await GcpTriggerService.getGcpTrigger({ - workspace: workspace!, - path: path - }) + } else if (kind === 'gcp') { + const { gcp_resource_path, script_path, is_flow } = await GcpTriggerService.getGcpTrigger({ + workspace: workspace!, + path: path + }) result = retrieveKindsValues({ resource_path: gcp_resource_path, script_path, is_flow }) - } - else if (kind === 'websockets') { + } else if (kind === 'websockets') { const { script_path, is_flow, url, initial_messages } = await WebsocketTriggerService.getWebsocketTrigger({ workspace: workspace!, @@ -546,12 +530,17 @@ export async function getTriggerDependency(kind: TriggerKind, path: string, work } }) } else if (kind === 'routes') { - const { script_path, is_flow, authentication_resource_path} = await HttpTriggerService.getHttpTrigger({ - workspace: workspace!, - path: path - }) + const { script_path, is_flow, authentication_resource_path } = + await HttpTriggerService.getHttpTrigger({ + workspace: workspace!, + path: path + }) - result = retrieveKindsValues({ script_path, is_flow, resource_path: authentication_resource_path}) + result = retrieveKindsValues({ + script_path, + is_flow, + resource_path: authentication_resource_path + }) } else if (kind === 'schedules') { const { script_path, is_flow } = await ScheduleService.getSchedule({ workspace: workspace!, diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index f5dd7abfe2..fe46d90921 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -70,8 +70,9 @@ $: $page.url && onQueryChange() function onQueryChangeUserSettings() { - if (userSettings && $page.url.hash === USER_SETTINGS_HASH) { - userSettings.openDrawer() + if (userSettings && $page.url.hash.startsWith(USER_SETTINGS_HASH)) { + const mcpMode = $page.url.hash.includes('-mcp') + userSettings.openDrawer(mcpMode) } } @@ -348,7 +349,7 @@ - + {#if $page.status == 404}
diff --git a/frontend/src/routes/(root)/(logged)/gcp_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/gcp_triggers/+page.svelte index c5fadefa4e..9fcaa0f202 100644 --- a/frontend/src/routes/(root)/(logged)/gcp_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/gcp_triggers/+page.svelte @@ -11,7 +11,8 @@ displayDate, getLocalSetting, sendUserToast, - storeLocalSetting + storeLocalSetting, + removeTriggerKindIfUnused } from '$lib/utils' import { base } from '$app/paths' import CenteredPage from '$lib/components/CenteredPage.svelte' @@ -21,7 +22,7 @@ import SharedBadge from '$lib/components/SharedBadge.svelte' import ShareModal from '$lib/components/ShareModal.svelte' import Toggle from '$lib/components/Toggle.svelte' - import { enterpriseLicense, userStore, workspaceStore } from '$lib/stores' + import { enterpriseLicense, usedTriggerKinds, userStore, workspaceStore } from '$lib/stores' import { Code, Eye, Pen, Plus, Share, Trash, Circle, FileUp, ClipboardCopy } from 'lucide-svelte' import { goto } from '$lib/navigation' import SearchItems from '$lib/components/SearchItems.svelte' @@ -64,6 +65,7 @@ return { canWrite: canWrite(x.path, x.extra_perms!, $userStore), ...x } } ) + $usedTriggerKinds = removeTriggerKindIfUnused(triggers.length, 'gcp', $usedTriggerKinds) loading = false } diff --git a/frontend/src/routes/(root)/(logged)/kafka_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/kafka_triggers/+page.svelte index 14a9c4b6f5..5d8c6ba86e 100644 --- a/frontend/src/routes/(root)/(logged)/kafka_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/kafka_triggers/+page.svelte @@ -10,7 +10,8 @@ displayDate, getLocalSetting, sendUserToast, - storeLocalSetting + storeLocalSetting, + removeTriggerKindIfUnused } from '$lib/utils' import { base } from '$app/paths' import CenteredPage from '$lib/components/CenteredPage.svelte' @@ -20,7 +21,13 @@ import SharedBadge from '$lib/components/SharedBadge.svelte' import ShareModal from '$lib/components/ShareModal.svelte' import Toggle from '$lib/components/Toggle.svelte' - import { userStore, workspaceStore, userWorkspaces, enterpriseLicense } from '$lib/stores' + import { + userStore, + workspaceStore, + userWorkspaces, + enterpriseLicense, + usedTriggerKinds + } from '$lib/stores' import { Code, Eye, Pen, Plus, Share, Trash, Circle, FileUp } from 'lucide-svelte' import { goto } from '$lib/navigation' import SearchItems from '$lib/components/SearchItems.svelte' @@ -62,6 +69,7 @@ return { canWrite: canWrite(x.path, x.extra_perms!, $userStore), ...x } } ) + $usedTriggerKinds = removeTriggerKindIfUnused(triggers.length, 'kafka', $usedTriggerKinds) loading = false } @@ -160,15 +168,15 @@ (x) => x.path.startsWith(ownerFilter + '/') && filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) - ) + ) : triggers?.filter( (x) => x.script_path.startsWith(ownerFilter + '/') && filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) - ) + ) : triggers?.filter((x) => filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) - ) + ) $: if ($workspaceStore) { ownerFilter = undefined @@ -178,10 +186,10 @@ selectedFilterKind === 'trigger' ? Array.from( new Set(filteredItems?.map((x) => x.path.split('/').slice(0, 2).join('/')) ?? []) - ).sort() + ).sort() : Array.from( new Set(filteredItems?.map((x) => x.script_path.split('/').slice(0, 2).join('/')) ?? []) - ).sort() + ).sort() $: items = filter !== '' ? filteredItems : preFilteredItems @@ -380,7 +388,7 @@ ? { icon: Pen } : { icon: Eye - }} + }} color="dark" > {canWrite ? 'Edit' : 'View'} @@ -427,7 +435,7 @@ }) } } - ] + ] : []), { displayName: 'Audit logs', diff --git a/frontend/src/routes/(root)/(logged)/mqtt_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/mqtt_triggers/+page.svelte index 7c2cc2a87e..4a78f7e35e 100644 --- a/frontend/src/routes/(root)/(logged)/mqtt_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/mqtt_triggers/+page.svelte @@ -10,7 +10,8 @@ displayDate, getLocalSetting, sendUserToast, - storeLocalSetting + storeLocalSetting, + removeTriggerKindIfUnused } from '$lib/utils' import { base } from '$app/paths' import CenteredPage from '$lib/components/CenteredPage.svelte' @@ -20,7 +21,7 @@ import SharedBadge from '$lib/components/SharedBadge.svelte' import ShareModal from '$lib/components/ShareModal.svelte' import Toggle from '$lib/components/Toggle.svelte' - import { enterpriseLicense, userStore, workspaceStore } from '$lib/stores' + import { enterpriseLicense, usedTriggerKinds, userStore, workspaceStore } from '$lib/stores' import { Code, Eye, Pen, Plus, Share, Trash, Circle, FileUp } from 'lucide-svelte' import { goto } from '$lib/navigation' import SearchItems from '$lib/components/SearchItems.svelte' @@ -62,6 +63,7 @@ return { canWrite: canWrite(x.path, x.extra_perms!, $userStore), ...x } } ) + $usedTriggerKinds = removeTriggerKindIfUnused(triggers.length, 'mqtt', $usedTriggerKinds) loading = false } @@ -160,15 +162,15 @@ (x) => x.path.startsWith(ownerFilter + '/') && filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) - ) + ) : triggers?.filter( (x) => x.script_path.startsWith(ownerFilter + '/') && filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) - ) + ) : triggers?.filter((x) => filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) - ) + ) $: if ($workspaceStore) { ownerFilter = undefined @@ -178,10 +180,10 @@ selectedFilterKind === 'trigger' ? Array.from( new Set(filteredItems?.map((x) => x.path.split('/').slice(0, 2).join('/')) ?? []) - ).sort() + ).sort() : Array.from( new Set(filteredItems?.map((x) => x.script_path.split('/').slice(0, 2).join('/')) ?? []) - ).sort() + ).sort() $: items = filter !== '' ? filteredItems : preFilteredItems @@ -360,7 +362,7 @@ ? { icon: Pen } : { icon: Eye - }} + }} color="dark" > {canWrite ? 'Edit' : 'View'} @@ -407,7 +409,7 @@ }) } } - ] + ] : []), { displayName: 'Audit logs', diff --git a/frontend/src/routes/(root)/(logged)/nats_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/nats_triggers/+page.svelte index 54d8496cb3..9e18225165 100644 --- a/frontend/src/routes/(root)/(logged)/nats_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/nats_triggers/+page.svelte @@ -10,7 +10,8 @@ displayDate, getLocalSetting, sendUserToast, - storeLocalSetting + storeLocalSetting, + removeTriggerKindIfUnused } from '$lib/utils' import { base } from '$app/paths' import CenteredPage from '$lib/components/CenteredPage.svelte' @@ -20,7 +21,13 @@ import SharedBadge from '$lib/components/SharedBadge.svelte' import ShareModal from '$lib/components/ShareModal.svelte' import Toggle from '$lib/components/Toggle.svelte' - import { userStore, workspaceStore, userWorkspaces, enterpriseLicense } from '$lib/stores' + import { + userStore, + workspaceStore, + userWorkspaces, + enterpriseLicense, + usedTriggerKinds + } from '$lib/stores' import { Code, Eye, Pen, Plus, Share, Trash, Circle, FileUp } from 'lucide-svelte' import { goto } from '$lib/navigation' import SearchItems from '$lib/components/SearchItems.svelte' @@ -61,6 +68,7 @@ return { canWrite: canWrite(x.path, x.extra_perms!, $userStore), ...x } } ) + $usedTriggerKinds = removeTriggerKindIfUnused(triggers.length, 'nats', $usedTriggerKinds) loading = false } @@ -159,15 +167,15 @@ (x) => x.path.startsWith(ownerFilter + '/') && filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) - ) + ) : triggers?.filter( (x) => x.script_path.startsWith(ownerFilter + '/') && filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) - ) + ) : triggers?.filter((x) => filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) - ) + ) $: if ($workspaceStore) { ownerFilter = undefined @@ -177,10 +185,10 @@ selectedFilterKind === 'trigger' ? Array.from( new Set(filteredItems?.map((x) => x.path.split('/').slice(0, 2).join('/')) ?? []) - ).sort() + ).sort() : Array.from( new Set(filteredItems?.map((x) => x.script_path.split('/').slice(0, 2).join('/')) ?? []) - ).sort() + ).sort() $: items = filter !== '' ? filteredItems : preFilteredItems @@ -379,7 +387,7 @@ ? { icon: Pen } : { icon: Eye - }} + }} color="dark" > {canWrite ? 'Edit' : 'View'} @@ -426,7 +434,7 @@ }) } } - ] + ] : []), { displayName: 'Audit logs', diff --git a/frontend/src/routes/(root)/(logged)/postgres_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/postgres_triggers/+page.svelte index 7af681c309..a23c23381e 100644 --- a/frontend/src/routes/(root)/(logged)/postgres_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/postgres_triggers/+page.svelte @@ -10,7 +10,8 @@ displayDate, getLocalSetting, sendUserToast, - storeLocalSetting + storeLocalSetting, + removeTriggerKindIfUnused } from '$lib/utils' import { base } from '$app/paths' import CenteredPage from '$lib/components/CenteredPage.svelte' @@ -20,7 +21,7 @@ import SharedBadge from '$lib/components/SharedBadge.svelte' import ShareModal from '$lib/components/ShareModal.svelte' import Toggle from '$lib/components/Toggle.svelte' - import { enterpriseLicense, userStore, workspaceStore } from '$lib/stores' + import { enterpriseLicense, usedTriggerKinds, userStore, workspaceStore } from '$lib/stores' import { Code, Eye, Pen, Plus, Share, Trash, Circle, Database, FileUp } from 'lucide-svelte' import { goto } from '$lib/navigation' import SearchItems from '$lib/components/SearchItems.svelte' @@ -61,6 +62,7 @@ ).map((x) => { return { canWrite: canWrite(x.path, x.extra_perms!, $userStore), ...x } }) + $usedTriggerKinds = removeTriggerKindIfUnused(triggers.length, 'postgres', $usedTriggerKinds) loading = false } diff --git a/frontend/src/routes/(root)/(logged)/resources/+page.svelte b/frontend/src/routes/(root)/(logged)/resources/+page.svelte index a5157c4f7c..5816ba85f9 100644 --- a/frontend/src/routes/(root)/(logged)/resources/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/resources/+page.svelte @@ -60,6 +60,8 @@ import EditableSchemaWrapper from '$lib/components/schema/EditableSchemaWrapper.svelte' import ResourceEditorDrawer from '$lib/components/ResourceEditorDrawer.svelte' import GfmMarkdown from '$lib/components/GfmMarkdown.svelte' + import DbManagerDrawerButton from '$lib/components/DBManagerDrawerButton.svelte' + import { isDbType } from '$lib/components/apps/components/display/dbtable/utils' type ResourceW = ListableResource & { canWrite: boolean; marked?: string } type ResourceTypeW = ResourceType & { canWrite: boolean } @@ -136,23 +138,23 @@ x.resource_type !== 'state' && x.resource_type !== 'cache' : tab === 'states' - ? x.resource_type === 'state' - : tab === 'cache' - ? x.resource_type === 'cache' - : tab === 'theme' - ? x.resource_type === 'app_theme' - : true - }) + ? x.resource_type === 'state' + : tab === 'cache' + ? x.resource_type === 'cache' + : tab === 'theme' + ? x.resource_type === 'app_theme' + : true + }) : preFilteredItemsOwners?.filter((x) => { return ( x.resource_type === typeFilter && (tab === 'workspace' ? x.resource_type !== 'app_theme' && - x.resource_type !== 'state' && - x.resource_type !== 'cache' + x.resource_type !== 'state' && + x.resource_type !== 'cache' : true) ) - }) + }) async function loadResources(): Promise { resources = await loadResourceInternal(undefined, 'cache,state') @@ -488,10 +490,7 @@ >
@@ -569,7 +568,8 @@
@@ -603,422 +603,435 @@ f={(x) => x.path + ' ' + x.resource_type + ' ' + x.description + ' '} /> -{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find(_ => _.id === $workspaceStore)?.operator_settings?.resources} - -{:else} - - -
- - -
- -
- { - if (e.detail == 'cache') { - loading.resources = true - loadCache() - } else if (e.detail == 'states') { - loading.resources = true - loadState() - } - }} - > - -
- - Workspace -
- - -
- Resource Types - - Every resource has a Resource Type attached to it which contains its schema and make it - easy in scripts and flows to accept only resources of a specific resource type. - -
- - -
- States - - States are actually resources (but excluded from the Workspace tab for clarity). States - are used by scripts to keep data persistent between runs of the same script by the same - trigger (schedule or user) - -
- - -
- Cache - - Cached results are actually resources (but excluded from the Workspace tab for clarity). - Cache are used by flows's step to cache result to avoid recomputing unnecessarily - -
- - -
- Theme - - Theme are actually resources (but excluded from the Workspace tab for clarity). Theme - are used by the apps to customize their look and feel. - -
- - -
-
+{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.resources} + - {#if tab == 'workspace' || tab == 'states' || tab == 'cache' || tab == 'theme'} -
- -
- - {#if tab != 'states' && tab != 'cache'} - - {:else} -
- {/if} - -
- {#if loading.resources} - - {#each new Array(6) as _} - - {/each} - {:else if filteredItems?.length == 0} -
-
No resources found
-
- Try changing the filters or creating a new resource +{:else} + + +
+ + +
+ +
+ { + if (e.detail == 'cache') { + loading.resources = true + loadCache() + } else if (e.detail == 'states') { + loading.resources = true + loadState() + } + }} + > + +
+ + Workspace
-
+ + +
+ Resource Types + + Every resource has a Resource Type attached to it which contains its schema and make + it easy in scripts and flows to accept only resources of a specific resource type. + +
+ + +
+ States + + States are actually resources (but excluded from the Workspace tab for clarity). + States are used by scripts to keep data persistent between runs of the same script by + the same trigger (schedule or user) + +
+ + +
+ Cache + + Cached results are actually resources (but excluded from the Workspace tab for + clarity). Cache are used by flows's step to cache result to avoid recomputing + unnecessarily + +
+ + +
+ Theme + + Theme are actually resources (but excluded from the Workspace tab for clarity). Theme + are used by the apps to customize their look and feel. + +
+ + +
+
+
+ {#if tab == 'workspace' || tab == 'states' || tab == 'cache' || tab == 'theme'} +
+ +
+ + {#if tab != 'states' && tab != 'cache'} + {:else} - - - - - Path - Resource type - Description - - - - - - {#if filteredItems} - {#each filteredItems as { path, description, resource_type, extra_perms, canWrite, is_oauth, is_linked, account, refresh_error, is_expired, marked, is_refreshed }} - - - - - - resourceEditor?.initEdit?.(path)} - >{#if marked}{@html marked}{:else}{path}{/if} - - - { - const linkedRt = resourceTypes?.find((rt) => rt.name === resource_type) - if (linkedRt) { - resourceTypeViewerObj = { - rt: linkedRt.name, - //@ts-ignore - schema: linkedRt.schema, - description: linkedRt.description ?? '', - formatExtension: linkedRt.format_extension +
+ {/if} + +
+ {#if loading.resources} + + {#each new Array(6) as _} + + {/each} + {:else if filteredItems?.length == 0} +
+
No resources found
+
+ Try changing the filters or creating a new resource +
+
+ {:else} + + + + + Path + Resource type + Description + + + + + + {#if filteredItems} + {#each filteredItems as { path, description, resource_type, extra_perms, canWrite, is_oauth, is_linked, account, refresh_error, is_expired, marked, is_refreshed }} + + + + + + resourceEditor?.initEdit?.(path)} + >{#if marked}{@html marked}{:else}{path}{/if} + + + { + const linkedRt = resourceTypes?.find((rt) => rt.name === resource_type) + if (linkedRt) { + resourceTypeViewerObj = { + rt: linkedRt.name, + //@ts-ignore + schema: linkedRt.schema, + description: linkedRt.description ?? '', + formatExtension: linkedRt.format_extension + } + resourceTypeViewer.openDrawer?.() + } else { + sendUserToast( + `Resource type ${resource_type} not found in workspace.`, + true + ) } - resourceTypeViewer.openDrawer?.() - } else { - sendUserToast( - `Resource type ${resource_type} not found in workspace.`, - true - ) - } - }} - > - - - - - - {removeMarkdown(truncate(description ?? '', 30))} - - - -
-
- {#if is_linked} - - -
- This resource is linked with a variable of the same path. They are - deleted and renamed together. -
- - {/if} -
-
- {#if is_refreshed} - - -
- The OAuth token will be kept up-to-date in the background by Windmill - using its refresh token -
- - {/if} -
- - {#if is_oauth} -
- {#if refresh_error} + }} + > + + + + + + {removeMarkdown(truncate(description ?? '', 30))} + + + +
+
+ {#if is_linked} - +
- Latest exchange of the refresh token did not succeed. Error: {refresh_error} -
- - {:else if is_expired} - - - -
- The access_token is expired, it will get renewed the next time this - variable is fetched or you can request is to be refreshed in the - dropdown on the right. -
- - {:else} - - -
- The resource was connected through OAuth and the token is not - expired. + This resource is linked with a variable of the same path. They are + deleted and renamed together.
{/if}
+
+ {#if is_refreshed} + + +
+ The OAuth token will be kept up-to-date in the background by + Windmill using its refresh token +
+ + {/if} +
+ + {#if is_oauth} +
+ {#if refresh_error} + + +
+ Latest exchange of the refresh token did not succeed. Error: {refresh_error} +
+ + {:else if is_expired} + + + +
+ The access_token is expired, it will get renewed the next time + this variable is fetched or you can request is to be refreshed in + the dropdown on the right. +
+ + {:else} + + +
+ The resource was connected through OAuth and the token is not + expired. +
+ + {/if} +
+ {/if} +
+ + + {#if path && isDbType(resource_type)} + {/if} -
- - - { - shareModal.openDrawer?.(path, 'resource') - } - }, - { - displayName: 'Edit', - icon: Pen, - disabled: !canWrite, - action: () => { - resourceEditor?.initEdit?.(path) - } - }, - ...(isDeployable('resource', path, deployUiSettings) - ? [ - { - displayName: 'Deploy to prod/staging', - icon: FileUp, - action: () => { - deploymentDrawer.openDrawer(path, 'resource') + { + shareModal.openDrawer?.(path, 'resource') + } + }, + { + displayName: 'Edit', + icon: Pen, + disabled: !canWrite, + action: () => { + resourceEditor?.initEdit?.(path) + } + }, + ...(isDeployable('resource', path, deployUiSettings) + ? [ + { + displayName: 'Deploy to prod/staging', + icon: FileUp, + action: () => { + deploymentDrawer.openDrawer(path, 'resource') + } } - } - ] - : []), - { - displayName: 'Delete', - disabled: !canWrite, - icon: Trash, - type: 'delete', - action: (event) => { - // TODO - // @ts-ignore - if (event?.shiftKey) { - deleteResource(path, account) - } else { - deleteConfirmedCallback = () => { + ] + : []), + { + displayName: 'Delete', + disabled: !canWrite, + icon: Trash, + type: 'delete', + action: (event) => { + // TODO + // @ts-ignore + if (event?.shiftKey) { deleteResource(path, account) + } else { + deleteConfirmedCallback = () => { + deleteResource(path, account) + } } } - } - }, - ...(account != undefined - ? [ - { - displayName: 'Refresh token', - icon: RotateCw, - action: async () => { - await OauthService.refreshToken({ - workspace: $workspaceStore ?? '', - id: account ?? 0, - requestBody: { - path - } - }) - sendUserToast('Token refreshed') - loadResources() + }, + ...(account != undefined + ? [ + { + displayName: 'Refresh token', + icon: RotateCw, + action: async () => { + await OauthService.refreshToken({ + workspace: $workspaceStore ?? '', + id: account ?? 0, + requestBody: { + path + } + }) + sendUserToast('Token refreshed') + loadResources() + } } - } - ] - : []) - ]} - /> - - - {/each} - {/if} - - - {/if} -
- {:else if tab == 'types'} - {#if loading.types} - - {#each new Array(6) as _} - - {/each} - {:else} -
- - - - Name - Description - - - - - {#if resourceTypes} - {#each resourceTypes as { name, description, schema, canWrite, format_extension }} - - - { - resourceTypeViewerObj = { - rt: name, - //@ts-ignore - schema: schema, - description: description ?? '', - formatExtension: format_extension - } - - resourceTypeViewer.openDrawer?.() - }} - > - - - - - - {removeMarkdown(truncate(description ?? '', 200))} - - - - {#if !canWrite} - - Shared globally - - This resource type is from the 'admins' workspace shared with all - workspaces - - - {:else if $userStore?.is_admin || $userStore?.is_super_admin} -
- - -
- {:else} - - Non Editable - - Since resource types are shared with the whole workspace, only admins can - edit/delete them - - - {/if} - - - {/each} - {/if} - - + ] + : []) + ]} + /> + + + {/each} + {/if} + + + {/if}
+ {:else if tab == 'types'} + {#if loading.types} + + {#each new Array(6) as _} + + {/each} + {:else} +
+ + + + Name + Description + + + + + {#if resourceTypes} + {#each resourceTypes as { name, description, schema, canWrite, format_extension }} + + + { + resourceTypeViewerObj = { + rt: name, + //@ts-ignore + schema: schema, + description: description ?? '', + formatExtension: format_extension + } + + resourceTypeViewer.openDrawer?.() + }} + > + + + + + + {removeMarkdown(truncate(description ?? '', 200))} + + + + {#if !canWrite} + + Shared globally + + This resource type is from the 'admins' workspace shared with all + workspaces + + + {:else if $userStore?.is_admin || $userStore?.is_super_admin} +
+ + +
+ {:else} + + Non Editable + + Since resource types are shared with the whole workspace, only admins + can edit/delete them + + + {/if} + + + {/each} + {/if} + + +
+ {/if} {/if} - {/if} - + {/if} diff --git a/frontend/src/routes/(root)/(logged)/routes/+page.svelte b/frontend/src/routes/(root)/(logged)/routes/+page.svelte index 1a33fae1be..89163b464a 100644 --- a/frontend/src/routes/(root)/(logged)/routes/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/routes/+page.svelte @@ -10,7 +10,8 @@ copyToClipboard, displayDate, getLocalSetting, - storeLocalSetting + storeLocalSetting, + removeTriggerKindIfUnused } from '$lib/utils' import { base } from '$app/paths' import CenteredPage from '$lib/components/CenteredPage.svelte' @@ -20,7 +21,13 @@ import SharedBadge from '$lib/components/SharedBadge.svelte' import ShareModal from '$lib/components/ShareModal.svelte' import Toggle from '$lib/components/Toggle.svelte' - import { userStore, workspaceStore, userWorkspaces, enterpriseLicense } from '$lib/stores' + import { + userStore, + workspaceStore, + userWorkspaces, + enterpriseLicense, + usedTriggerKinds + } from '$lib/stores' import { Route, Code, Eye, Pen, Plus, Share, Trash, FileUp, ClipboardCopy } from 'lucide-svelte' import { goto } from '$lib/navigation' import SearchItems from '$lib/components/SearchItems.svelte' @@ -60,6 +67,7 @@ return { canWrite: canWrite(x.path, x.extra_perms!, $userStore), ...x } } ) + $usedTriggerKinds = removeTriggerKindIfUnused(triggers.length, 'routes', $usedTriggerKinds) loading = false } @@ -115,15 +123,15 @@ (x) => x.path.startsWith(ownerFilter + '/') && filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) - ) + ) : triggers?.filter( (x) => x.script_path.startsWith(ownerFilter + '/') && filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) - ) + ) : triggers?.filter((x) => filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) - ) + ) $: if ($workspaceStore) { ownerFilter = undefined @@ -133,14 +141,14 @@ selectedFilterKind === 'trigger' ? Array.from( new Set(filteredItems?.map((x) => x.path.split('/').slice(0, 2).join('/')) ?? []) - ).sort() + ).sort() : Array.from( new Set( filteredItems ?.filter((x) => x.script_path) .map((x) => x.script_path.split('/').slice(0, 2).join('/')) ?? [] ) - ).sort() + ).sort() $: items = filter !== '' ? filteredItems : preFilteredItems @@ -299,7 +307,7 @@ ? { icon: Pen } : { icon: Eye - }} + }} color="dark" > {canWrite ? 'Edit' : 'View'} @@ -347,7 +355,7 @@ }) } } - ] + ] : []), { displayName: 'Audit logs', diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index 2160d7adc7..35f23fc9e7 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -81,13 +81,13 @@ import { json } from 'svelte-highlight/languages' import Toggle from '$lib/components/Toggle.svelte' import WorkflowTimeline from '$lib/components/WorkflowTimeline.svelte' - import ScheduleEditor from '$lib/components/ScheduleEditor.svelte' import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' import HighlightTheme from '$lib/components/HighlightTheme.svelte' import PreprocessedArgsDisplay from '$lib/components/runs/PreprocessedArgsDisplay.svelte' import ExecutionDuration from '$lib/components/ExecutionDuration.svelte' import CustomPopover from '$lib/components/CustomPopover.svelte' import { isWindmillTooBigObject } from '$lib/components/job_args' + import ScheduleEditor from '$lib/components/triggers/schedules/ScheduleEditor.svelte' let job: Job | undefined let jobUpdateLastFetch: Date | undefined diff --git a/frontend/src/routes/(root)/(logged)/schedules/+page.svelte b/frontend/src/routes/(root)/(logged)/schedules/+page.svelte index 75e65efb99..4c1b11b663 100644 --- a/frontend/src/routes/(root)/(logged)/schedules/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/schedules/+page.svelte @@ -1,7 +1,6 @@