From d6bf6f6b555bade81d1210de3648c09f95f0fde7 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 20 May 2025 12:52:28 +0200 Subject: [PATCH 01/45] Add Claude PR Assistant workflow (#5777) --- .github/workflows/claude.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/claude.yml diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml new file mode 100644 index 0000000000..75484f7aa0 --- /dev/null +++ b/.github/workflows/claude.yml @@ -0,0 +1,35 @@ +name: Claude Code + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + issues: + types: [opened, assigned] + pull_request_review: + types: [submitted] + +jobs: + claude: + if: | + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || + (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Run Claude Code + id: claude + uses: anthropics/claude-code-action@beta + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + From ba4c89e7dbfbe45c0ee5ad61658db78a308d013b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 20 May 2025 12:55:11 +0200 Subject: [PATCH 02/45] nit --- backend/src/monitor.rs | 5 ++++- backend/windmill-common/src/s3_helpers.rs | 9 +++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 24efd181c0..8b401e8acc 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -1126,7 +1126,10 @@ pub async fn reload_s3_cache_setting(db: &DB) { if let Err(e) = s3_client { tracing::error!("Error building s3 client from settings: {:?}", e) } else { - tracing::info!("Loaded object store {:?}", setting.unwrap().get_bucket()); + tracing::info!( + "Loaded object store {:?}", + setting.as_ref().unwrap().get_bucket() + ); *s3_cache_settings = Some(s3_client.unwrap()); } } diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 041ca1a5f8..698fad1c4d 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -415,13 +415,10 @@ pub enum ObjectSettings { } impl ObjectSettings { - pub fn get_bucket(&self) -> &str { + pub fn get_bucket(&self) -> Option<&String> { match self { - ObjectSettings::S3(s3_settings) => s3_settings - .bucket - .as_ref() - .unwrap_or_else(|| "missingbucket".to_string()), - ObjectSettings::Azure(azure_settings) => &azure_settings.container_name, + ObjectSettings::S3(s3_settings) => s3_settings.bucket.as_ref(), + ObjectSettings::Azure(azure_settings) => Some(&azure_settings.container_name), } } } From 7a43893616ce19f6565f475e7c709a671078a684 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 20 May 2025 13:23:02 +0200 Subject: [PATCH 03/45] nit --- backend/src/monitor.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 8b401e8acc..5c4d166beb 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -1122,14 +1122,13 @@ pub async fn reload_s3_cache_setting(db: &DB) { if let Err(e) = setting { tracing::error!("Error parsing s3 cache config: {:?}", e) } else { - let s3_client = build_object_store_from_settings(setting.unwrap()).await; + let setting = setting.unwrap(); + let bucket = setting.get_bucket().map(|b| b.to_string()); + let s3_client = build_object_store_from_settings(setting).await; if let Err(e) = s3_client { tracing::error!("Error building s3 client from settings: {:?}", e) } else { - tracing::info!( - "Loaded object store {:?}", - setting.as_ref().unwrap().get_bucket() - ); + tracing::info!("Loaded object store {:?}", bucket); *s3_cache_settings = Some(s3_client.unwrap()); } } From 29f92ea297b9eb113e191bd2fe59af3d6c26e030 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Tue, 20 May 2025 13:53:34 +0200 Subject: [PATCH 04/45] add claude instructions files (#5779) --- CLAUDE.md | 71 +++++++++++++ backend/CLAUDE.md | 104 +++++++++++++++++++ frontend/CLAUDE.md | 241 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 416 insertions(+) create mode 100644 CLAUDE.md create mode 100644 backend/CLAUDE.md create mode 100644 frontend/CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000..3b701f2536 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,71 @@ +# Windmill Overview + +Windmill is an open-source developer platform for building internal tools, API integrations, background jobs, workflows, and user interfaces. It offers a unified system where scripts are automatically turned into sharable UIs and can be composed into flows or embedded in custom applications. + +## Core Capabilities + +- **Script Development and Execution**: Write and run scripts in Python, TypeScript/JavaScript (Deno/Bun), Go, Bash, SQL, and other languages +- **Workflow Orchestration**: Compose scripts into multi-step flows with conditional logic, loops, and error handling +- **UI Generation**: Automatically generate UIs from scripts or build custom applications with a low-code editor +- **Job Scheduling**: Trigger scripts and flows on schedules, webhooks, or external events +- **Resource Management**: Securely store and use credentials, databases, and other connections + +## Platform Architecture + +The Windmill platform consists of several key components: + +- **Frontend UI**: Web-based interface for script and flow development, app building, and result visualization +- **API Server**: Central API that handles authentication, resource management, and job coordination +- **Workers**: Execute scripts in their respective environments with proper sandboxing +- **Database**: PostgreSQL database for storage of scripts, flows, resources, job results, and more +- **Job Queue**: Queue system for managing job execution, implemented in PostgreSQL +- **Client Libraries**: Libraries for interacting with Windmill from Python, TypeScript, or command line + +# Windmill Backend Architecture + +The Windmill backend is written in Rust and consists of several services working together. These services are designed for horizontal scaling with stateless API servers and workers that can be deployed across multiple machines. + +## Key Components + +- **API Server (`windmill-api`)**: Handles HTTP requests, authentication, and resource management +- **Queue Manager (`windmill-queue`)**: Manages the job queue in PostgreSQL +- **Worker System (`windmill-worker`)**: Executes jobs in sandboxed environments +- **Common Utilities (`windmill-common`)**: Shared code used by multiple services +- **Git Sync (`windmill-git-sync`)**: Synchronizes scripts with Git repositories + +## Job Execution System + +The job execution process follows these steps: + +1. The API server receives a request to run a script or flow and creates a job record in the database +2. The job is added to the queue system in PostgreSQL +3. Workers continuously poll the queue for jobs matching their capabilities +4. When a job is picked up, it's routed to the appropriate language executor +5. The script is executed in a sandboxed environment using NSJAIL for security +6. Results are processed and stored in the database +7. For flows, each step creates a new job that goes through the same process + +Windmill supports worker tags and groups to route jobs to workers with specific capabilities or resource access. + +# Windmill Frontend Architecture + +The Windmill frontend is built with Svelte and provides several key interfaces for interacting with the platform. + +## Key Components + +- **Script Builder**: Code editor with language support, schema inference, and dependency management +- **Flow Builder**: Visual editor for creating multi-step workflows with branching and looping +- **App Editor**: Grid-based editor for building custom UIs that integrate scripts and flows +- **Schema Form System**: Generates form interfaces from script parameters automatically +- **Result Viewer**: Visualizes job results, logs, and execution status + +The frontend uses the Monaco editor (same as VS Code) for code editing, with specialized language support for all supported script languages. + +## UI Framework + +The frontend is built with Svelte, providing a reactive and component-based architecture. Key frontend technologies include: + +- **Svelte/SvelteKit**: Core framework for UI components and routing +- **Monaco Editor**: Code editing experience similar to VS Code +- **Schema Form**: Automatic UI generation from TypeScript/JSON schemas +- **Tailwind CSS**: Utility-first CSS framework for styling diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md new file mode 100644 index 0000000000..382c7a04a9 --- /dev/null +++ b/backend/CLAUDE.md @@ -0,0 +1,104 @@ +# Windmill Backend - Rust Best Practices + +## Project Structure + +Windmill uses a workspace-based architecture with multiple crates: + +- **windmill-api**: API server functionality +- **windmill-worker**: Job execution +- **windmill-common**: Shared code used by all crates +- **windmill-queue**: Job & flow queuing +- **windmill-audit**: Audit logging +- Other specialized crates (git-sync, autoscaling, etc.) + +## Adding New Code + +### Module Organization + +- Place new code in the appropriate crate based on functionality +- For API endpoints, create or modify files in `windmill-api/src/` organized by domain +- For shared functionality, use `windmill-common/src/` +- Use the `_ee.rs` suffix for enterprise-only modules +- Follow existing patterns for file structure and organization + +### Error Handling + +- Use the custom `Error` enum from `windmill-common::error` +- Return `Result` or `JsonResult` for functions that can fail +- Use the `?` operator for error propagation +- Add location tracking to errors using `#[track_caller]` + +### Database Operations + +- Use `sqlx` for database operations with prepared statements +- Leverage existing database helper functions in `db.rs` modules +- Use transactions for multi-step operations +- Handle database errors properly + +### API Endpoints + +- Follow existing patterns in the `windmill-api` crate +- Use axum's routing system and extractors +- Group related routes together +- Use consistent response formats (JSON) +- Follow proper authentication and authorization patterns + +## Performance Optimizations + +When generating code, especially involving `serde`, `sqlx`, and `tokio`, prioritize performance by applying the following principles: + +### Serde Optimizations (Serialization & Deserialization) + +- **Specify Structure Explicitly:** When defining structs for Serde (`#[derive(Serialize, Deserialize)]`), use `#[serde(...` attributes extensively. This includes: + - `#[serde(rename = "...")]` or `#[serde(alias = "...")]` to map external names precisely, avoiding dynamic lookups. + - `#[serde(default)]` for optional fields with default values, reducing parsing complexity. + - `#[serde(skip_serializing_if = "...")]` to avoid writing fields that meet a certain condition (e.g., `Option::is_none()`, `Vec::is_empty()`, or a custom function), reducing output size and serialization work. + - `#[serde(skip_serializing)]` or `#[serde(skip_deserializing)]` for fields that should _not_ be included. +- **Prefer Borrowing:** Where possible and safe (data lifetime allows), use `Cow<'a, str>` or `&'a str` (with `#[serde(borrow)]`) instead of `String` for string fields during deserialization. This avoids allocating new strings, enabling zero-copy reading from the input buffer. Apply this principle to byte slices (`&'a [u8]` / `Cow<'a, [u8]>`) and potentially borrowed vectors as well. +- **Avoid Intermediate `Value`:** Unless the data structure is truly dynamic or unknown at compile time, deserialize directly into a well-defined struct or enum rather than into `serde_json::Value` (or equivalent for other formats). This avoids unnecessary heap allocations and type switching. + +### SQLx Optimizations (Database Interaction) + +- **Select Only Necessary Columns:** In `SELECT` queries, list specific column names rather than using `SELECT *`. This reduces data transferred from the database and the work needed for hydration/deserialization. +- **Batch Operations:** For multiple `INSERT`, `UPDATE`, or `DELETE` statements, prefer executing them in a single query if the database and driver support it efficiently (e.g., `INSERT INTO ... VALUES (...), (...), ...`). This minimizes round trips to the database. +- **Avoid N+1 Queries:** Do not loop through results of one query and execute a separate query for each item (e.g., fetching users, then querying for each user's profile in a loop). Instead, use JOINs or a single query with an `IN` clause to fetch related data efficiently. +- **Deserialize Directly:** Use `#[derive(FromRow)]` on structs and ensure the struct fields match the selected columns in the query. This allows SQLx to hydrate objects directly, avoiding intermediate data structures. +- **Parameterize Queries:** Always use SQLx's query methods (`.bind(...)`) to pass values as parameters rather than string formatting. This prevents SQL injection and allows the database to cache query plans, improving performance on repeated executions. + +### Tokio Optimizations (Asynchronous Runtime) + +- **Avoid Blocking Operations:** **Crucially**, never perform blocking operations (synchronous file I/O, `std::thread::sleep`, CPU-bound loops, `std::sync::Mutex::lock`, blocking network calls without `tokio::net`) directly within an `async fn` or a standard `tokio::spawn` task. Blocking pauses the entire worker thread, potentially starving other tasks. Use `tokio::task::spawn_blocking` for CPU-intensive work or blocking I/O. +- **Use Tokio's Async Primitives:** Prefer `tokio::sync` (channels, mutexes, semaphores), `tokio::io`, `tokio::net`, and `tokio::time` over their `std` counterparts in asynchronous contexts. These are designed to yield control back to the scheduler. +- **Manage Concurrency:** Be mindful of how many tasks are spawned. Creating a new task for every tiny piece of work can introduce overhead. Group related asynchronous operations where appropriate. +- **Handle Shared State Efficiently:** Use `Arc` for shared ownership in concurrent tasks. When shared state needs mutation, prefer `tokio::sync::Mutex` over `std::sync::Mutex` in `async` code. Consider `tokio::sync::RwLock` if reads significantly outnumber writes. Minimize the duration for which locks are held. +- **Understand `.await`:** Place `.await` strategically to allow the runtime to switch to other ready tasks. Ensure that `.await` points to genuinely asynchronous operations. +- **Backpressure:** If dealing with data streams or queues between tasks, implement backpressure mechanisms (e.g., bounded channels like `tokio::sync::mpsc::channel`) to prevent one component from overwhelming another or critical resources like the database. + +## Enterprise Features + +- Use feature flags for enterprise functionality +- Conditionally compile with `#[cfg(feature = "enterprise")]` +- Isolate enterprise code in separate modules + +## Code Style + +- Group imports by external and internal crates +- Place struct/enum definitions before implementations +- Group similar functionality together +- Use descriptive naming consistent with the codebase +- Follow existing patterns for async code using tokio + +## Testing + +- Write unit tests for core functionality +- Use the `#[cfg(test)]` module for test code +- For database tests, use the existing test utilities + +## Common Crates Used + +- **tokio**: For async runtime +- **axum**: For web server and routing +- **sqlx**: For database operations +- **serde**: For serialization/deserialization +- **tracing**: For logging and diagnostics +- **reqwest**: For HTTP client functionality diff --git a/frontend/CLAUDE.md b/frontend/CLAUDE.md new file mode 100644 index 0000000000..63f35ab820 --- /dev/null +++ b/frontend/CLAUDE.md @@ -0,0 +1,241 @@ +# Svelte 5 Best Practices + +This guide outlines best practices for developing with Svelte 5, incorporating the new Runes API and other modern Svelte features. These rules MUST NOT be applied on svelte 4 files unless explicitly asked to do so. + +## Reactivity with Runes + +Svelte 5 introduces Runes for more explicit and flexible reactivity. + +1. **Embrace Runes for State Management**: + + - Use `$state` for reactive local component state. + + ```svelte + + + + ``` + + - Use `$derived` for computed values based on other reactive state. + + ```svelte + + +

{count} * 2 = {doubled}

+ ``` + + - Use `$effect` for side effects that need to run when reactive values change (e.g., logging, manual DOM manipulation, data fetching). Remember `$effect` does not run on the server. + + ```svelte + + ``` + +2. **Props with `$props`**: + + - Declare component props using `$props()`. This offers better clarity and flexibility compared to `export let`. + + ```svelte + + +

Name: {name}

+

Age: {age}

+ ``` + + - For bindable props, use `$bindable`. + + ```svelte + + + + ``` + +## Event Handling + +- **Use direct event attributes**: Svelte 5 moves away from `on:` directives for DOM events. + - **Do**: `` + - **Don't**: `` +- **For component events, prefer callback props**: Instead of `createEventDispatcher`, pass functions as props. + + ```svelte + + + +

Message from child: {message}

+ + + + + ``` + +## Snippets for Content Projection + +- **Use `{#snippet ...}` and `{@render ...}` instead of slots**: Snippets are more powerful and flexible. + + ```svelte + + + + + {#snippet title()} + My Awesome Title + {/snippet} + {#snippet content()} +

Some interesting content here.

+ {/snippet} + + + + + +
+
{@render title()}
+
{@render content()}
+
+ ``` + +- Default content is passed via the `children` prop (which is a snippet). + ```svelte + + +
+ {@render children?.()} +
+ ``` + +## Component Design + +1. **Create Small, Reusable Components**: Break down complex UIs into smaller, focused components. Each component should have a single responsibility. This also aids performance by limiting the scope of reactivity updates. +2. **Descriptive Naming**: Use clear and descriptive names for variables, functions, and components. +3. **Minimize Logic in Components**: Move complex business logic to utility functions or services. Keep components focused on presentation and interaction. + +## State Management (Stores) + +1. **Segment Stores**: Avoid a single global store. Create multiple stores, each responsible for a specific piece of global state (e.g., `userStore.js`, `themeStore.js`). This can help limit reactivity updates to only the parts of the UI that depend on specific state segments. +2. **Use Custom Stores for Complex Logic**: For stores with related methods, create custom stores. + + ```javascript + // counterStore.js + import { writable } from 'svelte/store' + + function createCounter() { + const { subscribe, set, update } = writable(0) + + return { + subscribe, + increment: () => update((n) => n + 1), + decrement: () => update((n) => n - 1), + reset: () => set(0) + } + } + export const counter = createCounter() + ``` + +3. **Use Context API for Localized State**: For state shared within a component subtree, consider Svelte's context API (`setContext`, `getContext`) instead of global stores when the state doesn't need to be truly global. + +## Performance Optimizations (Svelte 5) + +When generating Svelte 5 code, prioritize frontend performance by applying the following principles: + +### General Svelte 5 Principles + +- **Leverage the Compiler:** Trust Svelte's compiler to generate optimized JavaScript. Avoid manual DOM manipulation (`document.querySelector`, etc.) unless absolutely necessary for integrating third-party libraries that lack Svelte adapters. +- **Keep Components Small and Focused:** Reinforcing from Component Design, smaller components lead to less complex reactivity graphs and more targeted, efficient updates. + +### Reactivity & State Management + +- **Optimize Computations with `$derived`:** Always use `$derived` for computed values that depend on other state. This ensures the computation only runs when its specific dependencies change, avoiding unnecessary work compared to recomputing derived values in `$effect` or less efficient methods. +- **Minimize `$effect` Usage:** Use `$effect` sparingly and only for true side effects that interact with the outside world or non-Svelte state. Avoid putting complex logic or state updates _within_ an `$effect` unless those updates are explicitly intended as a reaction to external changes or non-Svelte state. Excessive or complex effects can impact rendering performance. +- **Structure State for Fine-Grained Updates:** Design your `$state` objects or variables such that updates affect only the necessary parts of the UI. Avoid putting too much unrelated state into a single large object that gets frequently updated, as this can potentially trigger broader updates than necessary. Consider normalizing complex, nested state. + +### List Rendering (`{#each}`) + +- **Mandate `key` Attribute:** Always use a `key` attribute (`{#each items as item (item.id)}`) that refers to a unique, stable identifier for each item in a list. This is critical for allowing Svelte to efficiently update, reorder, add, or remove list items without destroying and re-creating unnecessary DOM elements and component instances. + +### Component Loading & Bundling + +- **Implement Lazy Loading/Code Splitting:** For routes, components, or modules that are not immediately needed on page load, use dynamic imports (`import(...)`) to split the code bundle. SvelteKit handles this automatically for routes, but it can be applied manually to components using helper patterns if needed. +- **Be Mindful of Third-Party Libraries:** When incorporating external libraries, import only the necessary functions or components to minimize the final bundle size. Prefer libraries designed to be tree-shakeable. + +### Rendering & DOM + +- **Use CSS for Animations/Transitions:** Prefer CSS animations or transitions where possible for performance. Svelte's built-in `transition:` directive is also highly optimized and should be used for complex state-driven transitions, but simple cases can often use plain CSS. +- **Optimize Image Loading:** Implement best practices for images: use optimized formats (WebP, AVIF), lazy loading (`loading="lazy"`), and responsive images (``, `srcset`) to avoid loading unnecessarily large images. + +### Server-Side Rendering (SSR) & Hydration + +- **Ensure SSR Compatibility:** Write components that can be rendered on the server for faster initial page loads. Avoid relying on browser-specific APIs (like `window` or `document`) in the main ` + +
+ Hello +
+ ``` + +5. **Stay Updated**: Keep Svelte and its related packages up to date to benefit from the latest features, performance improvements, and security fixes. From 5d5286d6279e2f7f01f68bc555887316d6fa1332 Mon Sep 17 00:00:00 2001 From: wendrul <53628737+wendrul@users.noreply.github.com> Date: Tue, 20 May 2025 14:25:10 +0200 Subject: [PATCH 05/45] Add missing trigger pages to quick access menu (Ctrl + K) (#5780) * Add extra menu items on ctrl+k for other triggers * Fix run search container It used to be truncated when showing the ee message --- .../search/GlobalSearchModal.svelte | 148 +++++++++++++++--- 1 file changed, 123 insertions(+), 25 deletions(-) diff --git a/frontend/src/lib/components/search/GlobalSearchModal.svelte b/frontend/src/lib/components/search/GlobalSearchModal.svelte index 9ca20adbe4..271389b007 100644 --- a/frontend/src/lib/components/search/GlobalSearchModal.svelte +++ b/frontend/src/lib/components/search/GlobalSearchModal.svelte @@ -18,13 +18,16 @@ BoxesIcon, CalendarIcon, Code2Icon, + Database, DollarSignIcon, HomeIcon, LayoutDashboardIcon, Loader2, PlayIcon, + Route, Search, - SearchCode + SearchCode, + Unplug } from 'lucide-svelte' import JobPreview from '../runs/JobPreview.svelte' import Portal from '$lib/components/Portal.svelte' @@ -33,13 +36,14 @@ import ContentSearchInner from '../ContentSearchInner.svelte' import { goto } from '$app/navigation' import QuickMenuItem from '../search/QuickMenuItem.svelte' - import { devopsRole, enterpriseLicense, workspaceStore } from '$lib/stores' + import { devopsRole, enterpriseLicense, userStore, workspaceStore } from '$lib/stores' import uFuzzy from '@leeoniya/ufuzzy' import BarsStaggered from '../icons/BarsStaggered.svelte' import { scroll_into_view_if_needed_polyfill } from '../multiselect/utils' import { Alert } from '../common' import Popover from '../Popover.svelte' import Logs from 'lucide-svelte/icons/logs' + import { AwsIcon, GoogleCloudIcon, KafkaIcon, MqttIcon, NatsIcon } from '../icons' let open: boolean = false @@ -63,43 +67,122 @@ action: () => void icon?: any shortcutKey?: string + disabled?: boolean } let switchModeItems: quickMenuItem[] = [ { search_id: 'switchto:run-search', - label: 'Search across completed runs', + label: 'Search across completed runs' + ($enterpriseLicense ? '' : ' (EE)'), action: () => switchMode('runs'), shortcutKey: RUNS_PREFIX, - icon: Search + icon: Search, + disabled: false }, { search_id: 'switchto:content-search', label: 'Search scripts/flows/apps based on content', action: () => switchMode('content'), shortcutKey: CONTENT_SEARCH_PREFIX, - icon: SearchCode + icon: SearchCode, + disabled: false } ] + + // These items are searchable but do not appear initially on the menu. + let hiddenMenuItems = [ + { + search_id: 'nav:http_routes', + label: 'Go to HTTP routes', + action: () => gotoPage('/routes'), + icon: Route, + disabled: $userStore?.operator + }, + { + search_id: 'nav:web_sockets', + label: 'Go to WebSockets', + action: () => gotoPage('/websocket_triggers'), + icon: Unplug, + disabled: $userStore?.operator + }, + { + search_id: 'nav:postgres_triggers', + label: 'Go to Postgres triggers', + action: () => gotoPage('/postgres_triggers'), + icon: Database, + disabled: $userStore?.operator + }, + { + search_id: 'nav:kafka_triggers', + label: 'Go to Kafka triggers' + ($enterpriseLicense ? '' : ' (EE)'), + action: () => gotoPage('/kafka_triggers'), + icon: KafkaIcon, + disabled: $userStore?.operator + }, + { + search_id: 'nav:nats_triggers', + label: 'Go to NATS triggers' + ($enterpriseLicense ? '' : ' (EE)'), + action: () => gotoPage('/nats_triggers'), + icon: NatsIcon, + disabled: $userStore?.operator + }, + { + search_id: 'nav:sqs_triggers', + label: 'Go to SQS triggers' + ($enterpriseLicense ? '' : ' (EE)'), + action: () => gotoPage('/sqs_triggers'), + icon: AwsIcon, + disabled: $userStore?.operator + }, + { + search_id: 'nav:gcp_pub_sub', + label: 'Go to GCP Pub/Sub' + ($enterpriseLicense ? '' : ' (EE)'), + action: () => gotoPage('/gcp_triggers'), + icon: GoogleCloudIcon, + disabled: $userStore?.operator + }, + { + search_id: 'nav:mqtt_triggers', + label: 'Go to MQTT triggers', + action: () => gotoPage('/mqtt_triggers'), + icon: MqttIcon, + disabled: $userStore?.operator + } + ] + let defaultMenuItems: quickMenuItem[] = [ - { search_id: 'nav:home', label: 'Go to Home', action: () => gotoPage('/'), icon: HomeIcon }, - { search_id: 'nav:runs', label: 'Go to Runs', action: () => gotoPage('/runs'), icon: PlayIcon }, + { + search_id: 'nav:home', + label: 'Go to Home', + action: () => gotoPage('/'), + icon: HomeIcon, + disabled: false + }, + { + search_id: 'nav:runs', + label: 'Go to Runs', + action: () => gotoPage('/runs'), + icon: PlayIcon, + disabled: false + }, { search_id: 'nav:variables', label: 'Go to Variables', action: () => gotoPage('/variables'), - icon: DollarSignIcon + icon: DollarSignIcon, + disabled: false }, { search_id: 'nav:resources', label: 'Go to Resources', action: () => gotoPage('/resources'), - icon: BoxesIcon + icon: BoxesIcon, + disabled: false }, { - search_id: 'nav:schedules', + search_id: 'nav:schedules_triggers', label: 'Go to Schedules', action: () => gotoPage('/schedules'), - icon: CalendarIcon + icon: CalendarIcon, + disabled: false }, ...switchModeItems, { @@ -107,10 +190,13 @@ label: 'Explore windmill service logs', action: () => gotoPage('/service_logs'), shortcutKey: LOGS_PREFIX, - icon: Logs + icon: Logs, + disabled: !$devopsRole } ] + let defaultMenuItemsWithHidden = [...defaultMenuItems, ...hiddenMenuItems] + let itemMap = { default: defaultMenuItems as any[], 'switch-mode': switchModeItems, @@ -152,6 +238,7 @@ let uf = new uFuzzy(opts) let defaultMenuItemLabels = defaultMenuItems.map((item) => item.label) + let defaultMenuItemAndHiddenLabels = defaultMenuItemsWithHidden.map((item) => item.label) let switchModeItemLabels = switchModeItems.map((item) => item.label) function fuzzyFilter(filter: string, items: any[], itemsPlainText: string[]) { @@ -210,7 +297,14 @@ } if (tab === 'default') { - itemMap['default'] = fuzzyFilter(searchTerm, defaultMenuItems, defaultMenuItemLabels) + if (searchTerm === '') + itemMap['default'] = fuzzyFilter(searchTerm, defaultMenuItems, defaultMenuItemLabels) + else + itemMap['default'] = fuzzyFilter( + searchTerm, + defaultMenuItemsWithHidden, + defaultMenuItemAndHiddenLabels + ) if (combinedItems) { itemMap['default'] = itemMap['default'].concat( fuzzyFilter( @@ -552,20 +646,24 @@
{#if tab === 'default' || tab === 'switch-mode'} - {@const items = (itemMap[tab] ?? []).filter((e) => defaultMenuItems.includes(e))} + {@const items = (itemMap[tab] ?? []).filter((e) => + defaultMenuItemsWithHidden.includes(e) + )} {#if items.length > 0}
{#each items as el} - (selectedItem = el)} - id={el?.search_id} - hovered={el?.search_id === selectedItem?.search_id} - label={el?.label} - icon={el?.icon} - shortcutKey={el?.shortcutKey} - bind:mouseMoved - /> + {#if !el.disabled} + (selectedItem = el)} + id={el?.search_id} + hovered={el?.search_id === selectedItem?.search_id} + label={el?.label} + icon={el?.icon} + shortcutKey={el?.shortcutKey} + bind:mouseMoved + /> + {/if} {/each}
{/if} @@ -718,7 +816,7 @@
{:else} -
+
{#if searchTerm === RUNS_PREFIX}
Enter your search terms
From 72c91ff17b2c01cb7a1512d074895ee85a7c9b41 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 20 May 2025 15:00:12 +0200 Subject: [PATCH 06/45] nit runs page filter reset --- .../src/lib/components/runs/RunsFilter.svelte | 20 +++---------------- .../(logged)/runs/[...path]/+page.svelte | 1 + 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/frontend/src/lib/components/runs/RunsFilter.svelte b/frontend/src/lib/components/runs/RunsFilter.svelte index 65860ccc53..9fa88a6c22 100644 --- a/frontend/src/lib/components/runs/RunsFilter.svelte +++ b/frontend/src/lib/components/runs/RunsFilter.svelte @@ -74,35 +74,25 @@ const dispatch = createEventDispatcher() - let autoSet = false - $: (path || user || folder || label || worker || concurrencyKey || tag || schedulePath) && autosetFilter() function autosetFilter() { if (path !== null && path !== '' && filterBy !== 'path') { - autoSet = true filterBy = 'path' } else if (user !== null && user !== '' && filterBy !== 'user') { - autoSet = true filterBy = 'user' } else if (folder !== null && folder !== '' && filterBy !== 'folder') { - autoSet = true filterBy = 'folder' } else if (label !== null && label !== '' && filterBy !== 'label') { - autoSet = true filterBy = 'label' } else if (concurrencyKey !== null && concurrencyKey !== '' && filterBy !== 'concurrencyKey') { - autoSet = true filterBy = 'concurrencyKey' } else if (tag !== null && tag !== '' && filterBy !== 'tag') { - autoSet = true filterBy = 'tag' } else if (schedulePath !== undefined && schedulePath !== '' && filterBy !== 'schedulePath') { - autoSet = true filterBy = 'schedulePath' } else if (worker !== null && worker !== '' && filterBy !== 'worker') { - autoSet = true filterBy = 'worker' } } @@ -136,8 +126,8 @@ Filter by { - if (!autoSet) { + on:selected={(e) => { + if (e.detail != filterBy) { path = null user = null folder = null @@ -145,8 +135,6 @@ concurrencyKey = null tag = null schedulePath = undefined - } else { - autoSet = false } }} let:item @@ -593,7 +581,7 @@ let:item bind:selected={filterBy} on:selected={() => { - if (!autoSet) { + if (e.detail != filterBy) { path = null user = null folder = null @@ -601,8 +589,6 @@ concurrencyKey = null tag = null schedulePath = undefined - } else { - autoSet = false } }} > diff --git a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte index 4f57d3d1ed..291ed588fd 100644 --- a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte @@ -371,6 +371,7 @@ lastFetchWentToEnd = false selectedManualDate = 0 selectedIds = [] + schedulePath = undefined batchReRunOptions = { flow: {}, script: {} } selectionMode = false selectedWorkspace = undefined From 70e52a5cf943d6c96bad71e75b9e7c23841d173c Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 20 May 2025 16:14:59 +0200 Subject: [PATCH 07/45] nit --- frontend/src/lib/components/runs/RunsFilter.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/components/runs/RunsFilter.svelte b/frontend/src/lib/components/runs/RunsFilter.svelte index 9fa88a6c22..844eca559a 100644 --- a/frontend/src/lib/components/runs/RunsFilter.svelte +++ b/frontend/src/lib/components/runs/RunsFilter.svelte @@ -580,7 +580,7 @@ { + on:selected={(e) => { if (e.detail != filterBy) { path = null user = null From 3bd36b8096ce289a0b3c9c4b02f88ce17d880691 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 21 May 2025 00:31:48 +0200 Subject: [PATCH 08/45] monaco fix --- frontend/src/lib/components/Editor.svelte | 6 +++++- frontend/src/lib/components/InputTransformForm.svelte | 4 +--- frontend/src/lib/components/SimpleEditor.svelte | 6 ++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 5b3f48aa0a..35c4576251 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -217,7 +217,6 @@ console.log('uri', uri) - function computeUri(filePath: string, scriptLang: string | undefined) { let file if (filePath.includes('.')) { @@ -265,7 +264,11 @@ } } + let valueAfterDispose: string | undefined = undefined export function getCode(): string { + if (valueAfterDispose != undefined) { + return valueAfterDispose + } return editor?.getValue() ?? '' } @@ -1489,6 +1492,7 @@ onDestroy(() => { console.log('destroying editor') + valueAfterDispose = getCode() destroyed = true disposeMethod && disposeMethod() websocketInterval && clearInterval(websocketInterval) diff --git a/frontend/src/lib/components/InputTransformForm.svelte b/frontend/src/lib/components/InputTransformForm.svelte index 3154e66c63..aa66e1d88b 100644 --- a/frontend/src/lib/components/InputTransformForm.svelte +++ b/frontend/src/lib/components/InputTransformForm.svelte @@ -573,9 +573,6 @@ { - dispatch('change', { argName, arg }) - }} {extraLib} lang="javascript" shouldBindKey={false} @@ -595,6 +592,7 @@ autoHeight loadAsync /> +
{#if !hideHelpButton} diff --git a/frontend/src/lib/components/SimpleEditor.svelte b/frontend/src/lib/components/SimpleEditor.svelte index b4947f7ab4..3cc814b855 100644 --- a/frontend/src/lib/components/SimpleEditor.svelte +++ b/frontend/src/lib/components/SimpleEditor.svelte @@ -79,6 +79,7 @@ let placeholderVisible = $state(false) let mounted = $state(false) + let valueAfterDispose: string | undefined = undefined let { lang, code = $bindable(), @@ -130,6 +131,9 @@ const uri = `file:///${hash}.${langToExt(lang)}` export function getCode(): string { + if (valueAfterDispose != undefined) { + return valueAfterDispose + } return editor?.getValue() ?? '' } @@ -406,6 +410,7 @@ editor.onDidBlurEditorText(() => { dispatch('blur') + code = getCode() }) @@ -535,6 +540,7 @@ onDestroy(() => { try { + valueAfterDispose = getCode() vimDisposable?.dispose() model && model.dispose() editor && editor.dispose() From c0d18eac0f3f90ba41064b1949017cf9b34e6a7e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 21 May 2025 00:36:28 +0200 Subject: [PATCH 09/45] template editor nit --- frontend/src/lib/components/TemplateEditor.svelte | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frontend/src/lib/components/TemplateEditor.svelte b/frontend/src/lib/components/TemplateEditor.svelte index 1d67ef9e6e..941c72fc74 100644 --- a/frontend/src/lib/components/TemplateEditor.svelte +++ b/frontend/src/lib/components/TemplateEditor.svelte @@ -399,7 +399,11 @@ } } + let valueAfterDispose: string | undefined = undefined export function getCode(): string { + if (valueAfterDispose != undefined) { + return valueAfterDispose + } return editor?.getValue() ?? '' } @@ -622,6 +626,7 @@ onDestroy(() => { try { + valueAfterDispose = getCode() jsLoader && clearTimeout(jsLoader) model && model.dispose() editor && editor.dispose() From f837dade92d6ea09b12baa7112803528f4b4a25a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 21 May 2025 11:45:51 +0200 Subject: [PATCH 10/45] whitelabel licenses --- .../src/lib/components/CenteredModal.svelte | 9 +++----- frontend/src/lib/stores.ts | 6 ++++++ .../src/routes/(root)/(logged)/+layout.svelte | 21 ++++++++++++++----- .../(logged)/user/(user)/login/+page.svelte | 10 +++++++-- 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/frontend/src/lib/components/CenteredModal.svelte b/frontend/src/lib/components/CenteredModal.svelte index 3548a20a8f..4380177888 100644 --- a/frontend/src/lib/components/CenteredModal.svelte +++ b/frontend/src/lib/components/CenteredModal.svelte @@ -1,20 +1,17 @@
- {#if (!disableLogo && !$enterpriseLicense) || !$enterpriseLicense?.endsWith('_whitelabel')} + {#if (!disableLogo && !$enterpriseLicense) || !$whitelabelNameStore} - @@ -258,12 +266,13 @@ invalidRelations(appendedRelations, { showError: true, trackSchemaTableError: false - }) === false + }) === '' ) { relations = appendedRelations } } }} + {disabled} >
diff --git a/frontend/src/lib/components/triggers/postgres/utils.ts b/frontend/src/lib/components/triggers/postgres/utils.ts index 6b4e3e91b5..8122da726d 100644 --- a/frontend/src/lib/components/triggers/postgres/utils.ts +++ b/frontend/src/lib/components/triggers/postgres/utils.ts @@ -1,6 +1,8 @@ -import type { Relations } from '$lib/gen' +import { PostgresTriggerService, type Relations } from '$lib/gen' import { sendUserToast } from '$lib/toast' import { emptyString } from '$lib/utils' +import type { Writable } from 'svelte/store' +import { get } from 'svelte/store' type RelationError = { schemaIndex: number @@ -18,7 +20,9 @@ export function invalidRelations( trackSchemaTableError?: boolean showError?: boolean } -): boolean { +): string { + let errorMessage: string = '' + let error: RelationError = { schemaIndex: -1, tableIndex: -1, @@ -82,8 +86,6 @@ export function invalidRelations( error.trackAllTablesInSchema && error.trackSpecificColumnsInTable) if ((options?.showError ?? false) && errorFound) { - let errorMessage: string = '' - if (error.schemaError) { errorMessage = `Schema Error: Please enter a name for schema number ${error.schemaIndex}` } else if (error.tableError) { @@ -95,8 +97,53 @@ export function invalidRelations( errorMessage = 'Configuration Error: Schema-level tracking and specific table tracking with column selection cannot be used together. Refer to the documentation for valid configurations.' } - sendUserToast(errorMessage, true) } - return errorFound + return errorMessage +} + +export async function savePostgresTriggerFromCfg( + initialPath: string, + config: Record, + edit: boolean, + workspace: string, + usedTriggerKinds: Writable +): Promise { + try { + const requestBody = { + path: config.path, + script_path: config.script_path, + is_flow: config.is_flow, + postgres_resource_path: config.postgres_resource_path, + replication_slot_name: config.replication_slot_name, + publication_name: config.publication_name, + publication: config.publication, + enabled: config.enabled + } + if (edit) { + await PostgresTriggerService.updatePostgresTrigger({ + workspace: workspace, + path: initialPath, + requestBody + }) + sendUserToast(`PostgresTrigger ${config.path} updated`) + } else { + await PostgresTriggerService.createPostgresTrigger({ + workspace: workspace, + requestBody: { + ...requestBody, + enabled: true + } + }) + sendUserToast(`PostgresTrigger ${config.path} created`) + } + + if (!get(usedTriggerKinds).includes('postgres')) { + usedTriggerKinds.update((t) => [...t, 'postgres']) + } + return true + } catch (error) { + sendUserToast(error.body || error.message, true) + return false + } } diff --git a/frontend/src/lib/components/triggers/schedules/ScheduleEditor.svelte b/frontend/src/lib/components/triggers/schedules/ScheduleEditor.svelte index f70e3a48cb..c75e215e79 100644 --- a/frontend/src/lib/components/triggers/schedules/ScheduleEditor.svelte +++ b/frontend/src/lib/components/triggers/schedules/ScheduleEditor.svelte @@ -2,7 +2,9 @@ import { tick } from 'svelte' import ScheduleEditorInner from './ScheduleEditorInner.svelte' - let open = false + let { onUpdate }: { onUpdate?: (path?: string) => void } = $props() + + let open = $state(false) export async function openEdit(ePath: string, isFlow: boolean) { open = true await tick() @@ -16,12 +18,12 @@ ) { open = true await tick() - drawer?.openNew(is_flow, initial_script_path, schedule_path) + drawer?.openNew(is_flow, initial_script_path, undefined, schedule_path) } - let drawer: ScheduleEditorInner + let drawer: ScheduleEditorInner | undefined = $state() {#if open} - + {/if} diff --git a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte index 69fe6c77d4..91dc02c2b0 100644 --- a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte +++ b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte @@ -23,77 +23,122 @@ import { enterpriseLicense, userStore, workspaceStore } from '$lib/stores' import { canWrite, emptyString, formatCron, sendUserToast, cronV1toV2 } from '$lib/utils' import { base } from '$lib/base' - import { createEventDispatcher } from 'svelte' import Section from '$lib/components/Section.svelte' import { List, Loader2, Save, AlertTriangle } from 'lucide-svelte' import autosize from '$lib/autosize' + import TriggerEditorToolbar from '$lib/components/triggers/TriggerEditorToolbar.svelte' + import { saveScheduleFromCfg } from '$lib/components/flows/scheduleUtils' import DateTimeInput from '$lib/components/DateTimeInput.svelte' import FlowRetries from '$lib/components/flows/content/FlowRetries.svelte' import Label from '$lib/components/Label.svelte' import WorkerTagPicker from '$lib/components/WorkerTagPicker.svelte' import { runScheduleNow } from '../scheduled/utils' + import { handleConfigChange } from '../utils' + + let { + useDrawer = true, + hideTarget = false, + docDescription = undefined, + allowDraft = false, + hasDraft = false, + isDraftOnly = false, + primary = false, + draftSchema = undefined, + customLabel = undefined, + isDeployed = false, + onUpdate = undefined, + onConfigChange = undefined, + onDelete = undefined, + onReset = undefined + } = $props() let optionTabSelected: 'error_handler' | 'recovery_handler' | 'success_handler' | 'retries' = - 'error_handler' - - let is_flow: boolean = false - let initialPath = '' - let edit = true - let schedule: string = '0 0 12 * *' - let cronVersion: string = 'v2' - let isLatestCron = true - let initialCronVersion: string = 'v2' + $state('error_handler') + let is_flow: boolean = $state(false) + let initialPath = $state('') + let edit = $state(true) + let schedule: string = $state('0 0 12 * *') + let cronVersion: string = $state('v2') + let isLatestCron = $state(true) + let initialCronVersion: string = $state('v2') let initialSchedule: string - let timezone: string = Intl.DateTimeFormat().resolvedOptions().timeZone - let paused_until: string | undefined = undefined + let timezone: string = $state(Intl.DateTimeFormat().resolvedOptions().timeZone) + let paused_until: string | undefined = $state(undefined) + let itemKind: 'flow' | 'script' = $state('script') + let errorHandleritemKind: 'flow' | 'script' = $state('script') + let wsErrorHandlerMuted: boolean = $state(false) + let errorHandlerPath: string | undefined = $state(undefined) + let errorHandlerCustomInitialPath: string | undefined = $state(undefined) + let errorHandlerSelected: 'custom' | 'slack' | 'teams' = $state('slack') + let errorHandlerExtraArgs: Record = $state({}) + let recoveryHandlerPath: string | undefined = $state(undefined) + let recoveryHandlerCustomInitialPath: string | undefined = $state(undefined) + let recoveryHandlerSelected: 'custom' | 'slack' | 'teams' = $state('slack') + let recoveryHandlerItemKind: 'flow' | 'script' = $state('script') + let recoveryHandlerExtraArgs: Record = $state({}) + let successHandlerPath: string | undefined = $state(undefined) + let successHandlerCustomInitialPath: string | undefined = $state(undefined) + let successHandlerSelected: 'custom' | 'slack' | 'teams' = $state('slack') + let successHandlerItemKind: 'flow' | 'script' = $state('script') + let successHandlerExtraArgs: Record = $state({}) + let failedTimes = $state(1) + let failedExact = $state(false) + let recoveredTimes = $state(1) + let retry: Retry | undefined = $state(undefined) + let script_path = $state('') + let initialScriptPath = $state('') + let runnable: Script | Flow | undefined = $state() + let args: Record = $state({}) + let loading = $state(false) + let drawerLoading = $state(true) + let showLoading = $state(false) + let initialConfig: Record | undefined = undefined + let extraPerms: Record = $state({}) + let can_write = $state(true) + let initNewPath = $state(false) + let path: string = $state('') + let enabled: boolean = $state(false) + let pathError = $state('') + let summary = $state('') + let description = $state('') + let no_flow_overlap = $state(false) + let tag: string | undefined = $state(undefined) + let validCRON = $state(true) + let isValid = $state(true) + let allowSchedule = $derived(isValid && validCRON && script_path != '') + let deploymentLoading = $state(false) - let itemKind: 'flow' | 'script' = 'script' - let errorHandleritemKind: 'flow' | 'script' = 'script' - let wsErrorHandlerMuted: boolean = false - let errorHandlerPath: string | undefined = undefined - let errorHandlerCustomInitialPath: string | undefined = undefined - let errorHandlerSelected: 'custom' | 'slack' | 'teams' = 'slack' - let errorHandlerExtraArgs: Record = {} - let recoveryHandlerPath: string | undefined = undefined - let recoveryHandlerCustomInitialPath: string | undefined = undefined - let recoveryHandlerSelected: 'custom' | 'slack' | 'teams' = 'slack' - let recoveryHandlerItemKind: 'flow' | 'script' = 'script' - let recoveryHandlerExtraArgs: Record = {} - let successHandlerPath: string | undefined = undefined - let successHandlerCustomInitialPath: string | undefined = undefined - let successHandlerSelected: 'custom' | 'slack' | 'teams' = 'slack' - let successHandlerItemKind: 'flow' | 'script' = 'script' - let successHandlerExtraArgs: Record = {} - let failedTimes = 1 - let failedExact = false - let recoveredTimes = 1 - let duplicate = false - let retry: Retry | undefined = undefined + const saveDisabled = $derived( + !allowSchedule || + pathError != '' || + emptyString(script_path) || + (errorHandlerSelected == 'slack' && + !emptyString(errorHandlerPath) && + emptyString(errorHandlerExtraArgs['channel'])) || + !can_write + ) + const scheduleCfg = $derived.by(getScheduleCfg) - let script_path = '' - let initialScriptPath = '' - - let runnable: Script | Flow | undefined - let args: Record = {} - - let loading = false - - let drawerLoading = true - export function openEdit(ePath: string, isFlow: boolean) { + export async function openEdit(ePath: string, isFlow: boolean, defaultCfg?: Record) { + let loadingTimeout = setTimeout(() => { + showLoading = true + }, 100) // Do not show loading spinner for the first 100ms drawerLoading = true try { drawer?.openDrawer() is_flow = isFlow initialPath = ePath itemKind = is_flow ? 'flow' : 'script' - if (path == ePath) { - loadSchedule() - } else { - path = ePath - } + path = defaultCfg?.path ?? ePath + await loadSchedule(defaultCfg) edit = true } finally { + if (!defaultCfg) { + initialConfig = structuredClone($state.snapshot(getScheduleCfg())) + } + clearTimeout(loadingTimeout) drawerLoading = false + showLoading = false } } @@ -147,8 +192,7 @@ successHandlerSelected = 'slack' successHandlerExtraArgs = {} } - } - else { + } else { let defaultErrorHandlerMaybe = undefined let defaultRecoveryHandlerMaybe = undefined let defaultSuccessHandlerMaybe = undefined @@ -221,8 +265,12 @@ export async function openNew( nis_flow: boolean, initial_script_path?: string, + defaultValues?: Schedule, schedule_path?: string ) { + let loadingTimeout = setTimeout(() => { + showLoading = true + }, 100) // Do not show loading spinner for the first 100ms drawerLoading = true try { let s: Schedule | undefined @@ -231,7 +279,9 @@ workspace: $workspaceStore!, path: schedule_path }) - duplicate = true + initNewPath = true + } else if (defaultValues) { + s = defaultValues } drawer?.openDrawer() runnable = undefined @@ -239,7 +289,7 @@ edit = false itemKind = is_flow ? 'flow' : 'script' initialScriptPath = initial_script_path ?? '' - path = duplicate === true ? '' : initialScriptPath + path = initNewPath ? '' : (defaultValues?.path ?? initialScriptPath) initialPath = path cronVersion = s?.cron_version ?? 'v2' @@ -264,7 +314,9 @@ await setScheduleHandler(s) } finally { + clearTimeout(loadingTimeout) drawerLoading = false + showLoading = false } } @@ -274,28 +326,18 @@ } } - $: (is_flow = itemKind == 'flow') && resetRetries() - - let isValid = true - - let path: string = '' - let enabled: boolean = false - let pathError = '' - let summary = '' - let description = '' - let no_flow_overlap = false - let tag: string | undefined = undefined - - let validCRON = true - $: allowSchedule = isValid && validCRON && script_path != '' + $effect(() => { + ;(is_flow = itemKind == 'flow') && resetRetries() + }) // set isValid to true when a script/flow without any properties is selected - $: runnable?.schema && - runnable.schema.properties && - Object.keys(runnable.schema.properties).length === 0 && - (isValid = true) + $effect(() => { + setDefaultValid(draftSchema ?? runnable?.schema) + }) - const dispatch = createEventDispatcher() + function setDefaultValid(schema: Record | undefined) { + isValid = schema?.properties && Object.keys(schema.properties).length === 0 + } async function loadScript(p: string | undefined): Promise { if (p) { @@ -395,129 +437,108 @@ } } - let can_write = true - async function loadSchedule(): Promise { - loading = true - try { - const s = await ScheduleService.getSchedule({ - workspace: $workspaceStore!, - path: initialPath - }) - is_flow = s.is_flow - cronVersion = s.cron_version ?? 'v2' - initialCronVersion = cronVersion - isLatestCron = cronVersion == 'v2' - enabled = s.enabled - schedule = s.schedule - initialSchedule = schedule - timezone = s.timezone - paused_until = s.paused_until - showPauseUntil = paused_until !== undefined - summary = s.summary ?? '' - description = s.description ?? '' - script_path = s.script_path ?? '' - args = s.args ?? {} - can_write = canWrite(s.path, s.extra_perms, $userStore) - tag = s.tag - - await loadScript(script_path) - - no_flow_overlap = s.no_flow_overlap ?? false - wsErrorHandlerMuted = s.ws_error_handler_muted ?? false - retry = s.retry - await setScheduleHandler(s) - } catch (err) { - sendUserToast(`Could not load schedule: ${err}`, true) + async function loadSchedule(defaultCfg?: Record): Promise { + if (!defaultCfg) { + try { + const s = await ScheduleService.getSchedule({ + workspace: $workspaceStore!, + path: initialPath + }) + await loadScheduleCfg(s) + } catch (err) { + sendUserToast(`Could not load schedule: ${err}`, true) + } + } else { + await loadScheduleCfg(defaultCfg) } + } + + async function loadScheduleCfg(cfg: Record): Promise { + loading = true + + cronVersion = cfg.cron_version ?? 'v2' + initialCronVersion = cronVersion + isLatestCron = cronVersion == 'v2' + enabled = cfg.enabled + schedule = cfg.schedule + initialSchedule = schedule + timezone = cfg.timezone + paused_until = cfg.paused_until + showPauseUntil = paused_until !== undefined + summary = cfg.summary ?? '' + description = cfg.description ?? '' + script_path = cfg.script_path ?? '' + await loadScript(script_path) + + is_flow = cfg.is_flow + no_flow_overlap = cfg.no_flow_overlap ?? false + wsErrorHandlerMuted = cfg.ws_error_handler_muted ?? false + retry = cfg.retry + if (cfg.on_failure) { + let splitted = cfg.on_failure.split('/') + errorHandleritemKind = splitted[0] as 'flow' | 'script' + errorHandlerPath = splitted.slice(1)?.join('/') + errorHandlerCustomInitialPath = errorHandlerPath + failedTimes = cfg.on_failure_times ?? 1 + failedExact = cfg.on_failure_exact ?? false + errorHandlerExtraArgs = cfg.on_failure_extra_args ?? {} + errorHandlerSelected = getHandlerType('error', errorHandlerPath ?? '') + } else { + errorHandlerPath = undefined + errorHandleritemKind = 'script' + errorHandlerCustomInitialPath = undefined + errorHandlerExtraArgs = {} + failedExact = false + failedTimes = 1 + errorHandlerSelected = 'slack' + } + if (cfg.on_recovery) { + let splitted = cfg.on_recovery.split('/') + recoveryHandlerItemKind = splitted[0] as 'flow' | 'script' + recoveryHandlerPath = splitted.slice(1)?.join('/') + recoveryHandlerCustomInitialPath = recoveryHandlerPath + recoveredTimes = cfg.on_recovery_times ?? 1 + recoveryHandlerExtraArgs = cfg.on_recovery_extra_args ?? {} + recoveryHandlerSelected = getHandlerType('recovery', recoveryHandlerPath ?? '') + } else { + recoveryHandlerPath = undefined + recoveryHandlerItemKind = 'script' + recoveryHandlerCustomInitialPath = undefined + recoveredTimes = 1 + recoveryHandlerSelected = 'slack' + recoveryHandlerExtraArgs = {} + } + if (cfg.on_success) { + let splitted = cfg.on_success.split('/') + successHandlerItemKind = splitted[0] as 'flow' | 'script' + successHandlerPath = splitted.slice(1)?.join('/') + successHandlerCustomInitialPath = successHandlerPath + successHandlerExtraArgs = cfg.on_success_extra_args ?? {} + successHandlerSelected = getHandlerType('success', successHandlerPath ?? '') + } else { + successHandlerPath = undefined + successHandlerItemKind = 'script' + successHandlerCustomInitialPath = undefined + successHandlerSelected = 'slack' + successHandlerExtraArgs = {} + } + args = cfg.args ?? {} + extraPerms = cfg.extra_perms ?? {} + can_write = canWrite(cfg.path, cfg.extra_perms, $userStore) + tag = cfg.tag + loading = false } async function scheduleScript(): Promise { - if (errorHandlerPath !== undefined && isSlackHandler('error', errorHandlerPath)) { - errorHandlerExtraArgs['slack'] = '$res:f/slack_bot/bot_token' - } else { - errorHandlerExtraArgs['slack'] = undefined + const scheduleCfg = getScheduleCfg() + deploymentLoading = true + const isSaved = await saveScheduleFromCfg(scheduleCfg, edit, $workspaceStore!) + if (isSaved) { + onUpdate?.(scheduleCfg.path) + drawer?.closeDrawer() } - if (recoveryHandlerPath !== undefined && isSlackHandler('recovery', recoveryHandlerPath)) { - recoveryHandlerExtraArgs['slack'] = '$res:f/slack_bot/bot_token' - } else { - recoveryHandlerExtraArgs['slack'] = undefined - } - if (successHandlerPath !== undefined && isSlackHandler('success', successHandlerPath)) { - successHandlerExtraArgs['slack'] = '$res:f/slack_bot/bot_token' - } else { - successHandlerExtraArgs['slack'] = undefined - } - if (edit) { - await ScheduleService.updateSchedule({ - workspace: $workspaceStore!, - path: initialPath, - requestBody: { - schedule: formatCron(schedule), - timezone, - args, - on_failure: errorHandlerPath ? `${errorHandleritemKind}/${errorHandlerPath}` : undefined, - on_failure_times: failedTimes, - on_failure_exact: failedExact, - on_failure_extra_args: errorHandlerPath ? errorHandlerExtraArgs : undefined, - on_recovery: recoveryHandlerPath - ? `${recoveryHandlerItemKind}/${recoveryHandlerPath}` - : undefined, - on_recovery_times: recoveredTimes, - on_recovery_extra_args: recoveryHandlerPath ? recoveryHandlerExtraArgs : {}, - on_success: successHandlerPath - ? `${successHandlerItemKind}/${successHandlerPath}` - : undefined, - on_success_extra_args: successHandlerPath ? successHandlerExtraArgs : {}, - ws_error_handler_muted: wsErrorHandlerMuted, - retry: retry, - summary: summary != '' ? summary : undefined, - description: description, - no_flow_overlap: no_flow_overlap, - tag: tag, - paused_until: paused_until, - cron_version: cronVersion - } - }) - sendUserToast(`Schedule ${path} updated`) - } else { - await ScheduleService.createSchedule({ - workspace: $workspaceStore!, - requestBody: { - path, - schedule: formatCron(schedule), - timezone, - script_path, - is_flow, - args, - enabled: true, - on_failure: errorHandlerPath ? `${errorHandleritemKind}/${errorHandlerPath}` : undefined, - on_failure_times: failedTimes, - on_failure_exact: failedExact, - on_failure_extra_args: errorHandlerPath ? errorHandlerExtraArgs : undefined, - on_recovery: recoveryHandlerPath - ? `${recoveryHandlerItemKind}/${recoveryHandlerPath}` - : undefined, - on_recovery_times: recoveredTimes, - on_recovery_extra_args: recoveryHandlerPath ? recoveryHandlerExtraArgs : {}, - on_success: successHandlerPath - ? `${successHandlerItemKind}/${successHandlerPath}` - : undefined, - on_success_extra_args: successHandlerPath ? successHandlerExtraArgs : {}, - ws_error_handler_muted: wsErrorHandlerMuted, - retry: retry, - summary: summary != '' ? summary : undefined, - description: description, - no_flow_overlap: no_flow_overlap, - tag: tag, - paused_until: paused_until, - cron_version: cronVersion - } - }) - sendUserToast(`Schedule ${path} created`) - } - dispatch('update') - drawer.closeDrawer() + deploymentLoading = false } function getHandlerType( @@ -562,21 +583,15 @@ } } - $: { - if ($workspaceStore) { - if (edit && path != '') { - loadSchedule() - } - } - } + let drawer: Drawer | undefined = $state() - let drawer: Drawer + let pathC: Path | undefined = $state() + let dirtyPath = $state(false) - let pathC: Path - let dirtyPath = false - - let showPauseUntil = false - $: !showPauseUntil && (paused_until = undefined) + let showPauseUntil = $state(false) + $effect(() => { + !showPauseUntil && (paused_until = undefined) + }) function onVersionChange() { cronVersion = isLatestCron ? 'v2' : 'v1' @@ -592,20 +607,99 @@ schedule = initialSchedule } } + + function getScheduleCfg(): Record { + let errorHadlerExtraArgsDerived = structuredClone($state.snapshot(errorHandlerExtraArgs)) + if (errorHandlerPath !== undefined && isSlackHandler('error', errorHandlerPath)) { + errorHadlerExtraArgsDerived['slack'] = '$res:f/slack_bot/bot_token' + } else { + errorHadlerExtraArgsDerived['slack'] = undefined + } + + let recoveryHandlerExtraArgsDerived = structuredClone($state.snapshot(recoveryHandlerExtraArgs)) + if (recoveryHandlerPath !== undefined && isSlackHandler('recovery', recoveryHandlerPath)) { + recoveryHandlerExtraArgsDerived['slack'] = '$res:f/slack_bot/bot_token' + } else { + recoveryHandlerExtraArgsDerived['slack'] = undefined + } + + let successHandlerExtraArgsDerived = structuredClone($state.snapshot(successHandlerExtraArgs)) + if (successHandlerPath !== undefined && isSlackHandler('success', successHandlerPath)) { + successHandlerExtraArgsDerived['slack'] = '$res:f/slack_bot/bot_token' + } else { + successHandlerExtraArgsDerived['slack'] = undefined + } + return { + path: path, + schedule: formatCron(schedule), + timezone: timezone, + script_path: script_path, + is_flow: is_flow, + args: args, + enabled: enabled, + on_failure: errorHandlerPath ? `${errorHandleritemKind}/${errorHandlerPath}` : undefined, + on_failure_times: failedTimes, + on_failure_exact: failedExact, + on_failure_extra_args: errorHandlerPath ? errorHadlerExtraArgsDerived : undefined, + on_recovery: recoveryHandlerPath + ? `${recoveryHandlerItemKind}/${recoveryHandlerPath}` + : undefined, + on_recovery_times: recoveredTimes, + on_recovery_extra_args: recoveryHandlerPath ? recoveryHandlerExtraArgsDerived : {}, + on_success: successHandlerPath + ? `${successHandlerItemKind}/${successHandlerPath}` + : undefined, + on_success_extra_args: successHandlerPath ? successHandlerExtraArgsDerived : {}, + ws_error_handler_muted: wsErrorHandlerMuted, + retry: retry, + summary: summary != '' ? summary : undefined, + description: description, + no_flow_overlap: no_flow_overlap, + tag: tag, + paused_until: paused_until, + cron_version: cronVersion, + extra_perms: extraPerms + } + } + + async function handleToggleEnabled(nEnabled: boolean) { + enabled = nEnabled + if (!isDraftOnly && !hasDraft) { + await ScheduleService.setScheduleEnabled({ + path: initialPath, + workspace: $workspaceStore ?? '', + requestBody: { enabled: nEnabled } + }) + sendUserToast(`${nEnabled ? 'enabled' : 'disabled'} schedule ${initialPath}`) + } + } + + $effect(() => { + if (!drawerLoading) { + handleConfigChange(scheduleCfg, initialConfig, saveDisabled, edit, onConfigChange) + } + }) - - - - {#if !drawerLoading} - {#if edit} +{#snippet saveButton()} + {#if !drawerLoading} + + {#snippet extra()} + {#if !drawerLoading && edit}
- {#if can_write} -
- { - await ScheduleService.setScheduleEnabled({ - path: initialPath, - workspace: $workspaceStore ?? '', - requestBody: { enabled: e.detail } - }) - dispatch('update') - sendUserToast(`${e.detail ? 'enabled' : 'disabled'} schedule ${initialPath}`) - }} - /> -
- {/if} {/if} - - {/if} - - {#if drawerLoading} - - {:else} -
-
-
-

Metadata

- -
- + {/snippet} + + {/if} +{/snippet} -
+
+
+

Metadata

+
- -
- - {#if cronVersion === 'v1'} - Schedules use CRON syntax. Seconds are mandatory. - {:else} - Schedules use extended CRON syntax. + {#if !edit && !primary} + + {:else} +
+ - {/if} - - {#if initialCronVersion !== 'v2'} -
- - + { + currentTarget.select() }} - size="xs" - bind:checked={isLatestCron} - on:change={onVersionChange} - disabled={!can_write} /> +
{/if} - + + +
+ +
+ {#snippet header()} + {#if cronVersion === 'v1'} + Schedules use CRON syntax. Seconds are mandatory. + {:else} + Schedules use extended CRON syntax. {/if} -
-
+ {/snippet} + {#if initialCronVersion !== 'v2'} +
+ + +
+ {/if} + + + {#if showPauseUntil} + + {/if} +
+ +
+ {#if !hideTarget} {#if !edit}

Pick a script or flow to be triggered by the schedule

{/if} -
- {#if !loading} - {#if runnable} - {#if runnable?.schema && runnable.schema.properties && Object.keys(runnable.schema.properties).length > 0} - {#await import('$lib/components/SchemaForm.svelte')} - - {:then Module} - - {/await} - {:else} -
- This {is_flow ? 'flow' : 'script'} takes no argument -
- {/if} - {:else if script_path != ''} -
- You cannot see the the {is_flow ? 'flow' : 'script'} input form as you do not have - access to it. -
- {:else} -
- Pick a {is_flow ? 'flow' : 'script'} and fill its argument here -
- {/if} - {:else} - - {/if} -
- - -
+ {/if} +
{#if !loading} - - Error Handler - Recovery Handler - Success Handler - {#if itemKind === 'script'} - Retries - Custom tag - {/if} - -
- {#if optionTabSelected === 'error_handler'} -
- -
- {#if !$enterpriseLicense}(ee only){/if} -
- - -
- defaults - saveAsDefaultErrorHandler(false) - }, - { - displayName: 'Override all existing', - type: 'delete', - action: () => saveAsDefaultErrorHandler(true) - } - ]} - > - - - Set as default - - -
- -
- 0} + {#await import('$lib/components/SchemaForm.svelte')} + + {:then Module} + + {/await} + {:else} +
+ This {is_flow ? 'flow' : 'script'} takes no argument
- - - - -
-
The following args will be passed to the error handler: -
    -
  • path: The path of the script or flow that failed.
    • -
  • is_flow: Whether the runnable is a flow.
    • -
  • schedule_path: The path of the schedule.
    • -
  • error: The error details.
    • -
  • failed_times: Minimum number of times the schedule failed - before calling the error handler.
    • -
  • started_at: The start datetime of the latest job that failed.
    • -
-
-
- - - -
-
-

- Triggered when schedule failed

- - -

time{failedTimes > 1 ? 's in a row' : ''}

-
-
-
- {:else if optionTabSelected === 'recovery_handler'} - {@const disabled = !can_write || emptyString($enterpriseLicense)} -
- -
- {#if !$enterpriseLicense}(ee only){/if} -
- - -
- defaults - saveAsDefaultRecoveryHandler(false) - }, - { - displayName: 'Override all existing', - type: 'delete', - action: () => saveAsDefaultRecoveryHandler(true) - } - ]} - > - - - Set as default - - -
- - - - -
-
The following args will be passed to the recovery handler: -
    -
  • path: The path of the script or flow that recovered.
    • -
  • is_flow: Whether the runnable is a flow.
    • -
  • schedule_path: The path of the schedule.
    • -
  • error: The error of the last job that errored
    • -
  • error_started_at: The start datetime of the last job that - errored
    • -
  • success_times: The number of times the schedule succeeded - before calling the recovery handler.
    • -
  • success_result: The result of the latest successful job
    • -
  • success_started_at: The start datetime of the latest - successful job
    • -
-
-
- - - -
-
-

Triggered when schedule recovered

- -

time{recoveredTimes > 1 ? 's in a row' : ''}

-
-
-
- {:else if optionTabSelected === 'success_handler'} - {@const disabled = !can_write || emptyString($enterpriseLicense)} -
- -
- {#if !$enterpriseLicense}(ee only){/if} -
- - -
- defaults - saveAsDefaultSuccessHandler(false) - }, - { - displayName: 'Override all existing', - type: 'delete', - action: () => saveAsDefaultSuccessHandler(true) - } - ]} - > - - - Set as default - - -
- - - - -
-
The following args will be passed to the success handler: -
    -
  • path: The path of the script or flow that succeeded.
    • -
  • is_flow: Whether the runnable is a flow.
    • -
  • schedule_path: The path of the schedule.
    • -
  • success_result: The result of the successful job
    • -
  • success_started_at: The start datetime of the successful job
    • -
-
-
- - - -
- {:else if optionTabSelected === 'retries'} - {@const disabled = !can_write || emptyString($enterpriseLicense)} -
- -
- {#if !$enterpriseLicense}(ee only){/if} -
- - If defined, upon error this schedule will be retried with a delay and a maximum - number of attempts as defined below. -
- This is only available for individual script. For flows, retries can be set on each - flow step in the flow editor. - - - -
- {:else if optionTabSelected === 'tag'} -
- -
+ {/if} + {:else if script_path != ''} +
+ You cannot see the the {is_flow ? 'flow' : 'script'} input form as you do not have access + to it. +
+ {:else} +
+ Pick a {is_flow ? 'flow' : 'script'} and fill its argument here +
{/if} {:else} {/if}
-
+ + +
+ {@render errorHandler()} +
+
+ {/if} +{/snippet} + +{#snippet errorHandler()} +
+ {#if !loading} + + Error Handler + Recovery Handler + Success Handler + {#if itemKind === 'script'} + Retries + Custom tag + {/if} + +
+ {#if optionTabSelected === 'error_handler'} +
+ {#snippet header()} +
+ {#if !$enterpriseLicense}(ee only){/if} +
+ {/snippet} + {#snippet action()} +
+ defaults + saveAsDefaultErrorHandler(false) + }, + { + displayName: 'Override all existing', + type: 'delete', + action: () => saveAsDefaultErrorHandler(true) + } + ]} + > + {#snippet children()} + + Set as default + {/snippet} + +
+ {/snippet} +
+ +
+ + + + + +
+
The following args will be passed to the error handler: +
    +
  • path: The path of the script or flow that failed.
    • +
  • is_flow: Whether the runnable is a flow.
    • +
  • schedule_path: The path of the schedule.
    • +
  • error: The error details.
    • +
  • failed_times: Minimum number of times the schedule failed before + calling the error handler.
    • +
  • started_at: The start datetime of the latest job that failed.
    • +
+
+
+ + + +
+
+

+ Triggered when schedule failed

+ + +

time{failedTimes > 1 ? 's in a row' : ''}

+
+
+
+ {:else if optionTabSelected === 'recovery_handler'} + {@const disabled = !can_write || emptyString($enterpriseLicense)} +
+ {#snippet header()} +
+ {#if !$enterpriseLicense}(ee only){/if} +
+ {/snippet} + {#snippet action()} +
+ defaults + saveAsDefaultRecoveryHandler(false) + }, + { + displayName: 'Override all existing', + type: 'delete', + action: () => saveAsDefaultRecoveryHandler(true) + } + ]} + > + {#snippet children()} + + Set as default + {/snippet} + +
+ {/snippet} + + + + +
+
The following args will be passed to the recovery handler: +
    +
  • path: The path of the script or flow that recovered.
    • +
  • is_flow: Whether the runnable is a flow.
    • +
  • schedule_path: The path of the schedule.
    • +
  • error: The error of the last job that errored
    • +
  • error_started_at: The start datetime of the last job that errored
    • +
  • success_times: The number of times the schedule succeeded before + calling the recovery handler.
    • +
  • success_result: The result of the latest successful job
    • +
  • success_started_at: The start datetime of the latest successful job
    • +
+
+
+ + + +
+
+

Triggered when schedule recovered

+ +

time{recoveredTimes > 1 ? 's in a row' : ''}

+
+
+
+ {:else if optionTabSelected === 'success_handler'} + {@const disabled = !can_write || emptyString($enterpriseLicense)} +
+ {#snippet header()} +
+ {#if !$enterpriseLicense}(ee only){/if} +
+ {/snippet} + {#snippet action()} +
+ defaults + saveAsDefaultSuccessHandler(false) + }, + { + displayName: 'Override all existing', + type: 'delete', + action: () => saveAsDefaultSuccessHandler(true) + } + ]} + > + {#snippet children()} + + Set as default + {/snippet} + +
+ {/snippet} + + + + +
+
The following args will be passed to the success handler: +
    +
  • path: The path of the script or flow that succeeded.
    • +
  • is_flow: Whether the runnable is a flow.
    • +
  • schedule_path: The path of the schedule.
    • +
  • success_result: The result of the successful job
    • +
  • success_started_at: The start datetime of the successful job
    • +
+
+
+ + + +
+ {:else if optionTabSelected === 'retries'} + {@const disabled = !can_write || emptyString($enterpriseLicense)} +
+ {#snippet header()} +
+ {#if !$enterpriseLicense}(ee only){/if} +
+ + If defined, upon error this schedule will be retried with a delay and a maximum number + of attempts as defined below. +
+ This is only available for individual script. For flows, retries can be set on each flow + step in the flow editor. + + {/snippet} + +
+ {:else if optionTabSelected === 'tag'} +
+ +
+ {/if} + {:else} + {/if} - - +
+{/snippet} + +{#if useDrawer} + + + +
+ {@render saveButton()} +
+ + {@render content()} + + +{:else} +
+ + {#if customLabel} + {@render customLabel()} + {/if} + + +
+ {@render saveButton()} +
+ + {#if docDescription} + {@render docDescription()} + {/if} + {@render content()} +
+{/if} diff --git a/frontend/src/lib/components/triggers/sqs/SqsCapture.svelte b/frontend/src/lib/components/triggers/sqs/SqsCapture.svelte new file mode 100644 index 0000000000..d61e663d0c --- /dev/null +++ b/frontend/src/lib/components/triggers/sqs/SqsCapture.svelte @@ -0,0 +1,52 @@ + + +{#if captureInfo} + + {#snippet description()} + {#if captureInfo.active} +

+ Listening to SQS messages... +

+ {:else} +

+ Start capturing to listen to SQS messages. +

+ {/if} + {/snippet} + +{/if} diff --git a/frontend/src/lib/components/triggers/sqs/SqsTriggerEditor.svelte b/frontend/src/lib/components/triggers/sqs/SqsTriggerEditor.svelte index 2de3e17567..2e1cc47727 100644 --- a/frontend/src/lib/components/triggers/sqs/SqsTriggerEditor.svelte +++ b/frontend/src/lib/components/triggers/sqs/SqsTriggerEditor.svelte @@ -2,7 +2,9 @@ import { tick } from 'svelte' import SqsTriggerEditorInner from './SqsTriggerEditorInner.svelte' - let open = false + let { onUpdate } = $props() + + let open = $state(false) export async function openEdit(ePath: string, isFlow: boolean) { open = true await tick() @@ -19,9 +21,9 @@ drawer?.openNew(is_flow, initial_script_path, defaultValues) } - let drawer: SqsTriggerEditorInner + let drawer: SqsTriggerEditorInner | undefined = $state() {#if open} - + {/if} diff --git a/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorConfigSection.svelte b/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorConfigSection.svelte index 3de41919fb..55282e8762 100644 --- a/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorConfigSection.svelte +++ b/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorConfigSection.svelte @@ -1,7 +1,5 @@
- {#if showCapture && captureInfo} - - {/if}
+ + {#if showTestingBadge} + + {/if} +
@@ -64,9 +53,13 @@ Select an AWS resource to authenticate your account.

- { - aws_resource_path = '' - }} let:item> + { + aws_resource_path = '' + }} + let:item + > diff --git a/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte index eee25de840..c23a683b2c 100644 --- a/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte @@ -5,40 +5,96 @@ import Path from '$lib/components/Path.svelte' import { usedTriggerKinds, userStore, workspaceStore } from '$lib/stores' import { canWrite, emptyString, sendUserToast } from '$lib/utils' - import { createEventDispatcher } from 'svelte' - import { Loader2, Save } from 'lucide-svelte' + import { Loader2 } from 'lucide-svelte' import Label from '$lib/components/Label.svelte' - import Toggle from '$lib/components/Toggle.svelte' import { SqsTriggerService, type AwsAuthResourceType } from '$lib/gen' import SqsTriggerEditorConfigSection from './SqsTriggerEditorConfigSection.svelte' import Section from '$lib/components/Section.svelte' import ScriptPicker from '$lib/components/ScriptPicker.svelte' import Required from '$lib/components/Required.svelte' + import type { Snippet } from 'svelte' + import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import { saveSqsTriggerFromCfg } from './utils' + import { handleConfigChange } from '../utils' - let drawer: Drawer - let is_flow: boolean = false - let initialPath = '' - let edit = true - let itemKind: 'flow' | 'script' = 'script' - let script_path = '' - let initialScriptPath = '' - let fixedScriptPath = '' - let path: string = '' - let pathError = '' - let enabled = false - let dirtyPath = false - let can_write = true - let drawerLoading = true - let aws_resource_path: string = '' - let queue_url = '' - let message_attributes: string[] = [] - let aws_auth_resource_type: AwsAuthResourceType = 'credentials' - let isValid = false - const dispatch = createEventDispatcher() + interface Props { + useDrawer?: boolean + description?: Snippet | undefined + hideTarget?: boolean + hideTooltips?: boolean + allowDraft?: boolean + hasDraft?: boolean + isDraftOnly?: boolean + isEditor?: boolean + customLabel?: Snippet + isDeployed?: boolean + cloudDisabled?: boolean + onConfigChange?: (cfg: Record, saveDisabled: boolean, updated: boolean) => void + onCaptureConfigChange?: (cfg: Record, isValid: boolean) => void + onUpdate?: (path?: string) => void + onDelete?: () => void + onReset?: () => void + } - $: is_flow = itemKind === 'flow' + let { + useDrawer = true, + description = undefined, + hideTarget = false, + hideTooltips = false, + allowDraft = false, + hasDraft = false, + isDraftOnly = false, + isEditor = false, + customLabel = undefined, + isDeployed = false, + cloudDisabled = false, + onConfigChange = undefined, + onCaptureConfigChange = undefined, + onUpdate = undefined, + onDelete = undefined, + onReset = undefined + }: Props = $props() - export async function openEdit(ePath: string, isFlow: boolean) { + let drawer: Drawer | undefined = $state(undefined) + let is_flow: boolean = $state(false) + let initialPath = $state('') + let edit = $state(true) + let itemKind: 'flow' | 'script' = $state('script') + let script_path = $state('') + let initialScriptPath = $state('') + let fixedScriptPath = $state('') + let path: string = $state('') + let pathError = $state('') + let enabled = $state(false) + let dirtyPath = $state(false) + let can_write = $state(true) + let drawerLoading = $state(true) + let showLoading = $state(false) + let aws_resource_path: string = $state('') + let queue_url = $state('') + let message_attributes: string[] = $state([]) + let aws_auth_resource_type: AwsAuthResourceType = $state('credentials') + let isValid = $state(false) + let initialConfig: Record | undefined = undefined + let deploymentLoading = $state(false) + + const sqsConfig = $derived.by(getSaveCfg) + const captureConfig = $derived.by(getCaptureConfig) + const saveDisabled = $derived( + pathError != '' || emptyString(script_path) || !isValid || !can_write + ) + $effect(() => { + is_flow = itemKind === 'flow' + }) + + export async function openEdit( + ePath: string, + isFlow: boolean, + defaultConfig?: Record + ) { + let loadingTimeout = setTimeout(() => { + showLoading = true + }, 100) // Do not show loading spinner for the first 100ms drawerLoading = true try { drawer?.openDrawer() @@ -46,11 +102,13 @@ itemKind = isFlow ? 'flow' : 'script' edit = true dirtyPath = false - await loadTrigger() + await loadTrigger(defaultConfig) } catch (err) { sendUserToast(`Could not load sqs trigger: ${err.body}`, true) } finally { + clearTimeout(loadingTimeout) drawerLoading = false + showLoading = false } } @@ -59,6 +117,9 @@ fixedScriptPath_?: string, defaultValues?: Record ) { + let loadingTimeout = setTimeout(() => { + showLoading = true + }, 100) drawerLoading = true try { drawer?.openDrawer() @@ -69,143 +130,207 @@ script_path = fixedScriptPath aws_resource_path = defaultValues?.aws_resource_path ?? '' queue_url = defaultValues?.queue_url ?? '' - path = '' + path = defaultValues?.path ?? '' message_attributes = defaultValues?.message_attributes ?? [] aws_auth_resource_type = defaultValues?.aws_auth_resource_type ?? 'credentials' initialPath = '' edit = false dirtyPath = false + enabled = defaultValues?.enabled ?? false } finally { + initialConfig = structuredClone($state.snapshot(getSaveCfg())) + clearTimeout(loadingTimeout) drawerLoading = false + showLoading = false } } - async function loadTrigger(): Promise { + async function loadTriggerConfig(cfg?: Record): Promise { try { - const s = await SqsTriggerService.getSqsTrigger({ - workspace: $workspaceStore!, - path: initialPath - }) - script_path = s.script_path - initialScriptPath = s.script_path - aws_resource_path = s.aws_resource_path - queue_url = s.queue_url - is_flow = s.is_flow - message_attributes = s.message_attributes ?? [] - path = s.path - enabled = s.enabled - aws_auth_resource_type = s.aws_auth_resource_type - can_write = canWrite(s.path, s.extra_perms, $userStore) + script_path = cfg?.script_path + initialScriptPath = cfg?.script_path + aws_resource_path = cfg?.aws_resource_path + queue_url = cfg?.queue_url + is_flow = cfg?.is_flow + message_attributes = cfg?.message_attributes ?? [] + path = cfg?.path + enabled = cfg?.enabled + aws_auth_resource_type = cfg?.aws_auth_resource_type + can_write = canWrite(cfg?.path, cfg?.extra_perms, $userStore) + } catch (error) { + sendUserToast(`Could not load SQS trigger config: ${error.body}`, true) + } + } + + async function loadTrigger(defaultConfig?: Record): Promise { + try { + if (defaultConfig) { + loadTriggerConfig(defaultConfig) + return + } else { + const s = await SqsTriggerService.getSqsTrigger({ + workspace: $workspaceStore!, + path: initialPath + }) + loadTriggerConfig(s) + } } catch (error) { sendUserToast(`Could not load SQS trigger: ${error.body}`, true) } } - async function updateTrigger(): Promise { - if (edit) { - await SqsTriggerService.updateSqsTrigger({ - workspace: $workspaceStore!, - path: initialPath, - requestBody: { - path, - script_path, - aws_auth_resource_type, - enabled, - is_flow, - queue_url, - aws_resource_path, - message_attributes - } - }) - sendUserToast(`SQS trigger ${path} updated`) - } else { - await SqsTriggerService.createSqsTrigger({ - workspace: $workspaceStore!, - requestBody: { - enabled: true, - aws_resource_path, - queue_url, - aws_auth_resource_type, - path, - script_path, - is_flow, - message_attributes - } - }) - sendUserToast(`SQS trigger ${path} created`) + function getSaveCfg(): Record { + return { + script_path, + is_flow, + path, + aws_resource_path, + queue_url, + message_attributes, + aws_auth_resource_type, + enabled } - - if (!$usedTriggerKinds.includes('sqs')) { - $usedTriggerKinds = [...$usedTriggerKinds, 'sqs'] - } - dispatch('update') - drawer.closeDrawer() } + + async function handleToggleEnabled(nEnabled: boolean) { + enabled = nEnabled + if (!isDraftOnly && !hasDraft) { + await SqsTriggerService.setSqsTriggerEnabled({ + path: initialPath, + workspace: $workspaceStore ?? '', + requestBody: { enabled: nEnabled } + }) + sendUserToast(`${nEnabled ? 'enabled' : 'disabled'} SQS trigger ${initialPath}`) + } + } + + async function updateTrigger(): Promise { + deploymentLoading = true + const cfg = getSaveCfg() + const isSaved = await saveSqsTriggerFromCfg( + initialPath, + cfg, + edit, + $workspaceStore!, + usedTriggerKinds + ) + if (isSaved) { + onUpdate?.(cfg.path) + drawer?.closeDrawer() + } + deploymentLoading = false + } + + function getCaptureConfig(): Record { + return { + aws_resource_path, + queue_url, + message_attributes, + aws_auth_resource_type, + path + } + } + + $effect(() => { + onCaptureConfigChange?.(captureConfig, isValid) + }) + + $effect(() => { + if (!drawerLoading) { + handleConfigChange(sqsConfig, initialConfig, saveDisabled, edit, onConfigChange) + } + }) - - - - {#if !drawerLoading && can_write} - {#if edit} -
- { - sendUserToast(`${e.detail ? 'enabled' : 'disabled'} sqs trigger ${initialPath}`) - }} - /> -
- {/if} - +{#if useDrawer} + + + + {@render actions()} + + {@render config()} + + +{:else} +
+ + {#if customLabel} + {@render customLabel()} {/if} - {#if drawerLoading} -
- -

Loading...

-
- {:else} -
- + + {@render actions()} + + {@render config()} +
+{/if} + +{#snippet actions()} + {#if !drawerLoading} + + {/if} +{/snippet} + +{#snippet config()} + {#if drawerLoading} + {#if showLoading} + + {/if} + {:else} +
+ {#if description} + {@render description()} + {/if} + {#if !hideTooltips} + {#if edit} Changes can take up to 30 seconds to take effect. {:else} - New postgres triggers can take up to 30 seconds to start listening. + New SQS triggers can take up to 30 seconds to start listening. {/if} + {/if} +
+
+
+
-
-
- -
+ {#if !hideTarget}

Pick a script or flow to be triggered @@ -218,30 +343,35 @@ allowFlow={true} bind:itemKind bind:scriptPath={script_path} - allowRefresh + allowRefresh={can_write} + allowEdit={!$userStore?.operator} /> {#if emptyString(script_path)} + Create from template + {/if}

+ {/if} - -
- {/if} - - + +
+ {/if} +{/snippet} diff --git a/frontend/src/lib/components/triggers/sqs/SqsTriggerPanel.svelte b/frontend/src/lib/components/triggers/sqs/SqsTriggerPanel.svelte index e981d7dedf..24bd1137f0 100644 --- a/frontend/src/lib/components/triggers/sqs/SqsTriggerPanel.svelte +++ b/frontend/src/lib/components/triggers/sqs/SqsTriggerPanel.svelte @@ -1,135 +1,69 @@ - { - loadTriggers() - }} - bind:this={sqsTriggerEditor} -/> + onMount(() => { + sqsTriggerEditor && openSqsTriggerEditor(isFlow, selectedTrigger.isDraft ?? false) + }) + + const cloudDisabled = $derived(isCloudHosted()) + {#if !$enterpriseLicense} SQS triggers are an enterprise only feature. -{:else if isCloudHosted()} - - SQS triggers are disabled in the multi-tenant cloud. - {:else}
- - SQS triggers allow your scripts/flows to process messages from Amazon Simple Queue Service - (SQS) in real time. Each trigger listens to an SQS queue and executes a script or a flow when - new messages arrive. - - - {#if !newItem && sqsTriggers && sqsTriggers.length > 0} -
-
-
- {#each sqsTriggers as sqsTriggers (sqsTriggers.path)} -
-
{sqsTriggers.path}
-
- {sqsTriggers.queue_url} -
-
- -
-
- {/each} -
-
-
- {/if} - - { - sqsTriggerEditor?.openNew(isFlow, path, e.detail.config) - }} - on:addPreprocessor - on:updateSchema - on:testWithArgs - cloudDisabled={false} - triggerType="sqs" - {isFlow} - {path} - {isEditor} - {canHavePreprocessor} - {hasPreprocessor} - {newItem} - {openForm} - bind:showCapture={dontCloseOnLoad} - /> + {#snippet description()} + {#if cloudDisabled} + + SQS triggers are disabled in the multi-tenant cloud. + + {:else} + + SQS triggers allow you to execute scripts and flows in response to messages in an AWS + SQS queue. They can be configured to filter messages based on message attributes. + + {/if} + {/snippet} +
{/if} diff --git a/frontend/src/lib/components/triggers/sqs/utils.ts b/frontend/src/lib/components/triggers/sqs/utils.ts new file mode 100644 index 0000000000..7ef1afa60b --- /dev/null +++ b/frontend/src/lib/components/triggers/sqs/utils.ts @@ -0,0 +1,46 @@ +import { SqsTriggerService } from '$lib/gen' +import { sendUserToast } from '$lib/toast' +import { get, type Writable } from 'svelte/store' + +export async function saveSqsTriggerFromCfg( + initialPath: string, + cfg: Record, + edit: boolean, + workspace: string, + usedTriggerKinds: Writable +): Promise { + const requestBody = { + path: cfg.path, + script_path: cfg.script_path, + is_flow: cfg.is_flow, + aws_resource_path: cfg.aws_resource_path, + queue_url: cfg.queue_url, + message_attributes: cfg.message_attributes, + aws_auth_resource_type: cfg.aws_auth_resource_type, + enabled: cfg.enabled + } + try { + if (edit) { + await SqsTriggerService.updateSqsTrigger({ + workspace, + path: initialPath, + requestBody + }) + sendUserToast(`SQS trigger ${cfg.path} updated`) + } else { + await SqsTriggerService.createSqsTrigger({ + workspace, + requestBody: { ...requestBody, enabled: true } + }) + sendUserToast(`SQS trigger ${cfg.path} created`) + } + + if (!get(usedTriggerKinds).includes('sqs')) { + usedTriggerKinds.update((t) => [...t, 'sqs']) + } + return true + } catch (error) { + sendUserToast(error.body || error.message, true) + return false + } +} diff --git a/frontend/src/lib/components/triggers/testingBadge.svelte b/frontend/src/lib/components/triggers/testingBadge.svelte new file mode 100644 index 0000000000..eb8d288fbf --- /dev/null +++ b/frontend/src/lib/components/triggers/testingBadge.svelte @@ -0,0 +1,9 @@ + + + + + Config used for creating a testing endpoint + diff --git a/frontend/src/lib/components/triggers/triggers.svelte.ts b/frontend/src/lib/components/triggers/triggers.svelte.ts new file mode 100644 index 0000000000..5f40fb6126 --- /dev/null +++ b/frontend/src/lib/components/triggers/triggers.svelte.ts @@ -0,0 +1,456 @@ +import { + KafkaTriggerService, + MqttTriggerService, + NatsTriggerService, + PostgresTriggerService, + ScheduleService, + SqsTriggerService, + WebsocketTriggerService, + type GcpTrigger, + type KafkaTrigger, + type PostgresTrigger, + type Schedule, + type TriggersCount, + type HttpTrigger, + HttpTriggerService, + GcpTriggerService +} from '$lib/gen' +import { getLightConfig, sortTriggers, updateTriggersCount, type Trigger } from './utils' +import type { Writable } from 'svelte/store' +import type { TriggerType } from './utils' +import type { UserExt } from '$lib/stores' +import type { ScheduleTrigger } from '../triggers' +import { canWrite, formatCron } from '$lib/utils' + +export class Triggers { + #triggers = $state([]) + #selectedTriggerIndex = $state(undefined) + #selectedTrigger = $derived( + this.#selectedTriggerIndex !== undefined + ? this.#triggers[this.#selectedTriggerIndex] + : undefined + ) + #updateDraftCallback: (() => void) | undefined = undefined + + constructor( + triggers: Trigger[] = [], + selectedIndex?: number, + updateDraftCallback?: (() => void) | undefined + ) { + this.#triggers = triggers + this.#selectedTriggerIndex = selectedIndex + this.#updateDraftCallback = updateDraftCallback + } + + get selectedTrigger(): Trigger | undefined { + return this.#selectedTrigger + } + + get selectedTriggerIndex(): number | undefined { + return this.#selectedTriggerIndex + } + + set selectedTriggerIndex(index: number | undefined) { + if (index === undefined || index < 0 || index >= this.#triggers.length) { + this.#selectedTriggerIndex = undefined + } else { + this.#selectedTriggerIndex = index + } + this.#updateDraftCallback?.() + } + + get triggers(): Trigger[] { + return this.#triggers + } + + setTriggers(triggers: Trigger[]) { + this.#triggers = triggers + this.#updateDraftCallback?.() + } + + setDraftConfig(triggerIndex: number, draftConfig: Record | undefined) { + if (triggerIndex === undefined || triggerIndex < 0 || triggerIndex >= this.#triggers.length) { + return + } + this.#triggers[triggerIndex].draftConfig = draftConfig + this.#updateDraftCallback?.() + } + + getDraftTriggersSnapshot(): Trigger[] | undefined { + const draftTriggers = this.#triggers.filter((t) => t.draftConfig) + return draftTriggers.length > 0 ? $state.snapshot(draftTriggers) : undefined + } + + getSelectedTriggerSnapshot(): number | undefined { + return $state.snapshot(this.#selectedTriggerIndex) + } + + addDraftTrigger( + triggersCountStore: Writable, + type: TriggerType, + path?: string, + draftCfg?: Record + ): number { + const primaryScheduleExists = this.#triggers.some((t) => t.type === 'schedule' && t.isPrimary) + + // Create the new draft trigger + const draftId = crypto.randomUUID() + const isPrimary = type === 'schedule' && !primaryScheduleExists + const newTrigger = { + id: draftId, + type, + path, + isPrimary, + isDraft: true, + draftConfig: draftCfg + } + + this.#triggers.push(newTrigger) + this.#updateDraftCallback?.() + + updateTriggersCount(triggersCountStore, type, 'add', newTrigger.draftConfig) + + return this.#triggers.length - 1 + } + + deleteTrigger( + triggersCountStore: Writable, + triggerIndex: number + ): void { + if (triggerIndex === undefined || triggerIndex < 0 || triggerIndex >= this.#triggers.length) { + return + } + const { type } = this.#triggers[triggerIndex] + + this.#triggers = this.#triggers.filter((_, index) => index !== triggerIndex) + + updateTriggersCount(triggersCountStore, type, 'remove') + this.#updateDraftCallback?.() + } + + updateTriggers( + remoteTriggers: any[], + type: TriggerType, + user: UserExt | undefined = undefined + ): number { + const currentTriggers = this.#triggers + // Identify triggers with draftConfig to preserve + const configuredTriggers = currentTriggers.filter( + (t) => t.type === type && !t.isDraft && t.draftConfig + ) + + const configMap = new Map }>() + + configuredTriggers.forEach((t) => { + configMap.set(t.path ?? '', { draftConfig: t.draftConfig! }) + }) + + const backendTriggers = remoteTriggers.map((trigger) => { + const { draftConfig } = configMap.get(trigger.path) ?? {} + return { + type: type as TriggerType, + path: trigger.path, + isPrimary: type === 'schedule' && trigger.path === trigger.script_path, + isDraft: false, + canWrite: canWrite(trigger.path, trigger.extra_perms, user), + draftConfig: draftConfig, + lightConfig: getLightConfig(type, trigger) + } + }) + + const filteredTriggers = currentTriggers.filter((t) => t.type !== type || t.isDraft) + const newTriggers = sortTriggers([...filteredTriggers, ...backendTriggers]) + this.#triggers = newTriggers + + this.#updateDraftCallback?.() + return newTriggers.filter((t) => t.type === type).length + } + + async fetchSchedules( + triggersCountStore: Writable, + workspaceId: string | undefined, + path: string, + isFlow: boolean, + primarySchedule?: ScheduleTrigger | undefined | false, + user: UserExt | undefined = undefined + ): Promise { + if (!workspaceId) return + try { + //First update the store with legacy primary schedule + if (primarySchedule && !this.#triggers.some((s) => s.isPrimary)) { + const primary = { + type: 'schedule' as TriggerType, + path, + isPrimary: true, + isDraft: false, + draftConfig: { + schedule: primarySchedule.cron ? formatCron(primarySchedule.cron) : undefined, + args: primarySchedule.args, + timezone: primarySchedule.timezone, + summary: primarySchedule.summary, + description: primarySchedule.description, + enabled: primarySchedule.enabled + } + } + this.#triggers = [...this.#triggers, primary] + } + + const allDeployedSchedules: Schedule[] = await ScheduleService.listSchedules({ + workspace: workspaceId, + path, + isFlow + }) + + const scheduleCount = this.updateTriggers(allDeployedSchedules, 'schedule', user) + const updatedPrimarySchedule = this.#triggers.find((s) => s.isPrimary) + triggersCountStore.update((triggersCount) => { + return { + ...(triggersCount ?? {}), + schedule_count: scheduleCount, + primary_schedule: { + schedule: + updatedPrimarySchedule?.draftConfig?.schedule ?? + updatedPrimarySchedule?.lightConfig?.schedule + } + } + }) + + return + } catch (error) { + console.error('Failed to fetch schedules:', error) + return + } + } + + async fetchWebsocketTriggers( + triggersCountStore: Writable, + workspaceId: string | undefined, + path: string, + isFlow: boolean, + user: UserExt | undefined = undefined + ): Promise { + if (!workspaceId) return + try { + const wsTriggers = await WebsocketTriggerService.listWebsocketTriggers({ + workspace: workspaceId, + path, + isFlow + }) + const wsCount = this.updateTriggers(wsTriggers, 'websocket', user) + triggersCountStore.update((triggersCount) => { + return { + ...(triggersCount ?? {}), + websocket_count: wsCount + } + }) + } catch (error) { + console.error('Failed to fetch Websocket triggers:', error) + } + } + + async fetchPostgresTriggers( + triggersCountStore: Writable, + workspaceId: string | undefined, + path: string, + isFlow: boolean, + user: UserExt | undefined = undefined + ): Promise { + if (!workspaceId) return + try { + const pgTriggers: PostgresTrigger[] = await PostgresTriggerService.listPostgresTriggers({ + workspace: workspaceId, + path, + isFlow + }) + const pgCount = this.updateTriggers(pgTriggers, 'postgres', user) + triggersCountStore.update((triggersCount) => { + return { + ...(triggersCount ?? {}), + postgres_count: pgCount + } + }) + } catch (error) { + console.error('Failed to fetch Postgres triggers:', error) + } + } + + async fetchKafkaTriggers( + triggersCountStore: Writable, + workspaceId: string | undefined, + path: string, + isFlow: boolean, + user: UserExt | undefined = undefined + ): Promise { + if (!workspaceId) return + try { + const kafkaTriggers: KafkaTrigger[] = await KafkaTriggerService.listKafkaTriggers({ + workspace: workspaceId, + path, + isFlow + }) + const kafkaCount = this.updateTriggers(kafkaTriggers, 'kafka', user) + triggersCountStore.update((triggersCount) => { + return { + ...(triggersCount ?? {}), + kafka_count: kafkaCount + } + }) + } catch (error) { + console.error('Failed to fetch Kafka triggers:', error) + } + } + + async fetchNatsTriggers( + triggersCountStore: Writable, + workspaceId: string | undefined, + path: string, + isFlow: boolean, + user: UserExt | undefined = undefined + ): Promise { + if (!workspaceId) return + try { + const natsTriggers = await NatsTriggerService.listNatsTriggers({ + workspace: workspaceId, + path, + isFlow + }) + const natsCount = this.updateTriggers(natsTriggers, 'nats', user) + triggersCountStore.update((triggersCount) => { + return { + ...(triggersCount ?? {}), + nats_count: natsCount + } + }) + } catch (error) { + console.error('Failed to fetch NATS triggers:', error) + } + } + + async fetchMqttTriggers( + triggersCountStore: Writable, + workspaceId: string | undefined, + path: string, + isFlow: boolean, + user: UserExt | undefined = undefined + ): Promise { + if (!workspaceId) return + try { + const mqttTriggers = await MqttTriggerService.listMqttTriggers({ + workspace: workspaceId, + path, + isFlow + }) + const mqttCount = this.updateTriggers(mqttTriggers, 'mqtt', user) + triggersCountStore.update((triggersCount) => { + return { + ...(triggersCount ?? {}), + mqtt_count: mqttCount + } + }) + } catch (error) { + console.error('Failed to fetch MQTT triggers:', error) + } + } + + async fetchSqsTriggers( + triggersCountStore: Writable, + workspaceId: string | undefined, + path: string, + isFlow: boolean, + user: UserExt | undefined = undefined + ): Promise { + if (!workspaceId) return + try { + const sqsTriggers = await SqsTriggerService.listSqsTriggers({ + workspace: workspaceId, + path, + isFlow + }) + const sqsCount = this.updateTriggers(sqsTriggers, 'sqs', user) + triggersCountStore.update((triggersCount) => { + return { + ...(triggersCount ?? {}), + sqs_count: sqsCount + } + }) + } catch (error) { + console.error('Failed to fetch SQS triggers:', error) + } + } + + async fetchGcpTriggers( + triggersCountStore: Writable, + workspaceId: string | undefined, + path: string, + isFlow: boolean, + user: UserExt | undefined = undefined + ): Promise { + if (!workspaceId) return + try { + const gcpTriggers: GcpTrigger[] = await GcpTriggerService.listGcpTriggers({ + workspace: workspaceId, + path, + isFlow + }) + const gcpCount = this.updateTriggers(gcpTriggers, 'gcp', user) + triggersCountStore.update((triggersCount) => { + return { + ...(triggersCount ?? {}), + gcp_count: gcpCount + } + }) + } catch (error) { + console.error('Failed to fetch GCP Pub/Sub triggers:', error) + } + } + + async fetchHttpTriggers( + triggersCountStore: Writable, + workspaceId: string | undefined, + path: string, + isFlow: boolean, + user: UserExt | undefined = undefined + ): Promise { + if (!workspaceId) return + try { + const httpTriggers: HttpTrigger[] = await HttpTriggerService.listHttpTriggers({ + workspace: workspaceId, + path, + isFlow + }) + const httpCount = this.updateTriggers(httpTriggers, 'http', user) + triggersCountStore.update((triggersCount) => { + return { + ...(triggersCount ?? {}), + http_routes_count: httpCount + } + }) + } catch (error) { + console.error('Failed to fetch HTTP triggers:', error) + } + } + + async fetchTriggers( + triggersCountStore: Writable, + workspaceId: string | undefined, + path: string, + isFlow: boolean, + primarySchedule: ScheduleTrigger | undefined | false = undefined, + user: UserExt | undefined = undefined + ): Promise { + if (!workspaceId) return + + // Fetch each type of trigger + await Promise.all([ + this.fetchSchedules(triggersCountStore, workspaceId, path, isFlow, primarySchedule, user), + this.fetchHttpTriggers(triggersCountStore, workspaceId, path, isFlow, user), + this.fetchWebsocketTriggers(triggersCountStore, workspaceId, path, isFlow, user), + this.fetchPostgresTriggers(triggersCountStore, workspaceId, path, isFlow, user), + this.fetchKafkaTriggers(triggersCountStore, workspaceId, path, isFlow, user), + this.fetchNatsTriggers(triggersCountStore, workspaceId, path, isFlow, user), + this.fetchMqttTriggers(triggersCountStore, workspaceId, path, isFlow, user), + this.fetchSqsTriggers(triggersCountStore, workspaceId, path, isFlow, user), + this.fetchGcpTriggers(triggersCountStore, workspaceId, path, isFlow, user) + ]) + } +} diff --git a/frontend/src/lib/components/triggers/utils.ts b/frontend/src/lib/components/triggers/utils.ts new file mode 100644 index 0000000000..7daf829366 --- /dev/null +++ b/frontend/src/lib/components/triggers/utils.ts @@ -0,0 +1,529 @@ +import { Webhook, Mail, Calendar, Route, Unplug, Database, Terminal } from 'lucide-svelte' +import KafkaIcon from '$lib/components/icons/KafkaIcon.svelte' +import NatsIcon from '$lib/components/icons/NatsIcon.svelte' +import MqttIcon from '$lib/components/icons/MqttIcon.svelte' +import AwsIcon from '$lib/components/icons/AwsIcon.svelte' +import GoogleCloudIcon from '$lib/components/icons/GoogleCloudIcon.svelte' +import type { CaptureTriggerKind, Flow, NewScript, TriggersCount } from '$lib/gen/types.gen' +import type { Writable } from 'svelte/store' +import SchedulePollIcon from '../icons/SchedulePollIcon.svelte' +import { type TriggerKind } from '$lib/components/triggers' +import { saveScheduleFromCfg } from '$lib/components/flows/scheduleUtils' +import { saveHttpRouteFromCfg } from './http/utils' +import { saveWebsocketTriggerFromCfg } from './websocket/utils' +import { savePostgresTriggerFromCfg } from './postgres/utils' +import { saveKafkaTriggerFromCfg } from './kafka/utils' +import { saveSqsTriggerFromCfg } from './sqs/utils' +import { saveNatsTriggerFromCfg } from './nats/utils' +import { saveMqttTriggerFromCfg } from './mqtt/utils' +import { saveGcpTriggerFromCfg } from './gcp/utils' +import type { Triggers } from './triggers.svelte' +import { emptyString } from '$lib/utils' + +export const CLOUD_DISABLED_TRIGGER_TYPES = [ + 'nats', + 'kafka', + 'sqs', + 'mqtt', + 'gcp', + 'websocket', + 'postgres' +] + +export type TriggerType = + | 'webhook' + | 'email' + | 'schedule' + | 'http' + | 'websocket' + | 'postgres' + | 'kafka' + | 'nats' + | 'mqtt' + | 'sqs' + | 'gcp' + | 'poll' + | 'cli' + +export type Trigger = { + type: TriggerType + path?: string + isDraft?: boolean + isPrimary?: boolean + canWrite?: boolean + id?: string + draftConfig?: Record + captureConfig?: Record + extra?: Record + lightConfig?: Record +} + +// Map of trigger kinds to icons +export const triggerIconMap = { + webhook: Webhook, + email: Mail, + schedule: Calendar, + http: Route, + websocket: Unplug, + postgres: Database, + kafka: KafkaIcon, + nats: NatsIcon, + mqtt: MqttIcon, + sqs: AwsIcon, + gcp: GoogleCloudIcon, + primary_schedule: Calendar, + poll: SchedulePollIcon, + cli: Terminal +} + +/** + * Converts a TriggerType to a CaptureTriggerKind when a mapping exists + * @param triggerType The trigger type to convert + * @returns The corresponding CaptureTriggerKind or undefined if no mapping exists + */ +export function triggerTypeToCaptureKind(triggerType: TriggerType): CaptureTriggerKind | undefined { + // Define types that can be mapped to CaptureTriggerKind + const capturableTriggerTypes: TriggerType[] = [ + 'webhook', + 'email', + 'http', + 'websocket', + 'postgres', + 'kafka', + 'nats', + 'mqtt', + 'sqs', + 'gcp', + 'cli' + ] + + if (capturableTriggerTypes.includes(triggerType)) { + return triggerType as CaptureTriggerKind + } + + return undefined +} + +export function updateTriggersCount( + triggersCountStore: Writable, + type: TriggerType, + action: 'add' | 'remove', + primaryCfg?: Record, + isPrimary?: boolean +) { + // Map trigger types to their corresponding count property names + const countPropertyMap: Record = { + webhook: undefined, + email: undefined, + schedule: 'schedule_count', + http: 'http_routes_count', + websocket: 'websocket_count', + postgres: 'postgres_count', + kafka: 'kafka_count', + nats: 'nats_count', + mqtt: 'mqtt_count', + sqs: 'sqs_count', + gcp: 'gcp_count', + poll: undefined, + cli: undefined + } + + const countProperty = countPropertyMap[type] + + triggersCountStore.update((triggersCount) => { + // Handle special case for schedule + if (type === 'schedule') { + if (action === 'add' && primaryCfg) { + return { + ...(triggersCount ?? {}), + schedule_count: (triggersCount?.schedule_count ?? 0) + 1, + primary_schedule: primaryCfg?.schedule + } + } else if (action === 'remove') { + return { + ...(triggersCount ?? {}), + schedule_count: (triggersCount?.schedule_count ?? 1) - 1, + primary_schedule: isPrimary ? undefined : triggersCount?.primary_schedule + } + } + } + + // Handle standard count updates + if (countProperty && action === 'add') { + return { + ...(triggersCount ?? {}), + [countProperty]: (triggersCount?.[countProperty] ?? 0) + 1 + } + } else if (countProperty && action === 'remove') { + return { + ...(triggersCount ?? {}), + [countProperty]: (triggersCount?.[countProperty] ?? 1) - 1 + } + } + + return triggersCount + }) +} + +// TODO: Remove this once we've migrated all the trigger kinds to the new TriggerType enum +export function triggerKindToTriggerType(kind: TriggerKind): TriggerType | undefined { + switch (kind) { + case 'webhooks': + return 'webhook' + case 'emails': + return 'email' + case 'schedules': + return 'schedule' + case 'routes': + return 'http' + case 'websockets': + return 'websocket' + case 'postgres': + return 'postgres' + case 'kafka': + return 'kafka' + case 'nats': + return 'nats' + case 'mqtt': + return 'mqtt' + case 'sqs': + return 'sqs' + case 'gcp': + return 'gcp' + case 'scheduledPoll': + return 'poll' + default: + throw new Error(`Unknown TriggerKind: ${kind}`) + } +} + +export async function deployTriggers( + triggersToDeploy: Trigger[], + workspaceId: string | undefined, + isAdmin: boolean, + usedTriggerKinds: Writable, + initialPath?: string, + isNew?: boolean +) { + if (!workspaceId) return + + if (isNew && initialPath) { + triggersToDeploy.forEach((trigger) => { + trigger.draftConfig = { + ...trigger.draftConfig, + script_path: initialPath + } + }) + } + + // Map of trigger types to their save functions + const triggerSaveFunctions: Record = { + webhook: undefined, + email: undefined, + schedule: (trigger: Trigger) => { + if (trigger.isPrimary && initialPath) { + trigger.draftConfig = { + ...trigger.draftConfig, + path: initialPath, + script_path: initialPath + } + } + return saveScheduleFromCfg(trigger.draftConfig ?? {}, !trigger.isDraft, workspaceId) + }, + http: (trigger: Trigger) => + saveHttpRouteFromCfg( + trigger.path ?? trigger.draftConfig?.path ?? '', + trigger.draftConfig ?? {}, + !trigger.isDraft, + workspaceId, + isAdmin, + usedTriggerKinds + ), + websocket: (trigger: Trigger) => + saveWebsocketTriggerFromCfg( + trigger.path ?? trigger.draftConfig?.path ?? '', + trigger.draftConfig ?? {}, + !trigger.isDraft, + workspaceId, + usedTriggerKinds + ), + postgres: (trigger: Trigger) => + savePostgresTriggerFromCfg( + trigger.path ?? trigger.draftConfig?.path ?? '', + trigger.draftConfig ?? {}, + !trigger.isDraft, + workspaceId, + usedTriggerKinds + ), + kafka: (trigger: Trigger) => + saveKafkaTriggerFromCfg( + trigger.path ?? trigger.draftConfig?.path ?? '', + trigger.draftConfig ?? {}, + !trigger.isDraft, + workspaceId, + usedTriggerKinds + ), + nats: (trigger: Trigger) => + saveNatsTriggerFromCfg( + trigger.path ?? trigger.draftConfig?.path ?? '', + trigger.draftConfig ?? {}, + !trigger.isDraft, + workspaceId, + usedTriggerKinds + ), + mqtt: (trigger: Trigger) => + saveMqttTriggerFromCfg( + trigger.path ?? trigger.draftConfig?.path ?? '', + trigger.draftConfig ?? {}, + !trigger.isDraft, + workspaceId, + usedTriggerKinds + ), + sqs: (trigger: Trigger) => + saveSqsTriggerFromCfg( + trigger.path ?? trigger.draftConfig?.path ?? '', + trigger.draftConfig ?? {}, + !trigger.isDraft, + workspaceId, + usedTriggerKinds + ), + gcp: (trigger: Trigger) => + saveGcpTriggerFromCfg( + trigger.path ?? trigger.draftConfig?.path ?? '', + trigger.draftConfig ?? {}, + !trigger.isDraft, + workspaceId, + usedTriggerKinds + ), + poll: undefined, + cli: undefined + } + + await Promise.all( + triggersToDeploy.map(async (trigger) => { + const saveFunction = triggerSaveFunctions[trigger.type] + if (saveFunction) { + await saveFunction(trigger) + } else { + console.warn(`No save function defined for trigger type: ${trigger.type}`) + } + }) + ) +} + +export function handleSelectTriggerFromKind( + triggersState: Triggers, + triggersCountStore: Writable, + initialPath: string | undefined, + triggerKind: TriggerKind +) { + const triggerType = triggerKindToTriggerType(triggerKind) + + if (!triggerType) { + return + } + + const existingTriggerIndex = triggersState.triggers.findIndex( + (trigger) => trigger.type === triggerType + ) + + if (existingTriggerIndex !== -1) { + triggersState.selectedTriggerIndex = existingTriggerIndex + } else { + const newTrigger = triggersState.addDraftTrigger( + triggersCountStore, + triggerType, + triggerType === 'schedule' ? initialPath : undefined + ) + triggersState.selectedTriggerIndex = newTrigger + } +} + +export function handleConfigChange( + nCfg: Record, + initialConfig: Record | undefined, + saveDisabled: boolean, + edit: boolean, + onConfigChange?: (cfg: Record, saveDisabled: boolean, updated: boolean) => void +) { + let updated = false + if (!edit || !initialConfig) { + updated = true + } else { + // We ignore changes to enabled + let newCfg = { ...nCfg } + if ('enabled' in newCfg) { + delete newCfg.enabled + } + let initialCfg = { ...initialConfig } + if ('enabled' in initialCfg) { + delete initialCfg.enabled + } + if (JSON.stringify(newCfg) !== JSON.stringify(initialCfg)) { + updated = true + } + } + + onConfigChange?.(nCfg, saveDisabled, updated) +} + +export function getLightConfig( + triggerType: TriggerType, + trigger: Record +): Record | undefined { + if (triggerType === 'schedule') { + return { schedule: trigger.schedule, enable: trigger.enable, summary: trigger.summary } + } else if (triggerType === 'http') { + return { route_path: trigger.route_path, http_method: trigger.http_method } + } else if (triggerType === 'websocket') { + return { url: trigger.url } + } else if (triggerType === 'postgres') { + return { postgres_resource_path: trigger.postgres_resource_path } + } else if (triggerType === 'kafka') { + return { kafka_resource_path: trigger.kafka_resource_path, topics: trigger.topics } + } else if (triggerType === 'nats') { + return { nats_resource_path: trigger.nats_resource_path, subjects: trigger.subjects } + } else if (triggerType === 'mqtt') { + return { + mqtt_resource_path: trigger.mqtt_resource_path, + subscribe_topics: trigger.subscribe_topics + } + } else if (triggerType === 'sqs') { + return { queue_url: trigger.queue_url } + } else if (triggerType === 'gcp') { + return { gcp_resource_path: trigger.gcp_resource_path, topic: trigger.topic } + } else { + return undefined + } +} + +export function getTriggerLabel(trigger: Trigger): string { + const { type, isDraft, draftConfig, lightConfig, path } = trigger + const config = draftConfig ?? lightConfig + + if (type === 'webhook') { + return 'Webhook' + } else if (type === 'email') { + return 'Email' + } else if (type === 'cli') { + return 'CLI' + } else if (type === 'http' && !emptyString(config?.route_path)) { + return `${(draftConfig?.http_method ?? lightConfig?.http_method ?? 'post').toUpperCase()} ${draftConfig?.route_path ?? lightConfig?.route_path}` + } else if (type === 'schedule' && config?.summary) { + return `${config?.summary}` + } else if (type === 'kafka' && config?.topics && config?.kafka_resource_path) { + return `${config?.kafka_resource_path} - ${config?.topics.join(', ')}` + } else if (type === 'nats' && config?.subjects && config?.nats_resource_path) { + return `${config?.nats_resource_path} - ${config?.subjects.join(', ')}` + } else if (type === 'mqtt' && config?.subscribe_topics && config?.mqtt_resource_path) { + const topics = config?.subscribe_topics.map((topic: any) => topic.topic).join(', ') + return `${config?.mqtt_resource_path} - ${topics}` + } else if (type === 'sqs' && config?.queue_url) { + return `${config?.queue_url}` + } else if (type === 'gcp' && config?.gcp_resource_path && config?.topic) { + return `${config?.gcp_resource_path} - ${config?.topic}` + } else if (type === 'websocket' && config?.url) { + return `${config?.url}` + } else if (isDraft && draftConfig?.path) { + return `${draftConfig?.path}` + } else if (isDraft) { + return `New ${type.replace(/s$/, '')} trigger` + } else { + return path ?? '' + } +} + +export function sortTriggers(triggers: Trigger[]): Trigger[] { + const triggerTypeOrder = [ + 'webhook', + 'cli', + 'email', + 'poll', + 'schedule', + 'http', + 'websocket', + 'postgres', + 'kafka', + 'nats', + 'mqtt', + 'sqs', + 'gcp' + ] + + return triggers.sort((a, b) => { + // Draft triggers always come last + if (a.isDraft && !b.isDraft) return 1 + if (!a.isDraft && b.isDraft) return -1 + + // If both are drafts or both are not drafts, sort by type order + if (a.isDraft === b.isDraft) { + const aIndex = triggerTypeOrder.indexOf(a.type) + const bIndex = triggerTypeOrder.indexOf(b.type) + + // If both types are in the order array, sort by their position + if (aIndex >= 0 && bIndex >= 0) { + return aIndex - bIndex + } + + // If only one type is in the order array, it comes first + if (aIndex >= 0) return -1 + if (bIndex >= 0) return 1 + + // If neither type is in the order array, maintain original order + return 0 + } + + return 0 + }) +} + +export type FlowWithDraftAndDraftTriggers = Flow & { + draft?: Flow & { + draft_triggers?: Trigger[] + } +} + +export type NewScriptWithDraftAndDraftTriggers = NewScript & { + draft?: NewScript & { draft_triggers?: Trigger[] } + hash: string +} + +// Get rid of deployed triggers from the saved flow in the case there is a match with a deployed trigger +export function filterDraftTriggers( + savedValue: FlowWithDraftAndDraftTriggers | NewScriptWithDraftAndDraftTriggers, + triggersState: Triggers +): FlowWithDraftAndDraftTriggers | NewScriptWithDraftAndDraftTriggers { + const deployedTriggers = triggersState.triggers.filter((t) => !t.draftConfig && !t.isDraft) + let newSavedValue = savedValue + + const filterMatchingTriggers = (savedTriggers: Trigger[], deployedTriggers: Trigger[]) => { + return savedTriggers.filter( + (savedTrigger) => + !deployedTriggers.some( + (deployedTrigger) => + deployedTrigger.path === savedTrigger.draftConfig?.path && + deployedTrigger.type === savedTrigger.type + ) + ) + } + + const savedDraftTriggersFiltered = filterMatchingTriggers( + newSavedValue?.draft?.draft_triggers ?? [], + deployedTriggers + ) + if (newSavedValue?.draft?.draft_triggers) { + newSavedValue = { + ...newSavedValue, + draft: { + ...newSavedValue.draft, + draft_triggers: + savedDraftTriggersFiltered.length > 0 ? savedDraftTriggersFiltered : undefined + } + } as typeof newSavedValue + } + triggersState.setTriggers([ + ...triggersState.triggers.filter((t) => !t.draftConfig), + ...savedDraftTriggersFiltered + ]) + return newSavedValue +} diff --git a/frontend/src/lib/components/triggers/webhook/WebhooksCapture.svelte b/frontend/src/lib/components/triggers/webhook/WebhooksCapture.svelte new file mode 100644 index 0000000000..b6392bf41b --- /dev/null +++ b/frontend/src/lib/components/triggers/webhook/WebhooksCapture.svelte @@ -0,0 +1,80 @@ + + +{#if captureInfo} + + {#snippet description()} + {#if captureInfo.active} +

+ Send a POST request to the URL below to simulate a webhook event. +

+ {:else} +

+ Start capturing to listen to webhook events on this test URL. +

+ {/if} + {/snippet} + + + + +{/if} diff --git a/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte b/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte index bd8bb7d044..1ee21df607 100644 --- a/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte +++ b/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte @@ -21,9 +21,6 @@ import { workspaceStore, userStore } from '$lib/stores' import UserSettings from '../../UserSettings.svelte' import { generateRandomString } from '$lib/utils' - import CopyableCodeBlock from '../../details/CopyableCodeBlock.svelte' - import CaptureSection, { type CaptureInfo } from '../CaptureSection.svelte' - import CaptureTable from '../CaptureTable.svelte' export let isFlow: boolean = false export let path: string = '' @@ -32,9 +29,6 @@ export let runnableArgs: any export let triggerTokens: TriggerTokens | undefined = undefined export let scopes: string[] = [] - export let showCapture: boolean = false - export let captureTable: CaptureTable | undefined = undefined - export let captureInfo: CaptureInfo | undefined = undefined let webhooks: { async: { @@ -198,17 +192,6 @@ function waitForJobCompletion(UUID) { return `${mainFunction}\n\n${triggerJobFunction}\n\n${waitForJobCompletionFunction}` } - let captureUrl = `${location.origin}/api/w/${$workspaceStore}/capture_u/webhook/${ - isFlow ? 'flow' : 'script' - }/${path}` - - function captureCurlCode() { - return `curl \\ --X POST ${captureUrl} \\ --H 'Content-Type: application/json' \\ --d '${JSON.stringify(cleanedRunnableArgs ?? {}, null, 2)}'` - } - function curlCode() { return `TOKEN='${token}' ${requestType !== 'get_path' ? `BODY='${JSON.stringify(cleanedRunnableArgs ?? {})}'` : ''} @@ -263,195 +246,162 @@ done` {scopes} /> -
- {#if showCapture && captureInfo} - - - -
+ {/if} +{/snippet} diff --git a/frontend/src/lib/components/triggers/websocket/WebsocketTriggersPanel.svelte b/frontend/src/lib/components/triggers/websocket/WebsocketTriggersPanel.svelte index 12142f8343..269d69bb94 100644 --- a/frontend/src/lib/components/triggers/websocket/WebsocketTriggersPanel.svelte +++ b/frontend/src/lib/components/triggers/websocket/WebsocketTriggersPanel.svelte @@ -1,127 +1,61 @@ - { - loadTriggers() - }} - bind:this={wsTriggerEditor} -/> - -{#if isCloudHosted()} - - WebSocket triggers are disabled in the multi-tenant cloud. - -{:else} -
- - WebSocket triggers allow real-time bidirectional communication between your scripts/flows and - external systems. Each trigger creates a unique WebSocket endpoint. - - - {#if !newItem && wsTriggers && wsTriggers.length > 0} -
-
-
- {#each wsTriggers as wsTriggers (wsTriggers.path)} -
-
{wsTriggers.path}
-
- {wsTriggers.url} -
-
- -
-
- {/each} -
-
-
- {/if} - - { - wsTriggerEditor?.openNew(isFlow, path, e.detail.config) - }} - on:addPreprocessor - on:updateSchema - on:testWithArgs - cloudDisabled={false} - triggerType="websocket" - {isFlow} - {path} - {isEditor} - {canHavePreprocessor} - {hasPreprocessor} - {newItem} - {openForm} - /> -
-{/if} +
+ + {#snippet description()} + {#if cloudDisabled} + + WebSocket triggers are disabled in the multi-tenant cloud. + + {:else} + + WebSocket triggers allow real-time bidirectional communication between your scripts/flows + and external systems. Each trigger creates a unique WebSocket endpoint. + + {/if} + {/snippet} + +
diff --git a/frontend/src/lib/components/triggers/websocket/utils.ts b/frontend/src/lib/components/triggers/websocket/utils.ts new file mode 100644 index 0000000000..c112cb0f1a --- /dev/null +++ b/frontend/src/lib/components/triggers/websocket/utils.ts @@ -0,0 +1,46 @@ +import { WebsocketTriggerService } from '$lib/gen' +import { sendUserToast } from '$lib/toast' +import type { Writable } from 'svelte/store' +import { get } from 'svelte/store' + +export async function saveWebsocketTriggerFromCfg( + initialPath: string, + triggerCfg: Record, + edit: boolean, + workspace: string, + usedTriggerKinds: Writable +): Promise { + const requestBody = { + path: triggerCfg.path, + script_path: triggerCfg.script_path, + is_flow: triggerCfg.is_flow, + url: triggerCfg.url, + filters: triggerCfg.filters, + initial_messages: triggerCfg.initial_messages, + url_runnable_args: triggerCfg.url_runnable_args, + can_return_message: triggerCfg.can_return_message + } + try { + if (edit) { + await WebsocketTriggerService.updateWebsocketTrigger({ + workspace: workspace, + path: initialPath, + requestBody: requestBody + }) + sendUserToast(`Websocket trigger ${triggerCfg.path} updated`) + } else { + await WebsocketTriggerService.createWebsocketTrigger({ + workspace: workspace, + requestBody: { ...requestBody, enabled: true } + }) + sendUserToast(`Websocket trigger ${triggerCfg.path} created`) + } + if (!get(usedTriggerKinds).includes('ws')) { + usedTriggerKinds.update((t) => [...t, 'ws']) + } + return true + } catch (error) { + sendUserToast(error.body || error.message, true) + return false + } +} diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index 2609eba53d..6c2bde6431 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -1161,6 +1161,7 @@ export type Item = { disabled?: boolean type?: 'action' | 'delete' hide?: boolean | undefined + extra?: Snippet } export function isObjectTooBig(obj: any): boolean { @@ -1245,6 +1246,15 @@ export function formatDateShort(dateString: string | undefined): string { }).format(date) } +export function toJsonStr(result: any) { + try { + // console.log(result) + return JSON.stringify(result ?? null, null, 4) ?? 'null' + } catch (e) { + return 'error stringifying object: ' + e.toString() + } +} + export function getOS() { const userAgent = window.navigator.userAgent const platform = window.navigator.platform @@ -1268,6 +1278,7 @@ export function getOS() { import { type ClassValue, clsx } from 'clsx' import { twMerge } from 'tailwind-merge' +import type { Snippet } from 'svelte' export function cn(...inputs: ClassValue[]) { return twMerge(clsx(inputs)) diff --git a/frontend/src/routes/(root)/(logged)/flows/add/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/add/+page.svelte index e291917235..056424922b 100644 --- a/frontend/src/routes/(root)/(logged)/flows/add/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/add/+page.svelte @@ -13,9 +13,9 @@ import { decodeState, emptySchema } from '$lib/utils' import { tick } from 'svelte' import { writable } from 'svelte/store' - import type { ScheduleTrigger } from '$lib/components/triggers' import type { GetInitialAndModifiedValues } from '$lib/components/common/confirmationModal/unsavedTypes' import { replaceScriptPlaceholderWithItsValues } from '$lib/hub' + import type { Trigger } from '$lib/components/triggers/utils' let nodraft = $page.url.searchParams.get('nodraft') @@ -43,6 +43,7 @@ initialArgs = $initialArgsStore $initialArgsStore = undefined } + let flowBuilder: FlowBuilder | undefined = undefined export const flowStore = writable({ summary: '', @@ -56,7 +57,8 @@ }) const flowStateStore = writable({}) - let savedPrimarySchedule: ScheduleTrigger | undefined = undefined + let draftTriggersFromUrl: Trigger[] | undefined = undefined + let selectedTriggerIndexFromUrl: number | undefined = undefined async function loadFlow() { loading = true let flow: Flow = { @@ -101,7 +103,10 @@ flow = state.flow pathStoreInit = state.path - savedPrimarySchedule = state.primarySchedule + draftTriggersFromUrl = state.draft_triggers + selectedTriggerIndexFromUrl = state.selected_trigger + flowBuilder?.setDraftTriggers(draftTriggersFromUrl) + flowBuilder?.setSelectedTriggerIndex(selectedTriggerIndexFromUrl) state?.selectedId && (selectedId = state?.selectedId) } else { if (templatePath) { @@ -153,7 +158,6 @@ loadFlow() let getSelectedId: (() => string) | undefined = undefined - let flowBuilder: FlowBuilder | undefined = undefined let getInitialAndModifiedValues: GetInitialAndModifiedValues | undefined = undefined @@ -181,7 +185,8 @@ {flowStateStore} {selectedId} {loading} - {savedPrimarySchedule} + {draftTriggersFromUrl} + {selectedTriggerIndexFromUrl} > diff --git a/frontend/src/routes/(root)/(logged)/flows/edit/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/edit/[...path]/+page.svelte index 51e0e9d6e3..8001ee0281 100644 --- a/frontend/src/routes/(root)/(logged)/flows/edit/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/edit/[...path]/+page.svelte @@ -15,6 +15,7 @@ import UnsavedConfirmationModal from '$lib/components/common/confirmationModal/UnsavedConfirmationModal.svelte' import type { ScheduleTrigger } from '$lib/components/triggers' import type { GetInitialAndModifiedValues } from '$lib/components/common/confirmationModal/unsavedTypes' + import type { Trigger } from '$lib/components/triggers/utils' let version: undefined | number = undefined let nodraft = $page.url.searchParams.get('nodraft') @@ -60,6 +61,9 @@ let savedPrimarySchedule: ScheduleTrigger | undefined = stateLoadedFromUrl?.primarySchedule + let draftTriggersFromUrl: Trigger[] | undefined = undefined + let selectedTriggerIndexFromUrl: number | undefined = undefined + let flowBuilder: FlowBuilder | undefined = undefined async function loadFlow(): Promise { @@ -82,12 +86,19 @@ }) const draftOrDeployed = cleanValueProperties(savedFlow?.draft || savedFlow) - const urlScript = cleanValueProperties(stateLoadedFromUrl.flow) + const urlScript = cleanValueProperties({ + ...stateLoadedFromUrl.flow, + draft_triggers: stateLoadedFromUrl.draft_triggers + }) flow = stateLoadedFromUrl.flow - savedPrimarySchedule = stateLoadedFromUrl.primarySchedule + draftTriggersFromUrl = stateLoadedFromUrl.draft_triggers + selectedTriggerIndexFromUrl = stateLoadedFromUrl.selected_trigger + flowBuilder?.setDraftTriggers(draftTriggersFromUrl) + flowBuilder?.setSelectedTriggerIndex(selectedTriggerIndexFromUrl) + const selectedId = stateLoadedFromUrl?.selectedId ?? 'settings-metadata' const reloadAction = () => { stateLoadedFromUrl = undefined - goto(`/flows/edit/${statePath}`) + goto(`/flows/edit/${statePath}?selected=${selectedId}`) loadFlow() } if (orderedJsonStringify(draftOrDeployed) === orderedJsonStringify(urlScript)) { @@ -133,15 +144,18 @@ ? { ...structuredClone(flowWithDraft.draft), path: flowWithDraft.draft.path ?? flowWithDraft.path // backward compatibility for old drafts missing path - } + } : undefined } as Flow & { - draft?: Flow + draft?: Flow & { + draft_triggers?: Trigger[] + } } if (flowWithDraft.draft != undefined && !nobackenddraft) { flow = flowWithDraft.draft savedPrimarySchedule = flowWithDraft?.draft?.['primary_schedule'] flowBuilder?.setPrimarySchedule(savedPrimarySchedule) + flowBuilder?.setDraftTriggers(flowWithDraft?.draft?.['draft_triggers']) if (!flowWithDraft.draft_only) { const deployed = cleanValueProperties(flowWithDraft) @@ -178,6 +192,7 @@ } } else { flow = flowWithDraft + flowBuilder?.setDraftTriggers(undefined) } } @@ -200,6 +215,7 @@ return } diffDrawer.closeDrawer() + stateLoadedFromUrl = undefined goto(`/flows/edit/${savedFlow.draft.path}`) loadFlow() } @@ -217,6 +233,7 @@ path: savedFlow.path }) } + stateLoadedFromUrl = undefined goto(`/flows/edit/${savedFlow.path}`) loadFlow() } @@ -251,6 +268,8 @@ bind:savedFlow {diffDrawer} {savedPrimarySchedule} + {draftTriggersFromUrl} + {selectedTriggerIndexFromUrl} bind:version bind:getInitialAndModifiedValues > diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index a20cfd427a..55bf1d5d1c 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -41,35 +41,26 @@ } from 'lucide-svelte' import DetailPageHeader from '$lib/components/details/DetailPageHeader.svelte' - import WebhooksPanel from '$lib/components/triggers/webhook/WebhooksPanel.svelte' - import CliHelpBox from '$lib/components/CliHelpBox.svelte' import FlowGraphViewer from '$lib/components/FlowGraphViewer.svelte' - import RunPageSchedules from '$lib/components/RunPageSchedules.svelte' import { createAppFromFlow } from '$lib/components/details/createAppFromScript' import { importStore } from '$lib/components/apps/store' import TimeAgo from '$lib/components/TimeAgo.svelte' - import ClipboardPanel from '$lib/components/details/ClipboardPanel.svelte' import FlowGraphViewerStep from '$lib/components/FlowGraphViewerStep.svelte' import GfmMarkdown from '$lib/components/GfmMarkdown.svelte' import FlowHistory from '$lib/components/flows/FlowHistory.svelte' - import EmailTriggerPanel from '$lib/components/details/EmailTriggerPanel.svelte' import Star from '$lib/components/Star.svelte' - import RoutesPanel from '$lib/components/triggers/http/RoutesPanel.svelte' import { Highlight } from 'svelte-highlight' import json from 'svelte-highlight/languages/json' import { writable } from 'svelte/store' - import TriggersBadge from '$lib/components/graph/renderers/triggers/TriggersBadge.svelte' import InputSelectedBadge from '$lib/components/schema/InputSelectedBadge.svelte' - import WebsocketTriggersPanel from '$lib/components/triggers/websocket/WebsocketTriggersPanel.svelte' - import KafkaTriggersPanel from '$lib/components/triggers/kafka/KafkaTriggersPanel.svelte' - import NatsTriggersPanel from '$lib/components/triggers/nats/NatsTriggersPanel.svelte' - import PostgresTriggersPanel from '$lib/components/triggers/postgres/PostgresTriggersPanel.svelte' import Toggle from '$lib/components/Toggle.svelte' - import MqttTriggersPanel from '$lib/components/triggers/mqtt/MqttTriggersPanel.svelte' - import SqsTriggerPanel from '$lib/components/triggers/sqs/SqsTriggerPanel.svelte' - import { onDestroy } from 'svelte' + import { onDestroy, tick } from 'svelte' import LogViewer from '$lib/components/LogViewer.svelte' - import GcpTriggerPanel from '$lib/components/triggers/gcp/GcpTriggerPanel.svelte' + import TriggersEditor from '$lib/components/triggers/TriggersEditor.svelte' + import type { TriggerContext } from '$lib/components/triggers' + import { setContext } from 'svelte' + import TriggersBadge from '$lib/components/graph/renderers/triggers/TriggersBadge.svelte' + import { Triggers } from '$lib/components/triggers/triggers.svelte' let flow: Flow | undefined let can_write = false @@ -85,9 +76,29 @@ let intervalId: NodeJS.Timeout | undefined = undefined + $: { + const cliTrigger = triggersState.triggers.find((t) => t.type === 'cli') + if (cliTrigger) { + cliTrigger.extra = { + cliCommand: `wmill flow run ${flow?.path} -d '${JSON.stringify(args)}'` + } + } + } + const triggersCount = writable(undefined) - $: cliCommand = `wmill flow run ${flow?.path} -d '${JSON.stringify(args)}'` + // Add triggers context store + const triggersState = new Triggers([ + { type: 'webhook', path: '', isDraft: false }, + { type: 'email', path: '', isDraft: false }, + { type: 'cli', path: '', isDraft: false } + ]) + setContext('TriggerContext', { + triggersCount, + simplifiedPoll: writable(false), + showCaptureHint: writable(undefined), + triggersState + }) let previousPath: string | undefined = undefined $: { @@ -96,6 +107,7 @@ previousPath = path loadFlow() loadTriggersCount() + loadTriggers() } } } @@ -124,6 +136,17 @@ }) } + async function loadTriggers(): Promise { + await triggersState.fetchTriggers( + triggersCount, + $workspaceStore, + path, + true, + undefined, + $userStore + ) + } + async function loadFlow(): Promise { flow = await FlowService.getFlowByPath({ workspace: $workspaceStore!, @@ -371,7 +394,7 @@ } } let stepDetail: FlowModule | string | undefined = undefined - let token = 'TOKEN_TO_CREATE' + let rightPaneSelected = 'saved_inputs' let savedInputsV2: SavedInputsV2 | undefined = undefined let flowHistory: FlowHistory | undefined = undefined @@ -398,7 +421,6 @@ {#if flow} { + onSelect={async (triggerIndex: number) => { rightPaneSelected = 'triggers' + await tick() + triggersState.selectedTriggerIndex = triggerIndex }} + small={false} /> {#if $workspaceStore} @@ -590,89 +616,23 @@
- -
- -
- - -
- -
- - -
- -
- - - -
- -
- - - -
- -
- - - -
- -
- - -
- -
- - - -
- -
- - - -
- -
- - - -
- -
- - -
- -
- - -
- - -
- - {#if stepDetail} {/if} + + + {/if} diff --git a/frontend/src/routes/(root)/(logged)/gcp_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/gcp_triggers/+page.svelte index 9fcaa0f202..2fd36a1cc1 100644 --- a/frontend/src/routes/(root)/(logged)/gcp_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/gcp_triggers/+page.svelte @@ -263,7 +263,7 @@ - + - + - + - + - + - + - + { @@ -122,7 +120,6 @@ searchParams={$page.url.searchParams} {script} {showMeta} - {savedPrimarySchedule} replaceStateFn={(path) => replaceState(path, $page.state)} > diff --git a/frontend/src/routes/(root)/(logged)/scripts/edit/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/scripts/edit/[...path]/+page.svelte index fbe85f83c0..803b29bce8 100644 --- a/frontend/src/routes/(root)/(logged)/scripts/edit/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/scripts/edit/[...path]/+page.svelte @@ -12,6 +12,7 @@ import UnsavedConfirmationModal from '$lib/components/common/confirmationModal/UnsavedConfirmationModal.svelte' import type { ScheduleTrigger } from '$lib/components/triggers' import type { GetInitialAndModifiedValues } from '$lib/components/common/confirmationModal/unsavedTypes' + import type { Trigger } from '$lib/components/triggers/utils' let initialState = window.location.hash != '' ? window.location.hash.slice(1) : undefined let initialArgs = {} @@ -26,7 +27,8 @@ let scriptLoadedFromUrl = initialState != undefined ? decodeState(initialState) : undefined - let script: NewScript | undefined = undefined + + let script: (NewScript & { draft_triggers?: Trigger[] }) | undefined = undefined let initialPath: string = '' @@ -37,7 +39,7 @@ let savedScript: NewScriptWithDraft | undefined = undefined let fullyLoaded = false - let savedPrimarySchedule: ScheduleTrigger | undefined = scriptLoadedFromUrl?.primarySchedule + let savedPrimarySchedule: ScheduleTrigger | undefined = undefined async function loadScript(): Promise { fullyLoaded = false @@ -98,10 +100,12 @@ savedScript = structuredClone(scriptWithDraft) if (scriptWithDraft.draft != undefined) { script = scriptWithDraft.draft + scriptBuilder?.setDraftTriggers(script.draft_triggers) if (script['primary_schedule']) { savedPrimarySchedule = script['primary_schedule'] scriptBuilder?.setPrimarySchedule(savedPrimarySchedule) } + if (!scriptWithDraft.draft_only) { reloadAction = async () => { scriptLoadedFromUrl = undefined @@ -153,6 +157,7 @@ if (script) { initialPath = script.path + scriptBuilder?.setDraftTriggers(script.draft_triggers) scriptBuilder?.setCode(script.content) if (topHash) { script.parent_hash = topHash @@ -176,6 +181,7 @@ } diffDrawer.closeDrawer() goto(`/scripts/edit/${savedScript.draft.path}`) + scriptLoadedFromUrl = undefined loadScript() } @@ -193,6 +199,7 @@ }) } goto(`/scripts/edit/${savedScript.path}`) + scriptLoadedFromUrl = undefined loadScript() } diff --git a/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte b/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte index 0a7d41a4dc..dd4f7fa15f 100644 --- a/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte @@ -21,7 +21,7 @@ import { enterpriseLicense, hubBaseUrlStore, userStore, workspaceStore } from '$lib/stores' import { isDeployable, ALL_DEPLOYABLE } from '$lib/utils_deployable' - import { onDestroy } from 'svelte' + import { onDestroy, setContext, tick } from 'svelte' import HighlightCode from '$lib/components/HighlightCode.svelte' import { Tabs, @@ -42,10 +42,8 @@ import DeployWorkspaceDrawer from '$lib/components/DeployWorkspaceDrawer.svelte' import SavedInputsV2 from '$lib/components/SavedInputsV2.svelte' - import WebhooksPanel from '$lib/components/triggers/webhook/WebhooksPanel.svelte' import DetailPageLayout from '$lib/components/details/DetailPageLayout.svelte' import DetailPageHeader from '$lib/components/details/DetailPageHeader.svelte' - import CliHelpBox from '$lib/components/CliHelpBox.svelte' import { Activity, Archive, @@ -68,30 +66,22 @@ import { scriptToHubUrl } from '$lib/hub' import SharedBadge from '$lib/components/SharedBadge.svelte' import ScriptVersionHistory from '$lib/components/ScriptVersionHistory.svelte' - import RunPageSchedules from '$lib/components/RunPageSchedules.svelte' import { createAppFromScript } from '$lib/components/details/createAppFromScript' import { importStore } from '$lib/components/apps/store' import TimeAgo from '$lib/components/TimeAgo.svelte' - import ClipboardPanel from '$lib/components/details/ClipboardPanel.svelte' import PersistentScriptDrawer from '$lib/components/PersistentScriptDrawer.svelte' import GfmMarkdown from '$lib/components/GfmMarkdown.svelte' - import EmailTriggerPanel from '$lib/components/details/EmailTriggerPanel.svelte' import Star from '$lib/components/Star.svelte' import LogViewer from '$lib/components/LogViewer.svelte' - import RoutesPanel from '$lib/components/triggers/http/RoutesPanel.svelte' import { Highlight } from 'svelte-highlight' import json from 'svelte-highlight/languages/json' import { writable } from 'svelte/store' - import TriggersBadge from '$lib/components/graph/renderers/triggers/TriggersBadge.svelte' - import WebsocketTriggersPanel from '$lib/components/triggers/websocket/WebsocketTriggersPanel.svelte' - import KafkaTriggersPanel from '$lib/components/triggers/kafka/KafkaTriggersPanel.svelte' - import NatsTriggersPanel from '$lib/components/triggers/nats/NatsTriggersPanel.svelte' - import PostgresTriggersPanel from '$lib/components/triggers/postgres/PostgresTriggersPanel.svelte' import Toggle from '$lib/components/Toggle.svelte' import InputSelectedBadge from '$lib/components/schema/InputSelectedBadge.svelte' - import MqttTriggersPanel from '$lib/components/triggers/mqtt/MqttTriggersPanel.svelte' - import SqsTriggerPanel from '$lib/components/triggers/sqs/SqsTriggerPanel.svelte' - import GcpTriggerPanel from '$lib/components/triggers/gcp/GcpTriggerPanel.svelte' + import type { TriggerContext } from '$lib/components/triggers' + import TriggersBadge from '$lib/components/graph/renderers/triggers/TriggersBadge.svelte' + import TriggersEditor from '$lib/components/triggers/TriggersEditor.svelte' + import { Triggers } from '$lib/components/triggers/triggers.svelte' let script: Script | undefined let topHash: string | undefined @@ -107,7 +97,14 @@ let inputSelected: 'saved' | 'history' | undefined = undefined let jsonView = false - $: cliCommand = `wmill script run ${script?.path} -d '${JSON.stringify(args)}'` + $: { + const cliTrigger = triggersState.triggers.find((t) => t.type === 'cli') + if (cliTrigger) { + cliTrigger.extra = { + cliCommand: `wmill script run ${script?.path} -d '${JSON.stringify(args)}'` + } + } + } $: loading = !script @@ -121,6 +118,19 @@ const triggersCount = writable(undefined) + // Add triggers context store + const triggersState = new Triggers([ + { type: 'webhook', path: '', isDraft: false }, + { type: 'email', path: '', isDraft: false }, + { type: 'cli', path: '', isDraft: false } + ]) + setContext('TriggerContext', { + triggersCount, + simplifiedPoll: writable(false), + showCaptureHint: writable(undefined), + triggersState + }) + async function deleteScript(hash: string): Promise { try { await ScriptService.deleteScriptByHash({ workspace: $workspaceStore!, hash }) @@ -167,11 +177,15 @@ } let starred: boolean | undefined = undefined - async function loadTriggersCount(path: string) { - $triggersCount = await ScriptService.getTriggersCountOfScript({ - workspace: $workspaceStore!, - path: path - }) + async function loadTriggers(path: string): Promise { + await triggersState.fetchTriggers( + triggersCount, + $workspaceStore, + path, + false, + undefined, + $userStore + ) } async function loadScript(hash: string): Promise { @@ -194,7 +208,7 @@ can_write = script.workspace_id == $workspaceStore && canWrite(script.path, script.extra_perms!, $userStore) - loadTriggersCount(script.path) + loadTriggers(script.path) if (script.path && script.archived) { const script_by_path = await ScriptService.getScriptByPath({ @@ -497,7 +511,6 @@ } } - let token = 'TOKEN_TO_CREATE' let rightPaneSelected = 'saved_inputs' let savedInputsV2: SavedInputsV2 | undefined = undefined @@ -533,11 +546,7 @@ {#key script.hash} - + { + rightPaneSelected = 'triggers' + }} > { - rightPaneSelected = 'triggers' + selected={rightPaneSelected === 'triggers'} + onSelect={async (triggerIndex: number) => { + if (rightPaneSelected !== 'triggers') { + rightPaneSelected = 'triggers' + } + await tick() + triggersState.selectedTriggerIndex = triggerIndex }} /> @@ -740,76 +757,18 @@ /> {/if} - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- -
+ +
@@ -878,12 +837,6 @@ {/if} - -
- - -
- {/key} {/if} diff --git a/frontend/src/routes/(root)/(logged)/sqs_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/sqs_triggers/+page.svelte index b5a5522652..f117702fba 100644 --- a/frontend/src/routes/(root)/(logged)/sqs_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/sqs_triggers/+page.svelte @@ -220,7 +220,7 @@ - + - + >({}) const selectedIdStore = writable('settings-metadata') - const primaryScheduleStore = writable(undefined) const triggersCount = writable(undefined) - const selectedTriggerStore = writable< - 'webhooks' | 'emails' | 'schedules' | 'cli' | 'routes' | 'websockets' | 'scheduledPoll' - >('webhooks') setContext('TriggerContext', { - primarySchedule: primaryScheduleStore, - selectedTrigger: selectedTriggerStore, triggersCount: triggersCount, simplifiedPoll: writable(false), - defaultValues: writable(undefined), - captureOn: writable(undefined), - showCaptureHint: writable(undefined) + showCaptureHint: writable(undefined), + triggersState: new Triggers() }) setContext('FlowEditorContext', { From 78d6a571aac8906a23a06ae1d26a0cf5a0599b11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Wicha?= Date: Wed, 21 May 2025 23:03:28 +0100 Subject: [PATCH 17/45] Allow maximum length of tld in email validation (#5792) Signed-off-by: Rafal Wicha --- frontend/src/lib/components/StringTypeNarrowing.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/components/StringTypeNarrowing.svelte b/frontend/src/lib/components/StringTypeNarrowing.svelte index 955ccd6fde..bbecfae658 100644 --- a/frontend/src/lib/components/StringTypeNarrowing.svelte +++ b/frontend/src/lib/components/StringTypeNarrowing.svelte @@ -67,7 +67,7 @@ $: { if (format == 'email') { - pattern = '^[\\w-+.]+@([\\w-]+\\.)+[\\w-]{2,4}$' + pattern = '^[\\w-+.]+@([\\w-]+\\.)+[\\w-]{2,63}$' } } From 145a63f3f80ef4aa7cd05ce59b4fd4757967290c Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Thu, 22 May 2025 00:16:02 +0200 Subject: [PATCH 18/45] internal: clean aider flows (#5788) * add shareable flow * clean existing flows * apply to linear * cleaning * fix * cleaning --- .github/workflows/aider-after-review.yaml | 200 ++------- .github/workflows/aider-common.yml | 480 ++++++++++++++++++++++ .github/workflows/aider.yaml | 364 +++------------- .github/workflows/linear-issue.yaml | 242 ++--------- 4 files changed, 616 insertions(+), 670 deletions(-) create mode 100644 .github/workflows/aider-common.yml diff --git a/.github/workflows/aider-after-review.yaml b/.github/workflows/aider-after-review.yaml index 649ef52724..2ba3483a6c 100644 --- a/.github/workflows/aider-after-review.yaml +++ b/.github/workflows/aider-after-review.yaml @@ -5,12 +5,14 @@ on: types: [submitted] jobs: - auto-fix-review: + check-and-prepare: if: github.event.review.state == 'changes_requested' && contains(github.event.pull_request.title, '[Aider PR]') - runs-on: ubicloud-standard-8 + runs-on: ubicloud-standard-2 permissions: contents: write pull-requests: write + outputs: + prompt_content: ${{ steps.prepare_prompt.outputs.prompt_content }} env: GEMINI_API_KEY: ${{ secrets.GOOGLE_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} @@ -19,66 +21,24 @@ jobs: WINDMILL_TOKEN: ${{ secrets.WINDMILL_TOKEN }} steps: - - name: Harden Runner - uses: step-security/harden-runner@v2 - with: - egress-policy: audit - - - name: Check out code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Configure Git User - run: | - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" - - - name: Checkout PR Branch + - name: Acknowledge Request env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} run: | - echo "PR review trigger: Checking out PR branch..." - PR_NUMBER=${{ github.event.pull_request.number }} - PR_HEAD_REF=$(gh pr view $PR_NUMBER --json headRefName -q .headRefName --repo $GITHUB_REPOSITORY) - if [[ -z "$PR_HEAD_REF" || "$PR_HEAD_REF" == "null" ]]; then - echo "::error::Could not determine PR head branch for PR #$PR_NUMBER via gh CLI." - exit 1 - fi - echo "Checking out PR head branch: $PR_HEAD_REF for PR #$PR_NUMBER" - git fetch origin "refs/heads/${PR_HEAD_REF}:refs/remotes/origin/${PR_HEAD_REF}" --no-tags - git checkout "$PR_HEAD_REF" - echo "Successfully checked out branch $(git rev-parse --abbrev-ref HEAD)" + echo "Commenting on PR #${{ github.event.pull_request.number }} to acknowledge the /aider command." + gh pr comment ${{ github.event.pull_request.number }} --body "🤖 Aider is starting to work on your request. Please be patient, this might take a few minutes." --repo $GITHUB_REPOSITORY - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Install Aider and Dependencies - run: | - python -m pip install aider-install; aider-install - pip install -U google-generativeai - sudo apt-get update && sudo apt-get install -y jq - - - name: Generate Prompt from Review - id: generate_prompt + - name: Prepare prompt for Aider + id: prepare_prompt shell: bash run: | - mkdir -p .github/aider - PROMPT_FILE_PATH=".github/aider/review-prompt.txt" - # Get PR review body REVIEW_BODY="${{ github.event.review.body }}" REVIEW_BODY_Q=$(printf '%q' "$REVIEW_BODY") PR_NUMBER="${{ github.event.pull_request.number }}" - # Get PR description for context NOT USED FOR NOW - # PR_DETAILS=$(gh pr view $PR_NUMBER --json title,body --repo $GITHUB_REPOSITORY) - # PR_TITLE=$(echo "$PR_DETAILS" | jq -r .title) - # PR_BODY=$(echo "$PR_DETAILS" | jq -r .body) - # Get all PR review comments ALL_REVIEW_COMMENTS=$(gh api \ -H "Accept: application/vnd.github+json" \ @@ -87,132 +47,20 @@ jobs: | jq '[.[] | {diff_hunk: .diff_hunk, path: .path, body: .body}]') BASE_PROMPT="Fix the following issues in the PR based on the review feedback. The review body is prepended with REVIEW. The review comments are prepended with REVIEW_COMMENTS. The review body and comments are separated by a blank line." - COMPLETE_PROMPT=$(printf "%s\nREVIEW:\n%s\nREVIEW_COMMENTS:\n%s" \ - "$BASE_PROMPT" "$REVIEW_BODY_Q" "$ALL_REVIEW_COMMENTS") - echo "$COMPLETE_PROMPT" > "$PROMPT_FILE_PATH" - echo "PROMPT_FILE_PATH=$PROMPT_FILE_PATH" >> $GITHUB_OUTPUT + printf -v COMPLETE_PROMPT "%s\nREVIEW:\n%s\nREVIEW_COMMENTS:\n%s" \ + "$BASE_PROMPT" "$REVIEW_BODY_Q" "$ALL_REVIEW_COMMENTS" - - name: Probe Chat for Relevant Files - id: probe_files - env: - PROMPT_CONTENT_FILE: ${{ steps.generate_prompt.outputs.PROMPT_FILE_PATH }} - run: | - echo "Running probe-chat to find relevant files..." - if [[ ! -f "$PROMPT_CONTENT_FILE" ]]; then - echo "::error::Prompt file $PROMPT_CONTENT_FILE not found!" - exit 1 - fi - PROMPT_CONTENT=$(cat "$PROMPT_CONTENT_FILE") - if [ -z "$PROMPT_CONTENT" ]; then - echo "::error::Prompt content is empty!" - exit 1 - fi + echo "$COMPLETE_PROMPT" - PROMPT_ESCAPED=$(jq -Rs . <<< "$PROMPT_CONTENT") + # Use the proper multi-line output format + echo "prompt_content<> $GITHUB_OUTPUT + echo "$COMPLETE_PROMPT" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT - MESSAGE_FOR_PROBE=$(jq -n --arg prompt_escaped "$PROMPT_ESCAPED" \ - '{ "message": "I'\''m giving you a request that needs to be implemented. Your role is ONLY to give me the files that are relevant to the request and nothing else. The request is prepended with the word REQUEST.\\nREQUEST: \($prompt_escaped). Give me all the files relevant to this request. Your output MUST be a single json array that can be parsed with programatic json parsing, with the relevant files. Files can be rust or typescript or javascript files. DO NOT INCLUDE ANY OTHER TEXT IN YOUR OUTPUT. ONLY THE JSON ARRAY. Example of output: [\"file1.py\", \"file2.py\"]" }' | jq -r .message) - - set -o pipefail - PROBE_OUTPUT=$(npx --yes @buger/probe-chat@latest --max-iterations 50 --model-name gemini-2.5-pro-preview-05-06 --message "$MESSAGE_FOR_PROBE") || { - echo "::error::probe-chat command failed. Output:" - echo "$PROBE_OUTPUT" - exit 1 - } - set +o pipefail - echo "Probe-chat raw output:" - echo "$PROBE_OUTPUT" - - JSON_FILES=$(echo "$PROBE_OUTPUT" | sed -n '/^\s*\[/,$p' | sed '/^\s*\]/q') - echo "Extracted JSON block:" - echo "$JSON_FILES" - - FILES_LIST=$(echo "$JSON_FILES" | jq -e -r '[.[] | select(type == "string" and . != "" and . != null and (endswith("/") | not))] | map(@sh) | join(" ")' || echo "") - - if [[ -z "$FILES_LIST" ]]; then - echo "::warning::probe-chat did not identify any relevant files." - exit 1 - fi - - echo "Formatted files list for aider: $FILES_LIST" - echo "FILES_TO_EDIT=$FILES_LIST" >> $GITHUB_ENV - - - name: Run Aider with review prompt - run: | - aider \ - --read CLAUDE.md \ - --read backend/CLAUDE.md \ - --read frontend/CLAUDE.md \ - ${{ env.FILES_TO_EDIT }} \ - --model gemini/gemini-2.5-pro-preview-05-06 \ - --message-file .github/aider/review-prompt.txt \ - --yes \ - --no-check-update \ - --auto-commits \ - --no-analytics \ - --no-gitignore \ - | tee .github/aider/aider-output.txt || true - echo "Aider command completed. Output saved to .github/aider/aider-output.txt" - # Check if there are any changes to commit - if [[ -z "$(git status --porcelain)" ]]; then - echo "No changes detected after running Aider." - echo "HAS_CHANGES=false" >> $GITHUB_OUTPUT - exit 0 - fi - - - name: Clean up prompt file - if: always() - run: rm -f .github/aider/review-prompt.txt - - - name: Commit and Push Changes - id: commit_and_push - if: ${{ success() }} - run: | - CURRENT_BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD) - echo "Attempting to push changes to PR branch $CURRENT_BRANCH_NAME for PR #${{ github.event.pull_request.number }}" - - # Pull latest changes to avoid rejection due to non-fast-forward - git config pull.rebase true - git pull origin $CURRENT_BRANCH_NAME - - if git push origin $CURRENT_BRANCH_NAME; then - echo "Push to $CURRENT_BRANCH_NAME successful." - echo "CHANGES_APPLIED=true" >> $GITHUB_OUTPUT - else - echo "::warning::Push to PR branch $CURRENT_BRANCH_NAME failed." - echo "CHANGES_APPLIED=false" >> $GITHUB_OUTPUT - fi - - - name: Comment on PR - if: success() - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUM: ${{ github.event.pull_request.number }} - run: | - # Create comment body in a temporary file to avoid command line length limits - if [[ "${{ steps.commit_and_push.outputs.CHANGES_APPLIED }}" == "true" ]]; then - cat > /tmp/pr-comment.md << EOL - 🤖 I've automatically addressed the feedback based on the review. - - ## Aider Output - \`\`\` - $(cat .github/aider/aider-output.txt || echo 'No output available') - \`\`\` - - Please review the changes and let me know if further adjustments are needed. - EOL - else - cat > /tmp/pr-comment.md << EOL - 🤖 I attempted to address the review feedback, but no modifications were made. - - ## Aider Output - \`\`\` - $(cat .github/aider/aider-output.txt || echo 'No output available') - \`\`\` - - Please review the output and provide additional guidance if needed. - EOL - fi - - # Use the file for comment body - gh pr comment $PR_NUM --body-file /tmp/pr-comment.md + run-aider: + needs: check-and-prepare + uses: ./.github/workflows/aider-common.yml + with: + needs_processing: false + base_prompt: ${{ needs.check-and-prepare.outputs.prompt_content }} + secrets: inherit diff --git a/.github/workflows/aider-common.yml b/.github/workflows/aider-common.yml new file mode 100644 index 0000000000..2e4e99c716 --- /dev/null +++ b/.github/workflows/aider-common.yml @@ -0,0 +1,480 @@ +name: Aider Common Steps + +on: + workflow_call: + inputs: + issue_title: + description: "Title of the issue or PR" + required: false + type: string + issue_body: + description: "Body of the issue or PR" + required: false + type: string + instruction: + description: "Instruction for Aider" + required: false + type: string + issue_id: + description: "ID of the issue or PR" + required: false + type: string + needs_processing: + description: "Whether the issue needs to be processed by the external API" + required: false + type: boolean + default: true + base_prompt: + description: "Base prompt for Aider" + required: false + type: string + default: "Try to fix the following issue based on the instruction given by the user. The issue is prepended with the word ISSUE. The instruction is prepended with the word INSTRUCTION. The issue and instruction are separated by a blank line." + probe_prompt: + description: "Prompt for probe-chat" + required: false + type: string + default: 'I''m giving you a request that needs to be implemented. Your role is ONLY to give me the files that are relevant to the request and nothing else. The request is prepended with the word REQUEST. REQUEST: $FINAL_PROMPT. Give me all the files relevant to this request. Your output MUST be a single json array that can be parsed with programatic json parsing, with the relevant files. Files can be rust or typescript or javascript files. DO NOT INCLUDE ANY OTHER TEXT IN YOUR OUTPUT. ONLY THE JSON ARRAY. Example of output: ["file1.py", "file2.py"]' + outputs: + files_to_edit: + description: "Files identified by probe-chat for editing" + value: ${{ jobs.common-steps.outputs.files_to_edit }} + final_prompt: + description: "Final prompt for Aider" + value: ${{ jobs.common-steps.outputs.final_prompt }} + pr_branch_name: + description: "Name of the branch used for PR" + value: ${{ jobs.common-steps.outputs.pr_branch_name }} + changes_applied_message: + description: "Message indicating changes were applied" + value: ${{ jobs.common-steps.outputs.changes_applied_message }} + changes_applied: + description: "Boolean indicating if changes were successfully applied" + value: ${{ jobs.common-steps.outputs.changes_applied }} + +jobs: + common-steps: + runs-on: ubicloud-standard-8 + outputs: + files_to_edit: ${{ steps.probe_files.outputs.files_to_edit }} + final_prompt: ${{ steps.create_prompt.outputs.final_prompt }} + pr_branch_name: ${{ steps.commit_and_push.outputs.PR_BRANCH_NAME }} + changes_applied_message: ${{ steps.commit_and_push.outputs.CHANGES_APPLIED_MESSAGE }} + changes_applied: ${{ steps.commit_and_push.outputs.CHANGES_APPLIED }} + env: + GEMINI_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WINDMILL_TOKEN: ${{ secrets.WINDMILL_TOKEN }} + LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }} + + steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + + - name: Check out code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Checkout PR Branch + id: checkout_pr + if: (github.event_name == 'issue_comment' && github.event.issue.pull_request) || (github.event_name == 'pull_request_review') + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "Issue comment trigger: Checking out PR branch..." + PR_NUMBER="" + if [ -n "${{ github.event.issue.number }}" ]; then + PR_NUMBER="${{ github.event.issue.number }}" + elif [ -n "${{ github.event.pull_request.number }}" ]; then + PR_NUMBER="${{ github.event.pull_request.number }}" + else + echo "::error::Could not determine PR number." + exit 1 + fi + PR_HEAD_REF=$(gh pr view $PR_NUMBER --json headRefName -q .headRefName --repo $GITHUB_REPOSITORY) + if [[ -z "$PR_HEAD_REF" || "$PR_HEAD_REF" == "null" ]]; then + echo "::error::Could not determine PR head branch for PR #$PR_NUMBER via gh CLI." + exit 1 + fi + echo "Checking out PR head branch: $PR_HEAD_REF for PR #$PR_NUMBER" + git fetch origin "refs/heads/${PR_HEAD_REF}:refs/remotes/origin/${PR_HEAD_REF}" --no-tags + git checkout "$PR_HEAD_REF" + echo "Successfully checked out branch $(git rev-parse --abbrev-ref HEAD)" + echo "PR_BRANCH=$PR_HEAD_REF" >> $GITHUB_OUTPUT + + - name: Configure Git User + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Cache Python dependencies + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt', '**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Cache Aider installation + id: cache-aider + uses: actions/cache@v3 + with: + path: ~/.local/bin/aider + key: ${{ runner.os }}-aider-install-${{ hashFiles('**/requirements.txt', '**/setup.py') }} + restore-keys: | + ${{ runner.os }}-aider-install- + + - name: Install Aider and Dependencies + run: | + if [ -f ~/.local/bin/aider ] && [ -x ~/.local/bin/aider ]; then + echo "Using cached Aider installation" + export PATH="$HOME/.local/bin:$PATH" + else + echo "Installing Aider..." + python -m pip install aider-install; aider-install + fi + pip install -U google-generativeai + sudo apt-get update && sudo apt-get install -y jq + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Create Prompt for Aider + id: create_prompt + shell: bash + env: + BASE_PROMPT_ENV: ${{ inputs.base_prompt }} + ISSUE_TITLE_ENV: ${{ inputs.issue_title }} + ISSUE_BODY_ENV: ${{ inputs.issue_body }} + INSTRUCTION_ENV: ${{ inputs.instruction }} + NEEDS_PROCESSING_ENV: ${{ inputs.needs_processing }} + WINDMILL_TOKEN: ${{ secrets.WINDMILL_TOKEN }} + run: | + set -e + FINAL_PROMPT_CONTENT="" + + if [[ "$ISSUE_TITLE_ENV" != "" && "$ISSUE_BODY_ENV" != "" ]]; then + echo "Processing issue with title: $ISSUE_TITLE_ENV" + if [[ "$NEEDS_PROCESSING_ENV" == "true" ]]; then + echo "Needs processing is true. Calling Windmill API..." + JSON_PAYLOAD=$(jq -n \ + --arg title "$ISSUE_TITLE_ENV" \ + --arg body "$ISSUE_BODY_ENV" \ + '{"body":{"issue_title":$title,"issue_body":$body}}') + + echo "Windmill JSON Payload: $JSON_PAYLOAD" + + API_RESULT_FILE=$(mktemp) + HTTP_CODE=$(curl -s -o "$API_RESULT_FILE" -w "%{http_code}" \ + -X POST "https://app.windmill.dev/api/w/windmill-labs/jobs/run_wait_result/p/f/ai/quiet_script" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $WINDMILL_TOKEN" \ + --data-binary "$JSON_PAYLOAD" \ + --max-time 90) + + BODY_CONTENT=$(cat "$API_RESULT_FILE") + rm -f "$API_RESULT_FILE" # Clean up temp file + + echo "Windmill API HTTP Code: $HTTP_CODE" + if [[ "$HTTP_CODE" -eq 200 ]]; then + PROCESSED_ISSUE_PROMPT=$(echo "$BODY_CONTENT" | jq -r '.effective_body // empty') + if [[ -z "$PROCESSED_ISSUE_PROMPT" || "$PROCESSED_ISSUE_PROMPT" == "null" ]]; then + echo "::warning::Windmill API returned 200 but effective_body was empty or null." + EFFECTIVE_ISSUE_CONTENT_FOR_PROMPT="$ISSUE_BODY_ENV" + else + echo "Successfully processed issue via Windmill API." + EFFECTIVE_ISSUE_CONTENT_FOR_PROMPT="$PROCESSED_ISSUE_PROMPT" + fi + FINAL_PROMPT_CONTENT=$(printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \ + "$BASE_PROMPT_ENV" "$EFFECTIVE_ISSUE_CONTENT_FOR_PROMPT" "$INSTRUCTION_ENV") + else + echo "::error::Windmill API call failed (HTTP $HTTP_CODE). Using raw issue content for prompt." + FINAL_PROMPT_CONTENT=$(printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \ + "$BASE_PROMPT_ENV" "$ISSUE_BODY_ENV" "$INSTRUCTION_ENV") + fi + else + echo "Needs processing is false. Using raw issue content for prompt." + FINAL_PROMPT_CONTENT=$(printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \ + "$BASE_PROMPT_ENV" "$ISSUE_BODY_ENV" "$INSTRUCTION_ENV") + fi + else + echo "No issue title or body given. Using base prompt." + FINAL_PROMPT_CONTENT="$BASE_PROMPT_ENV" + fi + + echo "Final prompt: $FINAL_PROMPT_CONTENT" + echo "final_prompt<> "$GITHUB_OUTPUT" + echo "$FINAL_PROMPT_CONTENT" >> "$GITHUB_OUTPUT" + echo "EOF_AIDER_PROMPT" >> "$GITHUB_OUTPUT" + + - name: Probe Chat for Relevant Files + id: probe_files + shell: bash + env: + FINAL_PROMPT: ${{ steps.create_prompt.outputs.final_prompt }} + run: | + echo "Running probe-chat to find relevant files..." + + # escape the final prompt + printf -v MESSAGE_FOR_PROBE 'I'\''m giving you a request that needs to be implemented. Your role is ONLY to give me the files that are relevant to the request and nothing else. The request is prepended with the word REQUEST.\nREQUEST: %s. Give me all the files relevant to this request. Your output MUST be a single json array that can be parsed with programatic json parsing, with the relevant files. Files can be rust or typescript or javascript files. DO NOT INCLUDE ANY OTHER TEXT IN YOUR OUTPUT. ONLY THE JSON ARRAY. Example of output: ["file1.py", "file2.py"]' "$FINAL_PROMPT" + + set -o pipefail + PROBE_OUTPUT=$(npx --yes @buger/probe-chat@latest --max-iterations 50 --model-name gemini-2.5-pro-preview-05-06 --message "$MESSAGE_FOR_PROBE") || { + echo "::error::probe-chat command failed. Output:" + echo "$PROBE_OUTPUT" + exit 1 + } + set +o pipefail + echo "Probe-chat raw output:" + echo "$PROBE_OUTPUT" + + JSON_FILES=$(echo "$PROBE_OUTPUT" | sed -n '/^\s*\[/,$p' | sed '/^\s*\]/q') + echo "Extracted JSON block:" + echo "$JSON_FILES" + + FILES_LIST=$(echo "$JSON_FILES" | jq -e -r '[.[] | select(type == "string" and . != "" and . != null and (endswith("/") | not))] | join(" ")' || echo "") + + if [[ -z "$FILES_LIST" ]]; then + echo "::warning::probe-chat did not identify any relevant files." + fi + + echo "Formatted files list for aider: $FILES_LIST" + echo "files_to_edit=$FILES_LIST" >> $GITHUB_OUTPUT + + - name: Cache Aider tags + uses: actions/cache@v3 + with: + path: .aider.tags.cache.v4 + key: ${{ runner.os }}-aider-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-aider- + + - name: Run Aider + id: run_aider + shell: bash + env: + FILES_TO_EDIT: ${{ steps.probe_files.outputs.files_to_edit }} + FINAL_PROMPT: ${{ steps.create_prompt.outputs.final_prompt }} + run: | + + echo "$FINAL_PROMPT" > .aider_final_prompt.txt + echo "FILES_TO_EDIT: $FILES_TO_EDIT" + + aider \ + --read .cursor/rules/rust-best-practices.mdc \ + --read .cursor/rules/svelte5-best-practices.mdc \ + --read .cursor/rules/windmill-overview.mdc \ + $FILES_TO_EDIT \ + --model gemini/gemini-2.5-pro-preview-05-06 \ + --message "create a test file in backend/test.txt with hello world in it" \ + --yes \ + --no-check-update \ + --auto-commits \ + --no-analytics \ + --no-gitignore \ + | tee .aider_output.txt || true + + echo "Aider command completed. Output saved to .aider_output.txt" + + - name: Cache Node.js dependencies + uses: actions/cache@v3 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json', '**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-node- + + - name: Commit and Push Changes + id: commit_and_push + env: + ISSUE_ID: ${{ inputs.issue_id }} + run: | + if [[ "$ISSUE_ID" != "" ]]; then + BRANCH_NAME="aider-fix-issue-${ISSUE_ID}" + + # Check if branch exists remotely + if git ls-remote --heads origin $BRANCH_NAME | grep -q $BRANCH_NAME; then + echo "Branch $BRANCH_NAME already exists remotely, fetching it" + git fetch origin $BRANCH_NAME + git checkout $BRANCH_NAME + git pull origin $BRANCH_NAME + else + echo "Creating new branch $BRANCH_NAME" + git checkout -b $BRANCH_NAME + fi + + echo "Created/checked out branch $BRANCH_NAME for issue #${ISSUE_ID}" + git push origin $BRANCH_NAME + echo "Pushed to branch $BRANCH_NAME" + echo "PR_BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_OUTPUT + echo "CHANGES_APPLIED_MESSAGE=Aider changes pushed to branch $BRANCH_NAME." >> $GITHUB_OUTPUT + else + # We're in a pull_request_review event + PR_NUMBER="${{ github.event.pull_request.number }}" + PR_HEAD_REF="${{ github.event.pull_request.head.ref }}" + + echo "Handling pull_request_review for PR #$PR_NUMBER on branch $PR_HEAD_REF" + + # Ensure we're on the correct branch + git config pull.rebase true + git fetch origin $PR_HEAD_REF + git checkout $PR_HEAD_REF + git pull origin $PR_HEAD_REF + + echo "Attempting to push changes to PR branch $PR_HEAD_REF for PR #$PR_NUMBER" + if git push origin $PR_HEAD_REF; then + echo "Push to $PR_HEAD_REF successful (or no new changes to push)." + echo "CHANGES_APPLIED_MESSAGE=Aider changes (if any) pushed to PR branch $PR_HEAD_REF." >> $GITHUB_OUTPUT + echo "PR_BRANCH_NAME=$PR_HEAD_REF" >> $GITHUB_OUTPUT + echo "CHANGES_APPLIED=true" >> $GITHUB_OUTPUT + else + echo "::warning::Push to PR branch $PR_HEAD_REF failed." + echo "CHANGES_APPLIED_MESSAGE=Aider ran, but failed to push changes to PR branch $PR_HEAD_REF." >> $GITHUB_OUTPUT + echo "CHANGES_APPLIED=false" >> $GITHUB_OUTPUT + fi + fi + + - name: Create Pull Request + if: always() && (github.event_name == 'issue_comment' || github.event_name == 'repository_dispatch') && !github.event.issue.pull_request && steps.commit_and_push.outputs.PR_BRANCH_NAME != '' + id: create_pr + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_BRANCH: ${{ steps.commit_and_push.outputs.PR_BRANCH_NAME }} + ISSUE_NUM: ${{ inputs.issue_id }} + ISSUE_TITLE: ${{ inputs.issue_title }} + run: | + # Debug: Check latest commit and branch status + echo "Checking latest commit on branch $PR_BRANCH" + git log -1 --pretty=format:"%h - %an, %ar : %s" + echo "Changes not yet committed:" + git status --porcelain + # Check if there are any changes to commit + if [[ -n $(git status --porcelain) ]]; then + echo "Found uncommitted changes, committing them" + git add . + git commit -m "Aider changes for issue #${ISSUE_NUM}" + git push origin $PR_BRANCH + fi + + # Create PR description in a temporary file to avoid command line length limits and ensure it stays under 40k chars + cat > /tmp/pr-description.md << EOL | head -c 40000 + This PR was created automatically by Aider to fix issue #${ISSUE_NUM}. + + ## Aider Output + \`\`\` + $(cat .aider_output.txt || echo "No output available") + \`\`\` + EOL + + # Create PR using the file for the body content, handle errors gracefully + set +e # Don't exit on error + gh pr create \ + --title "[Aider PR] Fix: ${ISSUE_TITLE}" \ + --body-file /tmp/pr-description.md \ + --head "$PR_BRANCH" \ + --base main + PR_CREATE_EXIT_CODE=$? + set -e # Re-enable exit on error + + if [ $PR_CREATE_EXIT_CODE -eq 0 ]; then + echo "PR created successfully" + PR_URL=$(gh pr view $PR_BRANCH --json url --jq .url) + echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT + echo "PR_CREATED=true" >> $GITHUB_OUTPUT + else + echo "Warning: Failed to create PR. Exit code: $PR_CREATE_EXIT_CODE" + echo "PR_CREATED=false" >> $GITHUB_OUTPUT + # Continue workflow despite PR creation failure + fi + + - name: Comment on PR with Aider Output + if: always() && github.event_name == 'pull_request_review' && steps.commit_and_push.outputs.CHANGES_APPLIED != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUM: ${{ github.event.pull_request.number }} + JOB_STATUS: ${{ job.status }} + run: | + # Create comment body in a temporary file to avoid command line length limits + if [[ "${{ steps.commit_and_push.outputs.CHANGES_APPLIED }}" == "true" ]]; then + if [[ "$JOB_STATUS" == "success" ]]; then + STATUS_PREFIX="🤖 I've automatically addressed the feedback based on the review." + else + STATUS_PREFIX="⚠️ I attempted to address the feedback, but encountered some issues." + fi + else + if [[ "$JOB_STATUS" == "success" ]]; then + STATUS_PREFIX="🤖 I attempted to address the review feedback, but no modifications were made." + else + STATUS_PREFIX="⚠️ I encountered issues while attempting to address the feedback, and no modifications were made." + fi + fi + + cat > /tmp/pr-comment.md << EOL + ${STATUS_PREFIX} + + ## Aider Output + \`\`\` + $(cat .aider_output.txt || echo 'No output available') + \`\`\` + + Please review the output and provide additional guidance if needed. + EOL + + # Use the file for comment body + gh pr comment $PR_NUM --body-file /tmp/pr-comment.md + + - name: Comment on issue/PR to let the user know Aider has finished working on the request + if: always() && github.event_name == 'issue_comment' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + JOB_STATUS: ${{ job.status }} + PR_CREATED: ${{ steps.create_pr.outputs.PR_CREATED }} + run: | + echo "Commenting on issue/PR #${{ github.event.issue.number }} to let the user know Aider has finished working on the request." + + if [[ "$JOB_STATUS" == "success" ]]; then + if [[ "$PR_CREATED" == "true" ]]; then + COMMENT_BODY="🤖 Aider has finished working on your request. A PR has been created." + else + COMMENT_BODY="🤖 Aider has finished working on your request, but was unable to create a PR." + fi + else + COMMENT_BODY="⚠️ Aider encountered issues while working on your request. Please check the workflow logs for details." + fi + + gh issue comment ${{ github.event.issue.number }} --body "$COMMENT_BODY" --repo $GITHUB_REPOSITORY + + - name: Comment on linear issue to let the user know Aider has finished working on the request + if: always() && github.event_name == 'repository_dispatch' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + JOB_STATUS: ${{ job.status }} + LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }} + PR_CREATED: ${{ steps.create_pr.outputs.PR_CREATED }} + run: | + echo "Commenting on linear issue #${{ github.event.client_payload.issue_id }} to let the user know Aider has finished working on the request." + + if [[ "$JOB_STATUS" == "success" ]]; then + if [[ "$PR_CREATED" == "true" ]]; then + COMMENT_BODY="🤖 Aider has finished working on your request. A PR has been created." + else + COMMENT_BODY="🤖 Aider has finished working on your request, but was unable to create a PR." + fi + else + COMMENT_BODY="⚠️ Aider encountered issues while working on your request. Please check the workflow logs for details." + fi + + curl -X POST \ + -H "Authorization: $LINEAR_API_KEY" \ + -H "Content-Type: application/json" \ + "https://api.linear.app/graphql" \ + -d "{\"query\":\"mutation { commentCreate(input: { issueId: \\\"${{ github.event.client_payload.issue_id }}\\\", body: \\\"${COMMENT_BODY}\\\" }) { success } }\"}" diff --git a/.github/workflows/aider.yaml b/.github/workflows/aider.yaml index e97b75bf22..fdfd7b4d39 100644 --- a/.github/workflows/aider.yaml +++ b/.github/workflows/aider.yaml @@ -5,8 +5,8 @@ on: types: [created] jobs: - auto-fix: - runs-on: ubicloud-standard-8 + check-and-prepare: + runs-on: ubicloud-standard-2 if: | github.event_name == 'issue_comment' && contains(github.event.comment.body, '/aider') && @@ -21,323 +21,95 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} WINDMILL_TOKEN: ${{ secrets.WINDMILL_TOKEN }} + outputs: + issue_title: ${{ steps.determine_inputs.outputs.ISSUE_TITLE }} + issue_body: ${{ steps.determine_inputs.outputs.ISSUE_BODY }} + comment_content: ${{ steps.determine_inputs.outputs.COMMENT_CONTENT }} + pr_branch: ${{ steps.checkout_pr.outputs.PR_BRANCH }} steps: - - name: Harden Runner - uses: step-security/harden-runner@v2 - with: - egress-policy: audit - - - name: Check out code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Configure Git User - run: | - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" - - - name: Checkout PR Branch - if: github.event_name == 'issue_comment' && github.event.issue.pull_request + - name: Acknowledge Request env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} run: | - echo "Issue comment trigger: Checking out PR branch..." - PR_NUMBER=${{ github.event.issue.number }} - PR_HEAD_REF=$(gh pr view $PR_NUMBER --json headRefName -q .headRefName --repo $GITHUB_REPOSITORY) - if [[ -z "$PR_HEAD_REF" || "$PR_HEAD_REF" == "null" ]]; then - echo "::error::Could not determine PR head branch for PR #$PR_NUMBER via gh CLI." - exit 1 - fi - echo "Checking out PR head branch: $PR_HEAD_REF for PR #$PR_NUMBER" - git fetch origin "refs/heads/${PR_HEAD_REF}:refs/remotes/origin/${PR_HEAD_REF}" --no-tags - git checkout "$PR_HEAD_REF" - echo "Successfully checked out branch $(git rev-parse --abbrev-ref HEAD)" + echo "Commenting on issue/PR #${{ github.event.issue.number }} to acknowledge the /aider command." + gh issue comment ${{ github.event.issue.number }} --body "🤖 Aider is starting to work on your request. I'll update you here once I have a PR ready. Please be patient, this might take a few minutes." --repo $GITHUB_REPOSITORY - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Install Aider and Dependencies - run: | - python -m pip install aider-install; aider-install - pip install -U google-generativeai - sudo apt-get update && sudo apt-get install -y jq - - - name: Determine Prompt for Aider - id: determine_prompt + - name: Determine inputs for Aider + id: determine_inputs shell: bash run: | - PROMPT_FILE_PATH=".github/aider/issue-prompt.txt" - mkdir -p .github/aider + echo "Determining inputs for Aider..." + ISSUE_TITLE_VAL="" + ISSUE_BODY_VAL="" - # Determine if this is a PR comment or regular issue comment if [[ ! -z "${{ github.event.issue.pull_request }}" ]]; then echo "This is a comment on a Pull Request" PR_NUMBER="${{ github.event.issue.number }}" - # Get PR description to check for issue references - PR_BODY=$(gh pr view $PR_NUMBER --json body -q .body --repo $GITHUB_REPOSITORY) - - # Extract issue number from PR description (looking for #123 or "fixes #123" patterns) - REFERENCED_ISSUE=$(echo "$PR_BODY" | grep -oE "#[0-9]+" | grep -oE "[0-9]+" | head -1) - - if [[ ! -z "$REFERENCED_ISSUE" ]]; then - echo "Found referenced issue #$REFERENCED_ISSUE in PR description" - - # Fetch the referenced issue details - ISSUE_DETAILS=$(gh issue view $REFERENCED_ISSUE --json title,body --repo $GITHUB_REPOSITORY) - ISSUE_TITLE=$(echo "$ISSUE_DETAILS" | jq -r .title) - ISSUE_BODY=$(echo "$ISSUE_DETAILS" | jq -r .body) - - # Store raw comment body in a file first to avoid shell interpretation issues - echo '${{ github.event.comment.body }}' > /tmp/raw_comment.txt - RAW_COMMENT_BODY=$(cat /tmp/raw_comment.txt) - # Remove the /aider prefix and trim whitespace - COMMENT_CONTENT=$(echo "$RAW_COMMENT_BODY" | sed 's|^/aider||' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') - - echo "Sending issue content and PR comment to external API…" - - ISSUE_TITLE_Q=$(printf '%q' "$ISSUE_TITLE") - ISSUE_BODY_Q=$(printf '%q' "$ISSUE_BODY") - - JSON_PAYLOAD=$(jq -n \ - --arg title "$ISSUE_TITLE_Q" \ - --arg body "$ISSUE_BODY_Q" \ - '{"body":{"issue_title":$title,"issue_body":$body}}') - - API_RESULT=$(curl -s -w "\n%{http_code}" \ - -X POST "https://app.windmill.dev/api/w/windmill-labs/jobs/run_wait_result/p/f/ai/quiet_script" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $WINDMILL_TOKEN" \ - --data-binary "$JSON_PAYLOAD" \ - --max-time 90) - - HTTP_CODE=$(echo "$API_RESULT" | tail -n1) - BODY=$(echo "$API_RESULT" | sed '$d') - - echo "$BODY" > /tmp/api_response.txt - - BASE_PROMPT="Try to fix the following issue based on the instruction given by the user. The issue is prepended with the word ISSUE. The instruction is prepended with the word INSTRUCTION. The issue and instruction are separated by a blank line." - if [[ "$HTTP_CODE" -eq 200 ]]; then - PROCESSED_ISSUE_PROMPT=$(jq -r '.effective_body // empty' /tmp/api_response.txt) - if [[ -z "$PROCESSED_ISSUE_PROMPT" || "$PROCESSED_ISSUE_PROMPT" == "null" ]]; then - PROCESSED_ISSUE_PROMPT="" - fi - printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \ - "$BASE_PROMPT" "$PROCESSED_ISSUE_PROMPT" "$COMMENT_CONTENT" > "$PROMPT_FILE_PATH" - else - echo "::warning::API call failed (HTTP $HTTP_CODE). Using PR comment with issue context." - printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \ - "$BASE_PROMPT" "$ISSUE_BODY_Q" "$COMMENT_CONTENT" > "$PROMPT_FILE_PATH" - fi - rm -f /tmp/api_response.txt + PR_BODY_JSON=$(GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" gh pr view "$PR_NUMBER" --json body --repo "$GITHUB_REPOSITORY") + if [[ $? -ne 0 ]]; then + echo "Error fetching PR body for PR #$PR_NUMBER" + PR_BODY_VAL="" else - echo "No referenced issue found in PR description, using comment content only" - # Use comment content directly as with regular issue comments - echo '${{ github.event.comment.body }}' > /tmp/raw_comment.txt - RAW_COMMENT_BODY=$(cat /tmp/raw_comment.txt) - COMMENT_CONTENT=$(echo "$RAW_COMMENT_BODY" | sed 's|^/aider||' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + PR_BODY_VAL=$(echo "$PR_BODY_JSON" | jq -r .body) + fi + + if [[ ! -z "$PR_BODY_VAL" ]]; then + REFERENCED_ISSUE=$(echo "$PR_BODY_VAL" | grep -oE "#[0-9]+" | grep -oE "[0-9]+" | head -1) - if [[ -z "$COMMENT_CONTENT" ]]; then - echo "::error::Comment with /aider provided, but no instruction found after it. Cannot proceed." - printf "Error: /aider command found but no instruction followed." > "$PROMPT_FILE_PATH" - exit 1 - else - echo "Using comment content as prompt." - printf '%s' "$COMMENT_CONTENT" > "$PROMPT_FILE_PATH" + if [[ ! -z "$REFERENCED_ISSUE" ]]; then + echo "Found referenced issue #$REFERENCED_ISSUE in PR description" + + ISSUE_DETAILS_JSON=$(GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" gh issue view "$REFERENCED_ISSUE" --json title,body --repo "$GITHUB_REPOSITORY") + if [[ $? -ne 0 ]]; then + echo "Error fetching issue details for #$REFERENCED_ISSUE" + else + ISSUE_TITLE_VAL=$(echo "$ISSUE_DETAILS_JSON" | jq -r .title) + ISSUE_BODY_VAL=$(echo "$ISSUE_DETAILS_JSON" | jq -r .body) + fi fi + else + echo "PR body is empty or could not be fetched." fi else echo "This is a comment on a regular issue" - - # Fetch the issue details ISSUE_NUMBER="${{ github.event.issue.number }}" - ISSUE_DETAILS=$(gh issue view $ISSUE_NUMBER --json title,body --repo $GITHUB_REPOSITORY) - ISSUE_TITLE=$(echo "$ISSUE_DETAILS" | jq -r .title) - ISSUE_BODY=$(echo "$ISSUE_DETAILS" | jq -r .body) - - # Store raw comment body in a file first to avoid shell interpretation issues - echo '${{ github.event.comment.body }}' > /tmp/raw_comment.txt - # Extract the command part safely - RAW_COMMENT_BODY=$(cat /tmp/raw_comment.txt) - # Remove the /aider prefix and trim whitespace - COMMENT_CONTENT=$(echo "$RAW_COMMENT_BODY" | sed 's|^/aider||' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') - - if [[ -z "$COMMENT_CONTENT" ]]; then - echo "::error::Comment with /aider provided, but no instruction found after it. Cannot proceed." - printf "Error: /aider command found but no instruction followed." > "$PROMPT_FILE_PATH" - exit 1 + ISSUE_DETAILS_JSON=$(GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" gh issue view "$ISSUE_NUMBER" --json title,body --repo "$GITHUB_REPOSITORY") + if [[ $? -ne 0 ]]; then + echo "Error fetching issue details for #$ISSUE_NUMBER" else - echo "Sending issue content and issue comment to external API…" - - ISSUE_TITLE_Q=$(printf '%q' "$ISSUE_TITLE") - ISSUE_BODY_Q=$(printf '%q' "$ISSUE_BODY") - COMMENT_CONTENT_Q=$(printf '%q' "$COMMENT_CONTENT") - - JSON_PAYLOAD=$(jq -n \ - --arg title "$ISSUE_TITLE_Q" \ - --arg body "$ISSUE_BODY_Q" \ - --arg comment "$COMMENT_CONTENT_Q" \ - '{"body":{"issue_title":$title,"issue_body":$body,"issue_comment":$comment}}') - - API_RESULT=$(curl -s -w "\n%{http_code}" \ - -X POST "https://app.windmill.dev/api/w/windmill-labs/jobs/run_wait_result/p/f/ai/quiet_script" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $WINDMILL_TOKEN" \ - --data-binary "$JSON_PAYLOAD" \ - --max-time 90) - - HTTP_CODE=$(echo "$API_RESULT" | tail -n1) - BODY=$(echo "$API_RESULT" | sed '$d') - - echo "$BODY" > /tmp/api_response.txt - - BASE_PROMPT="Try to fix the following issue based on the instruction given by the user. The issue is prepended with the word ISSUE. The instruction is prepended with the word INSTRUCTION. The issue and instruction are separated by a blank line." - if [[ "$HTTP_CODE" -eq 200 ]]; then - PROCESSED_ISSUE_PROMPT=$(jq -r '.effective_body // empty' /tmp/api_response.txt) - if [[ -z "$PROCESSED_ISSUE_PROMPT" || "$PROCESSED_ISSUE_PROMPT" == "null" ]]; then - PROCESSED_ISSUE_PROMPT="" - fi - printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \ - "$BASE_PROMPT" "$PROCESSED_ISSUE_PROMPT" "$COMMENT_CONTENT" > "$PROMPT_FILE_PATH" - else - echo "::warning::API call failed (HTTP $HTTP_CODE). Using PR comment with issue context." - printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \ - "$BASE_PROMPT" "$ISSUE_BODY_Q" "$COMMENT_CONTENT" > "$PROMPT_FILE_PATH" - fi - - rm -f /tmp/api_response.txt + ISSUE_TITLE_VAL=$(echo "$ISSUE_DETAILS_JSON" | jq -r .title) + ISSUE_BODY_VAL=$(echo "$ISSUE_DETAILS_JSON" | jq -r .body) fi fi - echo "Prompt determined and written to $PROMPT_FILE_PATH" - echo "PROMPT_FILE_PATH=$PROMPT_FILE_PATH" >> $GITHUB_OUTPUT - - name: Probe Chat for Relevant Files - id: probe_files + echo "Setting GITHUB_OUTPUT for ISSUE_TITLE..." + echo "ISSUE_TITLE<> "$GITHUB_OUTPUT" + echo "$ISSUE_TITLE_VAL" >> "$GITHUB_OUTPUT" + echo "EOF_AIDER_TITLE" >> "$GITHUB_OUTPUT" + + echo "Setting GITHUB_OUTPUT for ISSUE_BODY..." + echo "ISSUE_BODY<> "$GITHUB_OUTPUT" + echo "$ISSUE_BODY_VAL" >> "$GITHUB_OUTPUT" + echo "EOF_AIDER_BODY" >> "$GITHUB_OUTPUT" + + # Process COMMENT_CONTENT + printf -v COMMENT_CONTENT_VAL "%s" "$(echo "${{ github.event.comment.body }}" | sed 's|^/aider||' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + echo "COMMENT_CONTENT<> "$GITHUB_OUTPUT" + echo "$COMMENT_CONTENT_VAL" >> "$GITHUB_OUTPUT" + echo "EOF_AIDER_COMMENT" >> "$GITHUB_OUTPUT" + echo "Finished determining inputs." env: - PROMPT_CONTENT_FILE: ${{ steps.determine_prompt.outputs.PROMPT_FILE_PATH }} - run: | - echo "Running probe-chat to find relevant files..." - if [[ ! -f "$PROMPT_CONTENT_FILE" ]]; then - echo "::error::Prompt file $PROMPT_CONTENT_FILE not found!" - exit 1 - fi - PROMPT_CONTENT=$(cat "$PROMPT_CONTENT_FILE") - if [ -z "$PROMPT_CONTENT" ]; then - echo "::error::Prompt content is empty!" - exit 1 - fi + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Make sure gh cli has a token - PROMPT_ESCAPED=$(jq -Rs . <<< "$PROMPT_CONTENT") - - MESSAGE_FOR_PROBE=$(jq -n --arg prompt_escaped "$PROMPT_ESCAPED" \ - '{ "message": "I'\''m giving you a request that needs to be implemented. Your role is ONLY to give me the files that are relevant to the request and nothing else. The request is prepended with the word REQUEST.\\nREQUEST: \($prompt_escaped). Give me all the files relevant to this request. Your output MUST be a single json array that can be parsed with programatic json parsing, with the relevant files. Files can be rust or typescript or javascript files. DO NOT INCLUDE ANY OTHER TEXT IN YOUR OUTPUT. ONLY THE JSON ARRAY. Example of output: [\"file1.py\", \"file2.py\"]" }' | jq -r .message) - - set -o pipefail - PROBE_OUTPUT=$(npx --yes @buger/probe-chat@latest --max-iterations 50 --model-name gemini-2.5-pro-preview-05-06 --message "$MESSAGE_FOR_PROBE") || { - echo "::error::probe-chat command failed. Output:" - echo "$PROBE_OUTPUT" - exit 1 - } - set +o pipefail - echo "Probe-chat raw output:" - echo "$PROBE_OUTPUT" - - JSON_FILES=$(echo "$PROBE_OUTPUT" | sed -n '/^\s*\[/,$p' | sed '/^\s*\]/q') - echo "Extracted JSON block:" - echo "$JSON_FILES" - - FILES_LIST=$(echo "$JSON_FILES" | jq -e -r '[.[] | select(type == "string" and . != "" and . != null and (endswith("/") | not))] | map(@sh) | join(" ")' || echo "") - - if [[ -z "$FILES_LIST" ]]; then - echo "::warning::probe-chat did not identify any relevant files." - exit 1 - fi - - echo "Formatted files list for aider: $FILES_LIST" - echo "FILES_TO_EDIT=$FILES_LIST" >> $GITHUB_ENV - - - name: Run Aider with external prompt - run: | - echo "Files identified by probe-chat: ${{ env.FILES_TO_EDIT }}" - aider \ - --read CLAUDE.md \ - --read backend/CLAUDE.md \ - --read frontend/CLAUDE.md \ - ${{ env.FILES_TO_EDIT }} \ - --model gemini/gemini-2.5-pro-preview-05-06 \ - --message-file .github/aider/issue-prompt.txt \ - --yes \ - --no-check-update \ - --auto-commits \ - --no-analytics \ - --no-gitignore \ - | tee .github/aider/aider-output.txt || true - echo "Aider command completed. Output saved to .github/aider/aider-output.txt" - - - name: Clean up prompt file - if: always() - run: rm -f .github/aider/issue-prompt.txt - - - name: Commit and Push Changes - id: commit_and_push - if: ${{ success() }} - run: | - if [[ -z "${{ github.event.issue.pull_request }}" ]]; then - BRANCH_NAME="aider-fix-issue-${{ github.event.issue.number }}" - - # Check if branch exists remotely - if git ls-remote --heads origin $BRANCH_NAME | grep -q $BRANCH_NAME; then - echo "Branch $BRANCH_NAME already exists remotely, fetching it" - git fetch origin $BRANCH_NAME - git checkout $BRANCH_NAME - git pull origin $BRANCH_NAME - else - echo "Creating new branch $BRANCH_NAME" - git checkout -b $BRANCH_NAME - fi - - echo "Created/checked out branch $BRANCH_NAME for issue #${{ github.event.issue.number }}" - git push origin $BRANCH_NAME - echo "Pushed to branch $BRANCH_NAME" - echo "PR_BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_OUTPUT - echo "CHANGES_APPLIED_MESSAGE=Aider changes pushed to branch $BRANCH_NAME." >> $GITHUB_OUTPUT - else - CURRENT_BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD) - echo "Attempting to push changes to PR branch $CURRENT_BRANCH_NAME for PR #${{ github.event.issue.number }}" - if git push origin $CURRENT_BRANCH_NAME; then - echo "Push to $CURRENT_BRANCH_NAME successful (or no new changes to push)." - echo "CHANGES_APPLIED_MESSAGE=Aider changes (if any) pushed to PR branch $CURRENT_BRANCH_NAME." >> $GITHUB_OUTPUT - echo "PR_BRANCH_NAME=$CURRENT_BRANCH_NAME" >> $GITHUB_OUTPUT - else - echo "::warning::Push to PR branch $CURRENT_BRANCH_NAME failed." - echo "CHANGES_APPLIED_MESSAGE=Aider ran, but failed to push changes to PR branch $CURRENT_BRANCH_NAME." >> $GITHUB_OUTPUT - fi - fi - - - name: Create Pull Request - if: success() && github.event_name == 'issue_comment' && !github.event.issue.pull_request && steps.commit_and_push.outputs.PR_BRANCH_NAME != '' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_BRANCH: ${{ steps.commit_and_push.outputs.PR_BRANCH_NAME }} - ISSUE_NUM: ${{ github.event.issue.number }} - run: | - # Create PR description in a temporary file to avoid command line length limits - cat > /tmp/pr-description.md << EOL - This PR was created automatically by Aider to fix issue #${ISSUE_NUM}. - - ## Aider Output - \`\`\` - $(cat .github/aider/aider-output.txt || echo "No output available") - \`\`\` - EOL - - # Create PR using the file for the body content - gh pr create \ - --title "[Aider PR] Add fixes for issue #${ISSUE_NUM}" \ - --body-file /tmp/pr-description.md \ - --head "$PR_BRANCH" \ - --base main + run-aider: + needs: check-and-prepare + uses: ./.github/workflows/aider-common.yml + with: + issue_title: ${{ needs.check-and-prepare.outputs.issue_title }} + issue_body: ${{ needs.check-and-prepare.outputs.issue_body }} + instruction: ${{ needs.check-and-prepare.outputs.comment_content }} + issue_id: ${{ github.event.issue.number }} + secrets: inherit diff --git a/.github/workflows/linear-issue.yaml b/.github/workflows/linear-issue.yaml index 7dd217e141..1dce5b6c87 100644 --- a/.github/workflows/linear-issue.yaml +++ b/.github/workflows/linear-issue.yaml @@ -5,219 +5,65 @@ on: types: [external_issue_fix] jobs: - auto-fix: - runs-on: ubicloud-standard-8 + check-and-prepare: + runs-on: ubicloud-standard-2 permissions: contents: write pull-requests: write - issues: write + outputs: + issue_title: ${{ steps.determine_inputs.outputs.ISSUE_TITLE }} + issue_body: ${{ steps.determine_inputs.outputs.ISSUE_BODY }} + instruction: ${{ steps.determine_inputs.outputs.INSTRUCTION }} env: GEMINI_API_KEY: ${{ secrets.GOOGLE_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} WINDMILL_TOKEN: ${{ secrets.WINDMILL_TOKEN }} + LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }} steps: - - name: Harden Runner - uses: step-security/harden-runner@v2 - with: - egress-policy: audit - - - name: Check out code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Configure Git User + - name: Acknowledge Request + env: + LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }} run: | - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" + echo "Commenting on Linear issue #${{ github.event.client_payload.issue_id }} to acknowledge the request." + curl -X POST \ + -H "Authorization: $LINEAR_API_KEY" \ + -H "Content-Type: application/json" \ + "https://api.linear.app/graphql" \ + -d "{\"query\":\"mutation { commentCreate(input: { issueId: \\\"${{ github.event.client_payload.issue_id }}\\\", body: \\\"🤖 Aider is starting to work on your request. I'll update you here once I have a PR ready. Please be patient, this might take a few minutes.\\\" }) { success } }\"}" - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Install Aider and Dependencies - run: | - python -m pip install aider-install; aider-install - pip install -U google-generativeai - sudo apt-get update && sudo apt-get install -y jq - - - name: Create Prompt for Aider - id: create_prompt + - name: Determine inputs for Aider + id: determine_inputs shell: bash run: | - PROMPT_FILE_PATH=".github/aider/issue-prompt.txt" - mkdir -p .github/aider + echo "Determining inputs for Aider..." + ISSUE_TITLE_VAL="${{ github.event.client_payload.issue_title }}" + INSTRUCTION_VAL="${{ github.event.client_payload.instruction }}" + ISSUE_BODY_VAL=$(printf '%q' "${{ github.event.client_payload.issue_body }}") + echo "Setting GITHUB_OUTPUT for ISSUE_TITLE..." + echo "ISSUE_TITLE<> "$GITHUB_OUTPUT" + echo "$ISSUE_TITLE_VAL" >> "$GITHUB_OUTPUT" + echo "EOF_AIDER_TITLE" >> "$GITHUB_OUTPUT" - ISSUE_TITLE="${{ github.event.client_payload.issue_title }}" - INSTRUCTION="${{ github.event.client_payload.instruction }}" - ISSUE_BODY=$(printf '%q' "${{ github.event.client_payload.issue_body }}") + echo "Setting GITHUB_OUTPUT for ISSUE_BODY..." + echo "ISSUE_BODY<> "$GITHUB_OUTPUT" + echo "$ISSUE_BODY_VAL" >> "$GITHUB_OUTPUT" + echo "EOF_AIDER_BODY" >> "$GITHUB_OUTPUT" - echo "Processing issue with title: $ISSUE_TITLE" + echo "Setting GITHUB_OUTPUT for INSTRUCTION..." + echo "INSTRUCTION<> "$GITHUB_OUTPUT" + echo "$INSTRUCTION_VAL" >> "$GITHUB_OUTPUT" + echo "EOF_AIDER_INSTRUCTION" >> "$GITHUB_OUTPUT" + echo "Finished determining inputs." - JSON_PAYLOAD=$(jq -n \ - --arg title "$ISSUE_TITLE" \ - --arg body "$ISSUE_BODY" \ - '{"body":{"issue_title":$title,"issue_body":$body}}') - - API_RESULT=$(curl -s -w "\n%{http_code}" \ - -X POST "https://app.windmill.dev/api/w/windmill-labs/jobs/run_wait_result/p/f/ai/quiet_script" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $WINDMILL_TOKEN" \ - --data-binary "$JSON_PAYLOAD" \ - --max-time 90) - - HTTP_CODE=$(echo "$API_RESULT" | tail -n1) - BODY=$(echo "$API_RESULT" | sed '$d') - - echo "$BODY" > /tmp/api_response.txt - - BASE_PROMPT="Try to fix the following issue based on the instruction given. The issue is prepended with the word ISSUE. The instruction is prepended with the word INSTRUCTION. The issue and instruction are separated by a blank line." - if [[ "$HTTP_CODE" -eq 200 ]]; then - PROCESSED_ISSUE_PROMPT=$(jq -r '.effective_body // empty' /tmp/api_response.txt) - if [[ -z "$PROCESSED_ISSUE_PROMPT" || "$PROCESSED_ISSUE_PROMPT" == "null" ]]; then - PROCESSED_ISSUE_PROMPT="" - fi - printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \ - "$BASE_PROMPT" "$PROCESSED_ISSUE_PROMPT" "$INSTRUCTION" > "$PROMPT_FILE_PATH" - else - echo "::warning::API call failed (HTTP $HTTP_CODE). Using raw issue content." - printf "%s\nISSUE:\n%s\nINSTRUCTION:\n%s" \ - "$BASE_PROMPT" "$ISSUE_BODY" "$INSTRUCTION" > "$PROMPT_FILE_PATH" - fi - rm -f /tmp/api_response.txt - - echo "Prompt created and written to $PROMPT_FILE_PATH" - echo "PROMPT_FILE_PATH=$PROMPT_FILE_PATH" >> $GITHUB_OUTPUT - - # Store the issue title for PR creation - ISSUE_TITLE_SAFE=$(echo "$ISSUE_TITLE" | tr -d '\n' | sed 's/"/\\"/g') - echo "ISSUE_TITLE=$ISSUE_TITLE_SAFE" >> $GITHUB_OUTPUT - - # Generate unique branch name using timestamp and issue info - ISSUE_ID="${{ github.event.client_payload.issue_id }}" - BRANCH_NAME="aider-fix-linear-issue-$ISSUE_ID" - echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_OUTPUT - - - name: Probe Chat for Relevant Files - id: probe_files - env: - PROMPT_CONTENT_FILE: ${{ steps.create_prompt.outputs.PROMPT_FILE_PATH }} - run: | - echo "Running probe-chat to find relevant files..." - if [[ ! -f "$PROMPT_CONTENT_FILE" ]]; then - echo "::error::Prompt file $PROMPT_CONTENT_FILE not found!" - exit 1 - fi - PROMPT_CONTENT=$(cat "$PROMPT_CONTENT_FILE") - if [ -z "$PROMPT_CONTENT" ]; then - echo "::error::Prompt content is empty!" - exit 1 - fi - - PROMPT_ESCAPED=$(jq -Rs . <<< "$PROMPT_CONTENT") - - MESSAGE_FOR_PROBE=$(jq -n --arg prompt_escaped "$PROMPT_ESCAPED" \ - '{ "message": "I'\''m giving you a request that needs to be implemented. Your role is ONLY to give me the files that are relevant to the request and nothing else. The request is prepended with the word REQUEST.\\nREQUEST: \($prompt_escaped). Give me all the files relevant to this request. Your output MUST be a single json array that can be parsed with programatic json parsing, with the relevant files. Files can be rust or typescript or javascript files. DO NOT INCLUDE ANY OTHER TEXT IN YOUR OUTPUT. ONLY THE JSON ARRAY. Example of output: [\"file1.py\", \"file2.py\"]" }' | jq -r .message) - - set -o pipefail - PROBE_OUTPUT=$(npx --yes @buger/probe-chat@latest --max-iterations 50 --model-name gemini-2.5-pro-preview-05-06 --message "$MESSAGE_FOR_PROBE") || { - echo "::error::probe-chat command failed. Output:" - echo "$PROBE_OUTPUT" - exit 1 - } - set +o pipefail - echo "Probe-chat raw output:" - echo "$PROBE_OUTPUT" - - JSON_FILES=$(echo "$PROBE_OUTPUT" | sed -n '/^\s*\[/,$p' | sed '/^\s*\]/q') - echo "Extracted JSON block:" - echo "$JSON_FILES" - - FILES_LIST=$(echo "$JSON_FILES" | jq -e -r '[.[] | select(type == "string" and . != "" and . != null and (endswith("/") | not))] | map(@sh) | join(" ")' || echo "") - - if [[ -z "$FILES_LIST" ]]; then - echo "::warning::probe-chat did not identify any relevant files." - exit 1 - fi - - echo "Formatted files list for aider: $FILES_LIST" - echo "FILES_TO_EDIT=$FILES_LIST" >> $GITHUB_ENV - - - name: Run Aider with external prompt - run: | - echo "Files identified by probe-chat: ${{ env.FILES_TO_EDIT }}" - aider \ - --read CLAUDE.md \ - --read backend/CLAUDE.md \ - --read frontend/CLAUDE.md \ - ${{ env.FILES_TO_EDIT }} \ - --model gemini/gemini-2.5-pro-preview-05-06 \ - --message-file .github/aider/issue-prompt.txt \ - --yes \ - --no-check-update \ - --auto-commits \ - --no-analytics \ - --no-gitignore \ - | tee .github/aider/aider-output.txt || true - echo "Aider command completed. Output saved to .github/aider/aider-output.txt" - - - name: Clean up prompt file - if: always() - run: rm -f .github/aider/issue-prompt.txt - - - name: Commit and Push Changes - id: commit_and_push - if: ${{ success() }} - run: | - BRANCH_NAME="${{ steps.create_prompt.outputs.BRANCH_NAME }}" - - # Check if branch exists remotely - if git ls-remote --heads origin $BRANCH_NAME | grep -q $BRANCH_NAME; then - echo "Branch $BRANCH_NAME already exists remotely, fetching it" - git fetch origin $BRANCH_NAME - git checkout $BRANCH_NAME - git pull origin $BRANCH_NAME - else - echo "Creating new branch $BRANCH_NAME" - git checkout -b $BRANCH_NAME - fi - - # Check if there are any changes to commit - if git diff --quiet && git diff --staged --quiet; then - echo "No changes to commit" - else - git commit -am "Auto-fix using Aider for external issue [skip ci]" || echo "No changes to commit" - fi - - git push origin $BRANCH_NAME - echo "Pushed to branch $BRANCH_NAME" - echo "PR_BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_OUTPUT - - - name: Create Pull Request - if: success() && steps.commit_and_push.outputs.PR_BRANCH_NAME != '' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_BRANCH: ${{ steps.commit_and_push.outputs.PR_BRANCH_NAME }} - ISSUE_TITLE: ${{ steps.create_prompt.outputs.ISSUE_TITLE }} - ISSUE_ID: ${{ github.event.client_payload.issue_id }} - run: | - # Create PR description in a temporary file to avoid command line length limits - cat > /tmp/pr-description.md << EOL - This PR was created automatically by Aider to fix an external issue: ${ISSUE_TITLE} - - ## Aider Output - \`\`\` - $(cat .github/aider/aider-output.txt || echo "No output available") - \`\`\` - EOL - - # Create PR using the file for the body content - gh pr create \ - --title "[Aider PR] Fix: ${ISSUE_TITLE}" \ - --body-file /tmp/pr-description.md \ - --head "$PR_BRANCH" \ - --base main || echo "PR already exists or couldn't be created" + run-aider: + needs: check-and-prepare + uses: ./.github/workflows/aider-common.yml + with: + issue_title: ${{ needs.check-and-prepare.outputs.issue_title }} + issue_body: ${{ needs.check-and-prepare.outputs.issue_body }} + instruction: ${{ needs.check-and-prepare.outputs.instruction }} + issue_id: ${{ github.event.client_payload.issue_id }} + secrets: inherit From dfd8c4cd2aa8c448aa245065047c36c58fadc27b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 22 May 2025 01:07:29 +0200 Subject: [PATCH 19/45] more verbose docker wait errors --- backend/windmill-worker/src/bash_executor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 9683c0d8d5..a372094d89 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -305,7 +305,7 @@ async fn handle_docker_job( .await .map_err(|e| { tracing::error!("Error waiting for container: {:?}", e); - anyhow::anyhow!("Error waiting for container") + anyhow::anyhow!("Error waiting for container: {:?}", e) })?; let waited = wait.first().map(|x| x.status_code); Ok(waited) From 3c28abc7bd6f22dde40639b16739f36c50996a95 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Thu, 22 May 2025 09:24:31 +0200 Subject: [PATCH 20/45] internal: Restrict access to git workflows (#5795) * restrict access * Update .github/workflows/aider-after-review.yaml Co-authored-by: graphite-app[bot] <96075541+graphite-app[bot]@users.noreply.github.com> * Update .github/workflows/create-docs.yml Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> --------- Co-authored-by: graphite-app[bot] <96075541+graphite-app[bot]@users.noreply.github.com> Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> --- .github/workflows/aider-after-review.yaml | 32 +++++++++++++++++++++-- .github/workflows/aider.yaml | 32 +++++++++++++++++++++-- .github/workflows/create-docs.yml | 28 +++++++++++++++++++- 3 files changed, 87 insertions(+), 5 deletions(-) diff --git a/.github/workflows/aider-after-review.yaml b/.github/workflows/aider-after-review.yaml index 2ba3483a6c..abd6091a31 100644 --- a/.github/workflows/aider-after-review.yaml +++ b/.github/workflows/aider-after-review.yaml @@ -5,9 +5,36 @@ on: types: [submitted] jobs: - check-and-prepare: + check-membership: if: github.event.review.state == 'changes_requested' && contains(github.event.pull_request.title, '[Aider PR]') runs-on: ubicloud-standard-2 + outputs: + is_member: ${{ steps.check-membership.outputs.is_member }} + steps: + - name: Check organization membership + id: check-membership + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + REVIEWER: ${{ github.event.review.user.login }} + run: | + ORG="windmill-labs" + STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: token $GH_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/orgs/$ORG/members/$REVIEWER") + + if [ "$STATUS" -eq 204 ]; then + echo "is_member=true" >> $GITHUB_OUTPUT + else + echo "is_member=false" >> $GITHUB_OUTPUT + fi + + check-and-prepare: + needs: check-membership + if: github.event.review.state == 'changes_requested' && contains(github.event.pull_request.title, '[Aider PR]') && needs.check-membership.outputs.is_member == 'true' + runs-on: ubicloud-standard-2 permissions: contents: write pull-requests: write @@ -58,7 +85,8 @@ jobs: echo "EOF" >> $GITHUB_OUTPUT run-aider: - needs: check-and-prepare + needs: [check-membership, check-and-prepare] + if: github.event.review.state == 'changes_requested' && contains(github.event.pull_request.title, '[Aider PR]') && needs.check-membership.outputs.is_member == 'true' uses: ./.github/workflows/aider-common.yml with: needs_processing: false diff --git a/.github/workflows/aider.yaml b/.github/workflows/aider.yaml index fdfd7b4d39..50d9946094 100644 --- a/.github/workflows/aider.yaml +++ b/.github/workflows/aider.yaml @@ -5,12 +5,39 @@ on: types: [created] jobs: - check-and-prepare: + check-membership: runs-on: ubicloud-standard-2 if: | github.event_name == 'issue_comment' && contains(github.event.comment.body, '/aider') && !contains(github.event.comment.user.login, '[bot]') + outputs: + is_member: ${{ steps.check-membership.outputs.is_member }} + steps: + - name: Check organization membership + id: check-membership + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + COMMENTER: ${{ github.event.comment.user.login }} + run: | + ORG="windmill-labs" + STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: token $GH_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/orgs/$ORG/members/$COMMENTER") + + if [ "$STATUS" -eq 204 ]; then + echo "is_member=true" >> $GITHUB_OUTPUT + else + echo "is_member=false" >> $GITHUB_OUTPUT + fi + + check-and-prepare: + needs: check-membership + runs-on: ubicloud-standard-2 + if: needs.check-membership.outputs.is_member == 'true' permissions: contents: write pull-requests: write @@ -105,7 +132,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Make sure gh cli has a token run-aider: - needs: check-and-prepare + needs: [check-membership, check-and-prepare] + if: needs.check-membership.outputs.is_member == 'true' uses: ./.github/workflows/aider-common.yml with: issue_title: ${{ needs.check-and-prepare.outputs.issue_title }} diff --git a/.github/workflows/create-docs.yml b/.github/workflows/create-docs.yml index 6c280ae5f9..209883d24c 100644 --- a/.github/workflows/create-docs.yml +++ b/.github/workflows/create-docs.yml @@ -3,8 +3,34 @@ on: types: [created] jobs: + check-membership: + if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/docs') && github.event.comment.user.type != 'Bot' }} + runs-on: ubicloud-standard-2 + outputs: + is_member: ${{ steps.check-membership.outputs.is_member }} + steps: + - name: Check organization membership + id: check-membership + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMMENTER: ${{ github.event.comment.user.login }} + run: | + ORG="windmill-labs" + STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: token $GH_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/orgs/$ORG/members/$COMMENTER") + + if [ "$STATUS" -eq 204 ]; then + echo "is_member=true" >> $GITHUB_OUTPUT + else + echo "is_member=false" >> $GITHUB_OUTPUT + fi + trigger-docs: - if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/docs') }} + needs: check-membership + if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/docs') && needs.check-membership.outputs.is_member == 'true' }} uses: windmill-labs/windmilldocs/.github/workflows/create-docs.yml@main with: pr_number: ${{ github.event.issue.number }} From dee62e1518e4bb4be8a339ae5f9f864002922333 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Thu, 22 May 2025 11:58:32 +0200 Subject: [PATCH 21/45] internal: secure flows (#5796) * secure flows * add restriction to claude code --- .github/workflows/aider-after-review.yaml | 25 ++++++------- .github/workflows/aider.yaml | 45 +++++++++++++---------- .github/workflows/claude.yml | 37 ++++++++++++++++--- .github/workflows/linear-issue.yaml | 17 ++++----- 4 files changed, 78 insertions(+), 46 deletions(-) diff --git a/.github/workflows/aider-after-review.yaml b/.github/workflows/aider-after-review.yaml index abd6091a31..578c7d1b7d 100644 --- a/.github/workflows/aider-after-review.yaml +++ b/.github/workflows/aider-after-review.yaml @@ -17,10 +17,11 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REPOSITORY: ${{ github.repository }} REVIEWER: ${{ github.event.review.user.login }} + ORG_ACCESS_TOKEN: ${{ secrets.ORG_ACCESS_TOKEN }} run: | ORG="windmill-labs" STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ - -H "Authorization: token $GH_TOKEN" \ + -H "Authorization: token $ORG_ACCESS_TOKEN" \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/orgs/$ORG/members/$REVIEWER") @@ -59,27 +60,25 @@ jobs: - name: Prepare prompt for Aider id: prepare_prompt shell: bash + env: + GITHUB_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REVIEW_BODY: ${{ github.event.review.body }} run: | - # Get PR review body - REVIEW_BODY="${{ github.event.review.body }}" - REVIEW_BODY_Q=$(printf '%q' "$REVIEW_BODY") + REVIEW_BODY_ESCAPED="${REVIEW_BODY//\\/\\\\}" + REVIEW_BODY_ESCAPED="${REVIEW_BODY_ESCAPED//\"/\\\"}" - PR_NUMBER="${{ github.event.pull_request.number }}" - - # Get all PR review comments ALL_REVIEW_COMMENTS=$(gh api \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - /repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/comments \ - | jq '[.[] | {diff_hunk: .diff_hunk, path: .path, body: .body}]') + /repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/comments) + + FORMATTED_COMMENTS=$(jq -r '[.[] | {diff_hunk: .diff_hunk, path: .path, body: .body}]' <<< "$ALL_REVIEW_COMMENTS") BASE_PROMPT="Fix the following issues in the PR based on the review feedback. The review body is prepended with REVIEW. The review comments are prepended with REVIEW_COMMENTS. The review body and comments are separated by a blank line." - printf -v COMPLETE_PROMPT "%s\nREVIEW:\n%s\nREVIEW_COMMENTS:\n%s" \ - "$BASE_PROMPT" "$REVIEW_BODY_Q" "$ALL_REVIEW_COMMENTS" - echo "$COMPLETE_PROMPT" + COMPLETE_PROMPT="${BASE_PROMPT}"$'\n'"REVIEW:"$'\n'"${REVIEW_BODY_ESCAPED}"$'\n'"REVIEW_COMMENTS:"$'\n'"${FORMATTED_COMMENTS}" - # Use the proper multi-line output format echo "prompt_content<> $GITHUB_OUTPUT echo "$COMPLETE_PROMPT" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT diff --git a/.github/workflows/aider.yaml b/.github/workflows/aider.yaml index 50d9946094..0614627b35 100644 --- a/.github/workflows/aider.yaml +++ b/.github/workflows/aider.yaml @@ -20,10 +20,11 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REPOSITORY: ${{ github.repository }} COMMENTER: ${{ github.event.comment.user.login }} + ORG_ACCESS_TOKEN: ${{ secrets.ORG_ACCESS_TOKEN }} run: | ORG="windmill-labs" STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ - -H "Authorization: token $GH_TOKEN" \ + -H "Authorization: token $ORG_ACCESS_TOKEN" \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/orgs/$ORG/members/$COMMENTER") @@ -66,6 +67,11 @@ jobs: - name: Determine inputs for Aider id: determine_inputs shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMMENT_BODY: ${{ github.event.comment.body }} + ISSUE_NUMBER: ${{ github.event.issue.number }} + GITHUB_REPOSITORY: ${{ github.repository }} run: | echo "Determining inputs for Aider..." ISSUE_TITLE_VAL="" @@ -73,28 +79,31 @@ jobs: if [[ ! -z "${{ github.event.issue.pull_request }}" ]]; then echo "This is a comment on a Pull Request" - PR_NUMBER="${{ github.event.issue.number }}" + PR_NUMBER="$ISSUE_NUMBER" - PR_BODY_JSON=$(GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" gh pr view "$PR_NUMBER" --json body --repo "$GITHUB_REPOSITORY") + PR_BODY_JSON=$(gh pr view "$PR_NUMBER" --json body --repo "$GITHUB_REPOSITORY") if [[ $? -ne 0 ]]; then echo "Error fetching PR body for PR #$PR_NUMBER" PR_BODY_VAL="" else - PR_BODY_VAL=$(echo "$PR_BODY_JSON" | jq -r .body) + PR_BODY_VAL=$(jq -r '.body // ""' <<< "$PR_BODY_JSON") fi if [[ ! -z "$PR_BODY_VAL" ]]; then - REFERENCED_ISSUE=$(echo "$PR_BODY_VAL" | grep -oE "#[0-9]+" | grep -oE "[0-9]+" | head -1) + REFERENCED_ISSUE="" + if [[ "$PR_BODY_VAL" =~ \#([0-9]+) ]]; then + REFERENCED_ISSUE="${BASH_REMATCH[1]}" + fi if [[ ! -z "$REFERENCED_ISSUE" ]]; then echo "Found referenced issue #$REFERENCED_ISSUE in PR description" - ISSUE_DETAILS_JSON=$(GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" gh issue view "$REFERENCED_ISSUE" --json title,body --repo "$GITHUB_REPOSITORY") + ISSUE_DETAILS_JSON=$(gh issue view "$REFERENCED_ISSUE" --json title,body --repo "$GITHUB_REPOSITORY") if [[ $? -ne 0 ]]; then echo "Error fetching issue details for #$REFERENCED_ISSUE" else - ISSUE_TITLE_VAL=$(echo "$ISSUE_DETAILS_JSON" | jq -r .title) - ISSUE_BODY_VAL=$(echo "$ISSUE_DETAILS_JSON" | jq -r .body) + ISSUE_TITLE_VAL=$(jq -r '.title // ""' <<< "$ISSUE_DETAILS_JSON") + ISSUE_BODY_VAL=$(jq -r '.body // ""' <<< "$ISSUE_DETAILS_JSON") fi fi else @@ -102,34 +111,32 @@ jobs: fi else echo "This is a comment on a regular issue" - ISSUE_NUMBER="${{ github.event.issue.number }}" - ISSUE_DETAILS_JSON=$(GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" gh issue view "$ISSUE_NUMBER" --json title,body --repo "$GITHUB_REPOSITORY") + + ISSUE_DETAILS_JSON=$(gh issue view "$ISSUE_NUMBER" --json title,body --repo "$GITHUB_REPOSITORY") if [[ $? -ne 0 ]]; then echo "Error fetching issue details for #$ISSUE_NUMBER" else - ISSUE_TITLE_VAL=$(echo "$ISSUE_DETAILS_JSON" | jq -r .title) - ISSUE_BODY_VAL=$(echo "$ISSUE_DETAILS_JSON" | jq -r .body) + ISSUE_TITLE_VAL=$(jq -r '.title // ""' <<< "$ISSUE_DETAILS_JSON") + ISSUE_BODY_VAL=$(jq -r '.body // ""' <<< "$ISSUE_DETAILS_JSON") fi fi - echo "Setting GITHUB_OUTPUT for ISSUE_TITLE..." echo "ISSUE_TITLE<> "$GITHUB_OUTPUT" echo "$ISSUE_TITLE_VAL" >> "$GITHUB_OUTPUT" echo "EOF_AIDER_TITLE" >> "$GITHUB_OUTPUT" - echo "Setting GITHUB_OUTPUT for ISSUE_BODY..." echo "ISSUE_BODY<> "$GITHUB_OUTPUT" echo "$ISSUE_BODY_VAL" >> "$GITHUB_OUTPUT" echo "EOF_AIDER_BODY" >> "$GITHUB_OUTPUT" - # Process COMMENT_CONTENT - printf -v COMMENT_CONTENT_VAL "%s" "$(echo "${{ github.event.comment.body }}" | sed 's|^/aider||' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + CLEAN_COMMENT="${COMMENT_BODY/\/aider/}" + CLEAN_COMMENT="${CLEAN_COMMENT#"${CLEAN_COMMENT%%[![:space:]]*}"}" + CLEAN_COMMENT="${CLEAN_COMMENT%"${CLEAN_COMMENT##*[![:space:]]}"}" + echo "COMMENT_CONTENT<> "$GITHUB_OUTPUT" - echo "$COMMENT_CONTENT_VAL" >> "$GITHUB_OUTPUT" + echo "$CLEAN_COMMENT" >> "$GITHUB_OUTPUT" echo "EOF_AIDER_COMMENT" >> "$GITHUB_OUTPUT" echo "Finished determining inputs." - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Make sure gh cli has a token run-aider: needs: [check-membership, check-and-prepare] diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 77a96d6119..4dc170d8d1 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -11,12 +11,39 @@ on: types: [submitted] jobs: - claude-code-action: + check-membership: if: | - (github.event_name == 'issue_comment' && contains(github.event.comment.body, '/aider')) || - (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '/aider')) || - (github.event_name == 'pull_request_review' && contains(github.event.review.body, '/aider')) || - (github.event_name == 'issues' && contains(github.event.issue.body, '/aider')) + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '/aider') && !contains(github.event.comment.user.login, '[bot]')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '/aider') && !contains(github.event.comment.user.login, '[bot]')) || + (github.event_name == 'pull_request_review' && contains(github.event.review.body, '/aider') && !contains(github.event.review.user.login, '[bot]')) || + (github.event_name == 'issues' && contains(github.event.issue.body, '/aider') && !contains(github.event.issue.user.login, '[bot]')) + runs-on: ubicloud-standard-2 + outputs: + is_member: ${{ steps.check-membership.outputs.is_member }} + steps: + - name: Check organization membership + id: check-membership + env: + COMMENTER: ${{ github.event_name == 'issue_comment' || github.event_name == 'pull_request_review_comment' ? github.event.comment.user.login : github.event_name == 'pull_request_review' ? github.event.review.user.login : github.event.issue.user.login }} + ORG_ACCESS_TOKEN: ${{ secrets.ORG_ACCESS_TOKEN }} + run: | + ORG="windmill-labs" + STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: token $ORG_ACCESS_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/orgs/$ORG/members/$COMMENTER") + + if [ "$STATUS" -eq 204 ]; then + echo "is_member=true" >> $GITHUB_OUTPUT + else + echo "is_member=false" >> $GITHUB_OUTPUT + fi + + claude-code-action: + needs: check-membership + if: | + needs.check-membership.outputs.is_member == 'true' runs-on: ubicloud-standard-8 permissions: contents: read diff --git a/.github/workflows/linear-issue.yaml b/.github/workflows/linear-issue.yaml index 1dce5b6c87..22f9e53c46 100644 --- a/.github/workflows/linear-issue.yaml +++ b/.github/workflows/linear-issue.yaml @@ -37,24 +37,23 @@ jobs: - name: Determine inputs for Aider id: determine_inputs shell: bash + env: + ISSUE_TITLE: ${{ github.event.client_payload.issue_title }} + ISSUE_BODY: ${{ github.event.client_payload.issue_body }} + INSTRUCTION: ${{ github.event.client_payload.instruction }} run: | echo "Determining inputs for Aider..." - ISSUE_TITLE_VAL="${{ github.event.client_payload.issue_title }}" - INSTRUCTION_VAL="${{ github.event.client_payload.instruction }}" - ISSUE_BODY_VAL=$(printf '%q' "${{ github.event.client_payload.issue_body }}") - echo "Setting GITHUB_OUTPUT for ISSUE_TITLE..." + echo "ISSUE_TITLE<> "$GITHUB_OUTPUT" - echo "$ISSUE_TITLE_VAL" >> "$GITHUB_OUTPUT" + echo "$ISSUE_TITLE" >> "$GITHUB_OUTPUT" echo "EOF_AIDER_TITLE" >> "$GITHUB_OUTPUT" - echo "Setting GITHUB_OUTPUT for ISSUE_BODY..." echo "ISSUE_BODY<> "$GITHUB_OUTPUT" - echo "$ISSUE_BODY_VAL" >> "$GITHUB_OUTPUT" + echo "$ISSUE_BODY" >> "$GITHUB_OUTPUT" echo "EOF_AIDER_BODY" >> "$GITHUB_OUTPUT" - echo "Setting GITHUB_OUTPUT for INSTRUCTION..." echo "INSTRUCTION<> "$GITHUB_OUTPUT" - echo "$INSTRUCTION_VAL" >> "$GITHUB_OUTPUT" + echo "$INSTRUCTION" >> "$GITHUB_OUTPUT" echo "EOF_AIDER_INSTRUCTION" >> "$GITHUB_OUTPUT" echo "Finished determining inputs." From d662e18f97c2edc3d60df9496b0927901edb26a5 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 22 May 2025 12:07:21 +0200 Subject: [PATCH 22/45] add more labels to traces --- backend/windmill-queue/src/jobs.rs | 1 + backend/windmill-worker/src/handle_child.rs | 11 +++++++---- backend/windmill-worker/src/job_logger_ee.rs | 1 + 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 1ff103cd9a..f2b1e56153 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -980,6 +980,7 @@ pub async fn add_completed_job( is_flow_step = queued_job.is_flow_step(), language = ?queued_job.script_lang, scheduled_for = ?queued_job.scheduled_for, + workspace_id = ?queued_job.workspace_id, success, "inserted completed job: {} (success: {success})", queued_job.id diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index d82a72df98..c47af18073 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -134,7 +134,7 @@ pub async fn handle_child( let (tx, rx) = broadcast::channel::<()>(3); let mut rx2: broadcast::Receiver<()> = tx.subscribe(); - let output = child_joined_output_stream(&mut child, job_id.clone()); + let output = child_joined_output_stream(&mut child, job_id.clone(), w_id.to_string()); let job_id: Uuid = job_id.clone(); @@ -729,6 +729,7 @@ where fn child_joined_output_stream( child: &mut Child, job_id: Uuid, + w_id: String, ) -> impl stream::FusedStream> { let stderr = child .stderr @@ -743,8 +744,8 @@ fn child_joined_output_stream( let stdout = BufReader::new(stdout).lines(); let stderr = BufReader::new(stderr).lines(); stream::select( - lines_to_stream(stderr, true, job_id.clone()), - lines_to_stream(stdout, false, job_id), + lines_to_stream(stderr, true, job_id.clone(), w_id.clone(), path.clone()), + lines_to_stream(stdout, false, job_id, w_id, path), ) } @@ -752,11 +753,13 @@ pub fn lines_to_stream( mut lines: tokio::io::Lines, stderr: bool, job_id: Uuid, + w_id: String, + path: String, ) -> impl futures::Stream> { stream::poll_fn(move |cx| { std::pin::Pin::new(&mut lines) .poll_next_line(cx) - .map(|result| process_streaming_log_lines(result, stderr, &job_id)) + .map(|result| process_streaming_log_lines(result, stderr, &job_id, &w_id)) }) } diff --git a/backend/windmill-worker/src/job_logger_ee.rs b/backend/windmill-worker/src/job_logger_ee.rs index 4b1d34392c..22772878ee 100644 --- a/backend/windmill-worker/src/job_logger_ee.rs +++ b/backend/windmill-worker/src/job_logger_ee.rs @@ -36,6 +36,7 @@ pub(crate) fn process_streaming_log_lines( r: Result, io::Error>, _stderr: bool, _job_id: &Uuid, + _w_id: &str, ) -> Option> { r.transpose() } From 3fbebcdef57c75c7effde0755794e81b9722def8 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 22 May 2025 12:35:32 +0200 Subject: [PATCH 23/45] add more labels to traces --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 71591f02b5..36e0923e83 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -0f0df9dd99a44baf890f24323f0a2eb2ee1120ce \ No newline at end of file +7632fb040ac1dd340d7ef4ddd90304c6a06e71f1 \ No newline at end of file From d9bd80b280690bc07f1e3c125bf4d3486b8ef0c9 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Thu, 22 May 2025 12:35:44 +0200 Subject: [PATCH 24/45] internal: fix flows (#5797) * remove test line * fix claude --- .github/workflows/aider-common.yml | 2 +- .github/workflows/claude.yml | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/aider-common.yml b/.github/workflows/aider-common.yml index 2e4e99c716..dceea4a012 100644 --- a/.github/workflows/aider-common.yml +++ b/.github/workflows/aider-common.yml @@ -273,7 +273,7 @@ jobs: --read .cursor/rules/windmill-overview.mdc \ $FILES_TO_EDIT \ --model gemini/gemini-2.5-pro-preview-05-06 \ - --message "create a test file in backend/test.txt with hello world in it" \ + --message-file .aider_final_prompt.txt \ --yes \ --no-check-update \ --auto-commits \ diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 4dc170d8d1..7ac5d93802 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -24,10 +24,18 @@ jobs: - name: Check organization membership id: check-membership env: - COMMENTER: ${{ github.event_name == 'issue_comment' || github.event_name == 'pull_request_review_comment' ? github.event.comment.user.login : github.event_name == 'pull_request_review' ? github.event.review.user.login : github.event.issue.user.login }} ORG_ACCESS_TOKEN: ${{ secrets.ORG_ACCESS_TOKEN }} run: | ORG="windmill-labs" + + if [[ "${{ github.event_name }}" == "issue_comment" || "${{ github.event_name }}" == "pull_request_review_comment" ]]; then + COMMENTER="${{ github.event.comment.user.login }}" + elif [[ "${{ github.event_name }}" == "pull_request_review" ]]; then + COMMENTER="${{ github.event.review.user.login }}" + else + COMMENTER="${{ github.event.issue.user.login }}" + fi + STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ -H "Authorization: token $ORG_ACCESS_TOKEN" \ -H "Accept: application/vnd.github+json" \ From e3e25daee79380131f3f28ad326c4455b489f1d3 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 22 May 2025 14:12:10 +0200 Subject: [PATCH 25/45] fix --- backend/windmill-worker/src/handle_child.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index c47af18073..3eb9e95dc9 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -744,8 +744,8 @@ fn child_joined_output_stream( let stdout = BufReader::new(stdout).lines(); let stderr = BufReader::new(stderr).lines(); stream::select( - lines_to_stream(stderr, true, job_id.clone(), w_id.clone(), path.clone()), - lines_to_stream(stdout, false, job_id, w_id, path), + lines_to_stream(stderr, true, job_id.clone(), w_id.clone()), + lines_to_stream(stdout, false, job_id, w_id), ) } @@ -754,7 +754,6 @@ pub fn lines_to_stream( stderr: bool, job_id: Uuid, w_id: String, - path: String, ) -> impl futures::Stream> { stream::poll_fn(move |cx| { std::pin::Pin::new(&mut lines) From 21741e68bcf467b4f87390da51b076b7af91f494 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 22 May 2025 14:23:49 +0200 Subject: [PATCH 26/45] fix --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 36e0923e83..fec7d05006 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -7632fb040ac1dd340d7ef4ddd90304c6a06e71f1 \ No newline at end of file +0854c5c00f62751aa5eb44fd9e550052fdcf7884 \ No newline at end of file From 88482c3bd76ddad16738354f7531d16fa806ad2f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 22 May 2025 14:40:53 +0200 Subject: [PATCH 27/45] fix: improve app css consistency --- .../apps/components/buttons/AppButton.svelte | 6 +- .../apps/components/display/AppAlert.svelte | 9 +- .../apps/components/layout/AppModal.svelte | 4 +- .../apps/editor/component/components.ts | 3 +- .../componentsPanel/CssHelperPanel.svelte | 256 ++++++++++-------- .../apps/editor/componentsPanel/cssUtils.ts | 32 ++- 6 files changed, 184 insertions(+), 126 deletions(-) diff --git a/frontend/src/lib/components/apps/components/buttons/AppButton.svelte b/frontend/src/lib/components/apps/components/buttons/AppButton.svelte index ff3f34097b..5ad842ba6c 100644 --- a/frontend/src/lib/components/apps/components/buttons/AppButton.svelte +++ b/frontend/src/lib/components/apps/components/buttons/AppButton.svelte @@ -229,7 +229,8 @@ css?.button?.class ?? '', isMenuItem ? 'flex items-center justify-start' : '', isMenuItem ? '!border-0' : '', - 'wm-button' + 'wm-button', + `wm-button-${resolvedConfig.color}` )} variant={isMenuItem ? 'border' : 'contained'} style={css?.button?.style} @@ -237,7 +238,8 @@ css?.container?.class ?? '', resolvedConfig.fillContainer ? 'w-full h-full' : '', isMenuItem ? 'w-full' : '', - 'wm-button-container' + 'wm-button-container', + `wm-button-container-${resolvedConfig.color}` )} wrapperStyle={css?.container?.style} disabled={resolvedConfig.disabled} diff --git a/frontend/src/lib/components/apps/components/display/AppAlert.svelte b/frontend/src/lib/components/apps/components/display/AppAlert.svelte index 20fc60ba03..24dfa9ea51 100644 --- a/frontend/src/lib/components/apps/components/display/AppAlert.svelte +++ b/frontend/src/lib/components/apps/components/display/AppAlert.svelte @@ -10,6 +10,7 @@ import InitializeComponent from '../helpers/InitializeComponent.svelte' import { Alert } from '$lib/components/common' import AlignWrapper from '../helpers/AlignWrapper.svelte' + import { appendClass } from '../../editor/componentsPanel/cssUtils' export let id: string export let configuration: RichConfigurations @@ -63,13 +64,13 @@ tooltip={resolvedConfig.tooltip} size={resolvedConfig.size} collapsible={resolvedConfig.collapsible} - bgClass={css?.background?.class} + bgClass={appendClass(css?.background?.class, 'wm-alert-card-background')} bgStyle={css?.background?.style} - iconClass={css?.icon?.class} + iconClass={appendClass(css?.icon?.class, 'wm-alert-card-icon')} iconStyle={css?.icon?.style} - titleClass={css?.title?.class} + titleClass={appendClass(css?.title?.class, 'wm-alert-card-title')} titleStyle={css?.title?.style} - descriptionClass={css?.description?.class} + descriptionClass={appendClass(css?.description?.class, 'wm-alert-card-description')} descriptionStyle={css?.description?.style} isCollapsed={resolvedConfig.initiallyCollapsed} > diff --git a/frontend/src/lib/components/apps/components/layout/AppModal.svelte b/frontend/src/lib/components/apps/components/layout/AppModal.svelte index 3000afdedf..15f3eca206 100644 --- a/frontend/src/lib/components/apps/components/layout/AppModal.svelte +++ b/frontend/src/lib/components/apps/components/layout/AppModal.svelte @@ -179,7 +179,7 @@ >
{ e?.stopPropagation() if (!$connectingInput.opened) { diff --git a/frontend/src/lib/components/apps/editor/component/components.ts b/frontend/src/lib/components/apps/editor/component/components.ts index 2ff364f44a..27d0bfc3bf 100644 --- a/frontend/src/lib/components/apps/editor/component/components.ts +++ b/frontend/src/lib/components/apps/editor/component/components.ts @@ -3410,7 +3410,8 @@ See date-fns format for more information. By default, it is 'dd.MM.yyyy HH:mm' customCss: { button: { class: '', style: '' }, buttonContainer: { class: '', style: '' }, - popup: { class: '', style: '' } + popup: { class: '', style: '' }, + container: { class: '', style: '' } }, initialData: { horizontalAlignment: 'center', diff --git a/frontend/src/lib/components/apps/editor/componentsPanel/CssHelperPanel.svelte b/frontend/src/lib/components/apps/editor/componentsPanel/CssHelperPanel.svelte index fda2b9338a..05e6380238 100644 --- a/frontend/src/lib/components/apps/editor/componentsPanel/CssHelperPanel.svelte +++ b/frontend/src/lib/components/apps/editor/componentsPanel/CssHelperPanel.svelte @@ -20,20 +20,38 @@ const dispatch = createEventDispatcher() interface CustomCSSEntry { - type: CustomCSSType + type?: CustomCSSType name: string icon: any - ids: { id: string; forceStyle: boolean; forceClass: boolean }[] + ids?: { id: string; forceStyle: boolean; forceClass: boolean }[] + description?: string + order?: number } const { app } = getContext('AppViewerContext') + const descriptions = { + buttoncomponent: + 'The button component also has additional color specific classes to allow customizing classes by color. wm-button-wrapper-blue, wm-button-container-blue, ...' + } const entries: CustomCSSEntry[] = [ + { + name: 'Dark Mode', + icon: LayoutDashboardIcon, + description: + 'When in dark mode, the entire document has the .dark class applied to it. You can apply selective styling by using the .dark class: e.g. .dark .my-element { color: white; }', + order: 3 + }, { type: 'app', name: 'App', icon: LayoutDashboardIcon, - ids: ['viewer', 'grid', 'component'].map((id) => ({ id, forceStyle: true, forceClass: true })) + ids: ['viewer', 'grid', 'component'].map((id) => ({ + id, + forceStyle: true, + forceClass: true + })), + order: 2 }, { type: 'quillcomponent', @@ -51,11 +69,12 @@ id, forceStyle: v?.style != undefined, forceClass: v?.['class'] != undefined - })) + })), + description: descriptions[type as keyof typeof descriptions] })) ] - entries.sort((a, b) => a.name.localeCompare(b.name)) + entries.sort((a, b) => (b.order ?? 0) - (a.order ?? 0) + a.name.localeCompare(b.name)) let search = '' @@ -66,15 +85,15 @@
{#each search != '' ? entries.filter((x) => x.name .toLowerCase() - .includes(search.toLowerCase())) : entries as { type, name, icon, ids } (name + type)} - {#if ids.length > 0} + .includes(search.toLowerCase())) : entries as { type, name, icon, ids, description } (name + type)} + {#if description || (ids && ids.length > 0)} { if ($app.css != undefined) { - if (e.detail && $app.css[type] == undefined) { - $app.css[type] = Object.fromEntries(ids.map(({ id }) => [id, {}])) + if (type && e.detail && $app.css[type] == undefined) { + $app.css[type] = Object.fromEntries((ids ?? []).map(({ id }) => [id, {}])) } } }} @@ -85,115 +104,120 @@ {name}
-
- {#each customisationByComponent.filter( (c) => c.components.includes(type) ) as customisation (customisation.components.join('-'))} - {#if customisation.link} - -
- See documentation - -
- - {/if} - - - {#if customisation.selectors.length > 0} - - Selectors ({customisation.selectors.length}) - - {/if} - {#if customisation.variables.length > 0} - -
- Variables ({customisation.variables.length}) + {#if description} +
{description}
+ {/if} + {#if type} +
+ {#each customisationByComponent.filter( (c) => c.components.includes(type) ) as customisation (customisation.components.join('-'))} + {#if customisation.link} + +
+ See documentation +
- + {/if} -
- - - - - Selector - Comment - - - - {#each customisation.selectors as { selector, comment }} - - - {selector} - - - {#if comment} -
{comment}
- {/if} - - - - - - {/each} - - - - - - - Variable - Default value - Comment - - - - {#each customisation.variables as { variable, value, comment }} - - - {variable} - - - {value} - - - {#if comment} -
{comment}
- {/if} - - - - - - {/each} - - -
- - {/each} -
+ + + {#if customisation.selectors.length > 0} + + Selectors ({customisation.selectors.length}) + + {/if} + {#if customisation.variables.length > 0} + +
+ Variables ({customisation.variables.length}) +
+ + {/if} +
+ + + + + Selector + Comment + + + + {#each customisation.selectors as { selector, comment }} + + + {selector} + + + {#if comment} +
{comment}
+ {/if} + + + + + + {/each} + + + + + + + Variable + Default value + Comment + + + + {#each customisation.variables as { variable, value, comment }} + + + {variable} + + + {value} + + + {#if comment} +
{comment}
+ {/if} + + + + + + {/each} + + +
+ + {/each} +
+ {/if} {/if} {/each} diff --git a/frontend/src/lib/components/apps/editor/componentsPanel/cssUtils.ts b/frontend/src/lib/components/apps/editor/componentsPanel/cssUtils.ts index 6fc8344295..4634959ac0 100644 --- a/frontend/src/lib/components/apps/editor/componentsPanel/cssUtils.ts +++ b/frontend/src/lib/components/apps/editor/componentsPanel/cssUtils.ts @@ -185,7 +185,11 @@ export const customisationByComponent: Customisation[] = [ components: ['modalcomponent'], selectors: [ { selector: '.wm-modal', comment: 'main modal element', customCssKey: 'popup' }, - { selector: '.wm-modal-button', comment: 'button to open modal', customCssKey: 'button' }, + { + selector: '.wm-modal-container', + comment: 'container for modal', + customCssKey: 'container' + }, { selector: '.wm-modal-button-container', comment: 'container for button to open modal', @@ -826,6 +830,26 @@ export const customisationByComponent: Customisation[] = [ selector: 'wm-alert-card-container', comment: 'Alert container', customCssKey: 'container' + }, + { + selector: 'wm-alert-card-background', + comment: 'Alert background', + customCssKey: 'background' + }, + { + selector: 'wm-alert-card-icon', + comment: 'Alert icon', + customCssKey: 'icon' + }, + { + selector: 'wm-alert-card-title', + comment: 'Alert title', + customCssKey: 'title' + }, + { + selector: 'wm-alert-card-description', + comment: 'Alert description', + customCssKey: 'description' } ], variables: [] @@ -860,3 +884,9 @@ export function hasStyleValue(obj: ComponentCssProperty | undefined) { return obj.style !== '' } + +export function appendClass(className: string | undefined, customCssKey: string) { + if (!className) return customCssKey + + return `${className} ${customCssKey}` +} From 55ae76648475ce9ff14b2fa33b2a71b90fbd50a1 Mon Sep 17 00:00:00 2001 From: wendrul <53628737+wendrul@users.noreply.github.com> Date: Thu, 22 May 2025 15:23:45 +0200 Subject: [PATCH 28/45] feat: job search pagination + result count (#5789) * add tracing to get of authed client * fix: make disabled items not selectable with arrow keys * Invert showing EE message only when not in EE * Makea component for the Run Search part of the Search modal * Make the button to load more jobs * Add pagination for job search * fix missing bind to the openModal bool * Turn off spinner when aborting search results * fix typo in openapi.yaml * Update ee repo ref * Remove unused imports and vars --------- Co-authored-by: Ruben Fiszel --- backend/ee-repo-ref.txt | 2 +- backend/windmill-api/openapi.yaml | 26 +- .../search/GlobalSearchModal.svelte | 211 +++----------- .../lib/components/search/RunsSearch.svelte | 266 ++++++++++++++++++ 4 files changed, 321 insertions(+), 184 deletions(-) create mode 100644 frontend/src/lib/components/search/RunsSearch.svelte diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index fec7d05006..4d15f57a97 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -0854c5c00f62751aa5eb44fd9e550052fdcf7884 \ No newline at end of file +bea87fa885dc041fba83b2491609a4a2cdbbfa6f diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 5020067440..9895efce7e 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -12590,6 +12590,11 @@ paths: required: true schema: type: string + - name: pagination_offset + in: query + required: false + schema: + type: integer responses: "200": description: search results @@ -12602,15 +12607,26 @@ paths: description: a list of the terms that couldn't be parsed (and thus ignored) type: array items: - type: object - properties: - dancer: - type: string + type: string hits: description: the jobs that matched the query type: array items: $ref: "#/components/schemas/JobSearchHit" + hit_count: + description: how many jobs matched in total + type: number + index_metadata: + description: Metadata about the index current state + type: object + properties: + indexed_until: + description: Datetime of the most recently indexed job + type: string + format: date-time + lost_lock_ownership: + description: Is the current indexer service being replaced + type: boolean /srch/index/search/service_logs: get: @@ -16810,4 +16826,4 @@ components: channel_name: type: string description: Microsoft Teams channel name - minLength: 1 \ No newline at end of file + minLength: 1 diff --git a/frontend/src/lib/components/search/GlobalSearchModal.svelte b/frontend/src/lib/components/search/GlobalSearchModal.svelte index 271389b007..e684ce0d89 100644 --- a/frontend/src/lib/components/search/GlobalSearchModal.svelte +++ b/frontend/src/lib/components/search/GlobalSearchModal.svelte @@ -3,7 +3,6 @@ import { AppService, FlowService, - IndexSearchService, RawAppService, ScriptService, type Flow, @@ -11,8 +10,7 @@ type ListableRawApp, type Script } from '$lib/gen' - import { clickOutside, displayDateOnly, isMac, sendUserToast } from '$lib/utils' - import TimeAgo from '../TimeAgo.svelte' + import { clickOutside, isMac } from '$lib/utils' import { AlertTriangle, BoxesIcon, @@ -22,14 +20,12 @@ DollarSignIcon, HomeIcon, LayoutDashboardIcon, - Loader2, PlayIcon, Route, Search, SearchCode, Unplug } from 'lucide-svelte' - import JobPreview from '../runs/JobPreview.svelte' import Portal from '$lib/components/Portal.svelte' import { twMerge } from 'tailwind-merge' @@ -44,6 +40,7 @@ import Popover from '../Popover.svelte' import Logs from 'lucide-svelte/icons/logs' import { AwsIcon, GoogleCloudIcon, KafkaIcon, MqttIcon, NatsIcon } from '../icons' + import RunsSearch from './RunsSearch.svelte' let open: boolean = false @@ -72,7 +69,7 @@ let switchModeItems: quickMenuItem[] = [ { search_id: 'switchto:run-search', - label: 'Search across completed runs' + ($enterpriseLicense ? '' : ' (EE)'), + label: 'Search across completed runs' + (!$enterpriseLicense ? '' : ' (EE)'), action: () => switchMode('runs'), shortcutKey: RUNS_PREFIX, icon: Search, @@ -113,28 +110,28 @@ }, { search_id: 'nav:kafka_triggers', - label: 'Go to Kafka triggers' + ($enterpriseLicense ? '' : ' (EE)'), + label: 'Go to Kafka triggers' + (!$enterpriseLicense ? '' : ' (EE)'), action: () => gotoPage('/kafka_triggers'), icon: KafkaIcon, disabled: $userStore?.operator }, { search_id: 'nav:nats_triggers', - label: 'Go to NATS triggers' + ($enterpriseLicense ? '' : ' (EE)'), + label: 'Go to NATS triggers' + (!$enterpriseLicense ? '' : ' (EE)'), action: () => gotoPage('/nats_triggers'), icon: NatsIcon, disabled: $userStore?.operator }, { search_id: 'nav:sqs_triggers', - label: 'Go to SQS triggers' + ($enterpriseLicense ? '' : ' (EE)'), + label: 'Go to SQS triggers' + (!$enterpriseLicense ? '' : ' (EE)'), action: () => gotoPage('/sqs_triggers'), icon: AwsIcon, disabled: $userStore?.operator }, { search_id: 'nav:gcp_pub_sub', - label: 'Go to GCP Pub/Sub' + ($enterpriseLicense ? '' : ' (EE)'), + label: 'Go to GCP Pub/Sub' + (!$enterpriseLicense ? '' : ' (EE)'), action: () => gotoPage('/gcp_triggers'), icon: GoogleCloudIcon, disabled: $userStore?.operator @@ -264,12 +261,8 @@ return r } - let debounceTimeout: any = undefined - const debouncePeriod: number = 1000 - let loadingCompletedRuns: boolean = false let queryParseErrors: string[] = [] - let indexMetadata: any = {} async function handleSearch() { queryParseErrors = [] @@ -314,6 +307,7 @@ ) ) } + itemMap['default'] = itemMap['default'].filter((e) => !e.disabled) } if (tab === 'switch-mode') { itemMap['switch-mode'] = fuzzyFilter( @@ -323,26 +317,8 @@ ) } if (tab === 'runs') { - const s = removePrefix(searchTerm, RUNS_PREFIX) - clearTimeout(debounceTimeout) - loadingCompletedRuns = true - debounceTimeout = setTimeout(async () => { - clearTimeout(debounceTimeout) - let searchResults - try { - searchResults = await IndexSearchService.searchJobsIndex({ - searchQuery: s, - workspace: $workspaceStore! - }) - itemMap['runs'] = searchResults.hits - queryParseErrors = searchResults.query_parse_errors - indexMetadata = searchResults.index_metadata - } catch (e) { - sendUserToast(e.body, true) - } - loadingCompletedRuns = false - selectedItem = selectItem(0) - }, debouncePeriod) + await tick() + runsSearch?.handleRunSearch(removePrefix(searchTerm, RUNS_PREFIX)) } selectedItem = selectItem(0) } @@ -594,6 +570,8 @@ return 'max-h-[60vh]' } } + + let runsSearch: RunsSearch {#if open} @@ -652,18 +630,16 @@ {#if items.length > 0}
{#each items as el} - {#if !el.disabled} - (selectedItem = el)} - id={el?.search_id} - hovered={el?.search_id === selectedItem?.search_id} - label={el?.label} - icon={el?.icon} - shortcutKey={el?.shortcutKey} - bind:mouseMoved - /> - {/if} + (selectedItem = el)} + id={el?.search_id} + hovered={el?.search_id === selectedItem?.search_id} + label={el?.label} + icon={el?.icon} + shortcutKey={el?.shortcutKey} + bind:mouseMoved + /> {/each}
{/if} @@ -729,138 +705,17 @@ {/if}
{:else if tab === 'runs'} -
- {#if loadingCompletedRuns} -
-
- -
-
- {:else if itemMap['runs'] && itemMap['runs'].length > 0} -
- {#each itemMap['runs'] ?? [] as r} - { - selectedItem = r - selectedWorkspace = r?.document.workspace_id[0] - }} - on:keyboardOnlySelect={() => { - open = false - goto(`/run/${r?.document.id[0]}`) - }} - id={r?.document.id[0]} - hovered={selectedItem && r?.document.id[0] === selectedItem?.document.id[0]} - icon={r?.icon} - containerClass="rounded-md px-2 py-1 my-2" - bind:mouseMoved - > - -
-
-
-
{r?.document.script_path}
-
-
- {displayDateOnly(new Date(r?.document.created_at[0]))} -
-
- -
-
-
-
- - - {/each} -
-
- {#if selectedItem === undefined} - Select a result to preview - {:else} -
- -
- {/if} -
- {#if indexMetadata.indexed_until} - - Most recently indexed job was created at - - {/if} - {#if indexMetadata.lost_lock_ownership} - - - - The current indexer is no longer indexing new jobs. This is most likely - because of an ongoing deployment and indexing will resume once it's - complete. - - - {/if} -
-
- {:else} -
-
- {#if searchTerm === RUNS_PREFIX} -
Enter your search terms
-
Start typing to do full-text search across completed runs
- {:else} -
No runs found
-
There were no completed runs that match your query
- {/if} -
- Note that new runs might take a while to become searchable (by default ~5min) -
- {#if !$enterpriseLicense} -
- - - Full-text search on jobs is only available on EE. - - {/if} -
-
- {#if indexMetadata.indexed_until} - - Most recently indexed job was created at - - {/if} - {#if indexMetadata.lost_lock_ownership} - - - - The current indexer is no longer indexing new jobs. This is most likely - because of an ongoing deployment and indexing will resume once it's - complete. - - - {/if} -
-
- {/if} -
+ {/if}
diff --git a/frontend/src/lib/components/search/RunsSearch.svelte b/frontend/src/lib/components/search/RunsSearch.svelte new file mode 100644 index 0000000000..e955304786 --- /dev/null +++ b/frontend/src/lib/components/search/RunsSearch.svelte @@ -0,0 +1,266 @@ + + +
+ {#if loadingCompletedRuns} +
+
+ +
+
+ {:else if loadedRuns && loadedRuns.length > 0} +
+
+ {runSearchTotalCount} jobs matched the query +
+
+ {#each loadedRuns ?? [] as r} + {#if r.search_id === 'opt:load_more_jobs'} +
+ {#if loadingMoreJobs} +
+ +
+ {:else} + { + selectedItem = r + selectedWorkspace = undefined + const paginationOffset = runSearchTotalCount! - runSearchRemainingCount! + loadMoreJobs(searchTerm, paginationOffset) + }} + id={'opt:load_more_jobs'} + hovered={selectedItem && r?.search_id === selectedItem?.search_id} + containerClass="rounded-md px-2 py-1 my-2" + bind:mouseMoved + > + +
+ Some other {runSearchRemainingCount} jobs matched the query. Click to load more. + + +
+ + + {/if} + {:else} + { + selectedItem = r + selectedWorkspace = r?.document.workspace_id[0] + }} + on:keyboardOnlySelect={() => { + open = false + goto(`/run/${r?.document.id[0]}`) + }} + id={r?.document.id[0]} + hovered={selectedItem && r?.search_id === selectedItem?.search_id} + icon={r?.icon} + containerClass="rounded-md px-2 py-1 my-2" + bind:mouseMoved + > + +
+
+
+
{r?.document.script_path}
+
+
+ {displayDateOnly(new Date(r?.document.created_at[0]))} +
+
+ +
+
+
+
+ + + {/if} + {/each} +
+
+
+ {#if selectedItem === undefined} + Select a result to preview + {:else} +
+ +
+ {/if} +
+ {#if indexMetadata.indexed_until} + + Most recently indexed job was created at + + {/if} + {#if indexMetadata.lost_lock_ownership} + + + + The current indexer is no longer indexing new jobs. This is most likely because of an + ongoing deployment and indexing will resume once it's complete. + + + {/if} +
+
+ {:else} +
+
+ {#if searchTerm === ''} +
Enter your search terms
+
Start typing to do full-text search across completed runs
+ {:else} +
No runs found
+
There were no completed runs that match your query
+ {/if} +
+ Note that new runs might take a while to become searchable (by default ~5min) +
+ {#if !$enterpriseLicense} +
+ + + Full-text search on jobs is only available on EE. + + {/if} +
+
+ {#if indexMetadata.indexed_until} + + Most recently indexed job was created at + + {/if} + {#if indexMetadata.lost_lock_ownership} + + + + The current indexer is no longer indexing new jobs. This is most likely because of an + ongoing deployment and indexing will resume once it's complete. + + + {/if} +
+
+ {/if} +
From 6381cdf7d3823dfd246e3b6971e160d7e3ab8fe7 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 22 May 2025 18:26:41 +0200 Subject: [PATCH 29/45] improve service log select --- .../lib/components/ServiceLogsInner.svelte | 58 ++++++++++++++++--- .../apps/svelte-select/lib/Select.svelte | 13 +++-- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/frontend/src/lib/components/ServiceLogsInner.svelte b/frontend/src/lib/components/ServiceLogsInner.svelte index 9769e82fca..9b358adf80 100644 --- a/frontend/src/lib/components/ServiceLogsInner.svelte +++ b/frontend/src/lib/components/ServiceLogsInner.svelte @@ -15,6 +15,9 @@ import AnsiUp from 'ansi_up' import { scroll_into_view_if_needed_polyfill } from './multiselect/utils' import SplitPanesOrColumnOnMobile from './splitPanes/SplitPanesOrColumnOnMobile.svelte' + import Select from './apps/svelte-select/lib/Select.svelte' + import { SELECT_INPUT_DEFAULT_STYLE } from '$lib/defaults' + import DarkModeObserver from './DarkModeObserver.svelte' export let searchTerm: string export let queryParseErrors: string[] = [] @@ -319,10 +322,13 @@ const buckets = res['buckets'] sumOtherDocCount = res['sum_other_doc_count'] countsPerHost = new Map(buckets.map(({ key, doc_count }) => [key, doc_count])) - countsPerHost = buckets.reduce((acc: any, { key, doc_count }) => { - acc[key] = { doc_count } - return acc - }, {} as Record) + countsPerHost = buckets.reduce( + (acc: any, { key, doc_count }) => { + acc[key] = { doc_count } + return acc + }, + {} as Record + ) queryParseErrors = countLogsResponse.query_parse_errors ?? [] loadingLogCounts = false } @@ -376,7 +382,7 @@ let ret = {} for (const hk of Object.keys(countsPerHost)) { - let u = hk.split(",") + let u = hk.split(',') let [mode, wg, hn] = [u[0], u[1], u[2]] if (!ret[mode]) { @@ -392,8 +398,23 @@ return ret } + + function getSelectItems(allLogs: ByMode, countsPerHost: any): { label: string; value: any }[] { + return Object.entries(allLogsOrQueryResults(allLogs, countsPerHost)).flatMap(([mode, o1]) => + Object.entries(o1).flatMap(([wg, o2]) => + Object.keys(o2).map((hn) => ({ + label: hn, + value: [mode, wg, hn] + })) + ) + ) + } + + let darkMode = false + + @@ -436,7 +457,7 @@ month: '2-digit', hour: '2-digit', minute: '2-digit' - }) + }) : 'min datetime'} disabled /> @@ -476,7 +497,7 @@ month: '2-digit', hour: '2-digit', minute: '2-digit' - }) + }) : 'max datetime'} disabled /> @@ -548,6 +569,24 @@ >
{/if} +
+ + {:else if bucket_config.type === 'AwsOidc'} + + + {:else}
Unknown bucket type {bucket_config['type']}
{/if}