diff --git a/README.md b/README.md index a8f21b3442..d4d5cbb0ce 100644 --- a/README.md +++ b/README.md @@ -305,6 +305,9 @@ upcoming CLI tool. | DATABASE_CONNECTIONS | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool | All | | SUPERADMIN_SECRET | None | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev | Server | | TIMEOUT_WAIT_RESULT | 20 | The number of seconds to wait before timeout on the 'run_wait_result' endpoint | Worker | +| QUEUE_LIMIT_WAIT_RESULT | None | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit. | Worker | + + ## Run a local dev setup diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index c0addf1c2a..93336ef3b8 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -179,6 +179,7 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma "git" => "GitPython", "u" => "requests", "f" => "requests", + "shopify" => "ShopifyAPI", "seleniumwire" => "selenium-wire", }; diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 300eb4e786..78892088af 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -38,7 +38,7 @@ use crate::{ db::{UserDB, DB}, users::{require_owner_of_path, Authed}, variables::get_workspace_key, - BaseUrl, TimeoutWaitResult, + BaseUrl, QueueLimitWaitResult, TimeoutWaitResult, }; pub fn workspaced_service() -> Router { @@ -1298,12 +1298,13 @@ pub async fn run_wait_result_job_by_path( Extension(user_db): Extension, Extension(db): Extension, Extension(timeout): Extension>, + Extension(queue_limit): Extension>, Path((w_id, script_path)): Path<(String, StripPath)>, Query(run_query): Query, headers: HeaderMap, Json(args): Json>>, ) -> error::JsonResult { - check_queue_too_long(db, run_query.queue_limit).await?; + check_queue_too_long(db, queue_limit.0.or(run_query.queue_limit)).await?; let script_path = script_path.to_path(); let mut tx = user_db.clone().begin(&authed).await?; let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?; diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index c245754cf8..8db6297e78 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -54,6 +54,7 @@ pub struct CookieDomain(Option); pub struct CloudHosted(bool); pub struct ContentSecurityPolicy(String); pub struct TimeoutWaitResult(i32); +pub struct QueueLimitWaitResult(Option); pub use users::delete_expired_items_perdiodically; @@ -117,14 +118,18 @@ pub async fn run_server( .nest("/scripts", scripts::workspaced_service()) .nest( "/jobs", - jobs::workspaced_service().layer(Extension(Arc::new( - TimeoutWaitResult( + jobs::workspaced_service() + .layer(Extension(Arc::new(TimeoutWaitResult( std::env::var("TIMEOUT_WAIT_RESULT") .ok() .and_then(|x| x.parse().ok()) .unwrap_or(20), - ), - ))), + )))) + .layer(Extension(Arc::new(QueueLimitWaitResult( + std::env::var("QUEUE_LIMIT_WAIT_RESULT") + .ok() + .and_then(|x| x.parse().ok()), + )))), ) .nest( "/users", diff --git a/backend/windmill-worker/nsjail/run.bash.config.proto b/backend/windmill-worker/nsjail/run.bash.config.proto index e93e6b4557..9d6723e630 100644 --- a/backend/windmill-worker/nsjail/run.bash.config.proto +++ b/backend/windmill-worker/nsjail/run.bash.config.proto @@ -3,12 +3,11 @@ name: "bash run script" mode: ONCE hostname: "bash" log_level: ERROR -time_limit: 300 -rlimit_as: 2048 +rlimit_as: 4096 rlimit_cpu: 1000 -rlimit_fsize: 1024 -rlimit_nofile: 64 +rlimit_fsize: 1000 +rlimit_nofile: 10000 cwd: "/tmp" @@ -55,7 +54,7 @@ mount { dst: "/tmp" fstype: "tmpfs" rw: true - options: "size=500000000" + options: "size=800000000" } mount { @@ -87,4 +86,4 @@ iface_no_lo: true {SHARED_MOUNT} - +envar: "HOME=/tmp" diff --git a/backend/windmill-worker/nsjail/run.deno.config.proto b/backend/windmill-worker/nsjail/run.deno.config.proto index 9f106c231f..d9aa074840 100644 --- a/backend/windmill-worker/nsjail/run.deno.config.proto +++ b/backend/windmill-worker/nsjail/run.deno.config.proto @@ -3,12 +3,11 @@ name: "deno run script" mode: ONCE hostname: "deno" log_level: ERROR -time_limit: 300 rlimit_as: 16000 rlimit_cpu: 1000 -rlimit_fsize: 1024 -rlimit_nofile: 64 +rlimit_fsize: 1000 +rlimit_nofile: 10000 cwd: "/tmp" diff --git a/backend/windmill-worker/nsjail/run.go.config.proto b/backend/windmill-worker/nsjail/run.go.config.proto index 3af548d18c..7bc44b6a4c 100644 --- a/backend/windmill-worker/nsjail/run.go.config.proto +++ b/backend/windmill-worker/nsjail/run.go.config.proto @@ -3,12 +3,11 @@ name: "go run script" mode: ONCE hostname: "go" log_level: ERROR -time_limit: 300 -rlimit_as: 2048 +rlimit_as: 4096 rlimit_cpu: 1000 -rlimit_fsize: 1024 -rlimit_nofile: 64 +rlimit_fsize: 1000 +rlimit_nofile: 10000 cwd: "/tmp/go" diff --git a/backend/windmill-worker/nsjail/run.python3.config.proto b/backend/windmill-worker/nsjail/run.python3.config.proto index 5747d3cb3f..ef4e9a77d8 100644 --- a/backend/windmill-worker/nsjail/run.python3.config.proto +++ b/backend/windmill-worker/nsjail/run.python3.config.proto @@ -3,12 +3,11 @@ name: "python run script" mode: ONCE hostname: "python" log_level: ERROR -time_limit: 300 -rlimit_as: 2048 +rlimit_as: 4096 rlimit_cpu: 1000 -rlimit_fsize: 1024 -rlimit_nofile: 64 +rlimit_fsize: 1000 +rlimit_nofile: 10000 cwd: "/tmp" diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 8135a627e5..110d4140de 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -12,7 +12,10 @@ use itertools::Itertools; use lazy_static::lazy_static; use regex::Regex; use sqlx::{Pool, Postgres, Transaction}; -use std::{borrow::Borrow, collections::HashMap, io, panic, process::Stdio, time::Duration}; +use std::{ + borrow::Borrow, collections::HashMap, io, os::unix::process::ExitStatusExt, panic, + process::Stdio, time::Duration, +}; use tracing::{trace_span, Instrument}; use uuid::Uuid; @@ -2420,9 +2423,12 @@ async fn handle_child( } else if let Some(code) = status.code() { Err(error::Error::ExitStatus(code)) } else { - Err(error::Error::ExecutionErr( - "process terminated by signal".to_string(), - )) + Err(error::Error::ExecutionErr(format!( + "process terminated by signal: {:#?}, stopped_signal: {:#?}, core_dumped: {}", + status.signal(), + status.stopped_signal(), + status.core_dumped() + ))) } } Ok(Err(kill_reason)) => Err(Error::ExecutionErr(format!( diff --git a/docker/DockerfileHeavy b/docker/DockerfileHeavy index 9170330d9f..7a02470844 100644 --- a/docker/DockerfileHeavy +++ b/docker/DockerfileHeavy @@ -3,6 +3,9 @@ FROM ghcr.io/windmill-labs/windmill-ee RUN /usr/local/bin/python3 -m pip install nltk RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon +RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - &&\ + apt-get install -y nodejs + RUN apt-get update \ && apt-get install -y tesseract-ocr \ && rm -rf /var/lib/apt/lists/* diff --git a/frontend/package-lock.json b/frontend/package-lock.json index bf32b04a43..ca3e1b96f2 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -24,6 +24,7 @@ "monaco-yaml": "^4.0.2", "svelte-autosize": "^1.0.1", "svelte-chartjs": "^3.1.0", + "svelte-portal": "^2.2.0", "svelte-select": "^5.0.2", "vscode-ws-jsonrpc": "^2.0.1" }, @@ -6256,6 +6257,11 @@ "svelte": ">=3" } }, + "node_modules/svelte-portal": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/svelte-portal/-/svelte-portal-2.2.0.tgz", + "integrity": "sha512-jhtZWtD6cUE2nMw46dJ5VXWYiqnER+JH+V/BmNBQ5fNP/YdsJCpJi+DemUy9msklqGb0f+wLhJPBtKHLWQvzjg==" + }, "node_modules/svelte-preprocess": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/svelte-preprocess/-/svelte-preprocess-5.0.0.tgz", @@ -11548,6 +11554,11 @@ "dev": true, "requires": {} }, + "svelte-portal": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/svelte-portal/-/svelte-portal-2.2.0.tgz", + "integrity": "sha512-jhtZWtD6cUE2nMw46dJ5VXWYiqnER+JH+V/BmNBQ5fNP/YdsJCpJi+DemUy9msklqGb0f+wLhJPBtKHLWQvzjg==" + }, "svelte-preprocess": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/svelte-preprocess/-/svelte-preprocess-5.0.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index dd5bc7b8f4..910f271e1f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -21,8 +21,8 @@ "@sveltejs/package": "^1.0.1", "@tailwindcss/forms": "^0.5.3", "@tailwindcss/typography": "^0.5.8", - "@types/vscode": "~1.74.0", "@types/node": "^18.11.18", + "@types/vscode": "~1.74.0", "@typescript-eslint/eslint-plugin": "^5.46.1", "@typescript-eslint/parser": "^5.48.0", "@windmill-labs/svelvet": "^4.0.20", @@ -77,6 +77,7 @@ "monaco-yaml": "^4.0.2", "svelte-autosize": "^1.0.1", "svelte-chartjs": "^3.1.0", + "svelte-portal": "^2.2.0", "svelte-select": "^5.0.2", "vscode-ws-jsonrpc": "^2.0.1" }, diff --git a/frontend/src/lib/components/apps/components/HiddenComponent.svelte b/frontend/src/lib/components/apps/components/HiddenComponent.svelte new file mode 100644 index 0000000000..ab76911659 --- /dev/null +++ b/frontend/src/lib/components/apps/components/HiddenComponent.svelte @@ -0,0 +1,28 @@ + + + + + diff --git a/frontend/src/lib/components/apps/components/buttons/AppButton.svelte b/frontend/src/lib/components/apps/components/buttons/AppButton.svelte index 262961aa65..4908997a94 100644 --- a/frontend/src/lib/components/apps/components/buttons/AppButton.svelte +++ b/frontend/src/lib/components/apps/components/buttons/AppButton.svelte @@ -86,11 +86,11 @@ e?.stopPropagation() window.dispatchEvent(new Event('pointerup')) }} - on:click={(e) => { + on:click={async (e) => { e?.stopPropagation() e?.preventDefault() ownClick = true - runnableComponent?.runComponent() + await runnableComponent?.runComponent() if (recomputeIds) { recomputeIds.forEach((id) => { diff --git a/frontend/src/lib/components/apps/components/dateInputs/AppDateInput.svelte b/frontend/src/lib/components/apps/components/dateInputs/AppDateInput.svelte index dbfba0c3d8..fb7afee03a 100644 --- a/frontend/src/lib/components/apps/components/dateInputs/AppDateInput.svelte +++ b/frontend/src/lib/components/apps/components/dateInputs/AppDateInput.svelte @@ -4,6 +4,7 @@ import type { Output } from '../../rx' import type { AppEditorContext } from '../../types' import AlignWrapper from '../helpers/AlignWrapper.svelte' + import InputDefaultValue from '../helpers/InputDefaultValue.svelte' import InputValue from '../helpers/InputValue.svelte' export let id: string @@ -17,6 +18,7 @@ let labelValue: string = 'Title' let minValue: string = '' let maxValue: string = '' + let defaultValue: string | undefined = undefined $: outputs = $worldStore?.outputsById[id] as { result: Output @@ -25,11 +27,16 @@ function handleInput() { outputs?.result.set(input.value) } + + $: input && handleInput() + + + + import { Button, type ButtonType } from '$lib/components/common' + import { faUser } from '@fortawesome/free-solid-svg-icons' + import { getContext } from 'svelte' + import { Icon } from 'svelte-awesome' + import type { AppInput } from '../../inputType' + import type { Output } from '../../rx' + import type { AppEditorContext } from '../../types' + import AlignWrapper from '../helpers/AlignWrapper.svelte' + import InputValue from '../helpers/InputValue.svelte' + import type RunnableComponent from '../helpers/RunnableComponent.svelte' + import RunnableWrapper from '../helpers/RunnableWrapper.svelte' + import Portal from 'svelte-portal' + import Modal from '$lib/components/common/modal/Modal.svelte' + + export let id: string + export let componentInput: AppInput | undefined + export let configuration: Record + export let recomputeIds: string[] | undefined = undefined + export let extraQueryParams: Record = {} + export let horizontalAlignment: 'left' | 'center' | 'right' | undefined = undefined + export let verticalAlignment: 'top' | 'center' | 'bottom' | undefined = undefined + + export const staticOutputs: string[] = ['loading', 'result'] + + const { runnableComponents, worldStore } = getContext('AppEditorContext') + + let labelValue: string = 'Default label' + let color: ButtonType.Color + let size: ButtonType.Size + let runnableComponent: RunnableComponent + let disabled: boolean | undefined = undefined + + let isLoading: boolean = false + let ownClick: boolean = false + + let errors: Record = {} + $: errorsMessage = Object.values(errors) + .filter((x) => x != '') + .join('\n') + + $: noInputs = + componentInput?.type != 'runnable' || Object.keys(componentInput?.fields ?? {}).length == 0 + + $: outputs = $worldStore?.outputsById[id] as { + result: Output> + loading: Output + } + + $: if (outputs?.loading != undefined) { + outputs.loading.set(false, true) + } + + $: outputs?.loading.subscribe({ + next: (value) => { + isLoading = value + if (ownClick && !value) { + ownClick = false + } + } + }) + + $: loading = isLoading && ownClick + + let open: boolean = false + + + + + + + + + { + open = false + }} + on:confirmed={() => { + open = false + }} + > + +
+
+ {#if noInputs} +
+ Run forms are associated with a runnable that has user inputs. +
+ Once a script or flow is chosen, set some Runnable Inputs to + + User Input + + +
+ {/if} +
+
+ +
+
+ + + + + + {#if errorsMessage} +
{errorsMessage}
+ {/if} + + diff --git a/frontend/src/lib/components/apps/components/helpers/InputDefaultValue.svelte b/frontend/src/lib/components/apps/components/helpers/InputDefaultValue.svelte new file mode 100644 index 0000000000..6a04b4eada --- /dev/null +++ b/frontend/src/lib/components/apps/components/helpers/InputDefaultValue.svelte @@ -0,0 +1,22 @@ + diff --git a/frontend/src/lib/components/apps/components/numberInputs/AppNumberInput.svelte b/frontend/src/lib/components/apps/components/numberInputs/AppNumberInput.svelte index bf99c94ad5..7f5597b9cc 100644 --- a/frontend/src/lib/components/apps/components/numberInputs/AppNumberInput.svelte +++ b/frontend/src/lib/components/apps/components/numberInputs/AppNumberInput.svelte @@ -4,6 +4,7 @@ import type { Output } from '../../rx' import type { AppEditorContext } from '../../types' import AlignWrapper from '../helpers/AlignWrapper.svelte' + import InputDefaultValue from '../helpers/InputDefaultValue.svelte' import InputValue from '../helpers/InputValue.svelte' export let id: string @@ -12,31 +13,39 @@ export const staticOutputs: string[] = ['result'] const { worldStore } = getContext('AppEditorContext') - let value: number - let labelValue: string = 'Title' + let input: HTMLInputElement + + let defaultValue: number | undefined = undefined + let placeholder: string | undefined = undefined $: outputs = $worldStore?.outputsById[id] as { result: Output } - $: if (value || !value) { - // Disallow 'e' character in numbers - // if(value && value.toString().includes('e')) { - // value = +value.toString().replaceAll('e', '') - // } + + function handleInput() { + const value = input?.value const num = isNaN(+value) ? null : +value outputs?.result.set(num) } + + $: input && handleInput() - + + + { + e?.stopPropagation() + window.dispatchEvent(new Event('pointerup')) + }} type="number" inputmode="numeric" pattern="\d*" - placeholder="Type..." - class="h-full" + {placeholder} /> diff --git a/frontend/src/lib/components/apps/components/table/AppTable.svelte b/frontend/src/lib/components/apps/components/table/AppTable.svelte index 606962dc41..4d9f54a48e 100644 --- a/frontend/src/lib/components/apps/components/table/AppTable.svelte +++ b/frontend/src/lib/components/apps/components/table/AppTable.svelte @@ -65,6 +65,7 @@ result.length > 0 && // We need to wait until the component is mounted so the world is created mounted && + outputs && toggleRow({ original: result[0] }, 0) function setOptions(filteredResult: Array>) { diff --git a/frontend/src/lib/components/apps/components/textInputs/AppTextInput.svelte b/frontend/src/lib/components/apps/components/textInputs/AppTextInput.svelte index e94b170d6a..6b22e2a045 100644 --- a/frontend/src/lib/components/apps/components/textInputs/AppTextInput.svelte +++ b/frontend/src/lib/components/apps/components/textInputs/AppTextInput.svelte @@ -4,6 +4,7 @@ import type { Output } from '../../rx' import type { AppEditorContext } from '../../types' import AlignWrapper from '../helpers/AlignWrapper.svelte' + import InputDefaultValue from '../helpers/InputDefaultValue.svelte' import InputValue from '../helpers/InputValue.svelte' export let id: string @@ -16,6 +17,7 @@ let input: HTMLInputElement let placeholder: string | undefined = undefined + let defaultValue: string | undefined = undefined $: outputs = $worldStore?.outputsById[id] as { result: Output @@ -24,9 +26,13 @@ function handleInput() { outputs?.result.set(input.value) } + + $: input && handleInput() + + + {:else if component.type === 'formbuttoncomponent'} + {:else if component.type === 'checkboxcomponent'} {:else if component.type === 'textinputcomponent'} diff --git a/frontend/src/lib/components/apps/editor/GridEditor.svelte b/frontend/src/lib/components/apps/editor/GridEditor.svelte index f6dfab6164..eaba61912e 100644 --- a/frontend/src/lib/components/apps/editor/GridEditor.svelte +++ b/frontend/src/lib/components/apps/editor/GridEditor.svelte @@ -8,6 +8,7 @@ import RecomputeAllComponents from './RecomputeAllComponents.svelte' import type { Policy } from '$lib/gen' + import HiddenComponent from '../components/HiddenComponent.svelte' export let policy: Policy @@ -176,6 +177,20 @@ +{#if $app.hiddenInlineScripts} + {#each $app.hiddenInlineScripts as script, index} + {#if script} + + {/if} + {/each} +{/if} +