From 26670f62e5e9f7bd50895fae9dd98bc75b61bb44 Mon Sep 17 00:00:00 2001 From: Guillaume Bouvignies Date: Wed, 13 Dec 2023 07:08:07 +0100 Subject: [PATCH] feat: Add default timeout for instance and custom timeout for scripts (#2811) * feat: Add default timeout for instance and custom timeout for scripts * final * sqlx prepare * Logs a warning in the job logs * Fix merge * JOB_DEFAULT_TIMEOUT is an optional * Fix JOB_DEFAULT_TIMEOUT monitor loading --------- Co-authored-by: Ruben Fiszel --- README.md | 235 ++++++++++-------- ...039a6ea6697e5a49a633b767c052aa3e0a18.json} | 5 +- ...e47f57de0f073d3ce3bc7d21a7e404a83b5c.json} | 8 +- ...2a6342e25e815dcd8b9fd620ddb81063c5d76.json | 22 -- ...629e6ee2c053c7b1fa10f84462a327c739b3.json} | 7 +- ...7f6231a44b6ef5a52754074d136007f4f72a.json} | 10 +- ...433e1485324ff7dc802fe75d21c8c6db1d42.json} | 10 +- ...22991f51e1c945efc2924df6253d62b83bba.json} | 16 +- backend/src/main.rs | 15 +- backend/src/monitor.rs | 39 ++- backend/windmill-api/src/jobs.rs | 34 ++- backend/windmill-api/src/oauth2.rs | 1 + backend/windmill-api/src/scripts.rs | 10 +- .../windmill-common/src/global_settings.rs | 1 + backend/windmill-common/src/jobs.rs | 17 +- backend/windmill-common/src/lib.rs | 8 +- backend/windmill-queue/src/schedule.rs | 7 +- backend/windmill-worker/src/common.rs | 103 +++++--- backend/windmill-worker/src/worker.rs | 8 +- backend/windmill-worker/src/worker_flow.rs | 25 +- .../src/lib/components/ScriptBuilder.svelte | 30 ++- .../flows/common/FlowCardHeader.svelte | 4 +- .../flows/content/FlowModuleComponent.svelte | 2 +- .../flows/content/FlowModuleHeader.svelte | 152 ++++++----- .../src/lib/components/instanceSettings.ts | 8 + 25 files changed, 465 insertions(+), 312 deletions(-) rename backend/.sqlx/{query-a9915d3c72a278ff08fbe6571c2a4fc993507e255004107345b7151322e063de.json => query-0a686ca61444d7ad7484071727aa039a6ea6697e5a49a633b767c052aa3e0a18.json} (86%) rename backend/.sqlx/{query-f760667ffd6727443f023480627f62728bf9b6284d92e5fa9b97dc4e0b1c3a93.json => query-2f42460fdd8aa125c8fd46b3cd02e47f57de0f073d3ce3bc7d21a7e404a83b5c.json} (81%) delete mode 100644 backend/.sqlx/query-322038aae939d738f47b5bddaec2a6342e25e815dcd8b9fd620ddb81063c5d76.json rename backend/.sqlx/{query-7db59e88a10150ab6a3e81f164343e6bb2be6149fb690e63b8a0c376f39e9924.json => query-4a393e61e33c1204c01044254692629e6ee2c053c7b1fa10f84462a327c739b3.json} (85%) rename backend/.sqlx/{query-0234079d04372b82ec8f2990916433bd8771c21d22a1db02eb1e388a8bfb28ff.json => query-6b313cc9a57ae3c943bda4a3213f7f6231a44b6ef5a52754074d136007f4f72a.json} (85%) rename backend/.sqlx/{query-e541faa52c530c787d45c5ee70930687587c139606056bdc06cd860abeb0805b.json => query-b69891c25dd029b1a54e97ace292433e1485324ff7dc802fe75d21c8c6db1d42.json} (86%) rename backend/.sqlx/{query-c7febee7146a28481ce36f3570bceb38751a6527827525a582d9f8804981f552.json => query-ef132ac8d79579b08d7359789b6f22991f51e1c945efc2924df6253d62b83bba.json} (74%) diff --git a/README.md b/README.md index 8c569677a6..ba9f86637f 100644 --- a/README.md +++ b/README.md @@ -71,28 +71,37 @@ https://github.com/windmill-labs/windmill/assets/122811744/0b132cd1-ee67-4505-82 ## Main Concepts 1. Define a minimal and generic script in Python, TypeScript, Go or Bash that - solves a specific task. The code can be defined - in the [provided Web IDE](https://www.windmill.dev/docs/code_editor) or [synchronized with your own GitHub repo](https://www.windmill.dev/docs/advanced/cli/sync) (e.g. through [VS Code](https://www.windmill.dev/docs/cli_local_dev/vscode-extension) extension): + solves a specific task. The code can be defined in the + [provided Web IDE](https://www.windmill.dev/docs/code_editor) or + [synchronized with your own GitHub repo](https://www.windmill.dev/docs/advanced/cli/sync) + (e.g. through + [VS Code](https://www.windmill.dev/docs/cli_local_dev/vscode-extension) + extension): ![Step 1](./imgs/windmill-editor.png) -2. Your scripts parameters are automatically parsed and [generate a frontend](https://www.windmill.dev/docs/core_concepts/auto_generated_uis). +2. Your scripts parameters are automatically parsed and + [generate a frontend](https://www.windmill.dev/docs/core_concepts/auto_generated_uis). - ![Step 2](./imgs/windmill-run.png) - - ![Step 3](./imgs/windmill-result.png) +![Step 2](./imgs/windmill-run.png) -3. Make it [flow](https://www.windmill.dev/docs/flows/flow_editor)! You can chain your scripts or scripts made by the community - shared on [WindmillHub](https://hub.windmill.dev). +![Step 3](./imgs/windmill-result.png) + +3. Make it [flow](https://www.windmill.dev/docs/flows/flow_editor)! You can + chain your scripts or scripts made by the community shared on + [WindmillHub](https://hub.windmill.dev). ![Step 3](./imgs/windmill-flow.png) -4. Build [complex UIs](https://www.windmill.dev/docs/apps/app_editor) on top of your scripts and flows. +4. Build [complex UIs](https://www.windmill.dev/docs/apps/app_editor) on top of + your scripts and flows. ![Step 4](./imgs/windmill-builder.png) -Scripts and flows can also be triggered by a [cron schedule](https://www.windmill.dev/docs/core_concepts/scheduling) (e.g. '_/5 _ \* \* \*') or -through [webhooks](https://www.windmill.dev/docs/core_concepts/webhooks). +Scripts and flows can also be triggered by a +[cron schedule](https://www.windmill.dev/docs/core_concepts/scheduling) (e.g. +'_/5 _ \* \* \*') or through +[webhooks](https://www.windmill.dev/docs/core_concepts/webhooks). You can build your entire infra on top of Windmill! @@ -150,7 +159,8 @@ environment variables for the `wmill` client library to fetch resources and variables from your instance if necessary. See more: . -To develop & test locally scripts & flows, we recommend using the Windmill VS Code extension: . +To develop & test locally scripts & flows, we recommend using the Windmill VS +Code extension: . ## Stack @@ -173,9 +183,14 @@ To develop & test locally scripts & flows, we recommend using the Windmill VS Co ## Fastest Self-Hostable Workflow Engine -We have compared Windmill to other self-hostable workflow engines (Airflow, Prefect & Temporal) and Windmill is the most performant solution for both benchmarks: one flow composed of 40 lightweight tasks & one flow composed of 10 long-running tasks. +We have compared Windmill to other self-hostable workflow engines (Airflow, +Prefect & Temporal) and Windmill is the most performant solution for both +benchmarks: one flow composed of 40 lightweight tasks & one flow composed of 10 +long-running tasks. -All methodology & results on our [Benchmarks](https://www.windmill.dev/docs/misc/benchmarks/competitors#airflow-setup) page. +All methodology & results on our +[Benchmarks](https://www.windmill.dev/docs/misc/benchmarks/competitors#airflow-setup) +page. ![Fastest workflow engine](./imgs/fastest.png) @@ -220,7 +235,9 @@ compiling from source or using without a postgres super user, see ### Docker compose -Windmill can be deployed using 3 files: ([docker-compose.yml](./docker-compose.yml), [Caddyfile](./Caddyfile) and a [.env](./.env)) in a single command. +Windmill can be deployed using 3 files: +([docker-compose.yml](./docker-compose.yml), [Caddyfile](./Caddyfile) and a +[.env](./.env)) in a single command. Make sure Docker is started, and run: @@ -238,7 +255,8 @@ The default super-admin user is: admin@windmill.dev / changeme. From there, you can follow the setup app and create other users. -More details in [Self-Host Documention](https://www.windmill.dev/docs/advanced/self_host#docker). +More details in +[Self-Host Documention](https://www.windmill.dev/docs/advanced/self_host#docker). ### Kubernetes (k8s) and Helm charts @@ -262,22 +280,24 @@ wget "$ARTIFACT_URL" -O windmill Windmill Community Edition allows to configure the OAuth, SSO (including Google Workspace SSO, Microsoft/Azure and Okta) directly from the UI in the superadmin -settings. Do note that there is a limit of 50 SSO users on the community +settings. Do note that there is a limit of 10 SSO users on the community edition. [See documentation](https://www.windmill.dev/docs/misc/setup_oauth). ### Commercial license -To self-host Windmill, you must respect the terms of the [AGPLv3 license](https://www.gnu.org/licenses/agpl-3.0.en.html) which -you do not need to worry about for personal uses. For business uses, you should -be fine if you do not re-expose Windmill in any way to your users and are -comfortable with AGPLv3. +To self-host Windmill, you must respect the terms of the +[AGPLv3 license](https://www.gnu.org/licenses/agpl-3.0.en.html) which you do not +need to worry about for personal uses. For business uses, you should be fine if +you do not re-expose Windmill in any way to your users and are comfortable with +AGPLv3. -To [re-expose any Windmill parts to your users](https://www.windmill.dev/docs/misc/white_labelling) as a feature of your product, or -to build a feature on top of Windmill, to comply with AGPLv3 your product must -be AGPLv3 or you must get a commercial license. Contact us at - if you have any doubts. +To +[re-expose any Windmill parts to your users](https://www.windmill.dev/docs/misc/white_labelling) +as a feature of your product, or to build a feature on top of Windmill, to +comply with AGPLv3 your product must be AGPLv3 or you must get a commercial +license. Contact us at if you have any doubts. In addition, a commercial license grants you a dedicated engineer to transition your current infrastructure to Windmill, support with tight SLA, and our global @@ -286,91 +306,94 @@ nodes to 200+ nodes. ### Integrations -In Windmill, integrations are referred to as [resources and resource types](https://www.windmill.dev/docs/core_concepts/resources_and_types). Each Resource has a Resource Type that defines the schema that the resource needs to implement. +In Windmill, integrations are referred to as +[resources and resource types](https://www.windmill.dev/docs/core_concepts/resources_and_types). +Each Resource has a Resource Type that defines the schema that the resource +needs to implement. -On self-hosted instances, you might want to import all the approved resource types from -[WindmillHub](https://hub.windmill.dev). A setup script will prompt you to have -it being synced automatically everyday. +On self-hosted instances, you might want to import all the approved resource +types from [WindmillHub](https://hub.windmill.dev). A setup script will prompt +you to have it being synced automatically everyday. ## Environment Variables -| Environment Variable name | Default | Description | Api Server/Worker/All | -| --------------------------------------------- | ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | -| DATABASE_URL | | The Postgres database url. | All | -| WORKER_GROUP | default | The worker group the worker belongs to and get its configuration pulled from | Worker | -| MODE | standalone | The mode if the binary. Possible values: standalone, worker, server | All | -| SERVER_BIND_ADDR | 0.0.0.0 | IP Address on which to bind listening socket | Server | -| PORT | 8000 | Exposed port | Server | -| METRICS_ADDR | None | (ee only) The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001 | All | -| JSON_FMT | false | Output the logs in json format instead of logfmt | All | -| BASE_URL | http://localhost:8000 | The base url that is exposed publicly to access your instance. Is overriden by the instance settings if any. | Server | -| TIMEOUT | 60 _ 60 _ 24 \* 7 (1 week) | The maximum time of execution of a script. When reached, the job is failed as having timedout. | -| SCRIPT_TOKEN_EXPIRY | 900 | The default duration period of the ephemeral-token generated at the beginning of a script | Worker | -| ZOMBIE_JOB_TIMEOUT | 30 | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s) | Server | -| RESTART_ZOMBIE_JOBS | true | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed | Server | -| SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker | -| MAX_LOG_SIZE | 500000 | The maximum number of characters a job can emit (log + result) | Worker | -| DISABLE_NUSER | false | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting | Worker | -| KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker | -| LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker | -| S3_CACHE_BUCKET (EE only) | None | The S3 bucket to sync the cache of the workers to | Worker | -| TAR_CACHE_RATE (EE only) | 100 | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed). | Worker | -| SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server | -| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server | -| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker | -| PYTHON_PATH | /usr/local/bin/python3 | The path to the python binary. | Worker | -| GO_PATH | /usr/bin/go | The path to the go binary. | Worker | -| GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker | -| GOPROXY | | The GOPROXY env variable to use | Worker | -| NETRC | | The netrc content to use a private go registry | Worker | -| PIP_INDEX_URL | None | The index url to pass for pip. | Worker | -| PIP_EXTRA_INDEX_URL | None | The extra index url to pass to pip. | Worker | -| PIP_TRUSTED_HOST | None | The trusted host to pass to pip. | Worker | -| PATH | None | The path environment variable, usually inherited | Worker | -| HOME | None | The home directory to use for Go and Bash , usually inherited | Worker | -| DATABASE_CONNECTIONS | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool | All | -| SUPERADMIN_SECRET | None | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev | Server | -| TIMEOUT_WAIT_RESULT | 20 | The number of seconds to wait before timeout on the 'run_wait_result' endpoint | Worker | -| QUEUE_LIMIT_WAIT_RESULT | None | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit. | Worker | -| DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker | -| DENO_FLAGS | None | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json" | Worker | -| DENO_EXTRA_IMPORT_MAP | None | extra import map to use to run deno scripts (format: `key=value,key=value`) | Worker | -| NPM_CONFIG_REGISTRY | None | Registry to use for NPM dependencies, set if you have a private repository you need to use instead of the default public NPM registry | Worker | -| PIP_LOCAL_DEPENDENCIES | None | Specify dependencies that are installed locally and do not need to be solved nor installed again | | -| ADDITIONAL_PYTHON_PATHS | None | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill | Worker | -| INCLUDE_HEADERS | None | Whitelist of headers that are passed to jobs as args (separated by a comma) | Server | -| NUM_WORKERS | 1 | The number of worker per Worker instance (Set to 0 for API/Server instances, Set to 1 for normal workers, and > 1 for workers dedicated to native jobs) | Worker | -| INSTANCE_EVENTS_WEBHOOK | None | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails | -| GLOBAL_CACHE_INTERVAL | 10\*60 | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together. | Worker | -| WORKER_TAGS | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers | Worker | -| DEDICATED_WORKER | None | Unique script to run on that worker. Has to be in the form of `:` | Worker | -| CUSTOM_TAGS | None | The custom tags assignable to scripts. | Server | -| JOB_RETENTION_SECS | 60*60*24\*60 //60 days | **Overriden by the instance settings UI** The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete | -| WAIT_RESULT_FAST_POLL_INTERVAL_MS | 50 | The time in between polling for the run_wait_result endpoints in fast poll mode | Server | -| WAIT_RESULT_SLOW_POLL_INTERVAL_MS | 200 | The time in between polling for the run_wait_result endpoints in fast poll mode | Server | -| WAIT_RESULT_FAST_POLL_DURATION_SECS | 2 | The duration of fast poll mode before switching to slow poll | Server | -| EXIT_AFTER_NO_JOB_FOR_SECS | None | Exit worker if no job is received after duration in secs if defined | Worker | -| OAUTH_JSON_AS_BASE64 | None | Base64 encoded JSON of the OAuth configuration. e.g `OAUTH_JSON_AS_BASE64=$(base64 oauth.json \| tr -d '\n')` to encode it | Server | -| REQUEST_SIZE_LIMIT | 2097152 (2MB) | Max request size which impact the maximum size of resources and payload size of job args | Server | -| ACCEPT_INVALID_CERTS | false | Accept invalid certificates, including self-signed and expired certificates | Server | -| HTTP_PROXY | None | http_proxy | Server + Worker | -| HTTPS_PROXY | None | https_proxy | Server + Worker | -| NO_PROXY | None | no_proxy | Server + Worker | -| SMTP_FROM | None | the address to use as the from field for emails send | Server | -| SMTP_HOST | None | host for the smtp server to send invite emails | Server | -| SMTP_PORT | 587 | port for the smtp server to send invite emails | Server | -| SMTP_USERNAME | None | username for the smtp server to send invite emails | Server | -| SMTP_PASSWORD | None | password for the smtp server to send invite emails | Server | -| SMTP_TLS_IMPLICIT | false | https://docs.rs/mail-send/latest/mail_send/struct.SmtpClientBuilder.html#method.implicit_tlsemails | Server | -| CREATE_WORKSPACE_REQUIRE_SUPERADMIN | true | If true, only superadmin can create workspaces | Server | -| GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE | None | Path to a script or flow to run when a root job fails. The path needs to be prefixed with either `script/` or `flow/` to indicate the kind of error handler being used (assuming `script/` by default). The error handler will be run in and from the admins workspace | Server | -| WHITELIST_ENVS | None | List of envs variables, separated by a ',' that are whitelisted as being safe to passthrough the workers | Worker | -| SAML_METADATA | None | SAML Metadata URL to enable SAML SSO (EE only) | Server | -| SECRET_SALT | None | Secret Salt used for encryption and decryption of secrets. If defined, the secrets will not be decryptable unless the right salt is passed in, which is the case for the workers and the server | Server + Worker | -| OPENAI_AZURE_BASE_PATH | None | Azure OpenAI path to be used instead of the OpenAI path. All Windmill AI features will run on the specified deployed model. Format: `https://{your-resource-name}.openai.azure.com/openai/deployments/{deployment-id}` | Server | -| DISABLE_EMBEDDING | false | Disable local embedding search of hub scripts | Server | -| DISABLE_NSJAIL | true | Disable Nsjail Sandboxing | Worker | -| DISABLE_SERVER | false | Disable the external API, operate as a worker only instance | Worker | +| Environment Variable name | Default | Description | Api Server/Worker/All | +| --------------------------------------------- | ------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | +| DATABASE_URL | | The Postgres database url. | All | +| WORKER_GROUP | default | The worker group the worker belongs to and get its configuration pulled from | Worker | +| MODE | standalone | The mode if the binary. Possible values: standalone, worker, server | All | +| SERVER_BIND_ADDR | 0.0.0.0 | IP Address on which to bind listening socket | Server | +| PORT | 8000 | Exposed port | Server | +| METRICS_ADDR | None | (ee only) The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001 | All | +| JSON_FMT | false | Output the logs in json format instead of logfmt | All | +| BASE_URL | http://localhost:8000 | The base url that is exposed publicly to access your instance. Is overriden by the instance settings if any. | Server | +| TIMEOUT | 60 _ 60 _ 24 \* 7 (1 week) | The maximum time of execution of a script. When reached, the job is failed as having timedout. | +| SCRIPT_TOKEN_EXPIRY | 900 | The default duration period of the ephemeral-token generated at the beginning of a script | Worker | +| ZOMBIE_JOB_TIMEOUT | 30 | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s) | Server | +| RESTART_ZOMBIE_JOBS | true | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed | Server | +| SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker | +| MAX_LOG_SIZE | 500000 | The maximum number of characters a job can emit (log + result) | Worker | +| DISABLE_NUSER | false | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting | Worker | +| KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker | +| LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker | +| S3_CACHE_BUCKET (EE only) | None | The S3 bucket to sync the cache of the workers to | Worker | +| TAR_CACHE_RATE (EE only) | 100 | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed). | Worker | +| SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server | +| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server | +| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker | +| PYTHON_PATH | /usr/local/bin/python3 | The path to the python binary. | Worker | +| GO_PATH | /usr/bin/go | The path to the go binary. | Worker | +| GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker | +| GOPROXY | | The GOPROXY env variable to use | Worker | +| NETRC | | The netrc content to use a private go registry | Worker | +| PIP_INDEX_URL | None | The index url to pass for pip. | Worker | +| PIP_EXTRA_INDEX_URL | None | The extra index url to pass to pip. | Worker | +| PIP_TRUSTED_HOST | None | The trusted host to pass to pip. | Worker | +| PATH | None | The path environment variable, usually inherited | Worker | +| HOME | None | The home directory to use for Go and Bash , usually inherited | Worker | +| DATABASE_CONNECTIONS | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool | All | +| SUPERADMIN_SECRET | None | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev | Server | +| TIMEOUT_WAIT_RESULT | 20 | The number of seconds to wait before timeout on the 'run_wait_result' endpoint | Worker | +| QUEUE_LIMIT_WAIT_RESULT | None | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit. | Worker | +| DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker | +| DENO_FLAGS | None | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json" | Worker | +| DENO_EXTRA_IMPORT_MAP | None | extra import map to use to run deno scripts (format: `key=value,key=value`) | Worker | +| NPM_CONFIG_REGISTRY | None | Registry to use for NPM dependencies, set if you have a private repository you need to use instead of the default public NPM registry | Worker | +| PIP_LOCAL_DEPENDENCIES | None | Specify dependencies that are installed locally and do not need to be solved nor installed again | | +| ADDITIONAL_PYTHON_PATHS | None | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill | Worker | +| INCLUDE_HEADERS | None | Whitelist of headers that are passed to jobs as args (separated by a comma) | Server | +| NUM_WORKERS | 1 | The number of worker per Worker instance (Set to 0 for API/Server instances, Set to 1 for normal workers, and > 1 for workers dedicated to native jobs) | Worker | +| INSTANCE_EVENTS_WEBHOOK | None | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails | +| GLOBAL_CACHE_INTERVAL | 10\*60 | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together. | Worker | +| WORKER_TAGS | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers | Worker | +| DEDICATED_WORKER | None | Unique script to run on that worker. Has to be in the form of `:` | Worker | +| CUSTOM_TAGS | None | The custom tags assignable to scripts. | Server | +| JOB_RETENTION_SECS | 60*60*24\*60 //60 days | **Overriden by the instance settings UI** The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete | +| WAIT_RESULT_FAST_POLL_INTERVAL_MS | 50 | The time in between polling for the run_wait_result endpoints in fast poll mode | Server | +| WAIT_RESULT_SLOW_POLL_INTERVAL_MS | 200 | The time in between polling for the run_wait_result endpoints in fast poll mode | Server | +| WAIT_RESULT_FAST_POLL_DURATION_SECS | 2 | The duration of fast poll mode before switching to slow poll | Server | +| EXIT_AFTER_NO_JOB_FOR_SECS | None | Exit worker if no job is received after duration in secs if defined | Worker | +| OAUTH_JSON_AS_BASE64 | None | Base64 encoded JSON of the OAuth configuration. e.g `OAUTH_JSON_AS_BASE64=$(base64 oauth.json \| tr -d '\n')` to encode it | Server | +| REQUEST_SIZE_LIMIT | 2097152 (2MB) | Max request size which impact the maximum size of resources and payload size of job args | Server | +| ACCEPT_INVALID_CERTS | false | Accept invalid certificates, including self-signed and expired certificates | Server | +| HTTP_PROXY | None | http_proxy | Server + Worker | +| HTTPS_PROXY | None | https_proxy | Server + Worker | +| NO_PROXY | None | no_proxy | Server + Worker | +| SMTP_FROM | None | the address to use as the from field for emails send | Server | +| SMTP_HOST | None | host for the smtp server to send invite emails | Server | +| SMTP_PORT | 587 | port for the smtp server to send invite emails | Server | +| SMTP_USERNAME | None | username for the smtp server to send invite emails | Server | +| SMTP_PASSWORD | None | password for the smtp server to send invite emails | Server | +| SMTP_TLS_IMPLICIT | false | https://docs.rs/mail-send/latest/mail_send/struct.SmtpClientBuilder.html#method.implicit_tlsemails | Server | +| CREATE_WORKSPACE_REQUIRE_SUPERADMIN | true | If true, only superadmin can create workspaces | Server | +| GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE | None | Path to a script or flow to run when a root job fails. The path needs to be prefixed with either `script/` or `flow/` to indicate the kind of error handler being used (assuming `script/` by default). The error handler will be run in and from the admins workspace | Server | +| WHITELIST_ENVS | None | List of envs variables, separated by a ',' that are whitelisted as being safe to passthrough the workers | Worker | +| SAML_METADATA | None | SAML Metadata URL to enable SAML SSO (EE only) | Server | +| SECRET_SALT | None | Secret Salt used for encryption and decryption of secrets. If defined, the secrets will not be decryptable unless the right salt is passed in, which is the case for the workers and the server | Server + Worker | +| OPENAI_AZURE_BASE_PATH | None | Azure OpenAI path to be used instead of the OpenAI path. All Windmill AI features will run on the specified deployed model. Format: `https://{your-resource-name}.openai.azure.com/openai/deployments/{deployment-id}` | Server | +| DISABLE_EMBEDDING | false | Disable local embedding search of hub scripts | Server | +| DISABLE_NSJAIL | true | Disable Nsjail Sandboxing | Worker | +| DISABLE_SERVER | false | Disable the external API, operate as a worker only instance | Worker | ## Run a local dev setup diff --git a/backend/.sqlx/query-a9915d3c72a278ff08fbe6571c2a4fc993507e255004107345b7151322e063de.json b/backend/.sqlx/query-0a686ca61444d7ad7484071727aa039a6ea6697e5a49a633b767c052aa3e0a18.json similarity index 86% rename from backend/.sqlx/query-a9915d3c72a278ff08fbe6571c2a4fc993507e255004107345b7151322e063de.json rename to backend/.sqlx/query-0a686ca61444d7ad7484071727aa039a6ea6697e5a49a633b767c052aa3e0a18.json index 212a2a5f66..b1feff000e 100644 --- a/backend/.sqlx/query-a9915d3c72a278ff08fbe6571c2a4fc993507e255004107345b7151322e063de.json +++ b/backend/.sqlx/query-0a686ca61444d7ad7484071727aa039a6ea6697e5a49a633b767c052aa3e0a18.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH uuid_table as (\n select gen_random_uuid() as uuid from generate_series(1, $11)\n )\n INSERT INTO queue \n (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, concurrent_limit, concurrency_time_window_s)\n (SELECT uuid, $1, $2, $3, $4, ('{ \"uuid\": \"' || uuid || '\" }')::jsonb, $5, $6, $7, $8, $9, $10, $12, $13 FROM uuid_table) \n RETURNING id", + "query": "WITH uuid_table as (\n select gen_random_uuid() as uuid from generate_series(1, $11)\n )\n INSERT INTO queue \n (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, concurrent_limit, concurrency_time_window_s, timeout)\n (SELECT uuid, $1, $2, $3, $4, ('{ \"uuid\": \"' || uuid || '\" }')::jsonb, $5, $6, $7, $8, $9, $10, $12, $13, $14 FROM uuid_table) \n RETURNING id", "describe": { "columns": [ { @@ -66,6 +66,7 @@ "Varchar", "Int4", "Int4", + "Int4", "Int4" ] }, @@ -73,5 +74,5 @@ false ] }, - "hash": "a9915d3c72a278ff08fbe6571c2a4fc993507e255004107345b7151322e063de" + "hash": "0a686ca61444d7ad7484071727aa039a6ea6697e5a49a633b767c052aa3e0a18" } diff --git a/backend/.sqlx/query-f760667ffd6727443f023480627f62728bf9b6284d92e5fa9b97dc4e0b1c3a93.json b/backend/.sqlx/query-2f42460fdd8aa125c8fd46b3cd02e47f57de0f073d3ce3bc7d21a7e404a83b5c.json similarity index 81% rename from backend/.sqlx/query-f760667ffd6727443f023480627f62728bf9b6284d92e5fa9b97dc4e0b1c3a93.json rename to backend/.sqlx/query-2f42460fdd8aa125c8fd46b3cd02e47f57de0f073d3ce3bc7d21a7e404a83b5c.json index f32ede17a7..030a85c000 100644 --- a/backend/.sqlx/query-f760667ffd6727443f023480627f62728bf9b6284d92e5fa9b97dc4e0b1c3a93.json +++ b/backend/.sqlx/query-2f42460fdd8aa125c8fd46b3cd02e47f57de0f073d3ce3bc7d21a7e404a83b5c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use from script where path = $1 AND workspace_id = $2 AND\n created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)", + "query": "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, timeout FROM script where path = $1 AND workspace_id = $2 AND\n created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n deleted = false AND archived = false)", "describe": { "columns": [ { @@ -66,8 +66,8 @@ }, { "ordinal": 8, - "name": "delete_after_use", - "type_info": "Bool" + "name": "timeout", + "type_info": "Int4" } ], "parameters": { @@ -88,5 +88,5 @@ true ] }, - "hash": "f760667ffd6727443f023480627f62728bf9b6284d92e5fa9b97dc4e0b1c3a93" + "hash": "2f42460fdd8aa125c8fd46b3cd02e47f57de0f073d3ce3bc7d21a7e404a83b5c" } diff --git a/backend/.sqlx/query-322038aae939d738f47b5bddaec2a6342e25e815dcd8b9fd620ddb81063c5d76.json b/backend/.sqlx/query-322038aae939d738f47b5bddaec2a6342e25e815dcd8b9fd620ddb81063c5d76.json deleted file mode 100644 index 0e672f46a2..0000000000 --- a/backend/.sqlx/query-322038aae939d738f47b5bddaec2a6342e25e815dcd8b9fd620ddb81063c5d76.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT flow_status->'cleanup_module' as cleanup_module\n FROM queue\n WHERE id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "cleanup_module", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "322038aae939d738f47b5bddaec2a6342e25e815dcd8b9fd620ddb81063c5d76" -} diff --git a/backend/.sqlx/query-7db59e88a10150ab6a3e81f164343e6bb2be6149fb690e63b8a0c376f39e9924.json b/backend/.sqlx/query-4a393e61e33c1204c01044254692629e6ee2c053c7b1fa10f84462a327c739b3.json similarity index 85% rename from backend/.sqlx/query-7db59e88a10150ab6a3e81f164343e6bb2be6149fb690e63b8a0c376f39e9924.json rename to backend/.sqlx/query-4a393e61e33c1204c01044254692629e6ee2c053c7b1fa10f84462a327c739b3.json index 28167accf8..cac6b88592 100644 --- a/backend/.sqlx/query-7db59e88a10150ab6a3e81f164343e6bb2be6149fb690e63b8a0c376f39e9924.json +++ b/backend/.sqlx/query-4a393e61e33c1204c01044254692629e6ee2c053c7b1fa10f84462a327c739b3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25)", + "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26)", "describe": { "columns": [], "parameters": { @@ -63,10 +63,11 @@ "Bool", "Int2", "Bool", - "Bool" + "Bool", + "Int4" ] }, "nullable": [] }, - "hash": "7db59e88a10150ab6a3e81f164343e6bb2be6149fb690e63b8a0c376f39e9924" + "hash": "4a393e61e33c1204c01044254692629e6ee2c053c7b1fa10f84462a327c739b3" } diff --git a/backend/.sqlx/query-0234079d04372b82ec8f2990916433bd8771c21d22a1db02eb1e388a8bfb28ff.json b/backend/.sqlx/query-6b313cc9a57ae3c943bda4a3213f7f6231a44b6ef5a52754074d136007f4f72a.json similarity index 85% rename from backend/.sqlx/query-0234079d04372b82ec8f2990916433bd8771c21d22a1db02eb1e388a8bfb28ff.json rename to backend/.sqlx/query-6b313cc9a57ae3c943bda4a3213f7f6231a44b6ef5a52754074d136007f4f72a.json index 3f2301cd28..72c175aff7 100644 --- a/backend/.sqlx/query-0234079d04372b82ec8f2990916433bd8771c21d22a1db02eb1e388a8bfb28ff.json +++ b/backend/.sqlx/query-6b313cc9a57ae3c943bda4a3213f7f6231a44b6ef5a52754074d136007f4f72a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "select tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use from script where hash = $1 AND workspace_id = $2", + "query": "select tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where hash = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -63,6 +63,11 @@ "ordinal": 7, "name": "delete_after_use", "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "timeout", + "type_info": "Int4" } ], "parameters": { @@ -79,8 +84,9 @@ false, true, true, + true, true ] }, - "hash": "0234079d04372b82ec8f2990916433bd8771c21d22a1db02eb1e388a8bfb28ff" + "hash": "6b313cc9a57ae3c943bda4a3213f7f6231a44b6ef5a52754074d136007f4f72a" } diff --git a/backend/.sqlx/query-e541faa52c530c787d45c5ee70930687587c139606056bdc06cd860abeb0805b.json b/backend/.sqlx/query-b69891c25dd029b1a54e97ace292433e1485324ff7dc802fe75d21c8c6db1d42.json similarity index 86% rename from backend/.sqlx/query-e541faa52c530c787d45c5ee70930687587c139606056bdc06cd860abeb0805b.json rename to backend/.sqlx/query-b69891c25dd029b1a54e97ace292433e1485324ff7dc802fe75d21c8c6db1d42.json index b3297c8172..f95d9d95b2 100644 --- a/backend/.sqlx/query-e541faa52c530c787d45c5ee70930687587c139606056bdc06cd860abeb0805b.json +++ b/backend/.sqlx/query-b69891c25dd029b1a54e97ace292433e1485324ff7dc802fe75d21c8c6db1d42.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "select path, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use from script where hash = $1 AND workspace_id = $2", + "query": "select path, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where hash = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -68,6 +68,11 @@ "ordinal": 8, "name": "delete_after_use", "type_info": "Bool" + }, + { + "ordinal": 9, + "name": "timeout", + "type_info": "Int4" } ], "parameters": { @@ -85,8 +90,9 @@ false, true, true, + true, true ] }, - "hash": "e541faa52c530c787d45c5ee70930687587c139606056bdc06cd860abeb0805b" + "hash": "b69891c25dd029b1a54e97ace292433e1485324ff7dc802fe75d21c8c6db1d42" } diff --git a/backend/.sqlx/query-c7febee7146a28481ce36f3570bceb38751a6527827525a582d9f8804981f552.json b/backend/.sqlx/query-ef132ac8d79579b08d7359789b6f22991f51e1c945efc2924df6253d62b83bba.json similarity index 74% rename from backend/.sqlx/query-c7febee7146a28481ce36f3570bceb38751a6527827525a582d9f8804981f552.json rename to backend/.sqlx/query-ef132ac8d79579b08d7359789b6f22991f51e1c945efc2924df6253d62b83bba.json index 603bea689f..73af3f369c 100644 --- a/backend/.sqlx/query-c7febee7146a28481ce36f3570bceb38751a6527827525a582d9f8804981f552.json +++ b/backend/.sqlx/query-ef132ac8d79579b08d7359789b6f22991f51e1c945efc2924df6253d62b83bba.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority from script where path = $1 AND workspace_id = $2 AND\n created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n deleted = false AND archived = false)", + "query": "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where path = $1 AND workspace_id = $2 AND\n created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)", "describe": { "columns": [ { @@ -63,6 +63,16 @@ "ordinal": 7, "name": "priority", "type_info": "Int2" + }, + { + "ordinal": 8, + "name": "delete_after_use", + "type_info": "Bool" + }, + { + "ordinal": 9, + "name": "timeout", + "type_info": "Int4" } ], "parameters": { @@ -79,8 +89,10 @@ true, false, true, + true, + true, true ] }, - "hash": "c7febee7146a28481ce36f3570bceb38751a6527827525a582d9f8804981f552" + "hash": "ef132ac8d79579b08d7359789b6f22991f51e1c945efc2924df6253d62b83bba" } diff --git a/backend/src/main.rs b/backend/src/main.rs index 186751e8f8..d8a487f66d 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -24,9 +24,9 @@ use windmill_common::{ global_settings::{ BASE_URL_SETTING, CUSTOM_TAGS_SETTING, DISABLE_STATS_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, - REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RETENTION_PERIOD_SECS_SETTING, + JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, + NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING, + REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, }, stats::schedule_stats, utils::{rd_string, Mode}, @@ -42,9 +42,9 @@ use windmill_worker::{ use crate::monitor::{ initial_load, load_keep_job_dir, load_require_preexisting_user, monitor_db, monitor_pool, - reload_base_url_setting, reload_extra_pip_index_url_setting, reload_license_key, - reload_npm_config_registry_setting, reload_retention_period_setting, reload_server_config, - reload_worker_config, + reload_base_url_setting, reload_extra_pip_index_url_setting, + reload_job_default_timeout_setting, reload_license_key, reload_npm_config_registry_setting, + reload_retention_period_setting, reload_server_config, reload_worker_config, }; const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); @@ -410,6 +410,9 @@ Windmill Community Edition {GIT_VERSION} RETENTION_PERIOD_SECS_SETTING => { reload_retention_period_setting(&db).await }, + JOB_DEFAULT_TIMEOUT_SECS_SETTING => { + reload_job_default_timeout_setting(&db).await + }, EXTRA_PIP_INDEX_URL_SETTING => { reload_extra_pip_index_url_setting(&db).await }, diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index c3855c0abf..7879f8f056 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -22,9 +22,10 @@ use windmill_common::{ error, global_settings::{ BASE_URL_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, - EXTRA_PIP_INDEX_URL_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, - NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING, - REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, + LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, + REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, + RETENTION_PERIOD_SECS_SETTING, }, jobs::{JobKind, QueuedJob}, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, @@ -34,8 +35,8 @@ use windmill_common::{ BASE_URL, DB, METRICS_DEBUG_ENABLED, METRICS_ENABLED, }; use windmill_worker::{ - create_token_for_owner, handle_job_error, AuthedClient, KEEP_JOB_DIR, NPM_CONFIG_REGISTRY, - PIP_EXTRA_INDEX_URL, SCRIPT_TOKEN_EXPIRY, + create_token_for_owner, handle_job_error, AuthedClient, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, + NPM_CONFIG_REGISTRY, PIP_EXTRA_INDEX_URL, SCRIPT_TOKEN_EXPIRY, }; #[cfg(feature = "enterprise")] @@ -76,7 +77,6 @@ lazy_static::lazy_static! { ).unwrap(); static ref JOB_RETENTION_SECS: Arc> = Arc::new(RwLock::new(0)); - } pub async fn initial_load( @@ -274,7 +274,7 @@ pub async fn delete_expired_items(db: &DB) -> () { } pub async fn reload_extra_pip_index_url_setting(db: &DB) { - if let Err(e) = reload_option_string_setting( + if let Err(e) = reload_option_setting( db, EXTRA_PIP_INDEX_URL_SETTING, "PIP_EXTRA_INDEX_URL", @@ -287,7 +287,7 @@ pub async fn reload_extra_pip_index_url_setting(db: &DB) { } pub async fn reload_npm_config_registry_setting(db: &DB) { - if let Err(e) = reload_option_string_setting( + if let Err(e) = reload_option_setting( db, NPM_CONFIG_REGISTRY_SETTING, "NPM_CONFIG_REGISTRY", @@ -314,6 +314,19 @@ pub async fn reload_retention_period_setting(db: &DB) { } } +pub async fn reload_job_default_timeout_setting(db: &DB) { + if let Err(e) = reload_option_setting( + db, + JOB_DEFAULT_TIMEOUT_SECS_SETTING, + "JOB_DEFAULT_TIMEOUT_SECS", + JOB_DEFAULT_TIMEOUT.clone(), + ) + .await + { + tracing::error!("Error reloading job default timeout: {:?}", e) + } +} + pub async fn reload_request_size(db: &DB) { if let Err(e) = reload_setting( db, @@ -359,11 +372,11 @@ pub async fn reload_license_key(db: &DB) -> error::Result<()> { Ok(()) } -pub async fn reload_option_string_setting( +pub async fn reload_option_setting( db: &DB, setting_name: &str, std_env_var: &str, - lock: Arc>>, + lock: Arc>>, ) -> error::Result<()> { let q = sqlx::query!( "SELECT value FROM global_settings WHERE name = $1", @@ -372,10 +385,12 @@ pub async fn reload_option_string_setting( .fetch_optional(db) .await?; - let mut value = std::env::var(std_env_var).ok(); + let mut value = std::env::var(std_env_var) + .ok() + .and_then(|x| x.parse::().ok()); if let Some(q) = q { - if let Ok(v) = serde_json::from_value::(q.value.clone()) { + if let Ok(v) = serde_json::from_value::(q.value.clone()) { tracing::info!( "Loaded setting {setting_name} from db config: {:#?}", &q.value diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 933121d47d..341b07174d 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -330,9 +330,10 @@ pub async fn get_path_tag_limits_cache_for_hash( Option, Option, Option, + Option, )> { let script = sqlx::query!( - "select path, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use from script where hash = $1 AND workspace_id = $2", + "select path, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where hash = $1 AND workspace_id = $2", hash, w_id ) @@ -353,6 +354,7 @@ pub async fn get_path_tag_limits_cache_for_hash( script.dedicated_worker, script.priority, script.delete_after_use, + script.timeout, )) } @@ -1814,7 +1816,7 @@ pub async fn run_job_by_path( check_scopes(&authed, || format!("run:script/{script_path}"))?; - let (job_payload, tag, _delete_after_use) = + let (job_payload, tag, _delete_after_use, timeout) = script_path_to_payload(script_path, &db, &w_id).await?; let scheduled_for = run_query.get_scheduled_for(&db).await?; @@ -1840,7 +1842,7 @@ pub async fn run_job_by_path( None, !run_query.invisible_to_owner.unwrap_or(false), tag, - None, + timeout, None, None, ) @@ -2088,7 +2090,7 @@ pub async fn run_wait_result_job_by_path_get( let script_path = script_path.to_path(); check_scopes(&authed, || format!("run:script/{script_path}"))?; - let (job_payload, tag, delete_after_use) = + let (job_payload, tag, delete_after_use, timeout) = script_path_to_payload(script_path, &db, &w_id).await?; check_tag_available_for_workspace(&w_id, &tag).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq); @@ -2112,7 +2114,7 @@ pub async fn run_wait_result_job_by_path_get( None, !run_query.invisible_to_owner.unwrap_or(false), tag, - None, + timeout, None, None, ) @@ -2205,7 +2207,7 @@ async fn run_wait_result_script_by_path_internal( let script_path = script_path.to_path(); check_scopes(&authed, || format!("run:script/{script_path}"))?; - let (job_payload, tag, delete_after_use) = + let (job_payload, tag, delete_after_use, timeout) = script_path_to_payload(script_path, &db, &w_id).await?; check_tag_available_for_workspace(&w_id, &tag).await?; @@ -2230,7 +2232,7 @@ async fn run_wait_result_script_by_path_internal( None, !run_query.invisible_to_owner.unwrap_or(false), tag, - None, + timeout, None, None, ) @@ -2269,6 +2271,7 @@ pub async fn run_wait_result_script_by_hash( dedicated_worker, priority, delete_after_use, + timeout, ) = get_path_tag_limits_cache_for_hash(&db, &w_id, hash).await?; check_scopes(&authed, || format!("run:script/{path}"))?; @@ -2303,7 +2306,7 @@ pub async fn run_wait_result_script_by_hash( None, !run_query.invisible_to_owner.unwrap_or(false), tag, - None, + timeout, None, None, ) @@ -2483,6 +2486,7 @@ async fn add_batch_jobs( dedicated_worker, concurrent_limit, concurrent_time_window_s, + timeout, ) = match batch_info.kind.as_str() { "script" => { if let Some(path) = batch_info.path { @@ -2496,6 +2500,7 @@ async fn add_batch_jobs( dedicated_worker, _priority, _delete_after_use, + timeout, ) = get_latest_deployed_hash_for_path(&db, &w_id, &path).await?; ( Some(script_hash), @@ -2505,6 +2510,7 @@ async fn add_batch_jobs( dedicated_worker, concurrent_limit, concurrency_time_window_s, + timeout, ) } else { Err(anyhow::anyhow!( @@ -2563,7 +2569,7 @@ async fn add_batch_jobs( } return Ok(Json(uuids)); } - "noop" => (None, None, JobKind::Noop, None, None, None, None), + "noop" => (None, None, JobKind::Noop, None, None, None, None, None), _ => { return Err(error::Error::BadRequest(format!( "Invalid batch kind: {}", @@ -2589,8 +2595,8 @@ async fn add_batch_jobs( select gen_random_uuid() as uuid from generate_series(1, $11) ) INSERT INTO queue - (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, concurrent_limit, concurrency_time_window_s) - (SELECT uuid, $1, $2, $3, $4, ('{ "uuid": "' || uuid || '" }')::jsonb, $5, $6, $7, $8, $9, $10, $12, $13 FROM uuid_table) + (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, concurrent_limit, concurrency_time_window_s, timeout) + (SELECT uuid, $1, $2, $3, $4, ('{ "uuid": "' || uuid || '" }')::jsonb, $5, $6, $7, $8, $9, $10, $12, $13, $14 FROM uuid_table) RETURNING id"#, hash.map(|h| h.0), path, @@ -2604,7 +2610,8 @@ async fn add_batch_jobs( w_id, n, concurrent_limit, - concurrent_time_window_s + concurrent_time_window_s, + timeout ) .fetch_all(&db) .await?; @@ -2688,6 +2695,7 @@ pub async fn run_job_by_hash( dedicated_worker, priority, _delete_after_use, // not taken into account in async endpoints + timeout, ) = get_path_tag_limits_cache_for_hash(&db, &w_id, hash).await?; check_scopes(&authed, || format!("run:script/{path}"))?; @@ -2724,7 +2732,7 @@ pub async fn run_job_by_hash( None, !run_query.invisible_to_owner.unwrap_or(false), tag, - None, + timeout, None, None, ) diff --git a/backend/windmill-api/src/oauth2.rs b/backend/windmill-api/src/oauth2.rs index 60d14c8ae4..039f7c4add 100644 --- a/backend/windmill-api/src/oauth2.rs +++ b/backend/windmill-api/src/oauth2.rs @@ -874,6 +874,7 @@ async fn slack_command( dedicated_worker, priority, _delete_after_use, + _timeout, ) = windmill_common::get_latest_deployed_hash_for_path( &db, &settings.workspace_id, diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 1775777c7c..7831aef244 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -86,6 +86,8 @@ pub struct ScriptWDraft { pub restart_unless_cancelled: Option, #[serde(skip_serializing_if = "Option::is_none")] pub delete_after_use: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout: Option, } pub fn global_service() -> Router { @@ -474,8 +476,9 @@ async fn create_script( "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \ content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ - dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25)", + dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ + delete_after_use, timeout) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26)", &w_id, &hash.0, ns.path, @@ -501,6 +504,7 @@ async fn create_script( ns.priority, ns.restart_unless_cancelled, ns.delete_after_use, + ns.timeout, ) .execute(&mut tx) .await?; @@ -719,7 +723,7 @@ async fn get_script_by_path_w_draft( let mut tx = user_db.begin(&authed).await?; let script_o = sqlx::query_as::<_, ScriptWDraft>( - "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use FROM script LEFT JOIN draft ON + "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout FROM script LEFT JOIN draft ON script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script' WHERE script.path = $1 AND script.workspace_id = $2 \ AND script.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \ diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index c9dc284006..12e12e5bb8 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -3,6 +3,7 @@ pub const CUSTOM_TAGS_SETTING: &str = "custom_tags"; pub const BASE_URL_SETTING: &str = "base_url"; pub const OAUTH_SETTING: &str = "oauths"; pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs"; +pub const JOB_DEFAULT_TIMEOUT_SECS_SETTING: &str = "job_default_timeout"; pub const REQUEST_SIZE_LIMIT_SETTING: &str = "request_size_limit_mb"; pub const LICENSE_KEY_SETTING: &str = "license_key"; pub const NPM_CONFIG_REGISTRY_SETTING: &str = "npm_config_registry"; diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 0e1b47fb68..45dd4b0f01 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -335,12 +335,13 @@ pub async fn script_path_to_payload( script_path: &str, db: &DB, w_id: &str, -) -> error::Result<(JobPayload, Option, Option)> { - let (job_payload, tag, delete_after_use) = if script_path.starts_with("hub/") { +) -> error::Result<(JobPayload, Option, Option, Option)> { + let (job_payload, tag, delete_after_use, script_timeout) = if script_path.starts_with("hub/") { ( JobPayload::ScriptHub { path: script_path.to_owned() }, None, None, + None, ) } else { let ( @@ -353,6 +354,7 @@ pub async fn script_path_to_payload( dedicated_worker, priority, delete_after_use, + script_timeout, ) = get_latest_deployed_hash_for_path(db, w_id, script_path).await?; ( JobPayload::ScriptHash { @@ -367,9 +369,10 @@ pub async fn script_path_to_payload( }, tag, delete_after_use, + script_timeout, ) }; - Ok((job_payload, tag, delete_after_use)) + Ok((job_payload, tag, delete_after_use, script_timeout)) } pub async fn script_hash_to_tag_and_limits<'c>( @@ -385,9 +388,10 @@ pub async fn script_hash_to_tag_and_limits<'c>( Option, Option, Option, + Option, )> { let script = sqlx::query!( - "select tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use from script where hash = $1 AND workspace_id = $2", + "select tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where hash = $1 AND workspace_id = $2", script_hash.0, w_id ) @@ -407,6 +411,7 @@ pub async fn script_hash_to_tag_and_limits<'c>( script.dedicated_worker, script.priority, script.delete_after_use, + script.timeout, )) } @@ -415,7 +420,7 @@ pub async fn get_payload_tag_from_prefixed_path( db: &DB, w_id: &str, ) -> Result<(JobPayload, Option), Error> { - let (payload, tag, _) = if path.starts_with("script/") { + let (payload, tag, _, _) = if path.starts_with("script/") { script_path_to_payload(path.strip_prefix("script/").unwrap(), &db, w_id).await? } else if path.starts_with("flow/") { let path = path.strip_prefix("flow/").unwrap().to_string(); @@ -429,7 +434,7 @@ pub async fn get_payload_tag_from_prefixed_path( let (tag, dedicated_worker) = r .map(|x| (x.tag, x.dedicated_worker)) .unwrap_or_else(|| (None, None)); - (JobPayload::Flow { path, dedicated_worker }, tag, None) + (JobPayload::Flow { path, dedicated_worker }, tag, None, None) } else { return Err(Error::BadRequest(format!( "path must start with script/ or flow/ (got {})", diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 5eed4573eb..834b8e2cd5 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -229,9 +229,10 @@ pub async fn get_latest_deployed_hash_for_path( Option, Option, Option, + Option, )> { let r_o = sqlx::query!( - "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use from script where path = $1 AND workspace_id = $2 AND + "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where path = $1 AND workspace_id = $2 AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)", script_path, @@ -252,6 +253,7 @@ pub async fn get_latest_deployed_hash_for_path( script.dedicated_worker, script.priority, script.delete_after_use, + script.timeout, )) } @@ -268,9 +270,10 @@ pub async fn get_latest_hash_for_path<'c>( ScriptLang, Option, Option, + Option, )> { let r_o = sqlx::query!( - "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority from script where path = $1 AND workspace_id = $2 AND + "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, timeout FROM script where path = $1 AND workspace_id = $2 AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND deleted = false AND archived = false)", script_path, @@ -290,5 +293,6 @@ pub async fn get_latest_hash_for_path<'c>( script.language, script.dedicated_worker, script.priority, + script.timeout, )) } diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index f7dfc98477..bf58ee9fe3 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -71,7 +71,7 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>( } } - let (payload, tag) = if schedule.is_flow { + let (payload, tag, timeout) = if schedule.is_flow { let r = sqlx::query!( "SELECT tag, dedicated_worker from flow WHERE path = $1 and workspace_id = $2", &schedule.script_path, @@ -85,6 +85,7 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>( ( JobPayload::Flow { path: schedule.script_path, dedicated_worker }, tag, + None, ) } else { let ( @@ -96,6 +97,7 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>( language, dedicated_worker, priority, + timeout, ) = windmill_common::get_latest_hash_for_path( tx.transaction_mut(), &schedule.workspace_id, @@ -114,6 +116,7 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>( priority, }, tag, + timeout, ) }; @@ -144,7 +147,7 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>( None, true, tag, - None, + timeout, None, None, ) diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 6aad40165c..68de94dc83 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -46,8 +46,8 @@ use futures::{ }; use crate::{ - AuthedClient, AuthedClientBackgroundTask, MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGTERM, - ROOT_CACHE_DIR, TIMEOUT_DURATION, + AuthedClient, AuthedClientBackgroundTask, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, + MAX_TIMEOUT_DURATION, MAX_WAIT_FOR_SIGTERM, ROOT_CACHE_DIR, }; pub async fn build_args_map<'a>( @@ -513,39 +513,18 @@ pub async fn handle_child( Timeout, Cancelled, } + + let (timeout_duration, timeout_warn_msg) = + resolve_job_timeout(&db, _w_id, job_id, custom_timeout).await; + if let Some(msg) = timeout_warn_msg { + logs.push_str(msg.as_str()); + append_logs(job_id, msg.as_str(), db).await; + } + /* a future that completes when the child process exits */ let wait_on_child = async { let db = db.clone(); - #[cfg(not(feature = "enterprise"))] - let instance_timeout_duration = *TIMEOUT_DURATION; - - #[cfg(feature = "enterprise")] - let premium_workspace = *CLOUD_HOSTED - && sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", _w_id) - .fetch_one(&db) - .await - .map_err(|e| { - tracing::error!(%e, "error getting premium workspace for job {job_id}: {e}"); - }) - .unwrap_or(false); - - #[cfg(feature = "enterprise")] - let instance_timeout_duration = if premium_workspace { - *TIMEOUT_DURATION * 6 //30mins - } else { - *TIMEOUT_DURATION - }; - - let timeout_duration = if let Some(custom_timeout) = custom_timeout { - Duration::min( - instance_timeout_duration, - Duration::from_secs(custom_timeout as u64), - ) - } else { - instance_timeout_duration - }; - let kill_reason = tokio::select! { biased; result = child.wait() => return result.map(Ok), @@ -567,7 +546,7 @@ pub async fn handle_child( WHERE id = $2 "#, ) - .bind(format!("duration > {}", TIMEOUT_DURATION.as_secs())) + .bind(format!("duration > {}", timeout_duration.as_secs())) .bind(job_id) .execute(&db) .await @@ -604,6 +583,7 @@ pub async fn handle_child( } else { usize::MAX }; + /* log_remaining is zero when output limit was reached */ let mut log_remaining = max_log_size.saturating_sub(logs.chars().count()); let mut result = io::Result::Ok(()); @@ -682,7 +662,6 @@ pub async fn handle_child( if *set_too_many_logs.borrow() { break; } - } /* drop our end of the pipe */ @@ -732,6 +711,64 @@ pub async fn start_child_process(mut cmd: Command, executable: &str) -> Result, + w_id: &str, + job_id: Uuid, + custom_timeout_secs: Option, +) -> (Duration, Option) { + let mut warn_msg: Option = None; + #[cfg(feature = "enterprise")] + let cloud_premium_workspace = *CLOUD_HOSTED + && sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", w_id) + .fetch_one(db) + .await + .map_err(|e| { + tracing::error!(%e, "error getting premium workspace for job {job_id}: {e}"); + }) + .unwrap_or(false); + #[cfg(not(feature = "enterprise"))] + let cloud_premium_workspace = false; + + // compute global max timeout + let global_max_timeout_duration = if cloud_premium_workspace { + *MAX_TIMEOUT_DURATION * 6 //30mins + } else { + *MAX_TIMEOUT_DURATION + }; + + match custom_timeout_secs { + Some(timeout_secs) + if Duration::from_secs(timeout_secs as u64) < global_max_timeout_duration => + { + (Duration::from_secs(timeout_secs as u64), warn_msg) + } + Some(timeout_secs) => { + warn_msg = Some(format!("WARNING: Custom job timeout of {timeout_secs} seconds was greater than the maximum timeout. It will be ignored and the max timeout will be used instead")); + tracing::warn!(warn_msg); + (global_max_timeout_duration, warn_msg) + } + None => { + // fallback to default timeout or max if not set + let default_timeout = match JOB_DEFAULT_TIMEOUT.read().await.clone() { + None => global_max_timeout_duration, + Some(default_timeout_secs) + if Duration::from_secs(default_timeout_secs as u64) + < global_max_timeout_duration => + { + Duration::from_secs(default_timeout_secs as u64) + } + Some(default_timeout_secs) => { + warn_msg = Some(format!("WARNING: Default job timeout of {default_timeout_secs} seconds was greater than the maximum timeout. It will be ignored and the global max timeout will be used instead")); + tracing::warn!(warn_msg); + global_max_timeout_duration + } + }; + (default_timeout, warn_msg) + } + } +} + /// takes stdout and stderr from Child, panics if either are not present /// /// builds a stream joining both stdout and stderr each read line by line diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index cf1b79c9d4..6ea10164b4 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -239,9 +239,9 @@ lazy_static::lazy_static! { pub static ref GOPROXY: Option = std::env::var("GOPROXY").ok(); pub static ref NETRC: Option = std::env::var("NETRC").ok(); - pub static ref NPM_CONFIG_REGISTRY: Arc>> = Arc::new(RwLock::new(None)); pub static ref PIP_EXTRA_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); + pub static ref JOB_DEFAULT_TIMEOUT: Arc>> = Arc::new(RwLock::new(None)); pub static ref TAR_CACHE_RATE: i32 = std::env::var("TAR_CACHE_RATE") .ok() @@ -259,7 +259,7 @@ lazy_static::lazy_static! { "Total number of seconds since the worker has started" ); - static ref TIMEOUT: u64 = std::env::var("TIMEOUT") + static ref MAX_TIMEOUT: u64 = std::env::var("TIMEOUT") .ok() .and_then(|x| x.parse::().ok()) .unwrap_or_else(|| if *CLOUD_HOSTED { DEFAULT_CLOUD_TIMEOUT } else { DEFAULT_SELFHOSTED_TIMEOUT }); @@ -269,12 +269,12 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or_else(|| 5); - pub static ref TIMEOUT_DURATION: Duration = Duration::from_secs(*TIMEOUT); + pub static ref MAX_TIMEOUT_DURATION: Duration = Duration::from_secs(*MAX_TIMEOUT); pub static ref SCRIPT_TOKEN_EXPIRY: u64 = std::env::var("SCRIPT_TOKEN_EXPIRY") .ok() .and_then(|x| x.parse::().ok()) - .unwrap_or(*TIMEOUT); + .unwrap_or(*MAX_TIMEOUT); pub static ref GLOBAL_CACHE_INTERVAL: u64 = std::env::var("GLOBAL_CACHE_INTERVAL") .ok() diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index b9bcd3bffd..687cca6cd9 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -1844,7 +1844,7 @@ async fn push_next_flow_job } else { Some(flow_job.tag.clone()) }, - module.timeout, + payload_tag.timeout, Some(module.id.clone()), new_job_priority_override, ) @@ -2101,6 +2101,7 @@ struct JobPayloadWithTag { payload: JobPayload, tag: Option, delete_after_use: bool, + timeout: Option, } enum ContinuePayload { SingleJob(JobPayloadWithTag), @@ -2154,6 +2155,7 @@ async fn compute_next_flow_transform( payload: JobPayload::Identity, tag: None, delete_after_use: false, + timeout: None, }), NextStatus::NextStep, )); @@ -2164,6 +2166,7 @@ async fn compute_next_flow_transform( payload, tag: None, delete_after_use: false, + timeout: None, }), NextStatus::NextStep, )) @@ -2398,6 +2401,7 @@ async fn compute_next_flow_transform( }, tag: None, delete_after_use: delete_after_use, + timeout: None, }), NextStatus::NextLoopIteration { next: ns, @@ -2438,6 +2442,7 @@ async fn compute_next_flow_transform( }, tag: None, delete_after_use: delete_after_use, + timeout: None, } }; ContinuePayload::ForloopJobs { n: itered.len(), payload } @@ -2539,6 +2544,7 @@ async fn compute_next_flow_transform( }, tag: None, delete_after_use: delete_after_use, + timeout: None, }), NextStatus::BranchChosen(branch), )) @@ -2589,6 +2595,7 @@ async fn compute_next_flow_transform( }, tag: None, delete_after_use: delete_after_use, + timeout: None, } }) .collect(), @@ -2655,6 +2662,7 @@ async fn compute_next_flow_transform( }, tag: None, delete_after_use: delete_after_use, + timeout: None, }), NextStatus::NextBranchStep(NextBranch { status: branch_status, flow_jobs }), )) @@ -2723,12 +2731,13 @@ fn raw_script_to_payload( }), tag: tag.clone(), delete_after_use: *delete_after_use, + timeout: module.timeout, } } fn flow_to_payload(path: &str, delete_after_use: &bool) -> JobPayloadWithTag { let payload = JobPayload::Flow { path: path.to_string(), dedicated_worker: None }; - JobPayloadWithTag { payload, tag: None, delete_after_use: *delete_after_use } + JobPayloadWithTag { payload, tag: None, delete_after_use: *delete_after_use, timeout: None } } async fn script_to_payload( @@ -2738,7 +2747,7 @@ async fn script_to_payload( flow_job: &QueuedJob, module: &FlowModule, ) -> Result { - let (payload, tag, delete_after_use) = if script_hash.is_none() { + let (payload, tag, delete_after_use, script_timeout) = if script_hash.is_none() { script_path_to_payload(script_path, &db, &flow_job.workspace_id).await? } else { let hash = script_hash.clone().unwrap(); @@ -2752,6 +2761,7 @@ async fn script_to_payload( dedicated_worker, priority, delete_after_use, + script_timeout, ) = script_hash_to_tag_and_limits(&hash, &mut tx, &flow_job.workspace_id).await?; ( JobPayload::ScriptHash { @@ -2766,12 +2776,19 @@ async fn script_to_payload( }, tag, delete_after_use, + script_timeout, ) }; // the module value overrides the value set at the script level. Defaults to false if both are unset. let final_delete_after_user = module.delete_after_use.unwrap_or(false) || delete_after_use.unwrap_or(false); - Ok(JobPayloadWithTag { payload, tag, delete_after_use: final_delete_after_user }) + let flow_step_timeout = module.timeout.or(script_timeout); + Ok(JobPayloadWithTag { + payload, + tag, + delete_after_use: final_delete_after_user, + timeout: flow_step_timeout, + }) } async fn get_transform_context( diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index afdbeceae7..63a4d6d5b0 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -215,7 +215,8 @@ ws_error_handler_muted: script.ws_error_handler_muted, priority: script.priority, restart_unless_cancelled: script.restart_unless_cancelled, - delete_after_use: script.delete_after_use + delete_after_use: script.delete_after_use, + timeout: script.timeout } }) savedScript = cloneDeep(script) as NewScriptWithDraft @@ -291,7 +292,8 @@ ws_error_handler_muted: script.ws_error_handler_muted, priority: script.priority, restart_unless_cancelled: script.restart_unless_cancelled, - delete_after_use: script.delete_after_use + delete_after_use: script.delete_after_use, + timeout: script.timeout } }) } @@ -653,6 +655,30 @@ {/if} +
+
+ { + if (script.timeout && script.timeout != undefined) { + script.timeout = undefined + } else { + script.timeout = 300 + } + }} + options={{ + right: 'Add a custom timeout for this script' + }} + /> + Timeout duration + {#if script.timeout} + + {:else} + + {/if} +
+
{#if flowModule} @@ -102,7 +102,7 @@ {/if} {#if title} -
{title}
+
{title}
{/if}
diff --git a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte index a5a7cf5150..5722174927 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte @@ -193,7 +193,7 @@ on:toggleSleep={() => selectAdvanced('sleep')} on:toggleMock={() => selectAdvanced('mock')} on:toggleRetry={() => selectAdvanced('retries')} - on:toggleConcurrency={() => selectAdvanced('concurrency')} + on:toggleConcurrency={() => selectAdvanced('runtime')} on:toggleCache={() => selectAdvanced('cache')} on:toggleStopAfterIf={() => selectAdvanced('early-stop')} on:fork={async () => { diff --git a/frontend/src/lib/components/flows/content/FlowModuleHeader.svelte b/frontend/src/lib/components/flows/content/FlowModuleHeader.svelte index 80ec0edb8f..270881ef4d 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleHeader.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleHeader.svelte @@ -37,85 +37,79 @@ $: moduleRetry = module.retry?.constant || module.retry?.exponential -
+
{#if module.value.type === 'script' || module.value.type === 'rawscript' || module.value.type == 'flow'} - dispatch('toggleRetry')} - > - - Retries - - dispatch('toggleConcurrency')} - > - - Concurrency Limits - - dispatch('toggleCache')} - > - - Cache - - dispatch('toggleStopAfterIf')} - > - - Early stop/break - - dispatch('toggleSuspend')} - > - - Suspend - - dispatch('toggleSleep')} - > - - Sleep - - dispatch('toggleMock')} - > - - Mock - + {#if moduleRetry} + dispatch('toggleRetry')} + > + + Retries + + {/if} + {#if module?.value?.['concurrent_limit'] != undefined} + dispatch('toggleConcurrency')} + > + + Concurrency Limits + + {/if} + {#if module.cache_ttl != undefined} + dispatch('toggleCache')} + > + + Cache + + {/if} + {#if module.stop_after_if} + dispatch('toggleStopAfterIf')} + > + + Early stop/break + + {/if} + {#if module.suspend} + dispatch('toggleSuspend')} + > + + Suspend + + {/if} + {#if module.sleep} + dispatch('toggleSleep')} + > + + Sleep + + {/if} + {#if module.mock?.enabled} + dispatch('toggleMock')} + > + + Mock + + {/if} {/if} {#if module.value.type === 'script'}
@@ -149,7 +143,7 @@ Fork {/if} - +
{#if module.value.type === 'rawscript'} {#if $workerTags} {#if $workerTags?.length > 0} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 34827e3e56..1ce52950d3 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -55,6 +55,14 @@ export const settings: Record = { storage: 'setting', cloudonly: false }, + { + label: 'Default timeout', + key: 'job_default_timeout', + description: 'Default timeout for individual jobs', + fieldType: 'seconds', + storage: 'setting', + cloudonly: false + }, { label: 'Max Timeout for sync endpoints', key: 'timeout_wait_result',