diff --git a/README.md b/README.md index 51c9c99eb4..4df25439d3 100644 --- a/README.md +++ b/README.md @@ -288,57 +288,60 @@ it being synced automatically everyday. ## Environment Variables -| Environment Variable name | Default | Description | Api Server/Worker/All | -| ------------------------- | ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | -| DATABASE_URL | | The Postgres database url. | All | -| DISABLE_NSJAIL | true | Disable Nsjail Sandboxing | Worker | -| SERVER_BIND_ADDR | 0.0.0.0 | IP Address on which to bind listening socket | Server | -| PORT | 8000 | Exposed port | Server | -| NUM_WORKERS | 3 | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance) | Worker | -| DISABLE_SERVER | false | Binary would operate as a worker only instance | Worker | -| METRICS_ADDR | None | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001 | All | -| JSON_FMT | false | Output the logs in json format instead of logfmt | All | -| BASE_URL | http://localhost:8000 | The base url that is exposed publicly to access your instance | Server | -| BASE_INTERNAL_URL | http://localhost:8000 | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests. | Worker | -| TIMEOUT | 300 | The maximum time of execution of a script. When reached, the job is failed as having timedout. | Worker | -| ZOMBIE_JOB_TIMEOUT | 30 | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s) | Server | -| RESTART_ZOMBIE_JOBS | true | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed | Server | -| SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker | -| MAX_LOG_SIZE | 500000 | The maximum number of characters a job can emit (log + result) | Worker | -| DISABLE_NUSER | false | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting | Worker | -| KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker | -| LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker | -| S3_CACHE_BUCKET (EE only) | None | The S3 bucket to sync the cache of the workers to | Worker | -| TAR_CACHE_RATE (EE only) | 100 | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed). | Worker | -| SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server | -| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server | -| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker | -| PYTHON_PATH | /usr/local/bin/python3 | The path to the python binary. | Worker | -| GO_PATH | /usr/bin/go | The path to the go binary. | Worker | -| GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker | -| NETRC | | The netrc content to use a private go registry | Worker | -| PIP_INDEX_URL | None | The index url to pass for pip. | Worker | -| PIP_EXTRA_INDEX_URL | None | The extra index url to pass to pip. | Worker | -| PIP_TRUSTED_HOST | None | The trusted host to pass to pip. | Worker | -| PATH | None | The path environment variable, usually inherited | Worker | -| HOME | None | The home directory to use for Go and Bash , usually inherited | Worker | -| DATABASE_CONNECTIONS | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool | All | -| SUPERADMIN_SECRET | None | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev | Server | -| TIMEOUT_WAIT_RESULT | 20 | The number of seconds to wait before timeout on the 'run_wait_result' endpoint | Worker | -| QUEUE_LIMIT_WAIT_RESULT | None | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit. | Worker | -| DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker | -| DENO_FLAGS | None | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json" | Worker | -| NPM_CONFIG_REGISTRY | None | Registry to use for NPM dependencies, set if you have a private repository you need to use instead of the default public NPM registry | Worker | -| PIP_LOCAL_DEPENDENCIES | None | Specify dependencies that are installed locally and do not need to be solved nor installed again | | -| ADDITIONAL_PYTHON_PATHS | None | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill | Worker | -| INCLUDE_HEADERS | None | Whitelist of headers that are passed to jobs as args (separated by a comma) | Server | -| WHITELIST_WORKSPACES | None | Whitelist of workspaces this worker takes job from | Worker | -| BLACKLIST_WORKSPACES | None | Blacklist of workspaces this worker takes job from | Worker | -| INSTANCE_EVENTS_WEBHOOK | None | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails | -| GLOBAL_CACHE_INTERVAL | 10\*60 | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together. | Worker | -| WORKER_TAGS | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers | Worker | -| CUSTOM_TAGS | None | The custom tags assignable to scripts. | Server | -| JOB_RETENTION_SECS | 60*60*24\*60 //60 days | The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete | Server | +| Environment Variable name | Default | Description | Api Server/Worker/All | +| ----------------------------------- | ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | +| DATABASE_URL | | The Postgres database url. | All | +| DISABLE_NSJAIL | true | Disable Nsjail Sandboxing | Worker | +| SERVER_BIND_ADDR | 0.0.0.0 | IP Address on which to bind listening socket | Server | +| PORT | 8000 | Exposed port | Server | +| NUM_WORKERS | 3 | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance) | Worker | +| DISABLE_SERVER | false | Binary would operate as a worker only instance | Worker | +| METRICS_ADDR | None | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001 | All | +| JSON_FMT | false | Output the logs in json format instead of logfmt | All | +| BASE_URL | http://localhost:8000 | The base url that is exposed publicly to access your instance | Server | +| BASE_INTERNAL_URL | http://localhost:8000 | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests. | Worker | +| TIMEOUT | 300 | The maximum time of execution of a script. When reached, the job is failed as having timedout. | Worker | +| ZOMBIE_JOB_TIMEOUT | 30 | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s) | Server | +| RESTART_ZOMBIE_JOBS | true | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed | Server | +| SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker | +| MAX_LOG_SIZE | 500000 | The maximum number of characters a job can emit (log + result) | Worker | +| DISABLE_NUSER | false | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting | Worker | +| KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker | +| LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker | +| S3_CACHE_BUCKET (EE only) | None | The S3 bucket to sync the cache of the workers to | Worker | +| TAR_CACHE_RATE (EE only) | 100 | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed). | Worker | +| SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server | +| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server | +| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker | +| PYTHON_PATH | /usr/local/bin/python3 | The path to the python binary. | Worker | +| GO_PATH | /usr/bin/go | The path to the go binary. | Worker | +| GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker | +| NETRC | | The netrc content to use a private go registry | Worker | +| PIP_INDEX_URL | None | The index url to pass for pip. | Worker | +| PIP_EXTRA_INDEX_URL | None | The extra index url to pass to pip. | Worker | +| PIP_TRUSTED_HOST | None | The trusted host to pass to pip. | Worker | +| PATH | None | The path environment variable, usually inherited | Worker | +| HOME | None | The home directory to use for Go and Bash , usually inherited | Worker | +| DATABASE_CONNECTIONS | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool | All | +| SUPERADMIN_SECRET | None | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev | Server | +| TIMEOUT_WAIT_RESULT | 20 | The number of seconds to wait before timeout on the 'run_wait_result' endpoint | Worker | +| QUEUE_LIMIT_WAIT_RESULT | None | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit. | Worker | +| DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker | +| DENO_FLAGS | None | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json" | Worker | +| NPM_CONFIG_REGISTRY | None | Registry to use for NPM dependencies, set if you have a private repository you need to use instead of the default public NPM registry | Worker | +| PIP_LOCAL_DEPENDENCIES | None | Specify dependencies that are installed locally and do not need to be solved nor installed again | | +| ADDITIONAL_PYTHON_PATHS | None | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill | Worker | +| INCLUDE_HEADERS | None | Whitelist of headers that are passed to jobs as args (separated by a comma) | Server | +| WHITELIST_WORKSPACES | None | Whitelist of workspaces this worker takes job from | Worker | +| BLACKLIST_WORKSPACES | None | Blacklist of workspaces this worker takes job from | Worker | +| INSTANCE_EVENTS_WEBHOOK | None | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails | +| GLOBAL_CACHE_INTERVAL | 10\*60 | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together. | Worker | +| WORKER_TAGS | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers | Worker | +| CUSTOM_TAGS | None | The custom tags assignable to scripts. | Server | +| JOB_RETENTION_SECS | 60*60*24\*60 //60 days | The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete | +| WAIT_RESULT_FAST_POLL_INTERVAL_MS | 50 | The time in between polling for the run_wait_result endpoints in fast poll mode | Server | +| WAIT_RESULT_SLOW_POLL_INTERVAL_MS | 200 | The time in between polling for the run_wait_result endpoints in fast poll mode | Server | +| WAIT_RESULT_FAST_POLL_DURATION_SECS | 2 | The duration of fast poll mode before switching to slow poll | Server | ## Run a local dev setup diff --git a/backend/src/main.rs b/backend/src/main.rs index 80a321d523..59dec19995 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -166,6 +166,9 @@ Windmill Community Edition {GIT_VERSION} "WORKER_TAGS", "CUSTOM_TAGS", "JOB_RETENTION_SECS", + "WAIT_RESULT_FAST_POLL_DURATION_SECS", + "WAIT_RESULT_SLOW_POLL_INTERVAL_MS", + "WAIT_RESULT_FAST_POLL_INTERVAL_MS", ]); if server_mode || num_workers > 0 { diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index f41697add5..b2f274d5db 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -1367,12 +1367,13 @@ async fn run_wait_result( uuid: Uuid, Path((w_id, _)): Path<(String, T)>, ) -> error::JsonResult { - let mut result = None; - let iters = if timeout <= 0 { - 40 + let mut result; + let timeout_ms = if timeout <= 0 { + 2000 } else { - 20 + ((timeout - 1) * 5) + (timeout * 1000) as u64 }; + let mut g = Guard { done: false, id: uuid, @@ -1380,7 +1381,10 @@ async fn run_wait_result( db: user_db.clone(), authed: authed.clone(), }; - for i in 0..iters { + + let fast_poll_duration = *WAIT_RESULT_FAST_POLL_DURATION_SECS as u64 * 1000; + let mut accumulated_delay = 0 as u64; + loop { let mut tx = user_db.clone().begin(&authed).await?; result = sqlx::query_scalar!( "SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2", @@ -1395,8 +1399,17 @@ async fn run_wait_result( if result.is_some() { break; } + //for the first 10 seconds, we poll every 50ms, then every 200ms - let delay = if i < 200 { 50 } else { 200 }; + let delay = if accumulated_delay <= fast_poll_duration { + *WAIT_RESULT_FAST_POLL_INTERVAL_MS + } else { + *WAIT_RESULT_SLOW_POLL_INTERVAL_MS + }; + accumulated_delay += delay; + if accumulated_delay > timeout_ms { + break; + }; tokio::time::sleep(core::time::Duration::from_millis(delay)).await; } if let Some(result) = result { @@ -1436,6 +1449,18 @@ lazy_static::lazy_static! { .ok() .and_then(|x| x.parse().ok()) .unwrap_or(20); + pub static ref WAIT_RESULT_FAST_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_FAST_POLL_INTERVAL_MS") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(50); + pub static ref WAIT_RESULT_FAST_POLL_DURATION_SECS: u16 = std::env::var("WAIT_RESULT_FAST_POLL_DURATION_SECS") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(2); + pub static ref WAIT_RESULT_SLOW_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_SLOW_POLL_INTERVAL_MS") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(200); } pub async fn run_wait_result_job_by_path_get( diff --git a/frontend/Caddyfile b/frontend/Caddyfile index 90b4e57d45..7e9e9dfb66 100644 --- a/frontend/Caddyfile +++ b/frontend/Caddyfile @@ -1,4 +1,4 @@ -:8000 { +:80 { bind {$ADDRESS} reverse_proxy /api/* http://localhost:8000 reverse_proxy /* http://localhost:3000