starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-19 08:01:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix run_wait_result polling + make it configurable

Browse Source
This commit is contained in:
Ruben Fiszel
2023-04-27 07:30:21 +02:00
parent 95e52b2d74
commit 4387db275f
4 changed files with 89 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+54 -51
View File
@@ -288,57 +288,60 @@ it being synced automatically everyday.
## Environment Variables
| Environment Variable name | Default | Description | Api Server/Worker/All |
| ------------------------- | ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
| DATABASE_URL | | The Postgres database url. | All |
| DISABLE_NSJAIL | true | Disable Nsjail Sandboxing | Worker |
| SERVER_BIND_ADDR | 0.0.0.0 | IP Address on which to bind listening socket | Server |
| PORT | 8000 | Exposed port | Server |
| NUM_WORKERS | 3 | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance) | Worker |
| DISABLE_SERVER | false | Binary would operate as a worker only instance | Worker |
| METRICS_ADDR | None | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001 | All |
| JSON_FMT | false | Output the logs in json format instead of logfmt | All |
| BASE_URL | http://localhost:8000 | The base url that is exposed publicly to access your instance | Server |
| BASE_INTERNAL_URL | http://localhost:8000 | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests. | Worker |
| TIMEOUT | 300 | The maximum time of execution of a script. When reached, the job is failed as having timedout. | Worker |
| ZOMBIE_JOB_TIMEOUT | 30 | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s) | Server |
| RESTART_ZOMBIE_JOBS | true | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed | Server |
| SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker |
| MAX_LOG_SIZE | 500000 | The maximum number of characters a job can emit (log + result) | Worker |
| DISABLE_NUSER | false | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting | Worker |
| KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker |
| LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker |
| S3_CACHE_BUCKET (EE only) | None | The S3 bucket to sync the cache of the workers to | Worker |
| TAR_CACHE_RATE (EE only) | 100 | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed). | Worker |
| SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server |
| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server |
| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker |
| PYTHON_PATH | /usr/local/bin/python3 | The path to the python binary. | Worker |
| GO_PATH | /usr/bin/go | The path to the go binary. | Worker |
| GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker |
| NETRC | | The netrc content to use a private go registry | Worker |
| PIP_INDEX_URL | None | The index url to pass for pip. | Worker |
| PIP_EXTRA_INDEX_URL | None | The extra index url to pass to pip. | Worker |
| PIP_TRUSTED_HOST | None | The trusted host to pass to pip. | Worker |
| PATH | None | The path environment variable, usually inherited | Worker |
| HOME | None | The home directory to use for Go and Bash , usually inherited | Worker |
| DATABASE_CONNECTIONS | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool | All |
| SUPERADMIN_SECRET | None | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev | Server |
| TIMEOUT_WAIT_RESULT | 20 | The number of seconds to wait before timeout on the 'run_wait_result' endpoint | Worker |
| QUEUE_LIMIT_WAIT_RESULT | None | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit. | Worker |
| DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker |
| DENO_FLAGS | None | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json" | Worker |
| NPM_CONFIG_REGISTRY | None | Registry to use for NPM dependencies, set if you have a private repository you need to use instead of the default public NPM registry | Worker |
| PIP_LOCAL_DEPENDENCIES | None | Specify dependencies that are installed locally and do not need to be solved nor installed again | |
| ADDITIONAL_PYTHON_PATHS | None | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill | Worker |
| INCLUDE_HEADERS | None | Whitelist of headers that are passed to jobs as args (separated by a comma) | Server |
| WHITELIST_WORKSPACES | None | Whitelist of workspaces this worker takes job from | Worker |
| BLACKLIST_WORKSPACES | None | Blacklist of workspaces this worker takes job from | Worker |
| INSTANCE_EVENTS_WEBHOOK | None | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails |
| GLOBAL_CACHE_INTERVAL | 10\*60 | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together. | Worker |
| WORKER_TAGS | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers | Worker |
| CUSTOM_TAGS | None | The custom tags assignable to scripts. | Server |
| JOB_RETENTION_SECS | 60*60*24\*60 //60 days | The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete | Server |
| Environment Variable name | Default | Description | Api Server/Worker/All |
| ----------------------------------- | ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
| DATABASE_URL | | The Postgres database url. | All |
| DISABLE_NSJAIL | true | Disable Nsjail Sandboxing | Worker |
| SERVER_BIND_ADDR | 0.0.0.0 | IP Address on which to bind listening socket | Server |
| PORT | 8000 | Exposed port | Server |
| NUM_WORKERS | 3 | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance) | Worker |
| DISABLE_SERVER | false | Binary would operate as a worker only instance | Worker |
| METRICS_ADDR | None | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001 | All |
| JSON_FMT | false | Output the logs in json format instead of logfmt | All |
| BASE_URL | http://localhost:8000 | The base url that is exposed publicly to access your instance | Server |
| BASE_INTERNAL_URL | http://localhost:8000 | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests. | Worker |
| TIMEOUT | 300 | The maximum time of execution of a script. When reached, the job is failed as having timedout. | Worker |
| ZOMBIE_JOB_TIMEOUT | 30 | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s) | Server |
| RESTART_ZOMBIE_JOBS | true | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed | Server |
| SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker |
| MAX_LOG_SIZE | 500000 | The maximum number of characters a job can emit (log + result) | Worker |
| DISABLE_NUSER | false | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting | Worker |
| KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker |
| LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker |
| S3_CACHE_BUCKET (EE only) | None | The S3 bucket to sync the cache of the workers to | Worker |
| TAR_CACHE_RATE (EE only) | 100 | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed). | Worker |
| SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server |
| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server |
| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker |
| PYTHON_PATH | /usr/local/bin/python3 | The path to the python binary. | Worker |
| GO_PATH | /usr/bin/go | The path to the go binary. | Worker |
| GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker |
| NETRC | | The netrc content to use a private go registry | Worker |
| PIP_INDEX_URL | None | The index url to pass for pip. | Worker |
| PIP_EXTRA_INDEX_URL | None | The extra index url to pass to pip. | Worker |
| PIP_TRUSTED_HOST | None | The trusted host to pass to pip. | Worker |
| PATH | None | The path environment variable, usually inherited | Worker |
| HOME | None | The home directory to use for Go and Bash , usually inherited | Worker |
| DATABASE_CONNECTIONS | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool | All |
| SUPERADMIN_SECRET | None | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev | Server |
| TIMEOUT_WAIT_RESULT | 20 | The number of seconds to wait before timeout on the 'run_wait_result' endpoint | Worker |
| QUEUE_LIMIT_WAIT_RESULT | None | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit. | Worker |
| DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker |
| DENO_FLAGS | None | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json" | Worker |
| NPM_CONFIG_REGISTRY | None | Registry to use for NPM dependencies, set if you have a private repository you need to use instead of the default public NPM registry | Worker |
| PIP_LOCAL_DEPENDENCIES | None | Specify dependencies that are installed locally and do not need to be solved nor installed again | |
| ADDITIONAL_PYTHON_PATHS | None | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill | Worker |
| INCLUDE_HEADERS | None | Whitelist of headers that are passed to jobs as args (separated by a comma) | Server |
| WHITELIST_WORKSPACES | None | Whitelist of workspaces this worker takes job from | Worker |
| BLACKLIST_WORKSPACES | None | Blacklist of workspaces this worker takes job from | Worker |
| INSTANCE_EVENTS_WEBHOOK | None | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails |
| GLOBAL_CACHE_INTERVAL | 10\*60 | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together. | Worker |
| WORKER_TAGS | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers | Worker |
| CUSTOM_TAGS | None | The custom tags assignable to scripts. | Server |
| JOB_RETENTION_SECS | 60*60*24\*60 //60 days | The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete |
| WAIT_RESULT_FAST_POLL_INTERVAL_MS | 50 | The time in between polling for the run_wait_result endpoints in fast poll mode | Server |
| WAIT_RESULT_SLOW_POLL_INTERVAL_MS | 200 | The time in between polling for the run_wait_result endpoints in fast poll mode | Server |
| WAIT_RESULT_FAST_POLL_DURATION_SECS | 2 | The duration of fast poll mode before switching to slow poll | Server |
## Run a local dev setup
backend/src/main.rs
+3
View File
@@ -166,6 +166,9 @@ Windmill Community Edition {GIT_VERSION}
"WORKER_TAGS",
"CUSTOM_TAGS",
"JOB_RETENTION_SECS",
"WAIT_RESULT_FAST_POLL_DURATION_SECS",
"WAIT_RESULT_SLOW_POLL_INTERVAL_MS",
"WAIT_RESULT_FAST_POLL_INTERVAL_MS",
]);
if server_mode || num_workers > 0 {
backend/windmill-api/src/jobs.rs
+31 -6
View File
@@ -1367,12 +1367,13 @@ async fn run_wait_result<T>(
uuid: Uuid,
Path((w_id, _)): Path<(String, T)>,
) -> error::JsonResult<serde_json::Value> {
let mut result = None;
let iters = if timeout <= 0 {
40
let mut result;
let timeout_ms = if timeout <= 0 {
2000
} else {
20 + ((timeout - 1) * 5)
(timeout * 1000) as u64
};
let mut g = Guard {
done: false,
id: uuid,
@@ -1380,7 +1381,10 @@ async fn run_wait_result<T>(
db: user_db.clone(),
authed: authed.clone(),
};
for i in 0..iters {
let fast_poll_duration = *WAIT_RESULT_FAST_POLL_DURATION_SECS as u64 * 1000;
let mut accumulated_delay = 0 as u64;
loop {
let mut tx = user_db.clone().begin(&authed).await?;
result = sqlx::query_scalar!(
"SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2",
@@ -1395,8 +1399,17 @@ async fn run_wait_result<T>(
if result.is_some() {
break;
}
//for the first 10 seconds, we poll every 50ms, then every 200ms
let delay = if i < 200 { 50 } else { 200 };
let delay = if accumulated_delay <= fast_poll_duration {
*WAIT_RESULT_FAST_POLL_INTERVAL_MS
} else {
*WAIT_RESULT_SLOW_POLL_INTERVAL_MS
};
accumulated_delay += delay;
if accumulated_delay > timeout_ms {
break;
};
tokio::time::sleep(core::time::Duration::from_millis(delay)).await;
}
if let Some(result) = result {
@@ -1436,6 +1449,18 @@ lazy_static::lazy_static! {
.ok()
.and_then(|x| x.parse().ok())
.unwrap_or(20);
pub static ref WAIT_RESULT_FAST_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_FAST_POLL_INTERVAL_MS")
.ok()
.and_then(|x| x.parse().ok())
.unwrap_or(50);
pub static ref WAIT_RESULT_FAST_POLL_DURATION_SECS: u16 = std::env::var("WAIT_RESULT_FAST_POLL_DURATION_SECS")
.ok()
.and_then(|x| x.parse().ok())
.unwrap_or(2);
pub static ref WAIT_RESULT_SLOW_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_SLOW_POLL_INTERVAL_MS")
.ok()
.and_then(|x| x.parse().ok())
.unwrap_or(200);
}
pub async fn run_wait_result_job_by_path_get(
frontend/Caddyfile
+1 -1
View File
@@ -1,4 +1,4 @@
:8000 {
:80 {
bind {$ADDRESS}
reverse_proxy /api/* http://localhost:8000
reverse_proxy /* http://localhost:3000