starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-21 00:02:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

more feature flags (#4918)

Browse Source 
* more feature flags

* include python in tests

* smtp feature flag

* del

* add back file

* fix openapi int
This commit is contained in:
HugoCasa
2024-12-14 13:21:28 +01:00
committed by GitHub
parent dff89a03c2
commit 95f4d15ffe
22 changed files with 275 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/backend-test.yml
+1 -1
View File
@@ -49,4 +49,4 @@ jobs:
windmill-api/openapi-deref.yaml &&
DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill
DISABLE_EMBEDDING=true RUST_LOG=info cargo test --features
enterprise,deno_core --all -- --nocapture
enterprise,deno_core,python --all -- --nocapture
.github/workflows/build-publish-rh-image.yml
+2 -2
View File
@@ -64,7 +64,7 @@ jobs:
platforms: linux/amd64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql,csharp
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql,mssql,bigquery,websocket,python,smtp,csharp
secrets: |
rh_username=${{ secrets.RH_USERNAME }}
rh_password=${{ secrets.RH_PASSWORD }}
@@ -81,7 +81,7 @@ jobs:
platforms: linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql,csharp
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql,mssql,bigquery,websocket,python,smtp,csharp
secrets: |
rh_username=${{ secrets.RH_USERNAME }}
rh_password=${{ secrets.RH_PASSWORD }}
.github/workflows/build-staging-image.yml
+1 -1
View File
@@ -62,7 +62,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql,csharp
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql,mssql,bigquery,websocket,python,smtp,csharp
tags: |
${{ steps.meta-ee-public.outputs.tags }}
labels: |
.github/workflows/build_windows_worker_.yml
+1 -1
View File
@@ -45,7 +45,7 @@ jobs:
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
mkdir frontend/build && cd backend
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,php,mysql,csharp
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,php,mysql,mssql,bigquery,websocket,python,smtp,csharp
- name: Rename binary with corresponding architecture
run: |
.github/workflows/docker-image-rpi4.yml
+1 -1
View File
@@ -67,7 +67,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=embedding,parquet,openidconnect,deno_core,php,mysql,csharp
features=embedding,parquet,openidconnect,deno_core,php,mysql,mssql,bigquery,websocket,python,smtp,csharp
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
${{ steps.meta-public.outputs.tags }}
.github/workflows/docker-image.yml
+3 -3
View File
@@ -76,7 +76,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=embedding,parquet,openidconnect,jemalloc,deno_core,dind,php,mysql,csharp
features=embedding,parquet,openidconnect,jemalloc,deno_core,dind,php,mysql,mssql,bigquery,websocket,python,smtp,csharp
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
${{ steps.meta-public.outputs.tags }}
@@ -138,7 +138,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind,php,mysql,csharp
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind,php,mysql,mssql,bigquery,websocket,python,smtp,csharp
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}
${{ steps.meta-ee-public.outputs.tags }}
@@ -200,7 +200,7 @@ jobs:
platforms: linux/amd64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind,php,mysql,csharp
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind,php,mysql,mssql,bigquery,websocket,python,smtp,csharp
PYTHON_IMAGE=python:3.12.2-slim-bookworm
tags: |
${{ steps.meta-ee-public-py312.outputs.tags }}
.github/workflows/publish_windows_worker.yml
+1 -1
View File
@@ -47,7 +47,7 @@ jobs:
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
mkdir frontend/build && cd backend
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,php,mysql,csharp
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,php,mysql,mssql,bigquery,websocket,python,smtp,csharp
- name: Rename binary with corresponding architecture
run: |
backend/Cargo.lock
Generated
-2
View File
@@ -10887,7 +10887,6 @@ dependencies = [
"windmill-git-sync",
"windmill-indexer",
"windmill-parser",
"windmill-parser-py-imports",
"windmill-parser-ts",
"windmill-queue",
]
@@ -11304,7 +11303,6 @@ dependencies = [
"nix",
"object_store",
"once_cell",
"openidconnect",
"opentelemetry",
"pem 3.0.4",
"postgres-native-tls",
backend/Cargo.toml
+8 -3
View File
@@ -47,20 +47,20 @@ lto = "thin"
[features]
default = []
enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-indexer/enterprise"]
enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise"]
enterprise_saml = ["windmill-api/enterprise_saml"]
stripe = ["windmill-api/stripe"]
benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark", "windmill-common/benchmark"]
flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"]
loki = ["windmill-common/loki"]
embedding = ["windmill-api/embedding"]
parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "windmill-indexer/parquet", "dep:object_store"]
parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "dep:object_store"]
prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus"]
flow_testing = ["windmill-worker/flow_testing"]
openidconnect = ["windmill-api/openidconnect"]
cloud = ["windmill-queue/cloud", "windmill-worker/cloud"]
jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"]
tantivy = ["dep:windmill-indexer", "windmill-api/tantivy"]
tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet"]
sqlx = ["windmill-worker/sqlx"]
deno_core = ["windmill-worker/deno_core", "dep:deno_core"]
kafka = ["windmill-api/kafka"]
@@ -68,6 +68,11 @@ otel = ["windmill-common/otel", "windmill-worker/otel"]
dind = ["windmill-worker/dind"]
php = ["windmill-worker/php"]
mysql = ["windmill-worker/mysql"]
mssql = ["windmill-worker/mssql"]
bigquery = ["windmill-worker/bigquery"]
websocket = ["windmill-api/websocket"]
python = ["windmill-worker/python"]
smtp = ["windmill-api/smtp"]
csharp = ["windmill-worker/csharp"]
[dependencies]
backend/windmill-api/Cargo.toml
+5 -4
View File
@@ -10,7 +10,7 @@ path = "src/lib.rs"
[features]
default = []
enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:openidconnect"]
enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise"]
stripe = ["dep:async-stripe"]
enterprise_saml = ["dep:samael"]
benchmark = []
@@ -20,13 +20,14 @@ prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:pr
openidconnect = ["dep:openidconnect"]
tantivy = ["dep:windmill-indexer"]
kafka = ["dep:rdkafka"]
websocket = ["dep:tokio-tungstenite"]
smtp = ["dep:mail-parser"]
[dependencies]
windmill-queue.workspace = true
windmill-common = { workspace = true, default-features = false }
windmill-audit.workspace = true
windmill-parser.workspace = true
windmill-parser-py-imports.workspace = true
windmill-parser-ts.workspace = true
windmill-git-sync.workspace = true
windmill-indexer = { workspace = true, optional = true }
@@ -64,7 +65,7 @@ time.workspace = true
native-tls.workspace = true
tokio-native-tls.workspace = true
openssl.workspace = true
mail-parser = { workspace = true, features = ["serde_support"] }
mail-parser = { workspace = true, features = ["serde_support"], optional = true }
magic-crypt.workspace = true
tempfile.workspace = true
tokio-util.workspace = true
@@ -95,7 +96,7 @@ openidconnect = { workspace = true, optional = true}
url = { workspace = true, optional = true}
jsonwebtoken = { workspace = true }
matchit.workspace = true
tokio-tungstenite.workspace = true
tokio-tungstenite = { workspace = true, optional = true}
rdkafka = { workspace = true, optional = true }
const_format.workspace = true
backend/windmill-api/openapi.yaml
+1 -1
View File
@@ -1126,7 +1126,7 @@ paths:
in: query
required: false
schema:
type: int
type: integer
responses:
"200":
description: new token
backend/windmill-api/src/lib.rs
+31 -15
View File
@@ -12,6 +12,7 @@ use crate::ee::ExternalJwks;
#[cfg(feature = "embedding")]
use crate::embeddings::load_embeddings_db;
use crate::oauth2_ee::AllClients;
#[cfg(feature = "smtp")]
use crate::smtp_server_ee::SmtpServer;
use crate::tracing_init::MyOnFailure;
use crate::{
@@ -83,7 +84,8 @@ mod scim_ee;
mod scripts;
mod service_logs;
mod settings;
pub mod smtp_server_ee;
#[cfg(feature = "smtp")]
mod smtp_server_ee;
mod static_assets;
mod stripe_ee;
mod tracing_init;
@@ -93,6 +95,7 @@ mod users_ee;
mod utils;
mod variables;
mod webhook_util;
#[cfg(feature = "websocket")]
mod websocket_triggers;
mod workers;
mod workspaces;
@@ -214,14 +217,17 @@ pub async fn run_server(
#[cfg(feature = "embedding")]
load_embeddings_db(&db);
let smtp_server = Arc::new(SmtpServer {
db: db.clone(),
user_db: user_db,
auth_cache: auth_cache.clone(),
base_internal_url: base_internal_url.clone(),
});
if let Err(err) = smtp_server.start_listener_thread(addr).await {
tracing::error!("Error starting SMTP server: {err:#}");
#[cfg(feature = "smtp")]
{
let smtp_server = Arc::new(SmtpServer {
db: db.clone(),
user_db: user_db,
auth_cache: auth_cache.clone(),
base_internal_url: base_internal_url.clone(),
});
if let Err(err) = smtp_server.start_listener_thread(addr).await {
tracing::error!("Error starting SMTP server: {err:#}");
}
}
}
@@ -253,8 +259,11 @@ pub async fn run_server(
};
if !*CLOUD_HOSTED {
let ws_killpill_rx = rx.resubscribe();
websocket_triggers::start_websockets(db.clone(), ws_killpill_rx).await;
#[cfg(feature = "websocket")]
{
let ws_killpill_rx = rx.resubscribe();
websocket_triggers::start_websockets(db.clone(), ws_killpill_rx).await;
}
#[cfg(all(feature = "enterprise", feature = "kafka"))]
{
@@ -304,10 +313,17 @@ pub async fn run_server(
.nest("/workspaces", workspaces::workspaced_service())
.nest("/oidc", oidc_ee::workspaced_service())
.nest("/http_triggers", http_triggers::workspaced_service())
.nest(
"/websocket_triggers",
websocket_triggers::workspaced_service(),
)
.nest("/websocket_triggers", {
#[cfg(feature = "websocket")]
{
websocket_triggers::workspaced_service()
}
#[cfg(not(feature = "websocket"))]
{
Router::new()
}
})
.nest("/kafka_triggers", kafka_triggers_service),
)
.nest("/workspaces", workspaces::global_service())
backend/windmill-api/src/workers.rs
+1 -1
View File
@@ -185,4 +185,4 @@ async fn get_queue_counts(
require_super_admin(&db, &authed.email).await?;
let queue_counts = windmill_common::queue::get_queue_counts(&db).await;
Ok(Json(queue_counts))
}
}
backend/windmill-indexer/src/lib.rs
+1 -1
View File
@@ -1,3 +1,3 @@
pub mod completed_runs_ee;
pub mod service_logs_ee;
pub mod indexer_ee;
pub mod service_logs_ee;
backend/windmill-worker/Cargo.toml
+6 -4
View File
@@ -11,7 +11,9 @@ path = "src/lib.rs"
[features]
default = []
prometheus = ["dep:prometheus", "windmill-common/prometheus"]
enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:gcp_auth", "dep:pem", "dep:tiberius", "dep:tokio-util", "dep:openidconnect"]
enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:pem", "dep:tokio-util"]
mssql = ["dep:tiberius"]
bigquery = ["dep:gcp_auth"]
benchmark = ["windmill-queue/benchmark", "windmill-common/benchmark"]
flamegraph = []
parquet = ["windmill-common/parquet", "dep:object_store"]
@@ -24,6 +26,7 @@ otel = ["windmill-common/otel", "dep:opentelemetry"]
dind = ["dep:bollard"]
php = ["dep:windmill-parser-php"]
mysql = ["dep:mysql_async"]
python = ["dep:windmill-parser-py", "dep:windmill-parser-py-imports"]
csharp = ["dep:windmill-parser-csharp"]
[dependencies]
@@ -35,9 +38,9 @@ windmill-parser-ts.workspace = true
windmill-parser-go.workspace = true
windmill-parser-rust.workspace = true
windmill-parser-csharp = { workspace = true, optional = true }
windmill-parser-py.workspace = true
windmill-parser-py = { workspace = true, optional = true }
windmill-parser-yaml.workspace = true
windmill-parser-py-imports.workspace = true
windmill-parser-py-imports = { workspace = true, optional = true }
windmill-parser-bash.workspace = true
windmill-parser-sql.workspace = true
windmill-parser-graphql.workspace = true
@@ -93,7 +96,6 @@ reqwest.workspace = true
hex.workspace = true
tiberius = { workspace = true, optional = true }
tokio-util = { workspace = true, optional = true }
openidconnect = { workspace = true, optional = true}
tar.workspace = true
object_store = { workspace = true, optional = true}
convert_case.workspace = true
backend/windmill-worker/src/dedicated_worker.rs
+8
View File
@@ -679,6 +679,14 @@ async fn spawn_dedicated_worker(
if let Err(e) = match language {
Some(ScriptLang::Python3) => {
#[cfg(not(feature = "python"))]
{
tracing::error!("Python requires the python feature to be enabled");
killpill_tx.send(()).expect("send");
return;
}
#[cfg(feature = "python")]
crate::python_executor::start_worker(
lock,
&db,
backend/windmill-worker/src/job_logger_ee.rs
+1 -1
View File
@@ -37,4 +37,4 @@ pub(crate) fn process_streaming_log_lines(
_job_id: &Uuid,
) -> Option<Result<String, io::Error>> {
r.transpose()
}
}
backend/windmill-worker/src/lib.rs
+6 -3
View File
@@ -1,10 +1,11 @@
#[cfg(feature = "enterprise")]
#[cfg(all(feature = "enterprise", feature = "bigquery"))]
mod bigquery_executor;
#[cfg(feature = "enterprise")]
#[cfg(all(feature = "enterprise", feature = "mssql"))]
mod mssql_executor;
#[cfg(feature = "enterprise")]
mod snowflake_executor;
#[cfg(feature = "python")]
mod ansible_executor;
mod bash_executor;
@@ -19,18 +20,20 @@ mod go_executor;
mod graphql_executor;
mod handle_child;
mod job_logger;
mod job_logger_ee;
mod js_eval;
#[cfg(feature = "mysql")]
mod mysql_executor;
mod pg_executor;
#[cfg(feature = "php")]
mod php_executor;
#[cfg(feature = "python")]
mod python_executor;
mod result_processor;
mod rust_executor;
mod worker;
mod worker_flow;
mod worker_lockfiles;
mod job_logger_ee;
mod csharp_executor;
pub use worker::*;
backend/windmill-worker/src/php_executor.rs
-4
View File
@@ -192,13 +192,9 @@ pub async fn handle_php_job(
let main_override = get_main_override(job.args.as_ref());
let write_wrapper_f = async {
#[cfg(feature = "php")]
let args =
windmill_parser_php::parse_php_signature(inner_content, main_override.clone())?.args;
#[cfg(not(feature = "php"))]
let args: Vec<windmill_parser::Arg> = vec![];
let args_to_include = args
.iter()
.filter(|x| {
backend/windmill-worker/src/python_executor.rs
+10 -9
View File
@@ -1518,8 +1518,8 @@ pub async fn handle_python_reqs(
if mem < 0 {
tracing::warn!(
workspace_id = %w_id_2,
"Cannot get memory peak for pid: {:?}, job_id: {:?}, exit code: {mem}",
pid_o,
"Cannot get memory peak for pid: {:?}, job_id: {:?}, exit code: {mem}",
pid_o,
job_id_2
);
} else {
@@ -1536,12 +1536,12 @@ pub async fn handle_python_reqs(
} else {
tracing::debug!(
workspace_id = %w_id_2,
"Local mem_peak {:?}mb is smaller then global one {:?}mb, ignoring. job_id: {:?}",
"Local mem_peak {:?}mb is smaller then global one {:?}mb, ignoring. job_id: {:?}",
local_mem_peak / 1000,
*mem_peak_lock / 1000,
job_id_2
);
}
// Get the copy of value and drop lock itself, to release it as fast as possible
*mem_peak_lock
@@ -1576,28 +1576,27 @@ pub async fn handle_python_reqs(
if canceled {
tracing::info!(
// If there is listener on other side,
// If there is listener on other side,
workspace_id = %w_id_2,
"cancelling installations",
);
if let Err(ref e) = kill_tx.send(()){
tracing::error!(
// If there is listener on other side,
// If there is listener on other side,
workspace_id = %w_id_2,
"failed to send done: Probably receiving end closed too early or have not opened yet\n{}",
// If there is no listener, it will be dropped safely
e
);
}
}
}
}
// Once done_tx is dropped, this will be fired
_ = done_rx.recv() => break
}
}
});
}
// tl = total_length
@@ -1647,7 +1646,9 @@ pub async fn handle_python_reqs(
let total_time = std::time::Instant::now();
let has_work = req_with_penv.len() > 0;
for ((i, (req, venv_p)), mut kill_rx) in req_with_penv.iter().enumerate().zip(kill_rxs.into_iter()) {
for ((i, (req, venv_p)), mut kill_rx) in
req_with_penv.iter().enumerate().zip(kill_rxs.into_iter())
{
let permit = semaphore.clone().acquire_owned().await; // Acquire a permit
if let Err(_) = permit {
backend/windmill-worker/src/worker.rs
+53 -9
View File
@@ -90,7 +90,6 @@ use tokio::{
use rand::Rng;
use crate::{
ansible_executor::handle_ansible_job,
bash_executor::{handle_bash_job, handle_powershell_job},
bun_executor::handle_bun_job,
common::{
@@ -106,8 +105,6 @@ use crate::{
job_logger::NO_LOGS_AT_ALL,
js_eval::{eval_fetch_timeout, transpile_ts},
pg_executor::do_postgresql,
php_executor::handle_php_job,
python_executor::handle_python_job,
result_processor::{process_result, start_background_processor},
rust_executor::handle_rust_job,
worker_flow::{handle_flow, update_flow_status_in_progress},
@@ -116,6 +113,15 @@ use crate::{
},
};
#[cfg(feature = "php")]
use crate::php_executor::handle_php_job;
#[cfg(feature = "python")]
use crate::python_executor::handle_python_job;
#[cfg(feature = "python")]
use crate::ansible_executor::handle_ansible_job;
#[cfg(feature = "mysql")]
use crate::mysql_executor::do_mysql;
@@ -126,9 +132,13 @@ use backon::{BackoffBuilder, Retryable};
use crate::dedicated_worker::create_dedicated_worker_map;
#[cfg(feature = "enterprise")]
use crate::{
bigquery_executor::do_bigquery, mssql_executor::do_mssql, snowflake_executor::do_snowflake,
};
use crate::snowflake_executor::do_snowflake;
#[cfg(all(feature = "enterprise", feature = "mssql"))]
use crate::mssql_executor::do_mssql;
#[cfg(all(feature = "enterprise", feature = "bigquery"))]
use crate::bigquery_executor::do_bigquery;
#[cfg(feature = "benchmark")]
use windmill_common::bench::{benchmark_init, BenchmarkInfo, BenchmarkIter};
@@ -2376,7 +2386,9 @@ async fn handle_code_execution_job(
.await;
} else if language == Some(ScriptLang::Mysql) {
#[cfg(not(feature = "mysql"))]
return Err(Error::InternalErr("MySQL requires the mysql feature to be enabled".to_string()));
return Err(Error::InternalErr(
"MySQL requires the mysql feature to be enabled".to_string(),
));
#[cfg(feature = "mysql")]
return do_mysql(
@@ -2399,7 +2411,14 @@ async fn handle_code_execution_job(
));
}
#[cfg(feature = "enterprise")]
#[cfg(not(feature = "bigquery"))]
{
return Err(Error::InternalErr(
"Bigquery requires the bigquery feature to be enabled".to_string(),
));
}
#[cfg(all(feature = "enterprise", feature = "bigquery"))]
{
return do_bigquery(
job,
@@ -2445,7 +2464,14 @@ async fn handle_code_execution_job(
));
}
#[cfg(feature = "enterprise")]
#[cfg(not(feature = "mssql"))]
{
return Err(Error::InternalErr(
"Microsoft SQL server requires the mssql feature to be enabled".to_string(),
));
}
#[cfg(all(feature = "enterprise", feature = "mssql"))]
{
return do_mssql(
job,
@@ -2549,6 +2575,12 @@ mount {{
))?;
}
Some(ScriptLang::Python3) => {
#[cfg(not(feature = "python"))]
return Err(Error::InternalErr(
"Python requires the python feature to be enabled".to_string(),
));
#[cfg(feature = "python")]
handle_python_job(
requirements_o,
job_dir,
@@ -2660,6 +2692,12 @@ mount {{
.await
}
Some(ScriptLang::Php) => {
#[cfg(not(feature = "php"))]
return Err(Error::InternalErr(
"PHP requires the php feature to be enabled".to_string(),
));
#[cfg(feature = "php")]
handle_php_job(
requirements_o,
mem_peak,
@@ -2696,6 +2734,12 @@ mount {{
.await
}
Some(ScriptLang::Ansible) => {
#[cfg(not(feature = "python"))]
return Err(Error::InternalErr(
"Ansible requires the python feature to be enabled".to_string(),
));
#[cfg(feature = "python")]
handle_ansible_job(
requirements_o,
job_dir,
backend/windmill-worker/src/worker_lockfiles.rs
+134 -106
View File
@@ -23,21 +23,24 @@ use windmill_common::{
DB,
};
use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
#[cfg(feature = "python")]
use windmill_parser_py_imports::parse_relative_imports;
use windmill_parser_ts::parse_expr_for_imports;
use windmill_queue::{append_logs, CanceledBy, PushIsolationLevel};
use crate::common::OccupancyMetrics;
use crate::csharp_executor::generate_nuget_lockfile;
#[cfg(feature = "php")]
use crate::php_executor::{composer_install, parse_php_imports};
#[cfg(feature = "python")]
use crate::python_executor::{
create_dependencies_dir, handle_python_reqs, uv_pip_compile, USE_PIP_COMPILE, USE_PIP_INSTALL,
};
use crate::rust_executor::generate_cargo_lockfile;
use crate::{
bun_executor::gen_bun_lockfile,
deno_executor::generate_deno_lock,
bun_executor::gen_bun_lockfile, deno_executor::generate_deno_lock,
go_executor::install_go_dependencies,
php_executor::{composer_install, parse_php_imports},
};
pub async fn update_script_dependency_map(
@@ -199,6 +202,7 @@ pub fn extract_relative_imports(
language: &Option<ScriptLang>,
) -> Option<Vec<String>> {
match language {
#[cfg(feature = "python")]
Some(ScriptLang::Python3) => parse_relative_imports(&raw_code, script_path).ok(),
Some(ScriptLang::Bun) | Some(ScriptLang::Bunnative) => {
parse_bun_relative_imports(&raw_code, script_path).ok()
@@ -1566,6 +1570,7 @@ pub async fn handle_app_dependency_job(
}
}
#[cfg(feature = "python")]
async fn python_dep(
reqs: String,
job_id: &Uuid,
@@ -1643,95 +1648,111 @@ async fn capture_dependency_job(
) -> error::Result<String> {
match job_language {
ScriptLang::Python3 => {
let reqs = if raw_deps {
job_raw_code.to_string()
} else {
let mut already_visited = vec![];
#[cfg(not(feature = "python"))]
return Err(Error::InternalErr(
"Python requires the python feature to be enabled".to_string(),
));
windmill_parser_py_imports::parse_python_imports(
job_raw_code,
&w_id,
script_path,
&db,
&mut already_visited,
)
.await?
.join("\n")
};
#[cfg(feature = "python")]
{
let reqs = if raw_deps {
job_raw_code.to_string()
} else {
let mut already_visited = vec![];
let PythonAnnotations { no_uv, no_uv_install, no_uv_compile, .. } =
PythonAnnotations::parse(job_raw_code);
windmill_parser_py_imports::parse_python_imports(
job_raw_code,
&w_id,
script_path,
&db,
&mut already_visited,
)
.await?
.join("\n")
};
if no_uv || no_uv_install || no_uv_compile || *USE_PIP_COMPILE || *USE_PIP_INSTALL {
if let Err(e) = sqlx::query!(
r#"
INSERT INTO metrics (id, value)
VALUES ('no_uv_usage_py', $1)
"#,
serde_json::to_value("").map_err(to_anyhow)?
)
.execute(db)
.await
{
tracing::error!("Error inserting no_uv_usage_py to db: {:?}", e);
let PythonAnnotations { no_uv, no_uv_install, no_uv_compile, .. } =
PythonAnnotations::parse(job_raw_code);
if no_uv || no_uv_install || no_uv_compile || *USE_PIP_COMPILE || *USE_PIP_INSTALL {
if let Err(e) = sqlx::query!(
r#"
INSERT INTO metrics (id, value)
VALUES ('no_uv_usage_py', $1)
"#,
serde_json::to_value("").map_err(to_anyhow)?
)
.execute(db)
.await
{
tracing::error!("Error inserting no_uv_usage_py to db: {:?}", e);
}
}
}
python_dep(
reqs,
job_id,
mem_peak,
canceled_by,
job_dir,
db,
worker_name,
w_id,
worker_dir,
&mut Some(occupancy_metrics),
no_uv_compile | no_uv,
no_uv_install | no_uv,
)
.await
python_dep(
reqs,
job_id,
mem_peak,
canceled_by,
job_dir,
db,
worker_name,
w_id,
worker_dir,
&mut Some(occupancy_metrics),
no_uv_compile | no_uv,
no_uv_install | no_uv,
)
.await
}
}
ScriptLang::Ansible => {
if raw_deps {
return Err(Error::ExecutionErr(
"Raw dependencies not supported for ansible".to_string(),
));
}
let (_logs, reqs, _) = windmill_parser_yaml::parse_ansible_reqs(job_raw_code)?;
let reqs = reqs.map(|r| r.python_reqs.join("\n")).unwrap_or_default();
#[cfg(not(feature = "python"))]
return Err(Error::InternalErr(
"Ansible requires the python feature to be enabled".to_string(),
));
if *USE_PIP_COMPILE || *USE_PIP_INSTALL {
if let Err(e) = sqlx::query!(
r#"
INSERT INTO metrics (id, value)
VALUES ('no_uv_usage_ansible', $1)
"#,
serde_json::to_value("").map_err(to_anyhow)?
#[cfg(feature = "python")]
{
if raw_deps {
return Err(Error::ExecutionErr(
"Raw dependencies not supported for ansible".to_string(),
));
}
let (_logs, reqs, _) = windmill_parser_yaml::parse_ansible_reqs(job_raw_code)?;
let reqs = reqs.map(|r| r.python_reqs.join("\n")).unwrap_or_default();
if *USE_PIP_COMPILE || *USE_PIP_INSTALL {
if let Err(e) = sqlx::query!(
r#"
INSERT INTO metrics (id, value)
VALUES ('no_uv_usage_ansible', $1)
"#,
serde_json::to_value("").map_err(to_anyhow)?
)
.execute(db)
.await
{
tracing::error!("Error inserting no_uv_usage_ansible to db: {:?}", e);
};
}
python_dep(
reqs,
job_id,
mem_peak,
canceled_by,
job_dir,
db,
worker_name,
w_id,
worker_dir,
&mut Some(occupancy_metrics),
false,
false,
)
.execute(db)
.await
{
tracing::error!("Error inserting no_uv_usage_ansible to db: {:?}", e);
};
}
python_dep(
reqs,
job_id,
mem_peak,
canceled_by,
job_dir,
db,
worker_name,
w_id,
worker_dir,
&mut Some(occupancy_metrics),
false,
false,
)
.await
}
ScriptLang::Go => {
if raw_deps {
@@ -1822,33 +1843,40 @@ async fn capture_dependency_job(
Ok(req.unwrap_or_else(String::new))
}
ScriptLang::Php => {
let reqs = if raw_deps {
if job_raw_code.is_empty() {
return Ok("".to_string());
}
job_raw_code.to_string()
} else {
match parse_php_imports(job_raw_code)? {
Some(reqs) => reqs,
None => {
#[cfg(not(feature = "php"))]
return Err(Error::InternalErr(
"PHP requires the php feature to be enabled".to_string(),
));
#[cfg(feature = "php")]
{
let reqs = if raw_deps {
if job_raw_code.is_empty() {
return Ok("".to_string());
}
}
};
composer_install(
mem_peak,
canceled_by,
job_id,
w_id,
db,
job_dir,
worker_name,
reqs,
None,
occupancy_metrics,
)
.await
job_raw_code.to_string()
} else {
match parse_php_imports(job_raw_code)? {
Some(reqs) => reqs,
None => {
return Ok("".to_string());
}
}
};
composer_install(
mem_peak,
canceled_by,
job_id,
w_id,
db,
job_dir,
worker_name,
reqs,
None,
occupancy_metrics,
)
.await
}
}
ScriptLang::Rust => {
if raw_deps {