diff --git a/backend/COMPILATION_OPTIMIZATION.md b/backend/COMPILATION_OPTIMIZATION.md new file mode 100644 index 0000000000..4e055fcff1 --- /dev/null +++ b/backend/COMPILATION_OPTIMIZATION.md @@ -0,0 +1,97 @@ +# Backend Compilation Optimization + +## Summary + +Feature-gated heavy dependencies that were compiled by default but only used behind enterprise/EE feature flags. This reduces the default build from **761 crates to 609 crates** (20% reduction). + +## Changes + +### Root `Cargo.toml` +- Removed 12 unused direct dependencies: `kube`, `k8s-openapi`, `aws-sigv4`, `aws-sdk-config`, `opentelemetry-proto`, `systemstat`, `globset`, `libloading`, `bitflags`, `memchr`, `quote`, `pep440_rs` + +### `windmill-worker/Cargo.toml` +- Made optional (only needed for EE OTEL tracing proxy): `hudsucker`, `hyper-http-proxy`, `hyper-tls`, `hyper-util`, `rcgen`, `opentelemetry-proto`, `prost` +- Made optional (only needed for EE features): `aws-config`, `aws-credential-types`, `aws-smithy-types` +- Created `otel_proxy` feature to group the OTEL proxy deps +- Updated `private` feature to include `otel_proxy` + +### `windmill-common/Cargo.toml` +- Made optional: `aws-config`, `aws-credential-types`, `aws-smithy-types`, `systemstat`, `globset` +- Added AWS deps to `private`, `parquet`, `aws_auth`, `bedrock` features +- Added `systemstat` to `private` feature +- Added `globset` to `parquet` feature + +### `windmill-api/Cargo.toml` +- Made optional: `aws-sigv4`, `aws-sdk-config`, `aws-credential-types`, `aws-smithy-types`, `windmill-parser-py-imports`, `windmill-autoscaling` +- Added AWS deps to `parquet` and `bedrock` features +- Added `windmill-parser-py-imports` to `python` and `agent_worker_server` features +- Added `windmill-autoscaling` to `enterprise` feature + +### `windmill-autoscaling/Cargo.toml` +- Made optional: `kube`, `k8s-openapi` (only used in EE code) +- Added to `private` feature + +### `parsers/windmill-parser-py-imports/Cargo.toml` +- Removed unused direct dependencies: `malachite`, `malachite-bigint` (still available transitively via `rustpython-parser`) + +## Benchmarks + +### Default build (no features) + +| Metric | Before | After | +|---|---|---| +| Crates compiled | 761 | 609 | +| Notable deps eliminated | - | aws-sdk-config (9.5s), k8s-openapi (7.3s), zstd-sys (7.7s), kube-client (1.9s) | + +### Incremental compilation (stable-state, warm cache) + +| Scenario | Before | After | +|---|---|---| +| Touch `windmill-api/src/users.rs` | ~5.6s | ~5.4s | +| Touch `windmill-worker/src/worker.rs` | ~6.7s | ~6.2s | +| Touch `windmill-common/src/worker.rs` (cascade) | ~8.5s | ~8.5s | + +Incremental compilation improvement from feature-gating alone is modest because the bottleneck is the compilation of the windmill crates themselves (especially windmill-api at 90k LOC), not the dependencies. + +## Developer-Local Speed Tips + +These settings are **not committed** because they are developer-local preferences that depend on toolchain availability. Combined, they yield ~16% faster incremental compilation. + +### mold linker (~6% improvement) + +Install `mold` and add to `.cargo/config.toml`: + +```toml +[target.x86_64-unknown-linux-gnu] +linker = "clang" +rustflags = ["-C", "link-arg=-fuse-ld=mold"] +``` + +### Reduced debug info (~10% improvement) + +Add to `[profile.dev]` in `Cargo.toml`: + +```toml +split-debuginfo = "unpacked" +debug = "line-tables-only" +``` + +### SQLX offline mode (~4% improvement) + +If you're not modifying SQL queries: + +```bash +export SQLX_OFFLINE=true +``` + +### Combined effect + +| Scenario | Baseline | With all tips | +|---|---|---| +| Touch `windmill-api` file | 5.6s | **4.7s** | +| Touch `windmill-worker` file | 6.7s | **6.0s** | +| Touch `windmill-common` file (cascade) | 8.5s | **7.6s** | + +## What would help more (future work) + +The single biggest improvement would be **splitting `windmill-api`** (90k LOC) into smaller crates. Currently, any file change in the crate triggers re-analysis of all 90k lines. However, this requires significant refactoring due to tight coupling between the triggers subsystem, jobs, users, and the axum router initialization. diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 8a3bc32d5e..0b73574a82 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15490,11 +15490,8 @@ name = "windmill" version = "1.624.0" dependencies = [ "anyhow", - "aws-sdk-config", - "aws-sigv4", "axum 0.7.9", "base64 0.22.1", - "bitflags 2.9.4", "chrono", "constant_time_eq 0.3.1", "deno_core", @@ -15502,18 +15499,10 @@ dependencies = [ "futures", "gethostname", "git-version", - "globset", - "k8s-openapi", - "kube", "lazy_static", - "libloading 0.8.9", - "memchr", "object_store", "once_cell", - "opentelemetry-proto 0.29.0", - "pep440_rs", "prometheus", - "quote", "rand 0.9.0", "reqwest 0.13.1", "rustls 0.23.35", @@ -15526,7 +15515,6 @@ dependencies = [ "sql-builder", "sqlx", "strum 0.27.2", - "systemstat", "tempfile", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", @@ -16030,8 +16018,6 @@ dependencies = [ "async-recursion", "itertools 0.14.0", "lazy_static", - "malachite", - "malachite-bigint", "pep440_rs", "phf 0.11.3", "regex", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index c9f11920b0..47328ccbd9 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -150,21 +150,13 @@ deno_core = { workspace = true, optional = true } object_store = { workspace = true, optional = true } sha1 = { workspace = true, optional = true } constant_time_eq = { workspace = true, optional = true } -quote.workspace = true -memchr.workspace = true + + v8 = { workspace = true, optional = true } rustls.workspace = true -pep440_rs.workspace = true strum.workspace = true -aws-sigv4.workspace = true -aws-sdk-config.workspace = true -kube.workspace = true -k8s-openapi.workspace = true -libloading.workspace = true -bitflags.workspace = true -globset.workspace = true -opentelemetry-proto.workspace = true -systemstat.workspace = true + + [target.'cfg(windows)'.dependencies] windows-service = "0.7" diff --git a/backend/parsers/windmill-parser-py-imports/Cargo.toml b/backend/parsers/windmill-parser-py-imports/Cargo.toml index deea68c2ef..e53dcbc71f 100644 --- a/backend/parsers/windmill-parser-py-imports/Cargo.toml +++ b/backend/parsers/windmill-parser-py-imports/Cargo.toml @@ -18,8 +18,6 @@ regex.workspace = true windmill-parser.workspace = true windmill-common.workspace = true rustpython-parser.workspace = true -malachite.workspace = true -malachite-bigint.workspace = true phf.workspace = true itertools.workspace = true serde_json.workspace = true diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 16a0bc3a16..596c2b139a 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -11,13 +11,13 @@ path = "src/lib.rs" [features] default = [] private = ["windmill-audit/private", "windmill-common/private"] -enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise"] +enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise", "dep:windmill-autoscaling"] stripe = [] -agent_worker_server = [] +agent_worker_server = ["dep:windmill-parser-py-imports"] enterprise_saml = ["dep:samael", "dep:libxml"] benchmark = [] embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn"] -parquet = ["dep:datafusion", "dep:object_store", "windmill-common/parquet", "windmill-worker/parquet"] +parquet = ["dep:datafusion", "dep:object_store", "windmill-common/parquet", "windmill-worker/parquet", "dep:aws-sigv4", "dep:aws-sdk-config"] prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker/prometheus"] openidconnect = ["dep:openidconnect", "windmill-common/openidconnect"] tantivy = ["dep:windmill-indexer"] @@ -38,8 +38,8 @@ deno_core = ["dep:deno_core", "dep:deno_error"] gcp_trigger = ["dep:thiserror", "dep:google-cloud-pubsub", "dep:google-cloud-googleapis", "dep:tonic"] cloud = ["windmill-common/cloud"] mcp = ["dep:windmill-mcp", "windmill-mcp/server", "windmill-mcp/auth"] -bedrock = ["dep:aws-sdk-bedrock", "dep:aws-sdk-bedrockruntime", "windmill-common/bedrock", "dep:aws-config"] -python = [] +bedrock = ["dep:aws-sdk-bedrock", "dep:aws-sdk-bedrockruntime", "windmill-common/bedrock", "dep:aws-config", "dep:aws-sdk-config", "dep:aws-credential-types", "dep:aws-smithy-types"] +python = ["dep:windmill-parser-py-imports"] [dependencies] windmill-mcp = { workspace = true, optional = true } @@ -50,10 +50,10 @@ windmill-parser.workspace = true windmill-parser-sql.workspace = true windmill-parser-ts.workspace = true windmill-parser-py.workspace = true -windmill-parser-py-imports.workspace = true +windmill-parser-py-imports = { workspace = true, optional = true } windmill-git-sync.workspace = true windmill-indexer = { workspace = true, optional = true } -windmill-autoscaling.workspace = true +windmill-autoscaling = { workspace = true, optional = true } windmill-worker.workspace = true tokio.workspace = true tokio-stream.workspace = true @@ -152,13 +152,13 @@ aws-sdk-ssooidc = { workspace = true, optional = true } aws-sdk-sts = { workspace = true, optional = true } rustls = { workspace = true } -aws-sigv4.workspace = true -aws-sdk-config.workspace = true +aws-sigv4 = { workspace = true, optional = true } +aws-sdk-config = { workspace = true, optional = true } aws-config = { workspace = true, optional = true } -aws-credential-types.workspace = true +aws-credential-types = { workspace = true, optional = true } aws-sdk-bedrock = { workspace = true, optional = true } aws-sdk-bedrockruntime = { workspace = true, optional = true } -aws-smithy-types.workspace = true +aws-smithy-types = { workspace = true, optional = true } async-trait.workspace = true google-cloud-pubsub = { workspace = true, optional = true } google-cloud-googleapis = { workspace = true , optional = true } diff --git a/backend/windmill-autoscaling/Cargo.toml b/backend/windmill-autoscaling/Cargo.toml index fd5c5f4703..115ff9836b 100644 --- a/backend/windmill-autoscaling/Cargo.toml +++ b/backend/windmill-autoscaling/Cargo.toml @@ -10,7 +10,7 @@ path = "./src/lib.rs" [features] enterprise = ["windmill-queue/enterprise", "windmill-common/enterprise"] -private = [] +private = ["dep:kube", "dep:k8s-openapi"] default = [] [dependencies] @@ -22,8 +22,8 @@ tracing.workspace = true windmill-common = { workspace = true, default-features = false } windmill-queue.workspace = true anyhow.workspace = true -kube.workspace = true -k8s-openapi.workspace = true +kube = { workspace = true, optional = true } +k8s-openapi = { workspace = true, optional = true } tokio.workspace = true thiserror.workspace = true axum.workspace = true diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index bdd84663f6..3d5a0d7bc4 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -7,20 +7,20 @@ edition.workspace = true [features] default = [] enterprise = [] -private = ["dep:aws-sdk-rds"] +private = ["dep:aws-sdk-rds", "dep:systemstat", "dep:aws-config", "dep:aws-credential-types", "dep:aws-smithy-types"] jemalloc = ["dep:tikv-jemalloc-ctl"] tantivy = [] prometheus = ["dep:prometheus"] benchmark = [] -parquet = ["dep:object_store", "dep:aws-sdk-sts", "dep:aws-smithy-types-convert", "dep:datafusion"] -aws_auth = ["dep:aws-sdk-sts"] +parquet = ["dep:object_store", "dep:aws-sdk-sts", "dep:aws-smithy-types-convert", "dep:datafusion", "dep:aws-config", "dep:aws-credential-types", "dep:aws-smithy-types", "dep:globset"] +aws_auth = ["dep:aws-sdk-sts", "dep:aws-config", "dep:aws-credential-types", "dep:aws-smithy-types"] otel = ["dep:opentelemetry-semantic-conventions", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk", "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing", "dep:tonic", "dep:opentelemetry"] smtp = ["dep:mail-send"] scoped_cache = [] cloud = [] openidconnect = ["dep:openidconnect"] -bedrock = ["dep:aws-sdk-bedrockruntime"] +bedrock = ["dep:aws-sdk-bedrockruntime", "dep:aws-config", "dep:aws-credential-types", "dep:aws-smithy-types"] [lib] name = "windmill_common" @@ -63,10 +63,10 @@ cron.workspace = true magic-crypt.workspace = true object_store = { workspace = true, optional = true } prometheus = { workspace = true, optional = true } -aws-config.workspace = true +aws-config = { workspace = true, optional = true } aws-sdk-sts = { workspace = true, optional = true } -aws-credential-types.workspace = true -aws-smithy-types.workspace = true +aws-credential-types = { workspace = true, optional = true } +aws-smithy-types = { workspace = true, optional = true } aws-sdk-bedrockruntime = { workspace = true, optional = true } base64.workspace = true bitflags.workspace = true @@ -100,7 +100,7 @@ url.workspace = true urlencoding.workspace = true async-recursion.workspace = true pep440_rs.workspace = true -systemstat.workspace = true +systemstat = { workspace = true, optional = true } size.workspace = true semver.workspace = true @@ -109,7 +109,7 @@ quick_cache.workspace = true pin-project-lite.workspace = true futures.workspace = true tempfile.workspace = true -globset.workspace = true +globset = { workspace = true, optional = true } opentelemetry-semantic-conventions = { workspace = true, optional = true } opentelemetry-otlp = { workspace = true, optional = true } diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 8ba01c71ab..6a166dc2d4 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -private = [] +private = ["otel_proxy"] mcp = ["dep:windmill-mcp"] prometheus = ["dep:prometheus", "windmill-common/prometheus"] enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:pem", "dep:tokio-util"] @@ -25,7 +25,7 @@ deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net" "dep:deno_ast", "dep:deno_tls", "dep:deno_permissions", "dep:deno_io", "dep:deno_runtime", "dep:deno_telemetry", "dep:deno_error", "dep:winapi", "dep:rustls-pemfile", "quickjs"] libffi_mac = ["dep:libffi-sys"] -otel = ["windmill-common/otel", "dep:opentelemetry", "dep:tracing-opentelemetry"] +otel = ["windmill-common/otel", "dep:opentelemetry", "dep:tracing-opentelemetry", "dep:opentelemetry-proto", "dep:prost"] dind = ["dep:bollard"] php = ["dep:windmill-parser-php"] mysql = ["dep:mysql_async"] @@ -39,6 +39,7 @@ ruby = ["dep:windmill-parser-ruby"] duckdb = ["dep:libloading"] quickjs = ["dep:rquickjs"] bedrock = ["dep:aws-sdk-bedrockruntime", "windmill-common/bedrock"] +otel_proxy = ["dep:hudsucker", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util", "dep:rcgen", "otel"] [dependencies] windmill-queue.workspace = true @@ -63,9 +64,9 @@ windmill-parser-graphql.workspace = true windmill-parser-php = { workspace = true, optional = true } windmill-git-sync.workspace = true aws-sdk-bedrockruntime = { workspace = true, optional = true } -aws-config.workspace = true -aws-credential-types.workspace = true -aws-smithy-types.workspace = true +aws-config = { workspace = true, optional = true } +aws-credential-types = { workspace = true, optional = true } +aws-smithy-types = { workspace = true, optional = true } flume.workspace = true sqlx.workspace = true uuid.workspace = true @@ -140,19 +141,19 @@ process-wrap.workspace = true async-once-cell.workspace = true libloading = { workspace = true, optional = true } -opentelemetry-proto.workspace = true +opentelemetry-proto = { workspace = true, optional = true } opentelemetry = { workspace = true, optional = true } tracing-opentelemetry = { workspace = true, optional = true } -prost.workspace = true +prost = { workspace = true, optional = true } axum.workspace = true bollard = { workspace = true, optional = true } oracle = { workspace = true, optional = true } rquickjs = { workspace = true, optional = true } -hudsucker.workspace = true -hyper-http-proxy.workspace = true -hyper-tls.workspace = true -hyper-util.workspace = true -rcgen.workspace = true +hudsucker = { workspace = true, optional = true } +hyper-http-proxy = { workspace = true, optional = true } +hyper-tls = { workspace = true, optional = true } +hyper-util = { workspace = true, optional = true } +rcgen = { workspace = true, optional = true } [build-dependencies] deno_fetch = { workspace = true, optional = true }