From da8f8e11b207ae3e38ea8d4640e7e9cf50e2e597 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 9 Jan 2026 12:43:03 +0000 Subject: [PATCH] pg_embed --- backend/Cargo.lock | 267 ++++++++++++++++++++---- backend/Cargo.toml | 3 +- backend/README.md | 49 +++++ backend/src/main.rs | 28 +++ backend/windmill-common/Cargo.toml | 2 + backend/windmill-common/src/lib.rs | 2 + backend/windmill-common/src/pg_embed.rs | 144 +++++++++++++ 7 files changed, 458 insertions(+), 37 deletions(-) create mode 100644 backend/windmill-common/src/pg_embed.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 43cfa370e3..af15b75104 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1268,7 +1268,7 @@ dependencies = [ "hyper 1.8.1", "hyper-util", "itoa", - "matchit", + "matchit 0.7.3", "memchr", "mime", "multer", @@ -2823,7 +2823,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.12", ] [[package]] @@ -2837,7 +2837,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.12", ] [[package]] @@ -2894,7 +2894,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "parking_lot", + "parking_lot 0.12.5", "parquet", "rand 0.8.5", "regex", @@ -2929,7 +2929,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "parking_lot", + "parking_lot 0.12.5", "tokio", ] @@ -3102,7 +3102,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "parking_lot", + "parking_lot 0.12.5", "parquet", "rand 0.8.5", "tokio", @@ -3127,7 +3127,7 @@ dependencies = [ "futures", "log", "object_store", - "parking_lot", + "parking_lot 0.12.5", "rand 0.8.5", "tempfile", "url", @@ -3263,7 +3263,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-plan", - "parking_lot", + "parking_lot 0.12.5", "paste", ] @@ -3404,7 +3404,7 @@ dependencies = [ "indexmap 2.11.1", "itertools 0.14.0", "log", - "parking_lot", + "parking_lot 0.12.5", "pin-project-lite", "tokio", ] @@ -3429,7 +3429,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "parking_lot", + "parking_lot 0.12.5", "tokio", ] @@ -3558,7 +3558,7 @@ dependencies = [ "indexmap 2.11.1", "log", "once_cell", - "parking_lot", + "parking_lot 0.12.5", "serde", "serde_json", "sha2 0.10.9", @@ -3640,7 +3640,7 @@ dependencies = [ "indexmap 2.11.1", "libc", "memoffset", - "parking_lot", + "parking_lot 0.12.5", "percent-encoding", "pin-project", "serde", @@ -3885,7 +3885,7 @@ dependencies = [ "log", "once_cell", "os_pipe", - "parking_lot", + "parking_lot 0.12.5", "pin-project", "rand 0.8.5", "tokio", @@ -4265,7 +4265,7 @@ dependencies = [ "log", "node_resolver", "once_cell", - "parking_lot", + "parking_lot 0.12.5", "sys_traits", "thiserror 2.0.17", "url", @@ -4422,7 +4422,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6742a724e8becb372a74c650a1aefb8924a5b8107f7d75b3848763ea24b27a87" dependencies = [ "futures-util", - "parking_lot", + "parking_lot 0.12.5", "tokio", ] @@ -5667,7 +5667,7 @@ checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" dependencies = [ "futures-core", "lock_api", - "parking_lot", + "parking_lot 0.12.5", ] [[package]] @@ -6565,7 +6565,7 @@ dependencies = [ "ipconfig", "moka", "once_cell", - "parking_lot", + "parking_lot 0.12.5", "rand 0.9.0", "resolv-conf", "serde", @@ -7162,6 +7162,18 @@ dependencies = [ "generic-array", ] +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "integer-encoding" version = "3.0.4" @@ -7621,7 +7633,7 @@ dependencies = [ "json-patch", "k8s-openapi", "kube-client", - "parking_lot", + "parking_lot 0.12.5", "pin-project", "serde", "serde_json", @@ -8135,6 +8147,12 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "matchit" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f926ade0c4e170215ae43342bf13b9310a437609c81f29f86c5df6657582ef9" + [[package]] name = "md-5" version = "0.9.1" @@ -8341,7 +8359,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "equivalent", - "parking_lot", + "parking_lot 0.12.5", "portable-atomic", "smallvec", "tagptr", @@ -9059,7 +9077,7 @@ dependencies = [ "hyper 1.8.1", "itertools 0.14.0", "md-5 0.10.6", - "parking_lot", + "parking_lot 0.12.5", "percent-encoding", "quick-xml 0.37.5", "rand 0.9.0", @@ -9473,6 +9491,17 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -9480,7 +9509,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.12", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -9959,6 +10002,64 @@ dependencies = [ "uuid", ] +[[package]] +name = "postgresql_archive" +version = "0.18.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c1e2f490f528e4eb06be6e9494ad28ea2b3f376bac7a9ad7c99b9062a7358e" +dependencies = [ + "async-trait", + "flate2", + "futures-util", + "hex", + "num-format", + "regex-lite", + "reqwest 0.12.28", + "reqwest-middleware 0.4.2", + "reqwest-retry 0.7.0", + "reqwest-tracing", + "semver 1.0.27", + "serde", + "serde_json", + "sha2 0.10.9", + "tar", + "target-triple", + "tempfile", + "thiserror 2.0.17", + "tracing", + "url", +] + +[[package]] +name = "postgresql_commands" +version = "0.18.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56eae883251fb55f23ff8e9f16121cfb2b00a56ebe62c56183c67904af26fa1d" +dependencies = [ + "thiserror 2.0.17", + "tracing", +] + +[[package]] +name = "postgresql_embedded" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d56d311cb5868a3d91ba40dab5f87050dfd462ee81c39616735eb1f992cb17" +dependencies = [ + "anyhow", + "postgresql_archive", + "postgresql_commands", + "rand 0.9.0", + "semver 1.0.27", + "sqlx", + "target-triple", + "tempfile", + "thiserror 2.0.17", + "tokio", + "tracing", + "url", +] + [[package]] name = "potential_utf" version = "0.1.4" @@ -10150,7 +10251,7 @@ dependencies = [ "fnv", "lazy_static", "memchr", - "parking_lot", + "parking_lot 0.12.5", "thiserror 2.0.17", ] @@ -10319,7 +10420,7 @@ dependencies = [ "ahash 0.8.12", "equivalent", "hashbrown 0.16.1", - "parking_lot", + "parking_lot 0.12.5", ] [[package]] @@ -10608,6 +10709,15 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -10811,6 +10921,21 @@ dependencies = [ "web-sys", ] +[[package]] +name = "reqwest-middleware" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57f17d28a6e6acfe1733fe24bcd30774d13bffa4b8a22535b4c8c98423088d4e" +dependencies = [ + "anyhow", + "async-trait", + "http 1.4.0", + "reqwest 0.12.28", + "serde", + "thiserror 1.0.69", + "tower-service", +] + [[package]] name = "reqwest-middleware" version = "0.5.0" @@ -10826,6 +10951,28 @@ dependencies = [ "tower-service", ] +[[package]] +name = "reqwest-retry" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c73e4195a6bfbcb174b790d9b3407ab90646976c55de58a6515da25d851178" +dependencies = [ + "anyhow", + "async-trait", + "futures", + "getrandom 0.2.16", + "http 1.4.0", + "hyper 1.8.1", + "parking_lot 0.11.2", + "reqwest 0.12.28", + "reqwest-middleware 0.4.2", + "retry-policies 0.4.0", + "thiserror 1.0.69", + "tokio", + "tracing", + "wasm-timer", +] + [[package]] name = "reqwest-retry" version = "0.9.0" @@ -10839,20 +10986,45 @@ dependencies = [ "http 1.4.0", "hyper 1.8.1", "reqwest 0.13.1", - "reqwest-middleware", - "retry-policies", + "reqwest-middleware 0.5.0", + "retry-policies 0.5.0", "thiserror 2.0.17", "tokio", "tracing", "wasmtimer", ] +[[package]] +name = "reqwest-tracing" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d70ea85f131b2ee9874f0b160ac5976f8af75f3c9badfe0d955880257d10bd83" +dependencies = [ + "anyhow", + "async-trait", + "getrandom 0.2.16", + "http 1.4.0", + "matchit 0.8.6", + "reqwest 0.12.28", + "reqwest-middleware 0.4.2", + "tracing", +] + [[package]] name = "resolv-conf" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" +[[package]] +name = "retry-policies" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c" +dependencies = [ + "rand 0.8.5", +] + [[package]] name = "retry-policies" version = "0.5.0" @@ -13401,6 +13573,12 @@ dependencies = [ "xattr", ] +[[package]] +name = "target-triple" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac9aa371f599d22256307c24a9d748c041e548cbf599f35d890f9d365361790" + [[package]] name = "tempfile" version = "3.24.0" @@ -13707,7 +13885,7 @@ dependencies = [ "io-uring", "libc", "mio 1.1.1", - "parking_lot", + "parking_lot 0.12.5", "pin-project-lite", "signal-hook-registry", "slab", @@ -13772,7 +13950,7 @@ dependencies = [ "futures-channel", "futures-util", "log", - "parking_lot", + "parking_lot 0.12.5", "percent-encoding", "phf 0.11.3", "pin-project-lite", @@ -13798,7 +13976,7 @@ dependencies = [ "futures-channel", "futures-util", "log", - "parking_lot", + "parking_lot 0.12.5", "percent-encoding", "phf 0.11.3", "pin-project-lite", @@ -14074,7 +14252,7 @@ dependencies = [ "cookie 0.18.1", "futures-util", "http 1.4.0", - "parking_lot", + "parking_lot 0.12.5", "pin-project-lite", "tower-layer", "tower-service", @@ -14952,6 +15130,21 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-timer" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" +dependencies = [ + "futures", + "js-sys", + "parking_lot 0.11.2", + "pin-utils", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasm_dep_analyzer" version = "0.2.0" @@ -14970,7 +15163,7 @@ checksum = "1c598d6b99ea013e35844697fc4670d08339d5cda15588f193c6beedd12f644b" dependencies = [ "futures", "js-sys", - "parking_lot", + "parking_lot 0.12.5", "pin-utils", "slab", "wasm-bindgen", @@ -15048,7 +15241,7 @@ dependencies = [ "log", "naga", "once_cell", - "parking_lot", + "parking_lot 0.12.5", "profiling", "raw-window-handle", "ron", @@ -15090,7 +15283,7 @@ dependencies = [ "ndk-sys", "objc", "once_cell", - "parking_lot", + "parking_lot 0.12.5", "profiling", "range-alloc", "raw-window-handle", @@ -15213,6 +15406,7 @@ dependencies = [ "object_store", "once_cell", "pep440_rs", + "postgresql_embedded", "prometheus", "quote", "rand 0.9.0", @@ -15304,7 +15498,7 @@ dependencies = [ "libxml", "magic-crypt", "mail-parser", - "matchit", + "matchit 0.7.3", "mime_guess", "native-tls", "nkeys", @@ -15468,13 +15662,14 @@ dependencies = [ "phf 0.11.3", "pin-project-lite", "postgres-native-tls 0.5.1", + "postgresql_embedded", "prometheus", "quick_cache", "rand 0.9.0", "regex", "reqwest 0.13.1", - "reqwest-middleware", - "reqwest-retry", + "reqwest-middleware 0.5.0", + "reqwest-retry 0.9.0", "semver 1.0.27", "serde", "serde_json", @@ -15916,7 +16111,7 @@ dependencies = [ "rand 0.9.0", "regex", "reqwest 0.13.1", - "reqwest-middleware", + "reqwest-middleware 0.5.0", "rust_decimal", "serde", "serde_json", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 203e9aae6a..9808973f68 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -86,6 +86,7 @@ oauth2 = ["windmill-api/oauth2"] zip = ["windmill-api/zip"] static_frontend = ["windmill-api/static_frontend"] scoped_cache = ["windmill-common/scoped_cache"] +pg_embed = ["windmill-common/pg_embed", "dep:postgresql_embedded"] test_job_debouncing = [] # Languages python = ["windmill-worker/python", "windmill-api/python"] @@ -161,7 +162,7 @@ k8s-openapi.workspace = true libloading.workspace = true bitflags.workspace = true globset.workspace = true - +postgresql_embedded = { version = "0.18.1", optional = true, features = ["theseus"], default-features = false } [target.'cfg(windows)'.dependencies] windows-service = "0.7" diff --git a/backend/README.md b/backend/README.md index a184290c5d..dbf0b64459 100644 --- a/backend/README.md +++ b/backend/README.md @@ -14,6 +14,55 @@ contains files used to build the "root" binary. | [windmill-worker](./windmill-worker/) | The worker. Used to process and execute flows & jobs. | | [parsers](./parsers/) | Contains code to parse signatures in different langauges. | +## Features + +### Embedded PostgreSQL (`pg_embed`) + +The `pg_embed` feature allows Windmill to run with an embedded PostgreSQL database, eliminating the need for a separate PostgreSQL instance. This is useful for local development, testing, or single-machine deployments. + +**Building with pg_embed:** +```bash +cargo build --features pg_embed +``` + +**System Dependencies:** + +The embedded PostgreSQL requires certain system libraries to be installed: + +- **Arch Linux:** + ```bash + sudo pacman -S libxml2 icu openssl + ``` + +- **Ubuntu/Debian:** + ```bash + sudo apt-get install libxml2 libicu-dev libssl-dev + ``` + +- **RHEL/Fedora:** + ```bash + sudo dnf install libxml2 libicu openssl-libs + ``` + +**Usage:** + +When the `pg_embed` feature is enabled, if `DATABASE_URL` is not set, Windmill will automatically start an embedded PostgreSQL instance. + +You can customize the embedded PostgreSQL behavior with these environment variables: +- `PG_EMBED_DATA_DIR`: Directory for PostgreSQL data (default: `./postgresql_data`) +- `PG_EMBED_PORT`: Port for PostgreSQL (default: `5432`) +- `PG_EMBED_DATABASE`: Database name (default: `windmill`) +- `PG_EMBED`: Set to any value to force embedded PostgreSQL even if DATABASE_URL exists + +**Example:** +```bash +# Run with default embedded PostgreSQL +./windmill + +# Or with custom settings +PG_EMBED_DATA_DIR=/my/data PG_EMBED_PORT=5433 ./windmill +``` + ### Compile sqlx for offline ci ``` diff --git a/backend/src/main.rs b/backend/src/main.rs index 54fb29f272..03fec3edaa 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -332,6 +332,9 @@ fn print_help() { println!(" cache [hubPaths.json] Pre-cache hub scripts (default: ./hubPaths.json)"); println!(); println!("Environment variables (name = default):"); + #[cfg(feature = "pg_embed")] + println!(" DATABASE_URL = The Postgres database url (auto-generated with pg_embed if not set)."); + #[cfg(not(feature = "pg_embed"))] println!(" DATABASE_URL = The Postgres database url."); println!(" MODE = standalone Mode: standalone | worker | server | agent"); println!(" BASE_URL = http://localhost:8000 Public base URL of your instance (overridden by instance settings)"); @@ -345,10 +348,26 @@ fn print_help() { println!(" LICENSE_KEY = None (EE only) Enterprise license key (workers require valid key)"); println!(" RUN_UPDATE_CA_CERTIFICATE_AT_START = false Run system CA update at startup"); println!(" RUN_UPDATE_CA_CERTIFICATE_PATH = /usr/sbin/update-ca-certificates Path to CA update tool"); + #[cfg(feature = "pg_embed")] + { + println!(); + println!("Embedded PostgreSQL settings (pg_embed feature):"); + println!(" PG_EMBED = Force embedded PostgreSQL even if DATABASE_URL exists"); + println!(" PG_EMBED_DATA_DIR = ./postgresql_data Directory for embedded PostgreSQL data"); + println!(" PG_EMBED_PORT = 5432 Port for embedded PostgreSQL"); + println!(" PG_EMBED_DATABASE = windmill Database name for embedded PostgreSQL"); + println!(); + println!("System dependencies required for pg_embed:"); + println!(" Arch: sudo pacman -S libxml2 icu openssl"); + println!(" Ubuntu/Debian: sudo apt-get install libxml2 libicu-dev libssl-dev"); + println!(" RHEL/Fedora: sudo dnf install libxml2 libicu openssl-libs"); + } println!(); println!("Notes:"); println!("- Advanced and less commonly used settings are managed via the database and are omitted here."); println!("- At startup, Windmill logs currently set configuration keys for visibility."); + #[cfg(feature = "pg_embed")] + println!("- With pg_embed feature enabled, if DATABASE_URL is not set, an embedded PostgreSQL instance will be started automatically."); } async fn windmill_main() -> anyhow::Result<()> { @@ -470,6 +489,15 @@ async fn windmill_main() -> anyhow::Result<()> { IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)) }; + // Initialize embedded PostgreSQL if pg_embed feature is enabled + #[cfg(feature = "pg_embed")] + let _embedded_pg = if std::env::var("PG_EMBED").is_ok() || (mode != Mode::Agent && std::env::var("DATABASE_URL").is_err() && std::env::var("DATABASE_URL_FILE").is_err()) { + println!("DATABASE_URL not set, starting embedded PostgreSQL..."); + Some(windmill_common::pg_embed::init_embedded_postgres().await?) + } else { + None + }; + let (conn, first_suffix) = if mode == Mode::Agent { tracing::info!( "Creating http client for cluster using base internal url {}", diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 1acf9f4c6a..cf386f2c7b 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -20,6 +20,7 @@ smtp = ["dep:mail-send"] scoped_cache = [] cloud = [] openidconnect = ["dep:openidconnect"] +pg_embed = ["dep:postgresql_embedded"] [lib] name = "windmill_common" path = "src/lib.rs" @@ -114,6 +115,7 @@ opentelemetry = { workspace = true, optional = true } tracing-opentelemetry = { workspace = true, optional = true } opentelemetry-appender-tracing = { workspace = true, optional = true } tonic = { workspace = true, optional = true } +postgresql_embedded = { version = "0.18.1", optional = true, features = ["theseus"], default-features = false } [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemalloc-ctl = { optional = true, workspace = true } diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index a650f6bf0c..2edf51dee8 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -73,6 +73,8 @@ pub mod oidc_oss; #[cfg(feature = "private")] pub mod otel_ee; pub mod otel_oss; +#[cfg(feature = "pg_embed")] +pub mod pg_embed; pub mod queue; pub mod result_stream; pub mod runnable_settings; diff --git a/backend/windmill-common/src/pg_embed.rs b/backend/windmill-common/src/pg_embed.rs new file mode 100644 index 0000000000..4839509097 --- /dev/null +++ b/backend/windmill-common/src/pg_embed.rs @@ -0,0 +1,144 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use crate::error::Error; +use postgresql_embedded::{PostgreSQL, Settings}; +use std::sync::Arc; +use tokio::sync::RwLock; + +/// Manages an embedded PostgreSQL instance +pub struct EmbeddedPostgres { + postgresql: Arc>, + database_url: String, +} + +impl EmbeddedPostgres { + /// Initialize and start an embedded PostgreSQL instance + pub async fn new() -> Result { + tracing::info!("Initializing embedded PostgreSQL instance..."); + + // Configure the embedded PostgreSQL settings + let mut settings = Settings::default(); + + // Use environment variables if provided for customization + if let Ok(data_dir) = std::env::var("PG_EMBED_DATA_DIR") { + settings.installation_dir = data_dir.into(); + } + + if let Ok(port) = std::env::var("PG_EMBED_PORT") { + if let Ok(port_num) = port.parse::() { + settings.port = port_num; + } + } + + let mut postgresql = PostgreSQL::new(settings); + + // Setup the PostgreSQL instance + postgresql + .setup() + .await + .map_err(|e| { + let err_msg = format!("Failed to setup embedded PostgreSQL: {}", e); + if err_msg.contains("libxml2") || err_msg.contains("shared libraries") { + Error::InternalErr(format!( + "{}\n\n\ + System dependencies are required for embedded PostgreSQL.\n\ + On Arch Linux, install: sudo pacman -S libxml2 icu openssl\n\ + On Ubuntu/Debian: sudo apt-get install libxml2 libicu-dev libssl-dev\n\ + On RHEL/Fedora: sudo dnf install libxml2 libicu openssl-libs\n\n\ + Alternatively, set DATABASE_URL to use an external PostgreSQL instance.", + err_msg + )) + } else { + Error::InternalErr(err_msg) + } + })?; + + tracing::info!("Starting embedded PostgreSQL..."); + + // Start the PostgreSQL instance + postgresql + .start() + .await + .map_err(|e| Error::InternalErr(format!("Failed to start embedded PostgreSQL: {}", e)))?; + + tracing::info!("Embedded PostgreSQL started successfully"); + + // Get the database settings + let settings = postgresql.settings(); + + // Create the windmill database + let database_name = std::env::var("PG_EMBED_DATABASE") + .unwrap_or_else(|_| "windmill".to_string()); + + tracing::info!("Creating database: {}", database_name); + + postgresql + .create_database(&database_name) + .await + .map_err(|e| { + Error::InternalErr(format!("Failed to create database '{}': {}", database_name, e)) + })?; + + // Build the connection string + let database_url = format!( + "postgres://{}:{}@{}:{}/{}", + settings.username, + settings.password, + settings.host, + settings.port, + database_name + ); + + tracing::info!("Embedded PostgreSQL ready at: postgres://{}:{}@{}:{}/{}", + settings.username, + "***", // Don't log password + settings.host, + settings.port, + database_name + ); + + Ok(Self { + postgresql: Arc::new(RwLock::new(postgresql)), + database_url, + }) + } + + /// Get the database connection URL + pub fn database_url(&self) -> &str { + &self.database_url + } + + /// Stop the embedded PostgreSQL instance + pub async fn stop(&self) -> Result<(), Error> { + tracing::info!("Stopping embedded PostgreSQL..."); + + let pg = self.postgresql.write().await; + pg.stop() + .await + .map_err(|e| Error::InternalErr(format!("Failed to stop embedded PostgreSQL: {}", e)))?; + + tracing::info!("Embedded PostgreSQL stopped"); + Ok(()) + } +} + +/// Initialize an embedded PostgreSQL instance and set the DATABASE_URL environment variable +pub async fn init_embedded_postgres() -> Result { + let embedded_pg = EmbeddedPostgres::new().await?; + + // Set the DATABASE_URL environment variable so that the rest of the application + // can use it transparently + unsafe { + std::env::set_var("DATABASE_URL", embedded_pg.database_url()); + } + + tracing::info!("DATABASE_URL set to embedded PostgreSQL instance"); + + Ok(embedded_pg) +}