From 3d6fb15a90f4d5aa18a6d6158760a380295e3d9e Mon Sep 17 00:00:00 2001 From: Guillaume Bouvignies Date: Wed, 29 Nov 2023 16:30:45 +0100 Subject: [PATCH] fix: minor fixes to private NPM and python registries to get everything working (#2728) --- backend/windmill-worker/src/bun_executor.rs | 16 ++-------------- backend/windmill-worker/src/python_executor.rs | 10 ++++++++-- .../private-package-registry-tls/README.md | 6 ++++-- .../docker-compose.yml | 8 ++++++-- .../helloworld_python_module/setup.py | 4 ++-- .../verdaccio_conf/config.yaml | 1 + 6 files changed, 23 insertions(+), 22 deletions(-) diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 6aed35ddd2..1793a78595 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -16,7 +16,7 @@ use crate::{ start_child_process, write_file, write_file_binary, }, AuthedClientBackgroundTask, BUN_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, - NPM_CONFIG_REGISTRY, NSJAIL_PATH, PATH_ENV, TZ_ENV, + NSJAIL_PATH, PATH_ENV, TZ_ENV, }; use tokio::{ @@ -49,8 +49,6 @@ pub const EMPTY_FILE: &str = ""; lazy_static::lazy_static! { pub static ref TRUSTED_DEP: Regex = Regex::new(r"//\s?trustedDependencies:(.*)\n").unwrap(); - - static ref BUN_TLS_REJECT_UNAUTHORIZED: String = std::env::var("NODE_TLS_REJECT_UNAUTHORIZED").ok().unwrap_or_else(|| String::new()); } pub async fn gen_lockfile( @@ -514,7 +512,7 @@ plugin(p) } pub async fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap { - let mut bun_envs: HashMap = HashMap::from([ + let bun_envs: HashMap = HashMap::from([ (String::from("PATH"), PATH_ENV.clone()), (String::from("HOME"), HOME_ENV.clone()), (String::from("TZ"), TZ_ENV.clone()), @@ -531,16 +529,6 @@ pub async fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap 0 { - bun_envs.insert( - String::from("NODE_TLS_REJECT_UNAUTHORIZED"), - BUN_TLS_REJECT_UNAUTHORIZED.clone(), - ); - } return bun_envs; } diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index a19e0ae515..1e7fd91730 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -122,8 +122,8 @@ pub async fn pip_compile( if let Some(host) = PIP_TRUSTED_HOST.as_ref() { args.extend(["--trusted-host", host]); } - if let Some(host) = PIP_INDEX_CERT.as_ref() { - args.extend(["--cert", host]); + if let Some(cert_path) = PIP_INDEX_CERT.as_ref() { + args.extend(["--cert", cert_path]); } let mut child_cmd = Command::new("pip-compile"); @@ -609,6 +609,9 @@ pub async fn handle_python_reqs( if let Some(url) = PIP_INDEX_URL.as_ref() { vars.push(("INDEX_URL", url)); } + if let Some(cert_path) = PIP_INDEX_CERT.as_ref() { + vars.push(("PIP_INDEX_CERT", cert_path)); + } if let Some(host) = PIP_TRUSTED_HOST.as_ref() { vars.push(("TRUSTED_HOST", host)); } @@ -708,6 +711,9 @@ pub async fn handle_python_reqs( if let Some(url) = PIP_INDEX_URL.as_ref() { command_args.extend(["--index-url", url]); } + if let Some(cert_path) = PIP_INDEX_CERT.as_ref() { + command_args.extend(["--cert", cert_path]); + } if let Some(host) = PIP_TRUSTED_HOST.as_ref() { command_args.extend(["--trusted-host", &host]); } diff --git a/examples/deploy/private-package-registry-tls/README.md b/examples/deploy/private-package-registry-tls/README.md index 2c497f4a5d..e67b6ad247 100644 --- a/examples/deploy/private-package-registry-tls/README.md +++ b/examples/deploy/private-package-registry-tls/README.md @@ -80,14 +80,16 @@ python setup.py sdist twine upload --repository-url https://localhost/ dist/* --cert ../certs/windmill-root.crt # no username and password, just press enter. For the purpose of the demo we're running pypiserver completely unauthenticated ``` -You can check that the package is uploaded by visiting [https://localhost/simple](https://localhost/pypi/simple). +You can check that the package is uploaded by visiting [https://localhost/simple](https://localhost/simple). Go to Windmill at `http://localhost:8000`. Create a simple Python script: ```python +#requirements: +#windmill-helloworld==0.0.1 import windmill_helloworld def main(): - print(windmill_helloworld.say_hello("Windmill")) + print(windmill_helloworld.say_hello("Windmill")) ``` and execute it. It should return successfully with: ``` diff --git a/examples/deploy/private-package-registry-tls/docker-compose.yml b/examples/deploy/private-package-registry-tls/docker-compose.yml index 6d3b311763..6009a20093 100644 --- a/examples/deploy/private-package-registry-tls/docker-compose.yml +++ b/examples/deploy/private-package-registry-tls/docker-compose.yml @@ -48,11 +48,15 @@ services: environment: - DATABASE_URL=postgres://postgres:changeme@db/windmill?sslmode=disable - WORKER_TAGS=deno,go,python3,bash,flow,hub,dependency,nativets - - NPM_CONFIG_REGISTRY=https://caddy/npm/ + + # For DENO and REST scripts: + - NPM_CONFIG_REGISTRY=http://caddy/npm/ - DENO_CERT=/custom-certs/windmill-root.crt # this will make deno trust this RootCA for all sessions # - DENO_TLS_CA_STORE=system # alternatively, you can use this but you'll need to manually trust the RootCA at the host level, see README.md + + # For Python scripts: + - PIP_INDEX_URL=https://caddy/simple/ - PIP_INDEX_CERT=/custom-certs/windmill-root.crt # this will make pip trust this RootCA for all sessions - - BUN_TLS_REJECT_UNAUTHORIZED=0 # this will make bun ignore TLS errors. Bun does not support trusting an additional RootCA yet volumes: - ./certs:/custom-certs depends_on: diff --git a/examples/deploy/private-package-registry-tls/helloworld_python_module/setup.py b/examples/deploy/private-package-registry-tls/helloworld_python_module/setup.py index 81716d2f7d..da4c4807cd 100644 --- a/examples/deploy/private-package-registry-tls/helloworld_python_module/setup.py +++ b/examples/deploy/private-package-registry-tls/helloworld_python_module/setup.py @@ -7,12 +7,12 @@ module_path = os.path.join(os.path.dirname(__file__), "windmill_helloworld.py") setuptools.setup( name="windmill-helloworld", version="0.0.1", - url="https://github.com/windmill-labs/windmill/blob/exit()/examples/deploy/private-npm-registry-tls/README.md", + url="https://github.com/windmill-labs/windmill/blob//examples/deploy/private-package-registry-tls/README.md", author="WindmillLabs", author_email="contact@windmill.dev", description="Simple hello world python module to host on a private Pypi server", long_description=open("README.md").read(), - py_modules=["helloworld_python_module"], + py_modules=["windmill_helloworld"], zip_safe=False, platforms="any", install_requires=[], diff --git a/examples/deploy/private-package-registry-tls/verdaccio_conf/config.yaml b/examples/deploy/private-package-registry-tls/verdaccio_conf/config.yaml index a5faae480d..7b2893fc22 100644 --- a/examples/deploy/private-package-registry-tls/verdaccio_conf/config.yaml +++ b/examples/deploy/private-package-registry-tls/verdaccio_conf/config.yaml @@ -11,6 +11,7 @@ packages: access: $all publish: $authenticated "**": + access: $all proxy: npmjs listen: - http://localhost:4873