chore: Add examples on how to deploy private registries (#2719)

* chore: Add examples on how to deploy private registries

* chore: example for private pypiserver with https

* Allow DENO_CERT certificate with native workers

* Add BUN_TLS_REJECT_UNAUTHORIZED

backend/Cargo.lock

@@ -9901,6 +9901,7 @@ dependencies = [
  "deno_console",
  "deno_core",
  "deno_fetch",
+ "deno_tls",
  "deno_url",
  "deno_web",
  "deno_webidl",

backend/Cargo.toml

@@ -137,6 +137,7 @@ json-pointer = "^0"
 itertools = "^0"
 regex = "^1"
 deno_fetch = "0.139.0"
+deno_tls = "0.102.0"
 deno_console = "0.115.0"
 deno_url = "0.115.0"
 deno_webidl = "0.115.0"

backend/windmill-worker/Cargo.toml

@@ -60,6 +60,7 @@ deno_console.workspace = true
 deno_url.workspace = true
 deno_core.workspace = true
 deno_ast.workspace = true
+deno_tls.workspace = true
 postgres-native-tls.workspace = true
 native-tls.workspace = true
 mysql_async.workspace = true

backend/windmill-worker/src/bun_executor.rs

@@ -49,6 +49,8 @@ pub const EMPTY_FILE: &str = "<empty>";
 
 lazy_static::lazy_static! {
     pub static ref TRUSTED_DEP: Regex = Regex::new(r"//\s?trustedDependencies:(.*)\n").unwrap();
+
+    static ref BUN_TLS_REJECT_UNAUTHORIZED: String = std::env::var("NODE_TLS_REJECT_UNAUTHORIZED").ok().unwrap_or_else(|| String::new());
 }
 
 pub async fn gen_lockfile(
@@ -533,6 +535,12 @@ pub async fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap<String
     if let Some(ref s) = NPM_CONFIG_REGISTRY.read().await.clone() {
         bun_envs.insert(String::from("NPM_CONFIG_REGISTRY"), s.clone());
     }
+    if BUN_TLS_REJECT_UNAUTHORIZED.len() > 0 {
+        bun_envs.insert(
+            String::from("NODE_TLS_REJECT_UNAUTHORIZED"),
+            BUN_TLS_REJECT_UNAUTHORIZED.clone(),
+        );
+    }
     return bun_envs;
 }
 

backend/windmill-worker/src/js_eval.rs

@@ -6,16 +6,17 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::{cell::RefCell, collections::HashMap, rc::Rc, sync::Arc};
+use std::{cell::RefCell, collections::HashMap, rc::Rc, sync::Arc, env, io::{BufReader, self}};
 
 use deno_ast::{ParseParams, SourceTextInfo};
 use deno_core::{
     op, serde_v8,
     v8::IsolateHandle,
     v8::{self},
-    Extension, JsRuntime, Op, OpState, RuntimeOptions, Snapshot,
+    Extension, JsRuntime, Op, OpState, RuntimeOptions, Snapshot, error::AnyError,
 };
 use deno_fetch::FetchPermissions;
+use deno_tls::{rustls::RootCertStore, rustls_pemfile};
 use deno_web::{BlobStore, TimersPermission};
 use itertools::Itertools;
 use lazy_static::lazy_static;
@@ -38,6 +39,33 @@ pub struct IdContext {
     pub previous_id: String,
 }
 
+pub struct ContainerRootCertStoreProvider {
+    root_cert_store: RootCertStore,
+}
+
+impl ContainerRootCertStoreProvider {
+    fn new() -> ContainerRootCertStoreProvider {
+        return ContainerRootCertStoreProvider {
+            root_cert_store: deno_tls::create_default_root_cert_store(),
+        }
+    }
+
+    fn add_certificate(&mut self, cert_path: String) -> io::Result<()> {
+        let cert_file = std::fs::File::open(cert_path)?;
+        let mut reader = BufReader::new(cert_file);
+        let pem_file = rustls_pemfile::certs(&mut reader)?;
+
+        self.root_cert_store.add_parsable_certificates(&pem_file);
+        Ok(())
+    }
+}
+
+impl deno_tls::RootCertStoreProvider for ContainerRootCertStoreProvider {
+    fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> {
+        Ok(&self.root_cert_store)
+    }
+}
+
 pub struct PermissionsContainer;
 
 impl FetchPermissions for PermissionsContainer {
@@ -532,6 +560,18 @@ pub async fn eval_fetch_timeout(
                   ..Default::default()
                 };
 
+            let deno_fetch_options = if let Some(cert_path) = env::var("DENO_CERT").ok() {
+                let mut cert_store_provider = ContainerRootCertStoreProvider::new();
+                cert_store_provider.add_certificate(cert_path)?;
+
+                deno_fetch::Options {
+                    root_cert_store_provider: Some(Arc::new(cert_store_provider)),
+                    ..Default::default()
+                }
+            } else {
+                Default::default()
+            };
+
             let exts: Vec<Extension> = vec![
                 deno_webidl::deno_webidl::init_ops(),
                 deno_url::deno_url::init_ops(),
@@ -540,9 +580,7 @@ pub async fn eval_fetch_timeout(
                     Arc::new(BlobStore::default()),
                     None,
                 ),
-                deno_fetch::deno_fetch::init_ops::<PermissionsContainer>(
-                    Default::default(),
-                ),
+                deno_fetch::deno_fetch::init_ops::<PermissionsContainer>(deno_fetch_options),
                 ext
             ];
 

backend/windmill-worker/src/python_executor.rs

@@ -30,6 +30,7 @@ lazy_static::lazy_static! {
 
     static ref PIP_INDEX_URL: Option<String> = std::env::var("PIP_INDEX_URL").ok();
     static ref PIP_TRUSTED_HOST: Option<String> = std::env::var("PIP_TRUSTED_HOST").ok();
+    static ref PIP_INDEX_CERT: Option<String> = std::env::var("PIP_INDEX_CERT").ok();
 
 
     static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap();
@@ -121,6 +122,9 @@ pub async fn pip_compile(
     if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
         args.extend(["--trusted-host", host]);
     }
+    if let Some(host) = PIP_INDEX_CERT.as_ref() {
+        args.extend(["--cert", host]);
+    }
 
     let mut child_cmd = Command::new("pip-compile");
     child_cmd

examples/deploy/private-npm-registry-tls/README.md

@@ -1,104 +0,0 @@
-Private NPM registry with self-signed certificates
-==================================================
-
-Setup a private NPM registry with self-signed certificates.
-
-## Setup
-
-```bash
-# spin up the registry. You can go to http://localhost:4873 to verify it's up and running
-docker compose up -d
-
-# create your own user to be able to publish a package
-npm adduser --registry http://localhost:4873
-
-# publish a package
-cd helloworld
-npm publish --registry http://localhost:4873
-```
-
-### Setup
-
-1. Generate self signed certificates (or bring your own)
-```bash
-cd certs
-# feel free to read the file anc change the 
-./generate_certs.sh
-# choose a password for the RootCA key. You'll need to input it for pretty much all following steps
-```
-At the end of the script, you should have multiple files in the `certs/` folder. The most important ones are:
-- `windmill-root.key` (Root CA private key)
-- `windmill-root.crt` (Root CA certificate)
-- `npm_registry.key` (NPM registry server private key)
-- `npm_registry.crt` (NPM registry server certificate)
-
-2. Start the docker compose stack
-
-```bash
-docker compose up -d
-```
-
-This will start the private NPM registry, as well as a minimal Windmill stack composed of just one Windmill server/worker and the associated database. 
-
-For the latter, we invite you to refer to the latest [docker compose](/docker-compose.yml) at the root of this repository to setup a more evolved Windmill stack.
-
-For the former, it's using [Verdaccio](https://verdaccio.org/) as an easy-to-deploy NPM registry. We bring you attention to the fact that in addition to the config 
-file in `./verdaccio_conf/config.yaml`, we had to set both `VERDACCIO_PROTOCOL` and `VERDACCIO_PUBLIC_URL` in docker compose. See the official Verdaccio 
-documentation for more info on this.
-
-3. Upload the custom `helloworld` package to the private NPM registry
-
-```
-cd helloworld_package
-# you need to first create a registry user
-# you might need to run `npm config set strict-ssl false` if the below fails. If you do, then change it back to `true` after running the 2 commands
-npm adduser --registry https://0.0.0.0:4873/
-npm publish --registry https://0.0.0.0:4873/
-```
-
-4. Pull the custom NPM package from a deno script in Windmill
-
-Go to Windmill at `http://localhost:8000`. Create a simple deno script:
-```ts
-import * as testpackage from "npm:@windmill/helloworld@0.0.1"
-
-export async function main() {
-  console.log(testpackage.sayHello("Windmill"))
-}
-```
-and execute it. It should return successfully with:
-```
-Hello Windmill
-```
-
-
-### Remarks
-
-1. `DENO_TLS_CA_STORE` VS `DENO_CERT`
-Both works. `DENO_CERT` is better b/c you just have to set it to the path of the trusted Root CA certificate, and deno will trust this certificate.
-When using `DENO_TLS_CA_STORE=system`, you _have to_ make the server trust the custom Root CA certificate with the following commands:
-```bash
-# in the windmill-server container:
-cp /custom-certs/windmill-root.crt /usr/local/share/ca-certificates/
-update-ca-certificates # as root
-# the output should tell (among other things): " ... 1 added, 0 removed; done. ..."
-```
-
-2. Running deno scripts manually in the Windmill container
-This could be useful for debugging purposes.
-
-```bash
-# log into the Windmill Server container
-docker exec -it <WINDMILL_SERVER_CONTAINER> /bin/bash
-
-# Go into Deno REPL
-deno
-
-# try to import the package
-import * as testpackage from "npm:@windmill/helloworld@0.0.1"
-> undefined
-
-# if the above returns undefined, there's a good chance it's working. If you want to double check:
-testpackage.sayHello("Windmill")
-> Hello Windmill
-```

examples/deploy/private-npm-registry-tls/helloworld_package/.gitignore

@@ -1 +0,0 @@
-node_modules/

examples/deploy/private-package-registry-tls/Caddyfile

@@ -0,0 +1,16 @@
+{$BASE_URL} {
+        bind {$ADDRESS}
+        tls /certs/caddy.crt /certs/caddy.key
+        handle_path /static/* {
+            root * /static/
+            file_server {
+                browse
+            }
+        }
+        handle_path /npm/* {
+            reverse_proxy * http://npm_registry:4873
+        }
+        handle_path /* {
+            reverse_proxy * http://pypi_server:8080
+        }
+}

examples/deploy/private-package-registry-tls/README.md

@@ -0,0 +1,144 @@
+Private package registry with self-signed certificates
+==================================================
+
+Setup a private NPM registry and Pypi server behind a revers proxy (Caddy) exposing an HTTPS endpoint with self signed certificates
+
+### Setup
+
+1. Generate self signed certificates (or bring your own)
+```bash
+cd certs
+# feel free to read the file anc change the 
+./generate_certs.sh
+# choose a password for the RootCA key. You'll need to input it for pretty much all following steps
+```
+At the end of the script, you should have multiple files in the `certs/` folder. The most important ones are:
+- `windmill-root.key` (Root CA private key)
+- `windmill-root.crt` (Root CA certificate)
+- `caddy.key` (caddy server private key)
+- `caddy.crt` (caddy registry server certificate)
+
+2. Start the docker compose stack
+
+```bash
+docker compose up -d
+```
+
+This will start the private NPM registry, the private Pypi server and Caddy as a reverse proxy sitting in front of the two. Separately, for the purpose of proving an end to end setup, it will also start minimal Windmill stack composed of just one Windmill server/worker and the associated database. 
+
+For a more complete Windmill setup, we invite you to refer to the latest [docker compose](/docker-compose.yml) at the root of this repository to setup a more evolved Windmill stack.
+
+For the private registries/repositories, it's using [Verdaccio](https://verdaccio.org/) as an easy-to-deploy NPM registry and [pypiserver](https://pypi.org/project/pypiserver/) for Python.
+
+## Deno pulling package from private NPM registry
+
+Upload the custom `helloworld_npm_package` package to the private NPM registry and use it in a Windmill script
+
+```bash
+cd helloworld_npm_package
+# you need to first create a registry user
+# you might need to run `npm config set strict-ssl false` if the below fails. If you do, then change it back to `true` after running the 2 commands
+npm adduser --registry https://localhost/npm/
+npm publish --registry https://localhost/npm/
+```
+
+Go to Windmill at `http://localhost:8000`. Create a simple Deno script:
+```ts
+import * as testpackage from "npm:@windmill/helloworld@0.0.1"
+
+export async function main() {
+  console.log(testpackage.sayHello("Windmill"))
+}
+```
+and execute it. It should return successfully with:
+```
+Hello Windmill
+```
+
+Note: Native fetches to HTTPS endpoints with self signed certificates also takes into account the DENO_CERT environment variable
+
+Go to Windmill and create a new script of type "REST" with the following content:
+```ts
+export async function main() {
+  const res = await fetch("https://caddy/static/helloworld.json", {
+    headers: { "Content-Type": "application/json" },
+  });
+  return res.json();
+}
+```
+
+It will fetch a static json file from Caddy exposed behind its HTTPS endpoint with self signed certificate.
+
+## Python pulling package from private NPM registry
+
+Upload the custom `helloworld_python_module` python module to the private Pypi server
+
+```bash
+cd helloworld_python_module
+python setup.py sdist
+# using twine to upload the module here, get it with `pip install twine`
+twine upload --repository-url https://localhost/ dist/* --cert ../certs/windmill-root.crt
+# no username and password, just press enter. For the purpose of the demo we're running pypiserver completely unauthenticated
+```
+You can check that the package is uploaded by visiting [https://localhost/simple](https://localhost/pypi/simple).
+
+Go to Windmill at `http://localhost:8000`. Create a simple Python script:
+```python
+import windmill_helloworld
+
+def main():
+  print(windmill_helloworld.say_hello("Windmill"))
+```
+and execute it. It should return successfully with:
+```
+Hello Windmill
+```
+
+### MISC
+
+1. `DENO_TLS_CA_STORE` VS `DENO_CERT`
+Both works. `DENO_CERT` is better b/c you just have to set it to the path of the trusted Root CA certificate, and deno will trust this certificate.
+When using `DENO_TLS_CA_STORE=system`, you _have to_ make the server trust the custom Root CA certificate with the following commands:
+```bash
+# in the windmill-server container:
+cp /custom-certs/windmill-root.crt /usr/local/share/ca-certificates/
+update-ca-certificates # as root
+# the output should tell (among other things): " ... 1 added, 0 removed; done. ..."
+```
+
+2. Running Deno scripts manually in the Windmill container
+This could be useful for debugging purposes.
+
+```bash
+# log into the Windmill Server container
+docker exec -it <WINDMILL_SERVER_CONTAINER> /bin/bash
+
+# Go into Deno REPL
+deno
+
+# try to import the package
+import * as testpackage from "npm:@windmill/helloworld@0.0.1"
+> undefined
+
+# if the above returns undefined, there's a good chance it's working. If you want to double check:
+testpackage.sayHello("Windmill")
+> Hello Windmill
+
+# potentially inspect the env var available to DENO. Is you used DENO_CERT in the docker compose, check its value:
+console.log(Deno.env.get("DENO_CERT"))
+> /custom-certs/windmill-root.crt
+```
+
+3. Manually testing Pypi private server integration
+Can be useful to debug as well:
+
+```bash
+# install the package
+pip3 install --cert ../certs/windmill-root.crt -i https://localhost/simple/ windmill-helloworld
+
+# go to python CLI and try to import it and use it
+python
+>>> import windmill_helloworld
+>>> windmill_helloworld.say_hello("world")
+'Hello world'
+```

examples/deploy/private-package-registry-tls/certs/generate_certs.sh

@@ -8,14 +8,14 @@ STATE='Paris'
 CITY='Paris'
 ORGANIZATION='WindmillLabs'
 ROOT_CA_CN='WindmillRootCA'
-SERVER_CA_CN='npm_registry' # IMPORTANT: set this to the FQDN of the npm registry server. Here in the docker compose stack, it will be npm_registry
+SERVER_CA_CN='caddy' # IMPORTANT: set this to the FQDN of the caddy server. Here in the docker compose stack, it will be caddy
 
 echo "Generating RootCA key"
 openssl genrsa -aes256 -out $CANAME.key 4096
 echo "Generating RootCA certificate"
 openssl req -x509 -new -nodes -key $CANAME.key -sha256 -days 1826 -out $CANAME.crt -subj "/CN=${ROOT_CA_CN}/C=${COUNTRY}/ST=${STATE}/L=${CITY}/O=${ORGANIZATION}"
 
-CERTNAME=npm_registry
+CERTNAME=caddy
 echo "Generating server certificate private key and cert signing request"
 openssl req -new -nodes -out $CERTNAME.csr -newkey rsa:4096 -keyout $CERTNAME.key -subj "/CN=${SERVER_CA_CN}/C=${COUNTRY}/ST=${STATE}/L=${CITY}/O=${ORGANIZATION}"
 
@@ -27,7 +27,7 @@ basicConstraints=CA:FALSE
 keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
 subjectAltName = @alt_names
 [alt_names]
-DNS.1 = npm_registry
+DNS.1 = caddy
 DNS.2 = localhost
 EOF
 # ^ HERE ^ in the above alt_names, feel free to add any alternate CN

examples/deploy/private-package-registry-tls/docker-compose.yml

@@ -3,16 +3,31 @@ version: "3.7"
 services:
   npm_registry:
     image: verdaccio/verdaccio
-    ports:
-      - 4873:4873
     environment:
-      - VERDACCIO_PROTOCOL=https
-      - VERDACCIO_PUBLIC_URL=https://npm_registry:4873
+      - VERDACCIO_PROTOCOL=http
+      - VERDACCIO_PUBLIC_URL=http://caddy/npm/
     volumes:
       - ./verdaccio_conf:/verdaccio/conf
-      - ./certs:/verdaccio/certs
       - npm_registry_data:/verdaccio/storage
 
+  pypi_server:
+    image: pypiserver/pypiserver:latest
+    platform: linux/x86_64
+    volumes:
+      - pypi_data:/data/packages
+    command: run -P . -a . /data/packages
+
+  caddy:
+    image: caddy:2.5.2-alpine
+    volumes:
+      - ./Caddyfile:/etc/caddy/Caddyfile
+      - ./certs:/certs
+      - ./helloworld_static:/static/
+    ports:
+      - 443:443
+    environment:
+      - BASE_URL=":443"
+
   db:
     image: postgres:14
     volumes:
@@ -31,11 +46,13 @@ services:
     ports:
       - 8000:8000
     environment:
-      - WHITELIST_ENVS='DENO_CERT'
       - DATABASE_URL=postgres://postgres:changeme@db/windmill?sslmode=disable
-      - NPM_CONFIG_REGISTRY=https://npm_registry:4873
+      - WORKER_TAGS=deno,go,python3,bash,flow,hub,dependency,nativets
+      - NPM_CONFIG_REGISTRY=https://caddy/npm/
       - DENO_CERT=/custom-certs/windmill-root.crt # this will make deno trust this RootCA for all sessions
       # - DENO_TLS_CA_STORE=system # alternatively, you can use this but you'll need to manually trust the RootCA at the host level, see README.md
+      - PIP_INDEX_CERT=/custom-certs/windmill-root.crt # this will make pip trust this RootCA for all sessions
+      - BUN_TLS_REJECT_UNAUTHORIZED=0 # this will make bun ignore TLS errors. Bun does not support trusting an additional RootCA yet
     volumes:
       - ./certs:/custom-certs
     depends_on:
@@ -44,4 +61,5 @@ services:
 
 volumes:
   npm_registry_data: null
+  pypi_data: null
   db_data: null

examples/deploy/private-package-registry-tls/helloworld_npm_package/.gitignore

@@ -0,0 +1 @@
+node_modules/

examples/deploy/private-package-registry-tls/helloworld_npm_package/README.md

@@ -1,4 +1,4 @@
 Hello world test package
 ========================
 
-Dummy NPM package to test imports from a private NPM registry in Windmill
+Dummy NPM package to test imports from a private NPM registry in Windmill

examples/deploy/private-package-registry-tls/helloworld_npm_package/package.json

@@ -7,4 +7,4 @@
   "keywords": [],
   "author": "Windmill",
   "license": "Apache-2.0"
-}
+}

examples/deploy/private-package-registry-tls/helloworld_python_module/.gitignore

@@ -0,0 +1,3 @@
+dist/
+__pycache__/
+*.egg-info/

examples/deploy/private-package-registry-tls/helloworld_python_module/README.md

@@ -0,0 +1,4 @@
+Hello world test module
+=======================
+
+Dummy Python module to test imports from a private Pypi server in Windmill

examples/deploy/private-package-registry-tls/helloworld_python_module/setup.py

@@ -0,0 +1,30 @@
+import os
+
+import setuptools
+
+module_path = os.path.join(os.path.dirname(__file__), "windmill_helloworld.py")
+
+setuptools.setup(
+    name="windmill-helloworld",
+    version="0.0.1",
+    url="https://github.com/windmill-labs/windmill/blob/exit()/examples/deploy/private-npm-registry-tls/README.md",
+    author="WindmillLabs",
+    author_email="contact@windmill.dev",
+    description="Simple hello world python module to host on a private Pypi server",
+    long_description=open("README.md").read(),
+    py_modules=["helloworld_python_module"],
+    zip_safe=False,
+    platforms="any",
+    install_requires=[],
+    classifiers=[
+        "Development Status :: 2 - Pre-Alpha",
+        "Environment :: Web Environment",
+        "Intended Audience :: Developers",
+        "Operating System :: OS Independent",
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 2",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.3",
+    ],
+)

examples/deploy/private-package-registry-tls/helloworld_python_module/windmill_helloworld.py

@@ -0,0 +1,2 @@
+def say_hello(x: str) -> str:
+    return "Hello {}".format(x)

examples/deploy/private-package-registry-tls/helloworld_static/helloworld.json

@@ -0,0 +1,3 @@
+{
+    "hello": "world"
+}

examples/deploy/private-package-registry-tls/verdaccio_conf/config.yaml

@@ -7,15 +7,13 @@ uplinks:
   npmjs:
     url: https://registry.npmjs.org/
 packages:
-  '@*/*':
+  "@*/*":
     access: $all
     publish: $authenticated
-  '**':
+  "**":
     proxy: npmjs
 listen:
-- https://npm_registry:4873
-https:
-  key: /verdaccio/certs/npm_registry.key
-  cert: /verdaccio/certs/npm_registry.crt
-  ca: /verdaccio/certs/windmill-root.crt
+  - http://localhost:4873
+  - http://npm_registry:4873
+url_prefix: /npm/
 log: { type: stdout, format: pretty, level: http }