fix: Deno can talk to private NPM registries behind HTTPS (#2713)

backend/windmill-worker/src/deno_executor.rs

@@ -41,6 +41,8 @@ lazy_static::lazy_static! {
         .map(|x| format!(";{x}"))
         .unwrap_or_else(|| String::new());
 
+    static ref DENO_CERT: String = std::env::var("DENO_CERT").ok().unwrap_or_else(|| String::new());
+    static ref DENO_TLS_CA_STORE: String = std::env::var("DENO_TLS_CA_STORE").ok().unwrap_or_else(|| String::new());
 
 }
 async fn get_common_deno_proc_envs(
@@ -70,6 +72,12 @@ async fn get_common_deno_proc_envs(
     if let Some(ref s) = NPM_CONFIG_REGISTRY.read().await.clone() {
         deno_envs.insert(String::from("NPM_CONFIG_REGISTRY"), s.clone());
     }
+    if DENO_CERT.len() > 0 {
+        deno_envs.insert(String::from("DENO_CERT"), DENO_CERT.clone());
+    }
+    if DENO_TLS_CA_STORE.len() > 0 {
+        deno_envs.insert(String::from("DENO_TLS_CA_STORE"), DENO_TLS_CA_STORE.clone());
+    }
     return deno_envs;
 }
 

examples/deploy/private-npm-registry-tls/README.md

@@ -0,0 +1,104 @@
+Private NPM registry with self-signed certificates
+==================================================
+
+Setup a private NPM registry with self-signed certificates.
+
+## Setup
+
+```bash
+# spin up the registry. You can go to http://localhost:4873 to verify it's up and running
+docker compose up -d
+
+# create your own user to be able to publish a package
+npm adduser --registry http://localhost:4873
+
+# publish a package
+cd helloworld
+npm publish --registry http://localhost:4873
+```
+
+### Setup
+
+1. Generate self signed certificates (or bring your own)
+```bash
+cd certs
+# feel free to read the file anc change the 
+./generate_certs.sh
+# choose a password for the RootCA key. You'll need to input it for pretty much all following steps
+```
+At the end of the script, you should have multiple files in the `certs/` folder. The most important ones are:
+- `windmill-root.key` (Root CA private key)
+- `windmill-root.crt` (Root CA certificate)
+- `npm_registry.key` (NPM registry server private key)
+- `npm_registry.crt` (NPM registry server certificate)
+
+2. Start the docker compose stack
+
+```bash
+docker compose up -d
+```
+
+This will start the private NPM registry, as well as a minimal Windmill stack composed of just one Windmill server/worker and the associated database. 
+
+For the latter, we invite you to refer to the latest [docker compose](/docker-compose.yml) at the root of this repository to setup a more evolved Windmill stack.
+
+For the former, it's using [Verdaccio](https://verdaccio.org/) as an easy-to-deploy NPM registry. We bring you attention to the fact that in addition to the config 
+file in `./verdaccio_conf/config.yaml`, we had to set both `VERDACCIO_PROTOCOL` and `VERDACCIO_PUBLIC_URL` in docker compose. See the official Verdaccio 
+documentation for more info on this.
+
+3. Upload the custom `helloworld` package to the private NPM registry
+
+```
+cd helloworld_package
+# you need to first create a registry user
+# you might need to run `npm config set strict-ssl false` if the below fails. If you do, then change it back to `true` after running the 2 commands
+npm adduser --registry https://0.0.0.0:4873/
+npm publish --registry https://0.0.0.0:4873/
+```
+
+4. Pull the custom NPM package from a deno script in Windmill
+
+Go to Windmill at `http://localhost:8000`. Create a simple deno script:
+```ts
+import * as testpackage from "npm:@windmill/helloworld@0.0.1"
+
+export async function main() {
+  console.log(testpackage.sayHello("Windmill"))
+}
+```
+and execute it. It should return successfully with:
+```
+Hello Windmill
+```
+
+
+### Remarks
+
+1. `DENO_TLS_CA_STORE` VS `DENO_CERT`
+Both works. `DENO_CERT` is better b/c you just have to set it to the path of the trusted Root CA certificate, and deno will trust this certificate.
+When using `DENO_TLS_CA_STORE=system`, you _have to_ make the server trust the custom Root CA certificate with the following commands:
+```bash
+# in the windmill-server container:
+cp /custom-certs/windmill-root.crt /usr/local/share/ca-certificates/
+update-ca-certificates # as root
+# the output should tell (among other things): " ... 1 added, 0 removed; done. ..."
+```
+
+2. Running deno scripts manually in the Windmill container
+This could be useful for debugging purposes.
+
+```bash
+# log into the Windmill Server container
+docker exec -it <WINDMILL_SERVER_CONTAINER> /bin/bash
+
+# Go into Deno REPL
+deno
+
+# try to import the package
+import * as testpackage from "npm:@windmill/helloworld@0.0.1"
+> undefined
+
+# if the above returns undefined, there's a good chance it's working. If you want to double check:
+testpackage.sayHello("Windmill")
+> Hello Windmill
+```

examples/deploy/private-npm-registry-tls/certs/.gitignore

@@ -0,0 +1,5 @@
+*.crt
+*.csr
+*.key
+*.ext
+*.srl

examples/deploy/private-npm-registry-tls/certs/generate_certs.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+set eou -pipefail
+
+CANAME='windmill-root'
+
+COUNTRY='FR'
+STATE='Paris'
+CITY='Paris'
+ORGANIZATION='WindmillLabs'
+ROOT_CA_CN='WindmillRootCA'
+SERVER_CA_CN='npm_registry' # IMPORTANT: set this to the FQDN of the npm registry server. Here in the docker compose stack, it will be npm_registry
+
+echo "Generating RootCA key"
+openssl genrsa -aes256 -out $CANAME.key 4096
+echo "Generating RootCA certificate"
+openssl req -x509 -new -nodes -key $CANAME.key -sha256 -days 1826 -out $CANAME.crt -subj "/CN=${ROOT_CA_CN}/C=${COUNTRY}/ST=${STATE}/L=${CITY}/O=${ORGANIZATION}"
+
+CERTNAME=npm_registry
+echo "Generating server certificate private key and cert signing request"
+openssl req -new -nodes -out $CERTNAME.csr -newkey rsa:4096 -keyout $CERTNAME.key -subj "/CN=${SERVER_CA_CN}/C=${COUNTRY}/ST=${STATE}/L=${CITY}/O=${ORGANIZATION}"
+
+echo "Generating server certificate"
+cat > $CERTNAME.v3.ext << EOF
+[v3_req]
+authorityKeyIdentifier=keyid,issuer
+basicConstraints=CA:FALSE
+keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
+subjectAltName = @alt_names
+[alt_names]
+DNS.1 = npm_registry
+DNS.2 = localhost
+EOF
+# ^ HERE ^ in the above alt_names, feel free to add any alternate CN
+
+openssl x509 -req -in $CERTNAME.csr -CA $CANAME.crt -CAkey $CANAME.key -CAcreateserial -out $CERTNAME.crt -days 1826 -sha256 -extensions v3_req -extfile $CERTNAME.v3.ext

examples/deploy/private-npm-registry-tls/docker-compose.yml

@@ -0,0 +1,47 @@
+version: "3.7"
+
+services:
+  npm_registry:
+    image: verdaccio/verdaccio
+    ports:
+      - 4873:4873
+    environment:
+      - VERDACCIO_PROTOCOL=https
+      - VERDACCIO_PUBLIC_URL=https://npm_registry:4873
+    volumes:
+      - ./verdaccio_conf:/verdaccio/conf
+      - ./certs:/verdaccio/certs
+      - npm_registry_data:/verdaccio/storage
+
+  db:
+    image: postgres:14
+    volumes:
+      - db_data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_PASSWORD: changeme
+      POSTGRES_DB: windmill
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  windmill_server:
+    image: ghcr.io/windmill-labs/windmill:main
+    ports:
+      - 8000:8000
+    environment:
+      - WHITELIST_ENVS='DENO_CERT'
+      - DATABASE_URL=postgres://postgres:changeme@db/windmill?sslmode=disable
+      - NPM_CONFIG_REGISTRY=https://npm_registry:4873
+      - DENO_CERT=/custom-certs/windmill-root.crt # this will make deno trust this RootCA for all sessions
+      # - DENO_TLS_CA_STORE=system # alternatively, you can use this but you'll need to manually trust the RootCA at the host level, see README.md
+    volumes:
+      - ./certs:/custom-certs
+    depends_on:
+      db:
+        condition: service_healthy
+
+volumes:
+  npm_registry_data: null
+  db_data: null

examples/deploy/private-npm-registry-tls/helloworld_package/.gitignore

@@ -0,0 +1 @@
+node_modules/

examples/deploy/private-npm-registry-tls/helloworld_package/README.md

@@ -0,0 +1,4 @@
+Hello world test package
+========================
+
+Dummy NPM package to test imports from a private NPM registry in Windmill

examples/deploy/private-npm-registry-tls/helloworld_package/index.js

@@ -0,0 +1,3 @@
+exports.sayHello = function(x) {
+	return `Hello ${x}`
+}

examples/deploy/private-npm-registry-tls/helloworld_package/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "@windmill/helloworld",
+  "version": "0.0.1",
+  "description": "hello world test package",
+  "main": "index.js",
+  "scripts": {},
+  "keywords": [],
+  "author": "Windmill",
+  "license": "Apache-2.0"
+}

examples/deploy/private-npm-registry-tls/verdaccio_conf/config.default.yaml

@@ -0,0 +1,201 @@
+#
+# This is the default configuration file. It allows all users to do anything,
+# please read carefully the documentation and best practices to
+# improve security.
+#
+# Do not configure host and port under `listen` in this file
+# as it will be ignored when using docker.
+# see https://verdaccio.org/docs/en/docker#docker-and-custom-port-configuration
+#
+# Look here for more config file examples:
+# https://github.com/verdaccio/verdaccio/tree/5.x/conf
+#
+# Read about the best practices
+# https://verdaccio.org/docs/best
+
+# path to a directory with all packages
+storage: /verdaccio/storage/data
+# path to a directory with plugins to include
+plugins: /verdaccio/plugins
+
+# https://verdaccio.org/docs/webui
+web:
+  title: Verdaccio
+  # comment out to disable gravatar support
+  # gravatar: false
+  # by default packages are ordercer ascendant (asc|desc)
+  # sort_packages: asc
+  # convert your UI to the dark side
+  # darkMode: true
+  # html_cache: true
+  # by default all features are displayed
+  # login: true
+  # showInfo: true
+  # showSettings: true
+  # In combination with darkMode you can force specific theme
+  # showThemeSwitch: true
+  # showFooter: true
+  # showSearch: true
+  # showRaw: true
+  # showDownloadTarball: true
+  #  HTML tags injected after manifest <scripts/>
+  # scriptsBodyAfter:
+  #    - '<script type="text/javascript" src="https://my.company.com/customJS.min.js"></script>'
+  #  HTML tags injected before ends </head>
+  #  metaScripts:
+  #    - '<script type="text/javascript" src="https://code.jquery.com/jquery-3.5.1.slim.min.js"></script>'
+  #    - '<script type="text/javascript" src="https://browser.sentry-cdn.com/5.15.5/bundle.min.js"></script>'
+  #    - '<meta name="robots" content="noindex" />'
+  #  HTML tags injected first child at <body/>
+  #  bodyBefore:
+  #    - '<div id="myId">html before webpack scripts</div>'
+  #  Public path for template manifest scripts (only manifest)
+  #  publicPath: http://somedomain.org/
+
+# https://verdaccio.org/docs/configuration#authentication
+auth:
+  htpasswd:
+    file: /verdaccio/storage/htpasswd
+    # Maximum amount of users allowed to register, defaults to "+infinity".
+    # You can set this to -1 to disable registration.
+    # max_users: 1000
+    # Hash algorithm, possible options are: "bcrypt", "md5", "sha1", "crypt".
+    # algorithm: bcrypt # by default is crypt, but is recommended use bcrypt for new installations
+    # Rounds number for "bcrypt", will be ignored for other algorithms.
+    # rounds: 10
+
+# https://verdaccio.org/docs/configuration#uplinks
+# a list of other known repositories we can talk to
+uplinks:
+  npmjs:
+    url: https://registry.npmjs.org/
+
+# Learn how to protect your packages
+# https://verdaccio.org/docs/protect-your-dependencies/
+# https://verdaccio.org/docs/configuration#packages
+packages:
+  '@*/*':
+    # scoped packages
+    access: $all
+    publish: $authenticated
+    unpublish: $authenticated
+    proxy: npmjs
+
+  '**':
+    # allow all users (including non-authenticated users) to read and
+    # publish all packages
+    #
+    # you can specify usernames/groupnames (depending on your auth plugin)
+    # and three keywords: "$all", "$anonymous", "$authenticated"
+    access: $all
+
+    # allow all known users to publish/publish packages
+    # (anyone can register by default, remember?)
+    publish: $authenticated
+    unpublish: $authenticated
+
+    # if package is not available locally, proxy requests to 'npmjs' registry
+    proxy: npmjs
+
+# To improve your security configuration and  avoid dependency confusion
+# consider removing the proxy property for private packages
+# https://verdaccio.org/docs/best#remove-proxy-to-increase-security-at-private-packages
+
+# https://verdaccio.org/docs/configuration#server
+# You can specify HTTP/1.1 server keep alive timeout in seconds for incoming connections.
+# A value of 0 makes the http server behave similarly to Node.js versions prior to 8.0.0, which did not have a keep-alive timeout.
+# WORKAROUND: Through given configuration you can workaround following issue https://github.com/verdaccio/verdaccio/issues/301. Set to 0 in case 60 is not enough.
+server:
+  keepAliveTimeout: 60
+  # Allow `req.ip` to resolve properly when Verdaccio is behind a proxy or load-balancer
+  # See: https://expressjs.com/en/guide/behind-proxies.html
+  # trustProxy: '127.0.0.1'
+
+# https://verdaccio.org/docs/configuration#offline-publish
+# publish:
+#   allow_offline: false
+
+# https://verdaccio.org/docs/configuration#url-prefix
+# url_prefix: /verdaccio/
+# VERDACCIO_PUBLIC_URL='https://somedomain.org';
+# url_prefix: '/my_prefix'
+# // url -> https://somedomain.org/my_prefix/
+# VERDACCIO_PUBLIC_URL='https://somedomain.org';
+# url_prefix: '/'
+# // url -> https://somedomain.org/
+# VERDACCIO_PUBLIC_URL='https://somedomain.org/first_prefix';
+# url_prefix: '/second_prefix'
+# // url -> https://somedomain.org/second_prefix/'
+
+# https://verdaccio.org/docs/configuration#security
+# security:
+#   api:
+#     legacy: true
+#     jwt:
+#       sign:
+#         expiresIn: 29d
+#       verify:
+#         someProp: [value]
+#    web:
+#      sign:
+#        expiresIn: 1h # 1 hour by default
+#      verify:
+#         someProp: [value]
+
+# https://verdaccio.org/docs/configuration#user-rate-limit
+# userRateLimit:
+#   windowMs: 50000
+#   max: 1000
+
+# https://verdaccio.org/docs/configuration#max-body-size
+# max_body_size: 10mb
+
+# https://verdaccio.org/docs/configuration#listen-port
+# listen:
+# - localhost:4873            # default value
+# - http://localhost:4873     # same thing
+# - 0.0.0.0:4873              # listen on all addresses (INADDR_ANY)
+# - https://example.org:4873  # if you want to use https
+# - "[::1]:4873"                # ipv6
+# - unix:/tmp/verdaccio.sock    # unix socket
+
+# The HTTPS configuration is useful if you do not consider use a HTTP Proxy
+# https://verdaccio.org/docs/configuration#https
+# https:
+#   key: ./path/verdaccio-key.pem
+#   cert: ./path/verdaccio-cert.pem
+#   ca: ./path/verdaccio-csr.pem
+
+# https://verdaccio.org/docs/configuration#proxy
+# http_proxy: http://something.local/
+# https_proxy: https://something.local/
+
+# https://verdaccio.org/docs/configuration#notifications
+# notify:
+#   method: POST
+#   headers: [{ "Content-Type": "application/json" }]
+#   endpoint: https://usagge.hipchat.com/v2/room/3729485/notification?auth_token=mySecretToken
+#   content: '{"color":"green","message":"New package published: * {{ name }}*","notify":true,"message_format":"text"}'
+
+middlewares:
+  audit:
+    enabled: true
+
+# https://verdaccio.org/docs/logger
+# log settings
+log: { type: stdout, format: pretty, level: http }
+#experiments:
+#  # support for npm token command
+#  token: false
+#  # enable tarball URL redirect for hosting tarball with a different server, the tarball_url_redirect can be a template string
+#  tarball_url_redirect: 'https://mycdn.com/verdaccio/${packageName}/${filename}'
+#  # the tarball_url_redirect can be a function, takes packageName and filename and returns the url, when working with a js configuration file
+#  tarball_url_redirect(packageName, filename) {
+#    const signedUrl = // generate a signed url
+#    return signedUrl;
+#  }
+
+# translate your registry, api i18n not available yet
+# i18n:
+# list of the available translations https://github.com/verdaccio/verdaccio/blob/master/packages/plugins/ui-theme/src/i18n/ABOUT_TRANSLATIONS.md
+#   web: en-US

examples/deploy/private-npm-registry-tls/verdaccio_conf/config.yaml

@@ -0,0 +1,21 @@
+storage: /verdaccio/storage/data
+plugins: /verdaccio/plugins
+auth:
+  htpasswd:
+    file: /verdaccio/storage/htpasswd
+uplinks:
+  npmjs:
+    url: https://registry.npmjs.org/
+packages:
+  '@*/*':
+    access: $all
+    publish: $authenticated
+  '**':
+    proxy: npmjs
+listen:
+- https://npm_registry:4873
+https:
+  key: /verdaccio/certs/npm_registry.key
+  cert: /verdaccio/certs/npm_registry.crt
+  ca: /verdaccio/certs/windmill-root.crt
+log: { type: stdout, format: pretty, level: http }