diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 0389cdb841..0d8785e510 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -41,6 +41,8 @@ lazy_static::lazy_static! { .map(|x| format!(";{x}")) .unwrap_or_else(|| String::new()); + static ref DENO_CERT: String = std::env::var("DENO_CERT").ok().unwrap_or_else(|| String::new()); + static ref DENO_TLS_CA_STORE: String = std::env::var("DENO_TLS_CA_STORE").ok().unwrap_or_else(|| String::new()); } async fn get_common_deno_proc_envs( @@ -70,6 +72,12 @@ async fn get_common_deno_proc_envs( if let Some(ref s) = NPM_CONFIG_REGISTRY.read().await.clone() { deno_envs.insert(String::from("NPM_CONFIG_REGISTRY"), s.clone()); } + if DENO_CERT.len() > 0 { + deno_envs.insert(String::from("DENO_CERT"), DENO_CERT.clone()); + } + if DENO_TLS_CA_STORE.len() > 0 { + deno_envs.insert(String::from("DENO_TLS_CA_STORE"), DENO_TLS_CA_STORE.clone()); + } return deno_envs; } diff --git a/examples/deploy/private-npm-registry-tls/README.md b/examples/deploy/private-npm-registry-tls/README.md new file mode 100644 index 0000000000..3956361aaf --- /dev/null +++ b/examples/deploy/private-npm-registry-tls/README.md @@ -0,0 +1,104 @@ +Private NPM registry with self-signed certificates +================================================== + +Setup a private NPM registry with self-signed certificates. + +## Setup + +```bash +# spin up the registry. You can go to http://localhost:4873 to verify it's up and running +docker compose up -d + +# create your own user to be able to publish a package +npm adduser --registry http://localhost:4873 + +# publish a package +cd helloworld +npm publish --registry http://localhost:4873 +``` + +### Setup + +1. Generate self signed certificates (or bring your own) +```bash +cd certs +# feel free to read the file anc change the +./generate_certs.sh +# choose a password for the RootCA key. You'll need to input it for pretty much all following steps +``` +At the end of the script, you should have multiple files in the `certs/` folder. The most important ones are: +- `windmill-root.key` (Root CA private key) +- `windmill-root.crt` (Root CA certificate) +- `npm_registry.key` (NPM registry server private key) +- `npm_registry.crt` (NPM registry server certificate) + +2. Start the docker compose stack + +```bash +docker compose up -d +``` + +This will start the private NPM registry, as well as a minimal Windmill stack composed of just one Windmill server/worker and the associated database. + +For the latter, we invite you to refer to the latest [docker compose](/docker-compose.yml) at the root of this repository to setup a more evolved Windmill stack. + +For the former, it's using [Verdaccio](https://verdaccio.org/) as an easy-to-deploy NPM registry. We bring you attention to the fact that in addition to the config +file in `./verdaccio_conf/config.yaml`, we had to set both `VERDACCIO_PROTOCOL` and `VERDACCIO_PUBLIC_URL` in docker compose. See the official Verdaccio +documentation for more info on this. + +3. Upload the custom `helloworld` package to the private NPM registry + +``` +cd helloworld_package +# you need to first create a registry user +# you might need to run `npm config set strict-ssl false` if the below fails. If you do, then change it back to `true` after running the 2 commands +npm adduser --registry https://0.0.0.0:4873/ +npm publish --registry https://0.0.0.0:4873/ +``` + +4. Pull the custom NPM package from a deno script in Windmill + +Go to Windmill at `http://localhost:8000`. Create a simple deno script: +```ts +import * as testpackage from "npm:@windmill/helloworld@0.0.1" + +export async function main() { + console.log(testpackage.sayHello("Windmill")) +} +``` +and execute it. It should return successfully with: +``` +Hello Windmill +``` + + +### Remarks + +1. `DENO_TLS_CA_STORE` VS `DENO_CERT` +Both works. `DENO_CERT` is better b/c you just have to set it to the path of the trusted Root CA certificate, and deno will trust this certificate. +When using `DENO_TLS_CA_STORE=system`, you _have to_ make the server trust the custom Root CA certificate with the following commands: +```bash +# in the windmill-server container: +cp /custom-certs/windmill-root.crt /usr/local/share/ca-certificates/ +update-ca-certificates # as root +# the output should tell (among other things): " ... 1 added, 0 removed; done. ..." +``` + +2. Running deno scripts manually in the Windmill container +This could be useful for debugging purposes. + +```bash +# log into the Windmill Server container +docker exec -it /bin/bash + +# Go into Deno REPL +deno + +# try to import the package +import * as testpackage from "npm:@windmill/helloworld@0.0.1" +> undefined + +# if the above returns undefined, there's a good chance it's working. If you want to double check: +testpackage.sayHello("Windmill") +> Hello Windmill +``` diff --git a/examples/deploy/private-npm-registry-tls/certs/.gitignore b/examples/deploy/private-npm-registry-tls/certs/.gitignore new file mode 100644 index 0000000000..4700ea8b18 --- /dev/null +++ b/examples/deploy/private-npm-registry-tls/certs/.gitignore @@ -0,0 +1,5 @@ +*.crt +*.csr +*.key +*.ext +*.srl \ No newline at end of file diff --git a/examples/deploy/private-npm-registry-tls/certs/generate_certs.sh b/examples/deploy/private-npm-registry-tls/certs/generate_certs.sh new file mode 100755 index 0000000000..5e49f44aa2 --- /dev/null +++ b/examples/deploy/private-npm-registry-tls/certs/generate_certs.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set eou -pipefail + +CANAME='windmill-root' + +COUNTRY='FR' +STATE='Paris' +CITY='Paris' +ORGANIZATION='WindmillLabs' +ROOT_CA_CN='WindmillRootCA' +SERVER_CA_CN='npm_registry' # IMPORTANT: set this to the FQDN of the npm registry server. Here in the docker compose stack, it will be npm_registry + +echo "Generating RootCA key" +openssl genrsa -aes256 -out $CANAME.key 4096 +echo "Generating RootCA certificate" +openssl req -x509 -new -nodes -key $CANAME.key -sha256 -days 1826 -out $CANAME.crt -subj "/CN=${ROOT_CA_CN}/C=${COUNTRY}/ST=${STATE}/L=${CITY}/O=${ORGANIZATION}" + +CERTNAME=npm_registry +echo "Generating server certificate private key and cert signing request" +openssl req -new -nodes -out $CERTNAME.csr -newkey rsa:4096 -keyout $CERTNAME.key -subj "/CN=${SERVER_CA_CN}/C=${COUNTRY}/ST=${STATE}/L=${CITY}/O=${ORGANIZATION}" + +echo "Generating server certificate" +cat > $CERTNAME.v3.ext << EOF +[v3_req] +authorityKeyIdentifier=keyid,issuer +basicConstraints=CA:FALSE +keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment +subjectAltName = @alt_names +[alt_names] +DNS.1 = npm_registry +DNS.2 = localhost +EOF +# ^ HERE ^ in the above alt_names, feel free to add any alternate CN + +openssl x509 -req -in $CERTNAME.csr -CA $CANAME.crt -CAkey $CANAME.key -CAcreateserial -out $CERTNAME.crt -days 1826 -sha256 -extensions v3_req -extfile $CERTNAME.v3.ext diff --git a/examples/deploy/private-npm-registry-tls/docker-compose.yml b/examples/deploy/private-npm-registry-tls/docker-compose.yml new file mode 100644 index 0000000000..f12a1dd597 --- /dev/null +++ b/examples/deploy/private-npm-registry-tls/docker-compose.yml @@ -0,0 +1,47 @@ +version: "3.7" + +services: + npm_registry: + image: verdaccio/verdaccio + ports: + - 4873:4873 + environment: + - VERDACCIO_PROTOCOL=https + - VERDACCIO_PUBLIC_URL=https://npm_registry:4873 + volumes: + - ./verdaccio_conf:/verdaccio/conf + - ./certs:/verdaccio/certs + - npm_registry_data:/verdaccio/storage + + db: + image: postgres:14 + volumes: + - db_data:/var/lib/postgresql/data + environment: + POSTGRES_PASSWORD: changeme + POSTGRES_DB: windmill + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + + windmill_server: + image: ghcr.io/windmill-labs/windmill:main + ports: + - 8000:8000 + environment: + - WHITELIST_ENVS='DENO_CERT' + - DATABASE_URL=postgres://postgres:changeme@db/windmill?sslmode=disable + - NPM_CONFIG_REGISTRY=https://npm_registry:4873 + - DENO_CERT=/custom-certs/windmill-root.crt # this will make deno trust this RootCA for all sessions + # - DENO_TLS_CA_STORE=system # alternatively, you can use this but you'll need to manually trust the RootCA at the host level, see README.md + volumes: + - ./certs:/custom-certs + depends_on: + db: + condition: service_healthy + +volumes: + npm_registry_data: null + db_data: null diff --git a/examples/deploy/private-npm-registry-tls/helloworld_package/.gitignore b/examples/deploy/private-npm-registry-tls/helloworld_package/.gitignore new file mode 100644 index 0000000000..40b878db5b --- /dev/null +++ b/examples/deploy/private-npm-registry-tls/helloworld_package/.gitignore @@ -0,0 +1 @@ +node_modules/ \ No newline at end of file diff --git a/examples/deploy/private-npm-registry-tls/helloworld_package/README.md b/examples/deploy/private-npm-registry-tls/helloworld_package/README.md new file mode 100644 index 0000000000..b92d401744 --- /dev/null +++ b/examples/deploy/private-npm-registry-tls/helloworld_package/README.md @@ -0,0 +1,4 @@ +Hello world test package +======================== + +Dummy NPM package to test imports from a private NPM registry in Windmill \ No newline at end of file diff --git a/examples/deploy/private-npm-registry-tls/helloworld_package/index.js b/examples/deploy/private-npm-registry-tls/helloworld_package/index.js new file mode 100644 index 0000000000..811447ae24 --- /dev/null +++ b/examples/deploy/private-npm-registry-tls/helloworld_package/index.js @@ -0,0 +1,3 @@ +exports.sayHello = function(x) { + return `Hello ${x}` +} diff --git a/examples/deploy/private-npm-registry-tls/helloworld_package/package.json b/examples/deploy/private-npm-registry-tls/helloworld_package/package.json new file mode 100644 index 0000000000..d3af96f156 --- /dev/null +++ b/examples/deploy/private-npm-registry-tls/helloworld_package/package.json @@ -0,0 +1,10 @@ +{ + "name": "@windmill/helloworld", + "version": "0.0.1", + "description": "hello world test package", + "main": "index.js", + "scripts": {}, + "keywords": [], + "author": "Windmill", + "license": "Apache-2.0" +} \ No newline at end of file diff --git a/examples/deploy/private-npm-registry-tls/verdaccio_conf/config.default.yaml b/examples/deploy/private-npm-registry-tls/verdaccio_conf/config.default.yaml new file mode 100644 index 0000000000..a4cd315ab1 --- /dev/null +++ b/examples/deploy/private-npm-registry-tls/verdaccio_conf/config.default.yaml @@ -0,0 +1,201 @@ +# +# This is the default configuration file. It allows all users to do anything, +# please read carefully the documentation and best practices to +# improve security. +# +# Do not configure host and port under `listen` in this file +# as it will be ignored when using docker. +# see https://verdaccio.org/docs/en/docker#docker-and-custom-port-configuration +# +# Look here for more config file examples: +# https://github.com/verdaccio/verdaccio/tree/5.x/conf +# +# Read about the best practices +# https://verdaccio.org/docs/best + +# path to a directory with all packages +storage: /verdaccio/storage/data +# path to a directory with plugins to include +plugins: /verdaccio/plugins + +# https://verdaccio.org/docs/webui +web: + title: Verdaccio + # comment out to disable gravatar support + # gravatar: false + # by default packages are ordercer ascendant (asc|desc) + # sort_packages: asc + # convert your UI to the dark side + # darkMode: true + # html_cache: true + # by default all features are displayed + # login: true + # showInfo: true + # showSettings: true + # In combination with darkMode you can force specific theme + # showThemeSwitch: true + # showFooter: true + # showSearch: true + # showRaw: true + # showDownloadTarball: true + # HTML tags injected after manifest + # scriptsBodyAfter: + # - '' + # HTML tags injected before ends + # metaScripts: + # - '' + # - '' + # - '' + # HTML tags injected first child at + # bodyBefore: + # - '
html before webpack scripts
' + # Public path for template manifest scripts (only manifest) + # publicPath: http://somedomain.org/ + +# https://verdaccio.org/docs/configuration#authentication +auth: + htpasswd: + file: /verdaccio/storage/htpasswd + # Maximum amount of users allowed to register, defaults to "+infinity". + # You can set this to -1 to disable registration. + # max_users: 1000 + # Hash algorithm, possible options are: "bcrypt", "md5", "sha1", "crypt". + # algorithm: bcrypt # by default is crypt, but is recommended use bcrypt for new installations + # Rounds number for "bcrypt", will be ignored for other algorithms. + # rounds: 10 + +# https://verdaccio.org/docs/configuration#uplinks +# a list of other known repositories we can talk to +uplinks: + npmjs: + url: https://registry.npmjs.org/ + +# Learn how to protect your packages +# https://verdaccio.org/docs/protect-your-dependencies/ +# https://verdaccio.org/docs/configuration#packages +packages: + '@*/*': + # scoped packages + access: $all + publish: $authenticated + unpublish: $authenticated + proxy: npmjs + + '**': + # allow all users (including non-authenticated users) to read and + # publish all packages + # + # you can specify usernames/groupnames (depending on your auth plugin) + # and three keywords: "$all", "$anonymous", "$authenticated" + access: $all + + # allow all known users to publish/publish packages + # (anyone can register by default, remember?) + publish: $authenticated + unpublish: $authenticated + + # if package is not available locally, proxy requests to 'npmjs' registry + proxy: npmjs + +# To improve your security configuration and avoid dependency confusion +# consider removing the proxy property for private packages +# https://verdaccio.org/docs/best#remove-proxy-to-increase-security-at-private-packages + +# https://verdaccio.org/docs/configuration#server +# You can specify HTTP/1.1 server keep alive timeout in seconds for incoming connections. +# A value of 0 makes the http server behave similarly to Node.js versions prior to 8.0.0, which did not have a keep-alive timeout. +# WORKAROUND: Through given configuration you can workaround following issue https://github.com/verdaccio/verdaccio/issues/301. Set to 0 in case 60 is not enough. +server: + keepAliveTimeout: 60 + # Allow `req.ip` to resolve properly when Verdaccio is behind a proxy or load-balancer + # See: https://expressjs.com/en/guide/behind-proxies.html + # trustProxy: '127.0.0.1' + +# https://verdaccio.org/docs/configuration#offline-publish +# publish: +# allow_offline: false + +# https://verdaccio.org/docs/configuration#url-prefix +# url_prefix: /verdaccio/ +# VERDACCIO_PUBLIC_URL='https://somedomain.org'; +# url_prefix: '/my_prefix' +# // url -> https://somedomain.org/my_prefix/ +# VERDACCIO_PUBLIC_URL='https://somedomain.org'; +# url_prefix: '/' +# // url -> https://somedomain.org/ +# VERDACCIO_PUBLIC_URL='https://somedomain.org/first_prefix'; +# url_prefix: '/second_prefix' +# // url -> https://somedomain.org/second_prefix/' + +# https://verdaccio.org/docs/configuration#security +# security: +# api: +# legacy: true +# jwt: +# sign: +# expiresIn: 29d +# verify: +# someProp: [value] +# web: +# sign: +# expiresIn: 1h # 1 hour by default +# verify: +# someProp: [value] + +# https://verdaccio.org/docs/configuration#user-rate-limit +# userRateLimit: +# windowMs: 50000 +# max: 1000 + +# https://verdaccio.org/docs/configuration#max-body-size +# max_body_size: 10mb + +# https://verdaccio.org/docs/configuration#listen-port +# listen: +# - localhost:4873 # default value +# - http://localhost:4873 # same thing +# - 0.0.0.0:4873 # listen on all addresses (INADDR_ANY) +# - https://example.org:4873 # if you want to use https +# - "[::1]:4873" # ipv6 +# - unix:/tmp/verdaccio.sock # unix socket + +# The HTTPS configuration is useful if you do not consider use a HTTP Proxy +# https://verdaccio.org/docs/configuration#https +# https: +# key: ./path/verdaccio-key.pem +# cert: ./path/verdaccio-cert.pem +# ca: ./path/verdaccio-csr.pem + +# https://verdaccio.org/docs/configuration#proxy +# http_proxy: http://something.local/ +# https_proxy: https://something.local/ + +# https://verdaccio.org/docs/configuration#notifications +# notify: +# method: POST +# headers: [{ "Content-Type": "application/json" }] +# endpoint: https://usagge.hipchat.com/v2/room/3729485/notification?auth_token=mySecretToken +# content: '{"color":"green","message":"New package published: * {{ name }}*","notify":true,"message_format":"text"}' + +middlewares: + audit: + enabled: true + +# https://verdaccio.org/docs/logger +# log settings +log: { type: stdout, format: pretty, level: http } +#experiments: +# # support for npm token command +# token: false +# # enable tarball URL redirect for hosting tarball with a different server, the tarball_url_redirect can be a template string +# tarball_url_redirect: 'https://mycdn.com/verdaccio/${packageName}/${filename}' +# # the tarball_url_redirect can be a function, takes packageName and filename and returns the url, when working with a js configuration file +# tarball_url_redirect(packageName, filename) { +# const signedUrl = // generate a signed url +# return signedUrl; +# } + +# translate your registry, api i18n not available yet +# i18n: +# list of the available translations https://github.com/verdaccio/verdaccio/blob/master/packages/plugins/ui-theme/src/i18n/ABOUT_TRANSLATIONS.md +# web: en-US \ No newline at end of file diff --git a/examples/deploy/private-npm-registry-tls/verdaccio_conf/config.yaml b/examples/deploy/private-npm-registry-tls/verdaccio_conf/config.yaml new file mode 100644 index 0000000000..879af0e82a --- /dev/null +++ b/examples/deploy/private-npm-registry-tls/verdaccio_conf/config.yaml @@ -0,0 +1,21 @@ +storage: /verdaccio/storage/data +plugins: /verdaccio/plugins +auth: + htpasswd: + file: /verdaccio/storage/htpasswd +uplinks: + npmjs: + url: https://registry.npmjs.org/ +packages: + '@*/*': + access: $all + publish: $authenticated + '**': + proxy: npmjs +listen: +- https://npm_registry:4873 +https: + key: /verdaccio/certs/npm_registry.key + cert: /verdaccio/certs/npm_registry.crt + ca: /verdaccio/certs/windmill-root.crt +log: { type: stdout, format: pretty, level: http }