diff --git a/.env b/.env index 38e8bf85f1..ac44c197cd 100644 --- a/.env +++ b/.env @@ -1,2 +1,2 @@ DB_PASSWORD=changeme -WM_BASE_URL=windmill.localhost +WM_BASE_URL=localhost diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 6619c51fac..959d510a9b 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -26,7 +26,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Docker meta local - id: metalocal + id: meta-local uses: docker/metadata-action@v4 with: images: ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }} @@ -43,19 +43,31 @@ jobs: context: . push: true tags: | - ${{ steps.metalocal.outputs.tags }} + ${{ steps.meta-local.outputs.tags }} - labels: ${{ steps.metalocal.outputs.labels }} + labels: ${{ steps.meta-local.outputs.labels }} cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max - name: Docker meta if: github.event_name != 'pull_request' - id: meta + id: meta-slim uses: docker/metadata-action@v4 with: images: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Docker meta + if: github.event_name != 'pull_request' + id: meta-heavy + uses: docker/metadata-action@v4 + with: + images: | ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }} tags: | type=ref,event=branch @@ -72,6 +84,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Login to ECR + if: github.event_name != 'pull_request' uses: docker/login-action@v2 with: registry: ${{ env.ECR_REGISTRY }} @@ -86,10 +99,22 @@ jobs: push: true tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + labels: ${{ steps.meta-local.outputs.labels }} + cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache + cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max + + - name: Build and push publicly + uses: docker/build-push-action@v3 + if: github.event_name != 'pull_request' + with: + context: . + push: true + file: ./DockerfileHeavy + tags: | ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:latest - ${{ steps.metalocal.outputs.tags }} - ${{ steps.meta.outputs.tags }} - labels: ${{ steps.metalocal.outputs.labels }} + ${{ steps.meta-local.outputs.tags }} + ${{ steps.meta-heavy.outputs.tags }} + labels: ${{ steps.meta-local.outputs.labels }} cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max playwright: diff --git a/Dockerfile b/Dockerfile index 74aadafd9f..051d113f3b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.0-slim-buster as nsjail +FROM debian:buster-slim as nsjail WORKDIR /nsjail @@ -76,32 +76,21 @@ COPY .git/ .git/ RUN cargo build --release -FROM debian:buster-slim +FROM python:3.10-slim-buster + ARG APP=/usr/src/app RUN apt-get update \ - && apt-get install -y ca-certificates tzdata libpq5 \ - make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \ - libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \ - libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \ - libv8-dev tesseract-ocr \ + && apt-get install -y ca-certificates wget curl git jq libprotobuf-dev libnl-route-3-dev \ && rm -rf /var/lib/apt/lists/* -RUN wget https://golang.org/dl/go1.19.1.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz +RUN wget https://golang.org/dl/go1.19.1.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz && rm go1.19.1.linux-amd64.tar.gz ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go ENV TZ=Etc/UTC -ENV PYTHON_VERSION 3.10.4 - -RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \ - && tar -xf Python-${PYTHON_VERSION}.tgz && cd Python-${PYTHON_VERSION}/ && ./configure --enable-optimizations \ - && make -j 4 && make install - RUN /usr/local/bin/python3 -m pip install pip-tools -RUN /usr/local/bin/python3 -m pip install nltk -RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon COPY --from=builder /windmill/target/release/windmill ${APP}/windmill diff --git a/DockerfileHeavy b/DockerfileHeavy new file mode 100644 index 0000000000..6740ab9ae6 --- /dev/null +++ b/DockerfileHeavy @@ -0,0 +1,8 @@ +FROM ghcr.io/windmill-labs/windmill + +RUN /usr/local/bin/python3 -m pip install nltk +RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon + +RUN apt-get update \ + && apt-get install -y tesseract-ocr \ + && rm -rf /var/lib/apt/lists/* diff --git a/README.md b/README.md index ce427a3d14..610f33e21b 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,8 @@ compiling from source or using without a postgres super user, see `docker compose up` with the following docker-compose is sufficient: +Go to https://localhost et voilà :) + For older kernels < 4.18, set `DISABLE_NUSER=true` as env variable, otherwise nsjail will not be able to launch the isolated scripts. diff --git a/backend/windmill-worker/nsjail/run.python3.config.proto b/backend/windmill-worker/nsjail/run.python3.config.proto index 3ac95645a0..0efea2bd64 100644 --- a/backend/windmill-worker/nsjail/run.python3.config.proto +++ b/backend/windmill-worker/nsjail/run.python3.config.proto @@ -129,6 +129,7 @@ mount { iface_no_lo: true +envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" envar: "PYTHONPATH=/tmp/dependencies{ADDITIONAL_PYTHON_PATHS}" envar: "HOME=/tmp" diff --git a/docker-compose.yml b/docker-compose.yml index 0dda59edac..775e025695 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,14 +31,21 @@ services: - BASE_INTERNAL_URL=http://localhost:8000 - RUST_LOG=info - NUM_WORKERS=3 - - RUST_BACKTRACE=1 - DISABLE_NUSER=false + - DISABLE_NSJAIL=false + - KEEP_JOB_DIR=true - DENO_PATH=/usr/bin/deno - PYTHON_PATH=/usr/local/bin/python3 - NSJAIL_PATH=nsjail + - METRICS_ADDR=false depends_on: db: condition: service_healthy + # to mount the worker folder to debug,, KEEP_JOB_DIR=true and mount /tmp/windmill + volumes: + - worker_dependency_cache:/tmp/windmill/cache + # - ./oauth.json/:/usr/src/app/oauth.json + lsp: image: ghcr.io/windmill-labs/windmill-lsp:latest restart: unless-stopped @@ -57,3 +64,4 @@ services: volumes: db_data: null + worker_dependency_cache: null diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index 30fbce6b43..59408034d9 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -70,7 +70,7 @@
as JSON
{/if}{#if typeof result == 'object' && Object.keys(result).length > 0}
The result keys are: {truncate(Object.keys(result).join(', '), 50)}
{/if}{#if !forceJson && resultKind == 'table-col'}