From 152287ab9fa2fcecc45c25b952c600a26aca4eaa Mon Sep 17 00:00:00 2001
From: Ruben Fiszel <ruben@rubenfiszel.com>
Date: Fri, 12 Aug 2022 11:21:19 +0200
Subject: [PATCH] full backend ci tests

* attempt ci in container

* attempt ci in container

* psql

* psql

* psql

* psql

* privileged

* cache
---
 .github/DockerfileBackendTests     | 62 ++++++++++++++++++++++++++++++
 .github/workflows/backend-test.yml | 15 +++++---
 backend/src/worker.rs              |  8 ++--
 3 files changed, 74 insertions(+), 11 deletions(-)
 create mode 100644 .github/DockerfileBackendTests

diff --git a/.github/DockerfileBackendTests b/.github/DockerfileBackendTests
new file mode 100644
index 0000000000..40e08146f8
--- /dev/null
+++ b/.github/DockerfileBackendTests
@@ -0,0 +1,62 @@
+FROM python:3.10-slim-buster as nsjail
+
+WORKDIR /nsjail
+
+RUN apt-get -y update \
+    && apt-get install -y \
+    bison=2:3.3.* \
+    flex=2.6.* \
+    g++=4:8.3.* \
+    gcc=4:8.3.* \
+    git=1:2.20.* \
+    libprotobuf-dev=3.6.* \
+    libnl-route-3-dev=3.4.* \
+    make=4.2.* \
+    pkg-config=0.29-6 \
+    protobuf-compiler=3.6.*
+
+RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \
+    && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800
+RUN make
+
+
+FROM rust:slim-buster as builder
+
+RUN apt-get update && apt-get install -y git libssl-dev pkg-config
+
+RUN apt-get -y update \
+    && apt-get install -y \
+    curl lld
+
+ENV SQLX_OFFLINE=true
+
+COPY ./nsjail /nsjail
+
+RUN mkdir -p /frontend/build
+
+RUN apt-get update \
+    && apt-get install -y ca-certificates tzdata libpq5 \
+    make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
+    libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
+    libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \
+    libv8-dev tesseract-ocr \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV TZ=Etc/UTC
+
+ENV PYTHON_VERSION 3.10.4
+
+RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \
+    && tar -xf Python-${PYTHON_VERSION}.tgz && cd Python-${PYTHON_VERSION}/ && ./configure --enable-optimizations \
+    && make -j 4 && make install
+
+RUN /usr/local/bin/python3 -m pip install pip-tools
+RUN /usr/local/bin/python3 -m pip install nltk
+RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon
+
+COPY --from=nsjail /nsjail/nsjail /bin/nsjail
+
+COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
+
+RUN apt-get update \
+    && apt-get install -y postgresql-client
diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml
index 697d3d7b42..e903ad245e 100644
--- a/.github/workflows/backend-test.yml
+++ b/.github/workflows/backend-test.yml
@@ -16,6 +16,9 @@ on:
 jobs:
   cargo_test:
     runs-on: [self-hosted, new]
+    container:
+      image: ghcr.io/windmill-labs/backend-tests
+      options: --privileged
     services:
       postgres:
         image: postgres
@@ -23,8 +26,7 @@ jobs:
           POSTGRES_DB: windmill
           POSTGRES_USER: admin
           POSTGRES_PASSWORD: changeme
-        ports:
-          - 5434:5432
+
         options: >-
           --health-cmd pg_isready
           --health-interval 10s
@@ -32,9 +34,10 @@ jobs:
           --health-retries 5
     steps:
       - uses: actions/checkout@v3
-      - run: /root/.cargo/bin/rustup toolchain install stable --profile minimal
       - uses: Swatinem/rust-cache@v2
-      - run: psql postgres://admin:changeme@localhost:5434/windmill -c "CREATE ROLE app LOGIN PASSWORD 'changeme';"
-      - name: "cargo test"
+        with:
+          workspaces: backend -> target
+      - run: psql postgres://admin:changeme@postgres:5432/windmill -c "CREATE ROLE app LOGIN PASSWORD 'changeme';"
+      - name: cargo test
         timeout-minutes: 5
-        run: mkdir -p frontend/build && cd backend && SQLX_OFFLINE=true DATABASE_URL=postgres://admin:changeme@localhost:5434/windmill /root/.cargo/bin/cargo test
+        run: mkdir frontend/build && cd backend && DATABASE_URL=postgres://admin:changeme@postgres:5432/windmill cargo test
diff --git a/backend/src/worker.rs b/backend/src/worker.rs
index f5bb70704e..6a0c7e773f 100644
--- a/backend/src/worker.rs
+++ b/backend/src/worker.rs
@@ -1285,8 +1285,7 @@ mod tests {
         assert_eq!(result, serde_json::json!([2, 4, 6]));
     }
 
-    // CI is broken atm, todo
-    // #[sqlx::test(fixtures("base"))]
+    #[sqlx::test(fixtures("base"))]
     async fn test_python_flow(db: DB) {
         initialize_tracing().await;
 
@@ -1338,8 +1337,7 @@ mod tests {
         assert_eq!(result, serde_json::json!([2, 4, 6]));
     }
 
-    // ci is broken atm, TODO
-    // #[sqlx::test(fixtures("base"))]
+    #[sqlx::test(fixtures("base"))]
     async fn test_python_job(db: DB) {
         initialize_tracing().await;
 
@@ -1403,7 +1401,7 @@ def main():
             let sleep_queue: u64 = Default::default();
             let base_url: &str = Default::default();
             let disable_nuser = false;
-            let disable_nsjail = true;
+            let disable_nsjail = false;
 
             run_worker(
                 &db,