Separate duckdb crate to fix c++ build/link issues (#6551)

* call ffi

* remove duckdb dep

* rename windmill_duckdb_ffi_internal

* static lib

* ci

* back to dylib, bug isn't fixed in static

* feature flag and copy dynamic lib

* fix dynlib in docker

* load libwindmill_duckdb_ffi_internal at runtime on usage

* lazy static deadlocks

* Cache dynamic library handles

* update auto s3 path insert from editor bar

* Fix duckdb S3 freezing worker because of blocking task in tokio async context

* build dll windows GH workflow

* try fix windows build

* revert build.rs

* nit fixes CI

* Dockerfile update (not tested yet

* build dev sh for duckdb lib

* mistake

* attach windmill_duckdb_ffi_internal.so artefact

* rhel9

* docker fixes

* fix dockerfile

* better err msg

* forgot lib prefix .so

* add column_order

* fix column_order

.github/workflows/build-publish-rh-image.yml

@@ -97,6 +97,12 @@ jobs:
           image: ${{ steps.meta-ee-public.outputs.tags}}-amd64
           path: "/windmill/target/release/windmill"
 
+      - uses: shrink/actions-docker-extract@v3
+        id: extract-duckdb-ffi-internal
+        with:
+          image: ${{ steps.meta-ee-public.outputs.tags}}-amd64
+          path: "/usr/src/app/libwindmill_duckdb_ffi_internal.so"
+
       # - uses: shrink/actions-docker-extract@v3
       #   id: extract-ee-arm64
       #   with:
@@ -111,8 +117,12 @@ jobs:
       - uses: actions/upload-artifact@v4
         with:
           name: RHEL9-amd64 build
-          path: ${{ steps.extract-ee-amd64.outputs.destination
-            }}/windmill-ee-amd64-rhel9
+          path: ${{ steps.extract-ee-amd64.outputs.destination }}/windmill-ee-amd64-rhel9
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: RHEL9-amd64 dynamic libraries build
+          path: ${{ steps.extract-duckdb-ffi-internal.outputs.destination }}/libwindmill_duckdb_ffi_internal.so
 
       # - uses: actions/upload-artifact@v4
       #   with:

.github/workflows/build_windows_worker_.yml

@@ -41,7 +41,13 @@ jobs:
         run: |
           ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private
 
-      - name: Cargo build windows
+      - name: Cargo build dynamic libraries windows
+        timeout-minutes: 90
+        run: |
+          cd backend/windmill-duckdb-ffi-internal
+          cargo build --release -p windmill_duckdb_ffi_internal
+
+      - name: Cargo build binary windows
         timeout-minutes: 90
         run: |
           vcpkg.exe install openssl-windows:x64-windows
@@ -56,8 +62,14 @@ jobs:
         run: |
           Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe"
 
-      - name: Upload artifact
+      - name: Upload binary artifact
         uses: actions/upload-artifact@v4
         with:
           name: windmill-ee-binary
           path: ./backend/target/release/windmill-ee.exe
+
+      - name: Upload dynamic libraries artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: windmill_duckdb_ffi_internal.dll
+          path: ./backend/windmill-duckdb-ffi-internal/target/release/windmill_duckdb_ffi_internal.dll

.github/workflows/docker-image.yml

@@ -220,6 +220,12 @@ jobs:
           image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
           path: "/usr/src/app/windmill"
 
+      - uses: shrink/actions-docker-extract@v3
+        id: extract-duckdb-ffi-internal
+        with:
+          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
+          path: "/usr/src/app/libwindmill_duckdb_ffi_internal.so"
+
       - uses: shrink/actions-docker-extract@v3
         id: extract-ee
         with:
@@ -237,6 +243,7 @@ jobs:
           files: |
             ${{ steps.extract.outputs.destination }}/*
             ${{ steps.extract-ee.outputs.destination }}/*
+            ${{ steps.extract-duckdb-ffi-internal.outputs.destination }}/*
 
   # attach_arm64_binary_to_release:
   #   needs: [build, build_ee]

.github/workflows/publish_windows_worker.yml

@@ -43,6 +43,12 @@ jobs:
         run: |
           ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private
 
+      - name: Cargo build dynamic libraries windows
+        timeout-minutes: 90
+        run: |
+          cd backend/windmill-duckdb-ffi-internal
+          cargo build --release -p windmill_duckdb_ffi_internal
+
       - name: Cargo build windows
         timeout-minutes: 90
         run: |
@@ -63,3 +69,9 @@ jobs:
         with:
           files: |
             ./backend/target/release/windmill-ee.exe
+
+      - name: Attach dynamic libraries to release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: |
+            ./backend/windmill-duckdb-ffi-internal/target/release/windmill_duckdb_ffi_internal.dll