From 31c1af47b73330b3af28a2ad766c41f3e30b3198 Mon Sep 17 00:00:00 2001
From: BodoBolero <peterbendel@neon.tech>
Date: Tue, 5 Nov 2024 17:37:17 +0100
Subject: [PATCH] preparations for ingest benchmarking workflow - install
 pgcopydb into github runner

---
 .github/workflows/benchmarking.yml | 48 ++++++++++++++++++++++++------
 1 file changed, 39 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml
index 69b8bc5d70..9e4763d865 100644
--- a/.github/workflows/benchmarking.yml
+++ b/.github/workflows/benchmarking.yml
@@ -37,9 +37,9 @@ on:
         description: 'AWS-RDS and AWS-AURORA normally only run on Saturday. Set this to true to run them on every workflow_dispatch'
         required: false
         default: false
-      run_only_pgvector_tests:
+      run_only_ingest_tests:
         type: boolean
-        description: 'Run pgvector tests but no other tests. If not set, all tests including pgvector tests will be run'
+        description: 'Run ingest tests but no other tests. If not set, all tests including ingest tests will be run'
         required: false
         default: false
 
@@ -53,8 +53,37 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  ingest:
+    env:
+      PG_CONFIG: /tmp/neon/pg_install/v16/bin/pg_config
+    runs-on: [ self-hosted, us-east-2, x64 ]
+    container:
+      image: neondatabase/build-tools:pinned-bookworm
+      credentials:
+        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
+        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+      options: --init
+
+    steps:
+    - name: Download Neon artifact
+      uses: ./.github/actions/download
+      with:
+        name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
+        path: /tmp/neon/
+        prefix: latest
+
+    - name: invoke pg_config
+      run: |
+        /tmp/neon/pg_install/v16/bin/pg_config --bindir
+
+    - name: Install pgcopydb
+      run: |
+        apt update
+        apt-get install pgcopydb
+        pgcopydb --help
+
   bench:
-    if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }}
+    if: ${{ github.event.inputs.run_only_ingest_tests == 'false' || github.event.inputs.run_only_ingest_tests == null }}
     permissions:
       contents: write
       statuses: write
@@ -163,7 +192,7 @@ jobs:
         SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 
   replication-tests:
-    if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }}
+    if: ${{ github.event.inputs.run_only_ingest_tests == 'false' || github.event.inputs.run_only_ingest_tests == null }}
     permissions:
       contents: write
       statuses: write
@@ -253,7 +282,7 @@ jobs:
         SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 
   generate-matrices:
-    if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }}
+    if: ${{ github.event.inputs.run_only_ingest_tests == 'false' || github.event.inputs.run_only_ingest_tests == null }}
     # Create matrices for the benchmarking jobs, so we run benchmarks on rds only once a week (on Saturday)
     #
     # Available platforms:
@@ -352,7 +381,7 @@ jobs:
     secrets: inherit
 
   pgbench-compare:
-    if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }}
+    if: ${{ github.event.inputs.run_only_ingest_tests == 'false' || github.event.inputs.run_only_ingest_tests == null }}
     needs: [ generate-matrices, prepare_AWS_RDS_databases ]
     permissions:
       contents: write
@@ -502,6 +531,7 @@ jobs:
         SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 
   pgbench-pgvector:
+    if: ${{ github.event.inputs.run_only_ingest_tests == 'false' || github.event.inputs.run_only_ingest_tests == null }}
     permissions:
       contents: write
       statuses: write
@@ -642,7 +672,7 @@ jobs:
     #
     # *_CLICKBENCH_CONNSTR: Genuine ClickBench DB with ~100M rows
     # *_CLICKBENCH_10M_CONNSTR: DB with the first 10M rows of ClickBench DB
-    if: ${{ !cancelled() && (github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null) }}
+    if: ${{ !cancelled() && (github.event.inputs.run_only_ingest_tests == 'false' || github.event.inputs.run_only_ingest_tests == null) }}
     permissions:
       contents: write
       statuses: write
@@ -754,7 +784,7 @@ jobs:
     # We might change it after https://github.com/neondatabase/neon/issues/2900.
     #
     # *_TPCH_S10_CONNSTR: DB generated with scale factor 10 (~10 GB)
-    if: ${{ !cancelled() && (github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null) }}
+    if: ${{ !cancelled() && (github.event.inputs.run_only_ingest_tests == 'false' || github.event.inputs.run_only_ingest_tests == null) }}
     permissions:
       contents: write
       statuses: write
@@ -860,7 +890,7 @@ jobs:
         SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 
   user-examples-compare:
-    if: ${{ !cancelled() && (github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null) }}
+    if: ${{ !cancelled() && (github.event.inputs.run_only_ingest_tests == 'false' || github.event.inputs.run_only_ingest_tests == null) }}
     permissions:
       contents: write
       statuses: write