try first version of benchmark on lakebase

.github/actionlint.yml

@@ -27,6 +27,9 @@ config-variables:
   - HETZNER_CACHE_BUCKET
   - HETZNER_CACHE_ENDPOINT
   - HETZNER_CACHE_REGION
+  - LAKEBASE_API_HOST
+  - LAKEBASE_OAUTH_CLIENT_ID
+  - LAKEBASE_ORG_ID
   - NEON_DEV_AWS_ACCOUNT_ID
   - NEON_PROD_AWS_ACCOUNT_ID
   - PGREGRESS_PG16_PROJECT_ID

.github/actions/lakebase-project-create/action.yml

@@ -9,11 +9,11 @@ inputs:
     description: 'Organization ID, required'
     required: true
   api_host:
-    description: 'Lakebase API host, e.g. dbc-55e65913-66de.dev.databricks.com/ajax-api/2.0/lakebase-console'
+    description: 'Lakebase API host, e.g. dbc-55e65913-66de.dev.databricks.com/lakebase-console'
     required: true
   postgres_version:
     description: 'Postgres version; default is 16'
-    default: '16'
+    default: '17'
   compute_units:
     description: '[Min, Max] compute units'
     default: '[1, 1]'
@@ -33,6 +33,10 @@ inputs:
     description: 'Fixed hostname to use for connection URI'
     required: false
     default: 'k8s-dpingres-serverle-09ade1e9e9-0d7f675c53b35938.elb.us-west-2.amazonaws.com'
+  region_id:
+    description: 'Project region ID'
+    required: false
+    default: 'aws-us-east-2'
 
 outputs:
   dsn:

.github/workflows/lakebase_benchmarking.yml

@@ -0,0 +1,169 @@
+name: Lakebase Benchmarking
+
+on:
+  workflow_dispatch: # adds ability to run this manually
+    inputs:
+      postgres_version:
+        description: 'Postgres version'
+        required: false
+        default: '17'
+      save_perf_report:
+        type: boolean
+        description: 'Publish perf report'
+        required: false
+        default: false
+
+defaults:
+  run:
+    shell: bash -euxo pipefail {0}
+
+concurrency:
+  # Allow only one workflow per any non-`main` branch.
+  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }}
+  cancel-in-progress: true
+
+jobs:
+  lakebase-pgbench:
+    permissions:
+      contents: write
+      statuses: write
+      id-token: write # aws-actions/configure-aws-credentials
+
+    env:
+      TEST_PG_BENCH_DURATIONS_MATRIX: "60m"
+      TEST_PG_BENCH_SCALES_MATRIX: "10gb"
+      POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
+      PG_VERSION: ${{ github.event.inputs.postgres_version || '17' }}
+      TEST_OUTPUT: /tmp/test_output
+      BUILD_TYPE: remote
+      SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || false }}
+      PLATFORM: "lakebase-captest-new"
+
+    # TODO: for lakehouse test-shard which is probably deployed in US-West we need to change the runner
+    # to us-west to get correct OLTP latencies due to added speed of light latency
+    runs-on: [ self-hosted, us-east-2, x64 ]
+    container:
+      image: ghcr.io/neondatabase/build-tools:pinned-bookworm
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+      options: --init
+
+    # Increase timeout to 8h, default timeout is 6h
+    timeout-minutes: 480
+
+    steps:
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
+      with:
+        aws-region: eu-central-1
+        role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
+        role-duration-seconds: 18000 # 5 hours
+
+    - name: Download Neon artifact
+      uses: ./.github/actions/download
+      with:
+        name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
+        path: /tmp/neon/
+        prefix: latest
+        aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
+
+    - name: Create Lakebase Bearer Token
+      id: create-bearer-token
+      uses: ./.github/actions/lakebase_create_token
+      with:
+        client_id: ${{ vars.LAKEBASE_OAUTH_CLIENT_ID }}
+        client_secret: ${{ secrets.LAKEBASE_OAUTH_CLIENT_SECRET }}
+
+    ## TODO, currently we cannot really specify a small min max CU for lakebase project
+    ## and the semantic of a CU is different from Neon, so we need to carefully map
+    ## compute sizes before comparing results
+    - name: Create Lakebase Project
+      id: create-lakebase-project
+      uses: ./.github/actions/lakebase-project-create
+      with:
+        api_host: ${{ vars.LAKEBASE_API_HOST }}
+        org_id: ${{ vars.LAKEBASE_ORG_ID }}
+        postgres_version: ${{ env.PG_VERSION }}
+        compute_units: '[1, 1]'
+        curl_authorization_options: "--header 'X-CSRF-Token: ${{ secrets.LAKEBASE_X_CSRF_TOKEN }}' --cookie 'JSESSIONID=${{ secrets.LAKEBASE_JSESSIONID }}; DBAUTH=${{ secrets.LAKEBASE_DBAUTH }}'"
+
+    - name: Benchmark init
+      uses: ./.github/actions/run-python-test-set
+      with:
+        build_type: ${{ env.BUILD_TYPE }}
+        test_selection: performance
+        run_in_parallel: false
+        save_perf_report: ${{ env.SAVE_PERF_REPORT }}
+        extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_init
+        pg_version: ${{ env.PG_VERSION }}
+        aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
+      env:
+        BENCHMARK_CONNSTR: ${{ steps.create-lakebase-project.outputs.dsn }}
+        VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
+        PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
+
+    - name: Benchmark simple-update
+      uses: ./.github/actions/run-python-test-set
+      with:
+        build_type: ${{ env.BUILD_TYPE }}
+        test_selection: performance
+        run_in_parallel: false
+        save_perf_report: ${{ env.SAVE_PERF_REPORT }}
+        extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_simple_update
+        pg_version: ${{ env.PG_VERSION }}
+        aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
+      env:
+        BENCHMARK_CONNSTR: ${{ steps.create-lakebase-project.outputs.dsn }}
+        VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
+        PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
+
+    - name: Benchmark select-only
+      uses: ./.github/actions/run-python-test-set
+      with:
+        build_type: ${{ env.BUILD_TYPE }}
+        test_selection: performance
+        run_in_parallel: false
+        save_perf_report: ${{ env.SAVE_PERF_REPORT }}
+        extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_select_only
+        pg_version: ${{ env.PG_VERSION }}
+        aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
+      env:
+        BENCHMARK_CONNSTR: ${{ steps.create-lakebase-project.outputs.dsn }}
+        VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
+        PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
+
+    - name: Delete Lakebase Project
+      if: ${{ steps.create-lakebase-project.outputs.project_id && always() }}
+      uses: ./.github/actions/lakebase-project-delete
+      with:
+        api_host: ${{ vars.LAKEBASE_API_HOST }}
+        org_id: ${{ vars.LAKEBASE_ORG_ID }}
+        project_id: ${{ steps.create-lakebase-project.outputs.project_id }}
+        curl_authorization_options: "--header 'X-CSRF-Token: ${{ secrets.LAKEBASE_X_CSRF_TOKEN }}' --cookie 'JSESSIONID=${{ secrets.LAKEBASE_JSESSIONID }}; DBAUTH=${{ secrets.LAKEBASE_DBAUTH }}'"
+
+    - name: Create Allure report
+      id: create-allure-report
+      if: ${{ !cancelled() }}
+      uses: ./.github/actions/allure-report-generate
+      with:
+        aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
+
+    - name: Post to a Slack channel
+      if: ${{ failure() }}
+      uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
+      with:
+        channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
+        slack-message: |
+          Lakebase perf testing: ${{ job.status }}
+          <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>
+          <${{ steps.create-allure-report.outputs.report-url }}|Allure report>
+      env:
+        SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}