neon/.github/workflows/lakebase_benchmarking.yml

name: Lakebase Benchmarking

on:
  # uncomment to run on push for debugging your PR
  push:
    branches: [ bodobolero/lakebase_perf_tests ]

  workflow_dispatch: # adds ability to run this manually
    inputs:
      postgres_version:
        description: 'Postgres version'
        required: false
        default: '17'
      save_perf_report:
        type: boolean
        description: 'Publish perf report'
        required: false
        default: false

defaults:
  run:
    shell: bash -euxo pipefail {0}

concurrency:
  # Allow only one workflow per any non-`main` branch.
  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }}
  cancel-in-progress: true

jobs:
  lakebase-pgbench:
    permissions:
      contents: write
      statuses: write
      id-token: write # aws-actions/configure-aws-credentials

    env:
      TEST_PG_BENCH_DURATIONS_MATRIX: "60m"
      TEST_PG_BENCH_SCALES_MATRIX: "10gb"
      POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
      PG_VERSION: ${{ github.event.inputs.postgres_version || '17' }}
      TEST_OUTPUT: /tmp/test_output
      BUILD_TYPE: remote
      SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || false }}
      PLATFORM: "lakebase-captest-new"

    # TODO: for lakehouse test-shard which is probably deployed in US-West we need to change the runner
    # to us-west to get correct OLTP latencies due to added speed of light latency
    runs-on: [ self-hosted, us-east-2, x64 ]
    container:
      image: ghcr.io/neondatabase/build-tools:pinned-bookworm
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      options: --init

    # Increase timeout to 8h, default timeout is 6h
    timeout-minutes: 480

    steps:
    - name: Harden the runner (Audit all outbound calls)
      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
      with:
        egress-policy: audit

    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
      with:
        aws-region: eu-central-1
        role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
        role-duration-seconds: 18000 # 5 hours

    - name: Download Neon artifact
      uses: ./.github/actions/download
      with:
        name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
        path: /tmp/neon/
        prefix: latest
        aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}

    - name: Create Lakebase Bearer Token
      id: create-bearer-token
      uses: ./.github/actions/lakebase_create_token
      with:
        client_id: ${{ vars.LAKEBASE_OAUTH_CLIENT_ID }}
        client_secret: ${{ secrets.LAKEBASE_OAUTH_CLIENT_SECRET }}

    ## TODO, currently we cannot really specify a small min max CU for lakebase project
    ## and the semantic of a CU is different from Neon, so we need to carefully map
    ## compute sizes before comparing results
    - name: Create Lakebase Project
      id: create-lakebase-project
      uses: ./.github/actions/lakebase-project-create
      with:
        api_host: ${{ vars.LAKEBASE_API_HOST }}
        org_id: ${{ vars.LAKEBASE_ORG_ID }}
        postgres_version: ${{ env.PG_VERSION }}
        compute_units: '[1, 1]'
        curl_authorization_options: "--header 'X-CSRF-Token: ${{ secrets.LAKEBASE_X_CSRF_TOKEN }}' --cookie 'JSESSIONID=${{ secrets.LAKEBASE_JSESSIONID }}; DBAUTH=${{ secrets.LAKEBASE_DBAUTH }}'"

    - name: Benchmark init
      uses: ./.github/actions/run-python-test-set
      with:
        build_type: ${{ env.BUILD_TYPE }}
        test_selection: performance
        run_in_parallel: false
        save_perf_report: ${{ env.SAVE_PERF_REPORT }}
        extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_init
        pg_version: ${{ env.PG_VERSION }}
        aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
      env:
        BENCHMARK_CONNSTR: ${{ steps.create-lakebase-project.outputs.dsn }}
        VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
        PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"

    - name: Benchmark simple-update
      uses: ./.github/actions/run-python-test-set
      with:
        build_type: ${{ env.BUILD_TYPE }}
        test_selection: performance
        run_in_parallel: false
        save_perf_report: ${{ env.SAVE_PERF_REPORT }}
        extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_simple_update
        pg_version: ${{ env.PG_VERSION }}
        aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
      env:
        BENCHMARK_CONNSTR: ${{ steps.create-lakebase-project.outputs.dsn }}
        VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
        PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"

    - name: Benchmark select-only
      uses: ./.github/actions/run-python-test-set
      with:
        build_type: ${{ env.BUILD_TYPE }}
        test_selection: performance
        run_in_parallel: false
        save_perf_report: ${{ env.SAVE_PERF_REPORT }}
        extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_select_only
        pg_version: ${{ env.PG_VERSION }}
        aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
      env:
        BENCHMARK_CONNSTR: ${{ steps.create-lakebase-project.outputs.dsn }}
        VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
        PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"

    - name: Delete Lakebase Project
      if: ${{ steps.create-lakebase-project.outputs.project_id && always() }}
      uses: ./.github/actions/lakebase-project-delete
      with:
        api_host: ${{ vars.LAKEBASE_API_HOST }}
        org_id: ${{ vars.LAKEBASE_ORG_ID }}
        project_id: ${{ steps.create-lakebase-project.outputs.project_id }}
        curl_authorization_options: "--header 'X-CSRF-Token: ${{ secrets.LAKEBASE_X_CSRF_TOKEN }}' --cookie 'JSESSIONID=${{ secrets.LAKEBASE_JSESSIONID }}; DBAUTH=${{ secrets.LAKEBASE_DBAUTH }}'"

    - name: Create Allure report
      id: create-allure-report
      if: ${{ !cancelled() }}
      uses: ./.github/actions/allure-report-generate
      with:
        aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}

    - name: Post to a Slack channel
      if: ${{ failure() }}
      uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
      with:
        channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
        slack-message: |
          Lakebase perf testing: ${{ job.status }}
          <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>
          <${{ steps.create-allure-report.outputs.report-url }}|Allure report>
      env:
        SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}