From 3c8d67ae449d5f358eb645d628d23b5cc9e740ac Mon Sep 17 00:00:00 2001 From: BodoBolero Date: Wed, 30 Jul 2025 14:44:22 +0200 Subject: [PATCH] try first version of benchmark on lakebase --- .github/actionlint.yml | 3 + .../lakebase-project-create/action.yml | 8 +- .github/workflows/lakebase_benchmarking.yml | 169 ++++++++++++++++++ 3 files changed, 178 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/lakebase_benchmarking.yml diff --git a/.github/actionlint.yml b/.github/actionlint.yml index 8a4bcaf811..301f9168fc 100644 --- a/.github/actionlint.yml +++ b/.github/actionlint.yml @@ -27,6 +27,9 @@ config-variables: - HETZNER_CACHE_BUCKET - HETZNER_CACHE_ENDPOINT - HETZNER_CACHE_REGION + - LAKEBASE_API_HOST + - LAKEBASE_OAUTH_CLIENT_ID + - LAKEBASE_ORG_ID - NEON_DEV_AWS_ACCOUNT_ID - NEON_PROD_AWS_ACCOUNT_ID - PGREGRESS_PG16_PROJECT_ID diff --git a/.github/actions/lakebase-project-create/action.yml b/.github/actions/lakebase-project-create/action.yml index 4993fdb9f3..aa4d19e3da 100644 --- a/.github/actions/lakebase-project-create/action.yml +++ b/.github/actions/lakebase-project-create/action.yml @@ -9,11 +9,11 @@ inputs: description: 'Organization ID, required' required: true api_host: - description: 'Lakebase API host, e.g. dbc-55e65913-66de.dev.databricks.com/ajax-api/2.0/lakebase-console' + description: 'Lakebase API host, e.g. dbc-55e65913-66de.dev.databricks.com/lakebase-console' required: true postgres_version: description: 'Postgres version; default is 16' - default: '16' + default: '17' compute_units: description: '[Min, Max] compute units' default: '[1, 1]' @@ -33,6 +33,10 @@ inputs: description: 'Fixed hostname to use for connection URI' required: false default: 'k8s-dpingres-serverle-09ade1e9e9-0d7f675c53b35938.elb.us-west-2.amazonaws.com' + region_id: + description: 'Project region ID' + required: false + default: 'aws-us-east-2' outputs: dsn: diff --git a/.github/workflows/lakebase_benchmarking.yml b/.github/workflows/lakebase_benchmarking.yml new file mode 100644 index 0000000000..4a34b1626d --- /dev/null +++ b/.github/workflows/lakebase_benchmarking.yml @@ -0,0 +1,169 @@ +name: Lakebase Benchmarking + +on: + workflow_dispatch: # adds ability to run this manually + inputs: + postgres_version: + description: 'Postgres version' + required: false + default: '17' + save_perf_report: + type: boolean + description: 'Publish perf report' + required: false + default: false + +defaults: + run: + shell: bash -euxo pipefail {0} + +concurrency: + # Allow only one workflow per any non-`main` branch. + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }} + cancel-in-progress: true + +jobs: + lakebase-pgbench: + permissions: + contents: write + statuses: write + id-token: write # aws-actions/configure-aws-credentials + + env: + TEST_PG_BENCH_DURATIONS_MATRIX: "60m" + TEST_PG_BENCH_SCALES_MATRIX: "10gb" + POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install + PG_VERSION: ${{ github.event.inputs.postgres_version || '17' }} + TEST_OUTPUT: /tmp/test_output + BUILD_TYPE: remote + SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || false }} + PLATFORM: "lakebase-captest-new" + + # TODO: for lakehouse test-shard which is probably deployed in US-West we need to change the runner + # to us-west to get correct OLTP latencies due to added speed of light latency + runs-on: [ self-hosted, us-east-2, x64 ] + container: + image: ghcr.io/neondatabase/build-tools:pinned-bookworm + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --init + + # Increase timeout to 8h, default timeout is 6h + timeout-minutes: 480 + + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 + with: + egress-policy: audit + + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + with: + aws-region: eu-central-1 + role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} + role-duration-seconds: 18000 # 5 hours + + - name: Download Neon artifact + uses: ./.github/actions/download + with: + name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact + path: /tmp/neon/ + prefix: latest + aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} + + - name: Create Lakebase Bearer Token + id: create-bearer-token + uses: ./.github/actions/lakebase_create_token + with: + client_id: ${{ vars.LAKEBASE_OAUTH_CLIENT_ID }} + client_secret: ${{ secrets.LAKEBASE_OAUTH_CLIENT_SECRET }} + + ## TODO, currently we cannot really specify a small min max CU for lakebase project + ## and the semantic of a CU is different from Neon, so we need to carefully map + ## compute sizes before comparing results + - name: Create Lakebase Project + id: create-lakebase-project + uses: ./.github/actions/lakebase-project-create + with: + api_host: ${{ vars.LAKEBASE_API_HOST }} + org_id: ${{ vars.LAKEBASE_ORG_ID }} + postgres_version: ${{ env.PG_VERSION }} + compute_units: '[1, 1]' + curl_authorization_options: "--header 'X-CSRF-Token: ${{ secrets.LAKEBASE_X_CSRF_TOKEN }}' --cookie 'JSESSIONID=${{ secrets.LAKEBASE_JSESSIONID }}; DBAUTH=${{ secrets.LAKEBASE_DBAUTH }}'" + + - name: Benchmark init + uses: ./.github/actions/run-python-test-set + with: + build_type: ${{ env.BUILD_TYPE }} + test_selection: performance + run_in_parallel: false + save_perf_report: ${{ env.SAVE_PERF_REPORT }} + extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_init + pg_version: ${{ env.PG_VERSION }} + aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} + env: + BENCHMARK_CONNSTR: ${{ steps.create-lakebase-project.outputs.dsn }} + VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" + PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" + + - name: Benchmark simple-update + uses: ./.github/actions/run-python-test-set + with: + build_type: ${{ env.BUILD_TYPE }} + test_selection: performance + run_in_parallel: false + save_perf_report: ${{ env.SAVE_PERF_REPORT }} + extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_simple_update + pg_version: ${{ env.PG_VERSION }} + aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} + env: + BENCHMARK_CONNSTR: ${{ steps.create-lakebase-project.outputs.dsn }} + VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" + PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" + + - name: Benchmark select-only + uses: ./.github/actions/run-python-test-set + with: + build_type: ${{ env.BUILD_TYPE }} + test_selection: performance + run_in_parallel: false + save_perf_report: ${{ env.SAVE_PERF_REPORT }} + extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_select_only + pg_version: ${{ env.PG_VERSION }} + aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} + env: + BENCHMARK_CONNSTR: ${{ steps.create-lakebase-project.outputs.dsn }} + VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" + PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" + + - name: Delete Lakebase Project + if: ${{ steps.create-lakebase-project.outputs.project_id && always() }} + uses: ./.github/actions/lakebase-project-delete + with: + api_host: ${{ vars.LAKEBASE_API_HOST }} + org_id: ${{ vars.LAKEBASE_ORG_ID }} + project_id: ${{ steps.create-lakebase-project.outputs.project_id }} + curl_authorization_options: "--header 'X-CSRF-Token: ${{ secrets.LAKEBASE_X_CSRF_TOKEN }}' --cookie 'JSESSIONID=${{ secrets.LAKEBASE_JSESSIONID }}; DBAUTH=${{ secrets.LAKEBASE_DBAUTH }}'" + + - name: Create Allure report + id: create-allure-report + if: ${{ !cancelled() }} + uses: ./.github/actions/allure-report-generate + with: + aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} + + - name: Post to a Slack channel + if: ${{ failure() }} + uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1 + with: + channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream + slack-message: | + Lakebase perf testing: ${{ job.status }} + <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run> + <${{ steps.create-allure-report.outputs.report-url }}|Allure report> + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} \ No newline at end of file