mirror of
https://github.com/neondatabase/neon.git
synced 2025-12-22 21:59:59 +00:00
## Problem Extension tests were previously run sequentially, resulting in unnecessary wait time and underutilization of available CPU cores. ## Summary of changes Tests are now executed in a customizable number of parallel threads using separate database branches. This reduces overall test time by approximately 50% (e.g., on my laptop, parallel test lasts 173s, while sequential one lasts 340s) and increases the load on the pageserver, providing better test coverage. --------- Co-authored-by: Alexander Bayandin <alexander@neon.tech> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Alexey Masterov <alexey.masterov@databricks.com>
211 lines
6.3 KiB
YAML
211 lines
6.3 KiB
YAML
services:
|
|
minio:
|
|
restart: always
|
|
image: quay.io/minio/minio:RELEASE.2022-10-20T00-55-09Z
|
|
ports:
|
|
- 9000:9000
|
|
- 9001:9001
|
|
environment:
|
|
- MINIO_ROOT_USER=minio
|
|
- MINIO_ROOT_PASSWORD=password
|
|
command: server /data --address :9000 --console-address ":9001"
|
|
|
|
minio_create_buckets:
|
|
image: minio/mc
|
|
environment:
|
|
- MINIO_ROOT_USER=minio
|
|
- MINIO_ROOT_PASSWORD=password
|
|
entrypoint:
|
|
- "/bin/sh"
|
|
- "-c"
|
|
command:
|
|
- "until (/usr/bin/mc alias set minio http://minio:9000 $$MINIO_ROOT_USER $$MINIO_ROOT_PASSWORD) do
|
|
echo 'Waiting to start minio...' && sleep 1;
|
|
done;
|
|
/usr/bin/mc mb minio/neon --region=eu-north-1;
|
|
exit 0;"
|
|
depends_on:
|
|
- minio
|
|
|
|
pageserver:
|
|
restart: always
|
|
image: ${REPOSITORY:-ghcr.io/neondatabase}/neon:${TAG:-latest}
|
|
environment:
|
|
- AWS_ACCESS_KEY_ID=minio
|
|
- AWS_SECRET_ACCESS_KEY=password
|
|
#- RUST_BACKTRACE=1
|
|
ports:
|
|
#- 6400:6400 # pg protocol handler
|
|
- 9898:9898 # http endpoints
|
|
volumes:
|
|
- ./pageserver_config:/data/.neon/
|
|
depends_on:
|
|
- storage_broker
|
|
- minio_create_buckets
|
|
|
|
safekeeper1:
|
|
restart: always
|
|
image: ${REPOSITORY:-ghcr.io/neondatabase}/neon:${TAG:-latest}
|
|
environment:
|
|
- SAFEKEEPER_ADVERTISE_URL=safekeeper1:5454
|
|
- SAFEKEEPER_ID=1
|
|
- BROKER_ENDPOINT=http://storage_broker:50051
|
|
- AWS_ACCESS_KEY_ID=minio
|
|
- AWS_SECRET_ACCESS_KEY=password
|
|
#- RUST_BACKTRACE=1
|
|
ports:
|
|
#- 5454:5454 # pg protocol handler
|
|
- 7676:7676 # http endpoints
|
|
entrypoint:
|
|
- "/bin/sh"
|
|
- "-c"
|
|
command:
|
|
- "safekeeper --listen-pg=$$SAFEKEEPER_ADVERTISE_URL
|
|
--listen-http='0.0.0.0:7676'
|
|
--id=$$SAFEKEEPER_ID
|
|
--broker-endpoint=$$BROKER_ENDPOINT
|
|
-D /data
|
|
--remote-storage=\"{endpoint='http://minio:9000',
|
|
bucket_name='neon',
|
|
bucket_region='eu-north-1',
|
|
prefix_in_bucket='/safekeeper/'}\""
|
|
depends_on:
|
|
- storage_broker
|
|
- minio_create_buckets
|
|
|
|
safekeeper2:
|
|
restart: always
|
|
image: ${REPOSITORY:-ghcr.io/neondatabase}/neon:${TAG:-latest}
|
|
environment:
|
|
- SAFEKEEPER_ADVERTISE_URL=safekeeper2:5454
|
|
- SAFEKEEPER_ID=2
|
|
- BROKER_ENDPOINT=http://storage_broker:50051
|
|
- AWS_ACCESS_KEY_ID=minio
|
|
- AWS_SECRET_ACCESS_KEY=password
|
|
#- RUST_BACKTRACE=1
|
|
ports:
|
|
#- 5454:5454 # pg protocol handler
|
|
- 7677:7676 # http endpoints
|
|
entrypoint:
|
|
- "/bin/sh"
|
|
- "-c"
|
|
command:
|
|
- "safekeeper --listen-pg=$$SAFEKEEPER_ADVERTISE_URL
|
|
--listen-http='0.0.0.0:7676'
|
|
--id=$$SAFEKEEPER_ID
|
|
--broker-endpoint=$$BROKER_ENDPOINT
|
|
-D /data
|
|
--remote-storage=\"{endpoint='http://minio:9000',
|
|
bucket_name='neon',
|
|
bucket_region='eu-north-1',
|
|
prefix_in_bucket='/safekeeper/'}\""
|
|
depends_on:
|
|
- storage_broker
|
|
- minio_create_buckets
|
|
|
|
safekeeper3:
|
|
restart: always
|
|
image: ${REPOSITORY:-ghcr.io/neondatabase}/neon:${TAG:-latest}
|
|
environment:
|
|
- SAFEKEEPER_ADVERTISE_URL=safekeeper3:5454
|
|
- SAFEKEEPER_ID=3
|
|
- BROKER_ENDPOINT=http://storage_broker:50051
|
|
- AWS_ACCESS_KEY_ID=minio
|
|
- AWS_SECRET_ACCESS_KEY=password
|
|
#- RUST_BACKTRACE=1
|
|
ports:
|
|
#- 5454:5454 # pg protocol handler
|
|
- 7678:7676 # http endpoints
|
|
entrypoint:
|
|
- "/bin/sh"
|
|
- "-c"
|
|
command:
|
|
- "safekeeper --listen-pg=$$SAFEKEEPER_ADVERTISE_URL
|
|
--listen-http='0.0.0.0:7676'
|
|
--id=$$SAFEKEEPER_ID
|
|
--broker-endpoint=$$BROKER_ENDPOINT
|
|
-D /data
|
|
--remote-storage=\"{endpoint='http://minio:9000',
|
|
bucket_name='neon',
|
|
bucket_region='eu-north-1',
|
|
prefix_in_bucket='/safekeeper/'}\""
|
|
depends_on:
|
|
- storage_broker
|
|
- minio_create_buckets
|
|
|
|
storage_broker:
|
|
restart: always
|
|
image: ${REPOSITORY:-ghcr.io/neondatabase}/neon:${TAG:-latest}
|
|
ports:
|
|
- 50051:50051
|
|
command:
|
|
- "storage_broker"
|
|
- "--listen-addr=0.0.0.0:50051"
|
|
|
|
compute1:
|
|
restart: always
|
|
build:
|
|
context: ./compute_wrapper/
|
|
args:
|
|
- REPOSITORY=${REPOSITORY:-ghcr.io/neondatabase}
|
|
- COMPUTE_IMAGE=compute-node-v${PG_VERSION:-16}
|
|
- TAG=${COMPUTE_TAG:-${TAG:-latest}}
|
|
- http_proxy=${http_proxy:-}
|
|
- https_proxy=${https_proxy:-}
|
|
image: built-compute
|
|
environment:
|
|
- PG_VERSION=${PG_VERSION:-16}
|
|
- TENANT_ID=${TENANT_ID:-}
|
|
- TIMELINE_ID=${TIMELINE_ID:-}
|
|
#- RUST_BACKTRACE=1
|
|
# Mount the test files directly, for faster editing cycle.
|
|
volumes:
|
|
- ./compute_wrapper/var/db/postgres/configs/:/var/db/postgres/configs/
|
|
- ./compute_wrapper/shell/:/shell/
|
|
ports:
|
|
- 55433:55433 # pg protocol handler
|
|
- 3080:3080 # http endpoints
|
|
entrypoint:
|
|
- "/shell/compute.sh"
|
|
# Ad an alias for compute1 for compatibility
|
|
networks:
|
|
default:
|
|
aliases:
|
|
- compute
|
|
depends_on:
|
|
- safekeeper1
|
|
- safekeeper2
|
|
- safekeeper3
|
|
- pageserver
|
|
|
|
compute_is_ready:
|
|
image: postgres:latest
|
|
environment:
|
|
- PARALLEL_COMPUTES=1
|
|
entrypoint:
|
|
- "/bin/sh"
|
|
- "-c"
|
|
command:
|
|
- "for i in $(seq 1 $${PARALLEL_COMPUTES}); do
|
|
until pg_isready -h compute$$i -p 55433 -U cloud_admin ; do
|
|
sleep 1;
|
|
done;
|
|
done;
|
|
echo All computes are started"
|
|
depends_on:
|
|
- compute1
|
|
|
|
neon-test-extensions:
|
|
profiles: ["test-extensions"]
|
|
image: ${REPOSITORY:-ghcr.io/neondatabase}/neon-test-extensions-v${PG_TEST_VERSION:-${PG_VERSION:-16}}:${TEST_EXTENSIONS_TAG:-${TAG:-latest}}
|
|
environment:
|
|
- PGUSER=${PGUSER:-cloud_admin}
|
|
- PGPASSWORD=${PGPASSWORD:-cloud_admin}
|
|
entrypoint:
|
|
- "/bin/bash"
|
|
- "-c"
|
|
command:
|
|
- sleep 3600
|
|
depends_on:
|
|
- compute1
|