diff --git a/docker-compose/compute/shell/compute.sh b/docker-compose/compute/shell/compute.sh new file mode 100755 index 0000000000..cef2b485f3 --- /dev/null +++ b/docker-compose/compute/shell/compute.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -eux + +PG_VERSION=${PG_VERSION:-14} + +SPEC_FILE_ORG=/var/db/postgres/specs/spec.json +SPEC_FILE=/tmp/spec.json + +echo "Waiting pageserver become ready." +while ! nc -z pageserver 6400; do + sleep 1; +done +echo "Page server is ready." + +echo "Create a tenant and timeline" +PARAMS=( + -sb + -X POST + -H "Content-Type: application/json" + -d "{}" + http://pageserver:9898/v1/tenant/ +) +tenant_id=$(curl "${PARAMS[@]}" | sed 's/"//g') + +PARAMS=( + -sb + -X POST + -H "Content-Type: application/json" + -d "{\"tenant_id\":\"${tenant_id}\", \"pg_version\": ${PG_VERSION}}" + "http://pageserver:9898/v1/tenant/${tenant_id}/timeline/" +) +result=$(curl "${PARAMS[@]}") +echo $result | jq . + +echo "Overwrite tenant id and timeline id in spec file" +tenant_id=$(echo ${result} | jq -r .tenant_id) +timeline_id=$(echo ${result} | jq -r .timeline_id) + +sed "s/TENANT_ID/${tenant_id}/" ${SPEC_FILE_ORG} > ${SPEC_FILE} +sed -i "s/TIMELINE_ID/${timeline_id}/" ${SPEC_FILE} + +cat ${SPEC_FILE} + +echo "Start compute node" +/usr/local/bin/compute_ctl --pgdata /var/db/postgres/compute \ + -C "postgresql://cloud_admin@localhost:55433/postgres" \ + -b /usr/local/bin/postgres \ + -S ${SPEC_FILE} diff --git a/docker-compose/compute/var/db/postgres/specs/spec.json b/docker-compose/compute/var/db/postgres/specs/spec.json new file mode 100644 index 0000000000..10ae0b0ecf --- /dev/null +++ b/docker-compose/compute/var/db/postgres/specs/spec.json @@ -0,0 +1,141 @@ +{ + "format_version": 1.0, + + "timestamp": "2022-10-12T18:00:00.000Z", + "operation_uuid": "0f657b36-4b0f-4a2d-9c2e-1dcd615e7d8c", + + "cluster": { + "cluster_id": "docker_compose", + "name": "docker_compose_test", + "state": "restarted", + "roles": [ + { + "name": "cloud_admin", + "encrypted_password": "b093c0d3b281ba6da1eacc608620abd8", + "options": null + } + ], + "databases": [ + ], + "settings": [ + { + "name": "fsync", + "value": "off", + "vartype": "bool" + }, + { + "name": "wal_level", + "value": "replica", + "vartype": "enum" + }, + { + "name": "hot_standby", + "value": "on", + "vartype": "bool" + }, + { + "name": "wal_log_hints", + "value": "on", + "vartype": "bool" + }, + { + "name": "log_connections", + "value": "on", + "vartype": "bool" + }, + { + "name": "port", + "value": "55433", + "vartype": "integer" + }, + { + "name": "shared_buffers", + "value": "1MB", + "vartype": "string" + }, + { + "name": "max_connections", + "value": "100", + "vartype": "integer" + }, + { + "name": "listen_addresses", + "value": "0.0.0.0", + "vartype": "string" + }, + { + "name": "max_wal_senders", + "value": "10", + "vartype": "integer" + }, + { + "name": "max_replication_slots", + "value": "10", + "vartype": "integer" + }, + { + "name": "wal_sender_timeout", + "value": "5s", + "vartype": "string" + }, + { + "name": "wal_keep_size", + "value": "0", + "vartype": "integer" + }, + { + "name": "password_encryption", + "value": "md5", + "vartype": "enum" + }, + { + "name": "restart_after_crash", + "value": "off", + "vartype": "bool" + }, + { + "name": "synchronous_standby_names", + "value": "walproposer", + "vartype": "string" + }, + { + "name": "shared_preload_libraries", + "value": "neon", + "vartype": "string" + }, + { + "name": "neon.safekeepers", + "value": "safekeeper1:5454,safekeeper2:5454,safekeeper3:5454", + "vartype": "string" + }, + { + "name": "neon.timeline_id", + "value": "TIMELINE_ID", + "vartype": "string" + }, + { + "name": "neon.tenant_id", + "value": "TENANT_ID", + "vartype": "string" + }, + { + "name": "neon.pageserver_connstring", + "value": "host=pageserver port=6400", + "vartype": "string" + }, + { + "name": "max_replication_write_lag", + "value": "500MB", + "vartype": "string" + }, + { + "name": "max_replication_flush_lag", + "value": "10GB", + "vartype": "string" + } + ] + }, + + "delta_operations": [ + ] +} diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml new file mode 100644 index 0000000000..9ab775c3f9 --- /dev/null +++ b/docker-compose/docker-compose.yml @@ -0,0 +1,200 @@ +version: '3' + +services: + etcd: + image: quay.io/coreos/etcd:v3.5.4 + ports: + - 2379:2379 + - 2380:2380 + environment: + # This signifficantly speeds up etcd and we anyway don't data persistency there. + ETCD_UNSAFE_NO_FSYNC: "1" + command: + - "etcd" + - "--auto-compaction-mode=revision" + - "--auto-compaction-retention=1" + - "--name=etcd-cluster" + - "--initial-cluster-state=new" + - "--initial-cluster-token=etcd-cluster-1" + - "--initial-cluster=etcd-cluster=http://etcd:2380" + - "--initial-advertise-peer-urls=http://etcd:2380" + - "--advertise-client-urls=http://etcd:2379" + - "--listen-client-urls=http://0.0.0.0:2379" + - "--listen-peer-urls=http://0.0.0.0:2380" + - "--quota-backend-bytes=134217728" # 128 MB + + minio: + image: quay.io/minio/minio:RELEASE.2022-10-20T00-55-09Z + ports: + - 9000:9000 + - 9001:9001 + environment: + - MINIO_ROOT_USER=minio + - MINIO_ROOT_PASSWORD=password + command: server /data --address :9000 --console-address ":9001" + + minio_create_buckets: + image: minio/mc + environment: + - MINIO_ROOT_USER=minio + - MINIO_ROOT_PASSWORD=password + entrypoint: + - "/bin/sh" + - "-c" + command: + - "until (/usr/bin/mc alias set minio http://minio:9000 $$MINIO_ROOT_USER $$MINIO_ROOT_PASSWORD) do + echo 'Waiting to start minio...' && sleep 1; + done; + /usr/bin/mc mb minio/neon --region=eu-north-1; + exit 0;" + depends_on: + - minio + + pageserver: + image: neondatabase/neon:${TAG:-latest} + environment: + - BROKER_ENDPOINT='http://etcd:2379' + - AWS_ACCESS_KEY_ID=minio + - AWS_SECRET_ACCESS_KEY=password + #- RUST_BACKTRACE=1 + ports: + #- 6400:6400 # pg protocol handler + - 9898:9898 # http endpoints + entrypoint: + - "/bin/sh" + - "-c" + command: + - "/usr/local/bin/pageserver -D /data/.neon/ + -c \"broker_endpoints=[$$BROKER_ENDPOINT]\" + -c \"listen_pg_addr='0.0.0.0:6400'\" + -c \"listen_http_addr='0.0.0.0:9898'\" + -c \"remote_storage={endpoint='http://minio:9000', + bucket_name='neon', + bucket_region='eu-north-1', + prefix_in_bucket='/pageserver/'}\"" + depends_on: + - etcd + - minio_create_buckets + + safekeeper1: + image: neondatabase/neon:${TAG:-latest} + environment: + - SAFEKEEPER_ADVERTISE_URL=safekeeper1:5454 + - SAFEKEEPER_ID=1 + - BROKER_ENDPOINT=http://etcd:2379 + - AWS_ACCESS_KEY_ID=minio + - AWS_SECRET_ACCESS_KEY=password + #- RUST_BACKTRACE=1 + ports: + #- 5454:5454 # pg protocol handler + - 7676:7676 # http endpoints + entrypoint: + - "/bin/sh" + - "-c" + command: + - "safekeeper --listen-pg=$$SAFEKEEPER_ADVERTISE_URL + --listen-http='0.0.0.0:7676' + --id=$$SAFEKEEPER_ID + --broker-endpoints=$$BROKER_ENDPOINT + -D /data + --remote-storage=\"{endpoint='http://minio:9000', + bucket_name='neon', + bucket_region='eu-north-1', + prefix_in_bucket='/safekeeper/'}\"" + depends_on: + - etcd + - minio_create_buckets + + safekeeper2: + image: neondatabase/neon:${TAG:-latest} + environment: + - SAFEKEEPER_ADVERTISE_URL=safekeeper2:5454 + - SAFEKEEPER_ID=2 + - BROKER_ENDPOINT=http://etcd:2379 + - AWS_ACCESS_KEY_ID=minio + - AWS_SECRET_ACCESS_KEY=password + #- RUST_BACKTRACE=1 + ports: + #- 5454:5454 # pg protocol handler + - 7677:7676 # http endpoints + entrypoint: + - "/bin/sh" + - "-c" + command: + - "safekeeper --listen-pg=$$SAFEKEEPER_ADVERTISE_URL + --listen-http='0.0.0.0:7676' + --id=$$SAFEKEEPER_ID + --broker-endpoints=$$BROKER_ENDPOINT + -D /data + --remote-storage=\"{endpoint='http://minio:9000', + bucket_name='neon', + bucket_region='eu-north-1', + prefix_in_bucket='/safekeeper/'}\"" + depends_on: + - etcd + - minio_create_buckets + + safekeeper3: + image: neondatabase/neon:${TAG:-latest} + environment: + - SAFEKEEPER_ADVERTISE_URL=safekeeper3:5454 + - SAFEKEEPER_ID=3 + - BROKER_ENDPOINT=http://etcd:2379 + - AWS_ACCESS_KEY_ID=minio + - AWS_SECRET_ACCESS_KEY=password + #- RUST_BACKTRACE=1 + ports: + #- 5454:5454 # pg protocol handler + - 7678:7676 # http endpoints + entrypoint: + - "/bin/sh" + - "-c" + command: + - "safekeeper --listen-pg=$$SAFEKEEPER_ADVERTISE_URL + --listen-http='0.0.0.0:7676' + --id=$$SAFEKEEPER_ID + --broker-endpoints=$$BROKER_ENDPOINT + -D /data + --remote-storage=\"{endpoint='http://minio:9000', + bucket_name='neon', + bucket_region='eu-north-1', + prefix_in_bucket='/safekeeper/'}\"" + depends_on: + - etcd + - minio_create_buckets + + compute: + build: + context: ./image/compute + args: + - COMPUTE_IMAGE=compute-node-v${PG_VERSION:-14}:${TAG:-latest} + - http_proxy=$http_proxy + - https_proxy=$https_proxy + environment: + - PG_VERSION=${PG_VERSION:-14} + #- RUST_BACKTRACE=1 + volumes: + - ./compute/var/db/postgres/specs/:/var/db/postgres/specs/ + - ./compute/shell/:/shell/ + ports: + - 55433:55433 # pg protocol handler + - 3080:3080 # http endpoints + entrypoint: + - "/shell/compute.sh" + depends_on: + - safekeeper1 + - safekeeper2 + - safekeeper3 + - pageserver + + compute_is_ready: + image: postgres:latest + entrypoint: + - "/bin/bash" + - "-c" + command: + - "until pg_isready -h compute -p 55433 ; do + echo 'Waiting to start compute...' && sleep 1; + done" + depends_on: + - compute diff --git a/docker-compose/image/compute/Dockerfile b/docker-compose/image/compute/Dockerfile new file mode 100644 index 0000000000..1b9d8c4900 --- /dev/null +++ b/docker-compose/image/compute/Dockerfile @@ -0,0 +1,10 @@ +ARG COMPUTE_IMAGE=compute-node-v14:latest +FROM neondatabase/${COMPUTE_IMAGE} + +USER root +RUN apt-get update && \ + apt-get install -y curl \ + jq \ + netcat + +USER postgres diff --git a/docs/docker.md b/docs/docker.md index 100cdd248b..42f0048e6f 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -18,3 +18,67 @@ We build all images after a successful `release` tests run and push automaticall 1. `neondatabase/compute-tools` and `neondatabase/compute-node` 2. `neondatabase/neon` + +## Docker Compose example + +You can see a [docker compose](https://docs.docker.com/compose/) example to create a neon cluster in [/docker-compose/docker-compose.yml](/docker-compose/docker-compose.yml). It creates the following conatainers. + +- etcd x 1 +- pageserver x 1 +- safekeeper x 3 +- compute x 1 +- MinIO x 1 # This is Amazon S3 compatible object storage + +### How to use + +1. create containers + +You can specify version of neon cluster using following environment values. +- PG_VERSION: postgres version for compute (default is 14) +- TAG: the tag version of [docker image](https://registry.hub.docker.com/r/neondatabase/neon/tags) (default is latest), which is tagged in [CI test](/.github/workflows/build_and_test.yml) +``` +$ cd docker-compose/docker-compose.yml +$ docker-compose down # remove the conainers if exists +$ PG_VERSION=15 TAG=2221 docker-compose up --build -d # You can specify the postgres and image version +Creating network "dockercompose_default" with the default driver +Creating dockercompose_etcd3_1 ... +(...omit...) +``` + +2. connect compute node +``` +$ echo "localhost:55433:postgres:cloud_admin:cloud_admin" >> ~/.pgpass +$ psql -h localhost -p 55433 -U cloud_admin +postgres=# CREATE TABLE t(key int primary key, value text); +CREATE TABLE +postgres=# insert into t values(1,1); +INSERT 0 1 +postgres=# select * from t; + key | value +-----+------- + 1 | 1 +(1 row) +``` + +3. If you want to see the log, you can use `docker-compose logs` command. +``` +# check the container name you want to see +$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +d6968a5ae912 dockercompose_compute "/shell/compute.sh" 5 minutes ago Up 5 minutes 0.0.0.0:3080->3080/tcp, 0.0.0.0:55433->55433/tcp dockercompose_compute_1 +(...omit...) + +$ docker logs -f dockercompose_compute_1 +2022-10-21 06:15:48.757 GMT [56] LOG: connection authorized: user=cloud_admin database=postgres application_name=psql +2022-10-21 06:17:00.307 GMT [56] LOG: [NEON_SMGR] libpagestore: connected to 'host=pageserver port=6400' +(...omit...) +``` + +4. If you want to see durable data in MinIO which is s3 compatible storage + +Access http://localhost:9001 and sign in. + +- Username: `minio` +- Password: `password` + +You can see durable pages and WAL data in `neon` bucket. \ No newline at end of file diff --git a/scripts/docker-compose_test.sh b/scripts/docker-compose_test.sh new file mode 100755 index 0000000000..b4551365f8 --- /dev/null +++ b/scripts/docker-compose_test.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# this is a shortcut script to avoid duplication in CI +set -eux -o pipefail + +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +COMPOSE_FILE=$SCRIPT_DIR/../docker-compose/docker-compose.yml + +COMPUTE_CONTAINER_NAME=dockercompose_compute_1 +SQL="CREATE TABLE t(key int primary key, value text); insert into t values(1,1); select * from t;" +PSQL_OPTION="-h localhost -U cloud_admin -p 55433 -c '$SQL' postgres" + +cleanup() { + echo "show container information" + docker ps + docker-compose -f $COMPOSE_FILE logs + echo "stop containers..." + docker-compose -f $COMPOSE_FILE down +} + +echo "clean up containers if exists" +cleanup + +for pg_version in 14 15; do + echo "start containers (pg_version=$pg_version)." + PG_VERSION=$pg_version TAG=latest docker-compose -f $COMPOSE_FILE up --build -d + + echo "wait until the compute is ready. timeout after 60s. " + cnt=0 + while sleep 1; do + # check timeout + cnt=`expr $cnt + 1` + if [ $cnt -gt 60 ]; then + echo "timeout before the compute is ready." + cleanup + exit 1 + fi + + # check if the compute is ready + set +o pipefail + result=`docker-compose -f $COMPOSE_FILE logs "compute_is_ready" | grep "accepting connections" | wc -l` + set -o pipefail + if [ $result -eq 1 ]; then + echo "OK. The compute is ready to connect." + echo "execute simple queries." + docker exec -it $COMPUTE_CONTAINER_NAME /bin/bash -c "psql $PSQL_OPTION" + cleanup + break + fi + done +done