Fix the markdown rendering 004-durability.md RFC

.circleci/ansible/get_binaries.sh

@@ -6,7 +6,7 @@ RELEASE=${RELEASE:-false}
 
 # look at docker hub for latest tag for neon docker image
 if [ "${RELEASE}" = "true" ]; then
-    echo "search latest release tag"
+    echo "search latest relase tag"
     VERSION=$(curl -s https://registry.hub.docker.com/v1/repositories/neondatabase/neon/tags |jq -r -S '.[].name' | grep release | sed 's/release-//g' | grep -E '^[0-9]+$' | sort -n | tail -1)
     if [ -z "${VERSION}" ]; then
         echo "no any docker tags found, exiting..."
@@ -31,7 +31,7 @@ echo "found ${VERSION}"
 rm -rf neon_install postgres_install.tar.gz neon_install.tar.gz .neon_current_version
 mkdir neon_install
 
-# retrieve binaries from docker image
+# retrive binaries from docker image
 echo "getting binaries from docker image"
 docker pull --quiet neondatabase/neon:${TAG}
 ID=$(docker create neondatabase/neon:${TAG})

.circleci/ansible/neon-stress.hosts

@@ -1,19 +0,0 @@
-[pageservers]
-neon-stress-ps-1 console_region_id=1
-neon-stress-ps-2 console_region_id=1
-
-[safekeepers]
-neon-stress-sk-1 console_region_id=1
-neon-stress-sk-2 console_region_id=1
-neon-stress-sk-3 console_region_id=1
-
-[storage:children]
-pageservers
-safekeepers
-
-[storage:vars]
-console_mgmt_base_url = http://neon-stress-console.local
-bucket_name           = neon-storage-ireland
-bucket_region         = eu-west-1
-etcd_endpoints        = etcd-stress.local:2379
-safekeeper_enable_s3_offload = false

.circleci/ansible/production.hosts

@@ -1,6 +1,5 @@
 [pageservers]
-#zenith-1-ps-1 console_region_id=1
-zenith-1-ps-2 console_region_id=1
+zenith-1-ps-1 console_region_id=1
 
 [safekeepers]
 zenith-1-sk-1 console_region_id=1
@@ -16,3 +15,4 @@ console_mgmt_base_url = http://console-release.local
 bucket_name           = zenith-storage-oregon
 bucket_region         = us-west-2
 etcd_endpoints        = etcd-release.local:2379
+safekeeper_enable_s3_offload = true

.circleci/ansible/staging.hosts

@@ -3,9 +3,9 @@
 zenith-us-stage-ps-2 console_region_id=27
 
 [safekeepers]
+zenith-us-stage-sk-1 console_region_id=27
 zenith-us-stage-sk-4 console_region_id=27
 zenith-us-stage-sk-5 console_region_id=27
-zenith-us-stage-sk-6 console_region_id=27
 
 [storage:children]
 pageservers
@@ -16,3 +16,4 @@ console_mgmt_base_url = http://console-staging.local
 bucket_name           = zenith-staging-storage-us-east-1
 bucket_region         = us-east-1
 etcd_endpoints        = etcd-staging.local:2379
+safekeeper_enable_s3_offload = false

.circleci/ansible/systemd/pageserver.service

@@ -6,7 +6,7 @@ After=network.target auditd.service
 Type=simple
 User=pageserver
 Environment=RUST_BACKTRACE=1 ZENITH_REPO_DIR=/storage/pageserver LD_LIBRARY_PATH=/usr/local/lib
-ExecStart=/usr/local/bin/pageserver -c "pg_distrib_dir='/usr/local'" -c "listen_pg_addr='0.0.0.0:6400'" -c "listen_http_addr='0.0.0.0:9898'" -c "broker_endpoints=['{{ etcd_endpoints }}']" -D /storage/pageserver/data
+ExecStart=/usr/local/bin/pageserver -c "pg_distrib_dir='/usr/local'" -c "listen_pg_addr='0.0.0.0:6400'" -c "listen_http_addr='0.0.0.0:9898'" -D /storage/pageserver/data
 ExecReload=/bin/kill -HUP $MAINPID
 KillMode=mixed
 KillSignal=SIGINT

.circleci/ansible/systemd/safekeeper.service

@@ -6,7 +6,7 @@ After=network.target auditd.service
 Type=simple
 User=safekeeper
 Environment=RUST_BACKTRACE=1 ZENITH_REPO_DIR=/storage/safekeeper/data LD_LIBRARY_PATH=/usr/local/lib
-ExecStart=/usr/local/bin/safekeeper -l {{ inventory_hostname }}.local:6500 --listen-http {{ inventory_hostname }}.local:7676 -p {{ first_pageserver }}:6400 -D /storage/safekeeper/data --broker-endpoints={{ etcd_endpoints }} --remote-storage='{bucket_name="{{bucket_name}}", bucket_region="{{bucket_region}}", prefix_in_bucket="wal"}'
+ExecStart=/usr/local/bin/safekeeper -l {{ inventory_hostname }}.local:6500 --listen-http {{ inventory_hostname }}.local:7676 -p {{ first_pageserver }}:6400 -D /storage/safekeeper/data --broker-endpoints={{ etcd_endpoints }} --enable-s3-offload={{ safekeeper_enable_s3_offload }}
 ExecReload=/bin/kill -HUP $MAINPID
 KillMode=mixed
 KillSignal=SIGINT

.circleci/config.yml

@@ -11,6 +11,15 @@ executors:
       - image: zimg/rust:1.58
 
 jobs:
+  check-codestyle-rust:
+    executor: neon-xlarge-executor
+    steps:
+      - checkout
+      - run:
+          name: rustfmt
+          when: always
+          command: cargo fmt --all -- --check
+
   # A job to build postgres
   build-postgres:
     executor: neon-xlarge-executor
@@ -213,12 +222,6 @@ jobs:
           key: v2-python-deps-{{ checksum "poetry.lock" }}
           paths:
             - /home/circleci/.cache/pypoetry/virtualenvs
-      - run:
-          name: Print versions
-          when: always
-          command: |
-              poetry run python --version
-              poetry show
       - run:
           name: Run yapf to ensure code format
           when: always
@@ -352,7 +355,7 @@ jobs:
           when: always
           command: |
             du -sh /tmp/test_output/*
-            find /tmp/test_output -type f ! -name "*.log" ! -name "regression.diffs" ! -name "junit.xml" ! -name "*.filediff" ! -name "*.stdout" ! -name "*.stderr" ! -name "flamegraph.svg" ! -name "*.metrics" -delete
+            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "safekeeper.log" ! -name "regression.diffs" ! -name "junit.xml" ! -name "*.filediff" ! -name "*.stdout" ! -name "*.stderr" ! -name "flamegraph.svg" -delete
             du -sh /tmp/test_output/*
       - store_artifacts:
           path: /tmp/test_output
@@ -453,6 +456,9 @@ jobs:
       - checkout
       - setup_remote_docker:
           docker_layer_caching: true
+      # Build neondatabase/compute-tools:latest image and push it to Docker hub
+      # TODO: this should probably also use versioned tag, not just :latest.
+      # XXX: but should it? We build and use it only locally now.
       - run:
           name: Build and push compute-tools Docker image
           command: |
@@ -460,10 +466,7 @@ jobs:
             docker build \
               --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \
               --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \
-              --tag neondatabase/compute-tools:local \
-              --tag neondatabase/compute-tools:latest \
-              -f Dockerfile.compute-tools .
-            # Only push :latest image
+              --tag neondatabase/compute-tools:latest -f Dockerfile.compute-tools .
             docker push neondatabase/compute-tools:latest
       - run:
           name: Init postgres submodule
@@ -473,9 +476,7 @@ jobs:
           command: |
             echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
             DOCKER_TAG=$(git log --oneline|wc -l)
-            docker build --tag neondatabase/compute-node:${DOCKER_TAG} \
-              --tag neondatabase/compute-node:latest vendor/postgres \
-              --build-arg COMPUTE_TOOLS_TAG=local
+            docker build --tag neondatabase/compute-node:${DOCKER_TAG} --tag neondatabase/compute-node:latest vendor/postgres
             docker push neondatabase/compute-node:${DOCKER_TAG}
             docker push neondatabase/compute-node:latest
 
@@ -512,6 +513,9 @@ jobs:
       - checkout
       - setup_remote_docker:
           docker_layer_caching: true
+      # Build neondatabase/compute-tools:release image and push it to Docker hub
+      # TODO: this should probably also use versioned tag, not just :latest.
+      # XXX: but should it? We build and use it only locally now.
       - run:
           name: Build and push compute-tools Docker image
           command: |
@@ -519,10 +523,7 @@ jobs:
             docker build \
               --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \
               --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \
-              --tag neondatabase/compute-tools:release \
-              --tag neondatabase/compute-tools:local \
-              -f Dockerfile.compute-tools .
-            # Only push :release image
+              --tag neondatabase/compute-tools:release -f Dockerfile.compute-tools .
             docker push neondatabase/compute-tools:release
       - run:
           name: Init postgres submodule
@@ -532,9 +533,7 @@ jobs:
           command: |
             echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
             DOCKER_TAG="release-$(git log --oneline|wc -l)"
-            docker build --tag neondatabase/compute-node:${DOCKER_TAG} \
-              --tag neondatabase/compute-node:release vendor/postgres \
-              --build-arg COMPUTE_TOOLS_TAG=local
+            docker build --tag neondatabase/compute-node:${DOCKER_TAG} --tag neondatabase/compute-node:release vendor/postgres
             docker push neondatabase/compute-node:${DOCKER_TAG}
             docker push neondatabase/compute-node:release
 
@@ -588,56 +587,6 @@ jobs:
             helm upgrade neon-proxy       neondatabase/neon-proxy --install -f .circleci/helm-values/staging.proxy.yaml --set image.tag=${DOCKER_TAG} --wait
             helm upgrade neon-proxy-scram neondatabase/neon-proxy --install -f .circleci/helm-values/staging.proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait
 
-  deploy-neon-stress:
-    docker:
-      - image: cimg/python:3.10
-    steps:
-      - checkout
-      - setup_remote_docker
-      - run:
-          name: Setup ansible
-          command: |
-            pip install --progress-bar off --user ansible boto3
-      - run:
-          name: Redeploy
-          command: |
-            cd "$(pwd)/.circleci/ansible"
-
-            ./get_binaries.sh
-
-            echo "${TELEPORT_SSH_KEY}"  | tr -d '\n'| base64 --decode >ssh-key
-            echo "${TELEPORT_SSH_CERT}" | tr -d '\n'| base64 --decode >ssh-key-cert.pub
-            chmod 0600 ssh-key
-            ssh-add ssh-key
-            rm -f ssh-key ssh-key-cert.pub
-
-            ansible-playbook deploy.yaml -i neon-stress.hosts
-            rm -f neon_install.tar.gz .neon_current_version
-
-  deploy-neon-stress-proxy:
-    docker:
-      - image: cimg/base:2021.04
-    environment:
-      KUBECONFIG: .kubeconfig
-    steps:
-      - checkout
-      - run:
-          name: Store kubeconfig file
-          command: |
-            echo "${NEON_STRESS_KUBECONFIG_DATA}" | base64 --decode > ${KUBECONFIG}
-            chmod 0600 ${KUBECONFIG}
-      - run:
-          name: Setup helm v3
-          command: |
-            curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
-            helm repo add neondatabase https://neondatabase.github.io/helm-charts
-      - run:
-          name: Re-deploy proxy
-          command: |
-            DOCKER_TAG=$(git log --oneline|wc -l)
-            helm upgrade neon-stress-proxy       neondatabase/neon-proxy --install -f .circleci/helm-values/neon-stress.proxy.yaml --set image.tag=${DOCKER_TAG} --wait
-            helm upgrade neon-stress-proxy-scram neondatabase/neon-proxy --install -f .circleci/helm-values/neon-stress.proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait
-
   deploy-release:
     docker:
       - image: cimg/python:3.10
@@ -680,13 +629,12 @@ jobs:
           name: Setup helm v3
           command: |
             curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
-            helm repo add neondatabase https://neondatabase.github.io/helm-charts
+            helm repo add zenithdb https://neondatabase.github.io/helm-charts
       - run:
           name: Re-deploy proxy
           command: |
             DOCKER_TAG="release-$(git log --oneline|wc -l)"
-            helm upgrade neon-proxy       neondatabase/neon-proxy --install -f .circleci/helm-values/production.proxy.yaml --set image.tag=${DOCKER_TAG} --wait
-            helm upgrade neon-proxy-scram neondatabase/neon-proxy --install -f .circleci/helm-values/production.proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait
+            helm upgrade zenith-proxy zenithdb/zenith-proxy --install -f .circleci/helm-values/production.proxy.yaml --set image.tag=${DOCKER_TAG} --wait
 
   # Trigger a new remote CI job
   remote-ci-trigger:
@@ -735,6 +683,7 @@ jobs:
 workflows:
   build_and_test:
     jobs:
+      - check-codestyle-rust
       - check-codestyle-python
       - build-postgres:
           name: build-postgres-<< matrix.build_type >>
@@ -750,6 +699,7 @@ workflows:
             - build-postgres-<< matrix.build_type >>
       - run-pytest:
           name: pg_regress-tests-<< matrix.build_type >>
+          context: PERF_TEST_RESULT_CONNSTR
           matrix:
             parameters:
               build_type: ["debug", "release"]
@@ -821,25 +771,6 @@ workflows:
           requires:
             - docker-image
 
-      - deploy-neon-stress:
-          # Context gives an ability to login
-          context: Docker Hub
-          # deploy only for commits to main
-          filters:
-            branches:
-              only:
-                - main
-          requires:
-            - docker-image
-      - deploy-neon-stress-proxy:
-          # deploy only for commits to main
-          filters:
-            branches:
-              only:
-                - main
-          requires:
-            - docker-image
-
       - docker-image-release:
           # Context gives an ability to login
           context: Docker Hub

.circleci/helm-values/neon-stress.proxy-scram.yaml

@@ -1,26 +0,0 @@
-fullnameOverride: "neon-stress-proxy-scram"
-
-settings:
-  authBackend: "console"
-  authEndpoint: "http://neon-stress-console.local/management/api/v2"
-  domain: "*.stress.neon.tech"
-
-podLabels:
-  zenith_service: proxy-scram
-  zenith_env: staging
-  zenith_region: eu-west-1
-  zenith_region_slug: ireland
-
-exposedService:
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: '*.stress.neon.tech'
-
-metrics:
-  enabled: true
-  serviceMonitor:
-    enabled: true
-    selector:
-      release: kube-prometheus-stack

.circleci/helm-values/neon-stress.proxy.yaml

@@ -1,34 +0,0 @@
-fullnameOverride: "neon-stress-proxy"
-
-settings:
-  authEndpoint: "https://console.dev.neon.tech/authenticate_proxy_request/"
-  uri: "https://console.dev.neon.tech/psql_session/"
-
-# -- Additional labels for zenith-proxy pods
-podLabels:
-  zenith_service: proxy
-  zenith_env: staging
-  zenith_region: eu-west-1
-  zenith_region_slug: ireland
-
-service:
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internal
-    external-dns.alpha.kubernetes.io/hostname: neon-stress-proxy.local
-  type: LoadBalancer
-
-exposedService:
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: connect.dev.neon.tech
-
-metrics:
-  enabled: true
-  serviceMonitor:
-    enabled: true
-    selector:
-      release: kube-prometheus-stack

.circleci/helm-values/production.proxy-scram.yaml

@@ -1,24 +0,0 @@
-settings:
-  authBackend: "console"
-  authEndpoint: "http://console-release.local/management/api/v2"
-  domain: "*.cloud.neon.tech"
-
-podLabels:
-  zenith_service: proxy-scram
-  zenith_env: production
-  zenith_region: us-west-2
-  zenith_region_slug: oregon
-
-exposedService:
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: '*.cloud.neon.tech'
-
-metrics:
-  enabled: true
-  serviceMonitor:
-    enabled: true
-    selector:
-      release: kube-prometheus-stack

.circleci/helm-values/production.proxy.yaml

@@ -1,3 +1,9 @@
+# Helm chart values for zenith-proxy.
+# This is a YAML-formatted file.
+
+image:
+  repository: neondatabase/neon
+
 settings:
   authEndpoint: "https://console.neon.tech/authenticate_proxy_request/"
   uri: "https://console.neon.tech/psql_session/"
@@ -22,7 +28,7 @@ exposedService:
     service.beta.kubernetes.io/aws-load-balancer-type: external
     service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
     service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: connect.neon.tech,pg.neon.tech
+    external-dns.alpha.kubernetes.io/hostname: start.zenith.tech,connect.neon.tech,pg.neon.tech
 
 metrics:
   enabled: true

.github/workflows/benchmarking.yml

@@ -19,7 +19,7 @@ jobs:
   bench:
     # this workflow runs on self hosteed runner
     # it's environment is quite different from usual guthub runner
-    # probably the most important difference is that it doesn't start from clean workspace each time
+    # probably the most important difference is that it doesnt start from clean workspace each time
     # e g if you install system packages they are not cleaned up since you install them directly in host machine
     # not a container or something
     # See documentation for more info: https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners

.github/workflows/testing.yml

@@ -1,10 +1,6 @@
 name: Build and Test
 
-on:
-  push:
-    branches:
-    - main
-  pull_request:
+on: push
 
 jobs:
   regression-check:
@@ -12,7 +8,7 @@ jobs:
       matrix:
         # If we want to duplicate this job for different
         # Rust toolchains (e.g. nightly or 1.37.0), add them here.
-        rust_toolchain: [1.58]
+        rust_toolchain: [stable]
         os: [ubuntu-latest, macos-latest]
     timeout-minutes: 30
     name: run regression test suite
@@ -25,26 +21,22 @@ jobs:
           submodules: true
           fetch-depth: 2
 
-      - name: Install rust toolchain ${{ matrix.rust_toolchain }}
+      - name: install rust toolchain ${{ matrix.rust_toolchain }}
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
           toolchain: ${{ matrix.rust_toolchain }}
-          components: rustfmt, clippy
           override: true
 
-      - name: Check formatting
-        run: cargo fmt --all -- --check
-
       - name: Install Ubuntu postgres dependencies
         if: matrix.os == 'ubuntu-latest'
         run: |
           sudo apt update
-          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev libssl-dev
+          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev
 
-      - name: Install macOS postgres dependencies
+      - name: Install macOs postgres dependencies
         if: matrix.os == 'macos-latest'
-        run: brew install flex bison openssl
+        run: brew install flex bison
 
       - name: Set pg revision for caching
         id: pg_ver
@@ -58,27 +50,10 @@ jobs:
             tmp_install/
           key: ${{ runner.os }}-pg-${{ steps.pg_ver.outputs.pg_rev }}
 
-      - name: Set extra env for macOS
-        if: matrix.os == 'macos-latest'
-        run: |
-          echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV
-          echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV
-
       - name: Build postgres
         if: steps.cache_pg.outputs.cache-hit != 'true'
         run: make postgres
 
-      # Plain configure output can contain weird errors like 'error: C compiler cannot create executables'
-      # and the real cause will be inside config.log
-      - name: Print configure logs in case of failure
-        if: failure()
-        continue-on-error: true
-        run: |
-          echo '' && echo '=== config.log ===' && echo ''
-          cat tmp_install/build/config.log
-          echo '' && echo '=== configure.log ===' && echo ''
-          cat tmp_install/build/configure.log
-
       - name: Cache cargo deps
         id: cache_cargo
         uses: actions/cache@v2
@@ -87,7 +62,7 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('./Cargo.lock') }}-rust-${{ matrix.rust_toolchain }}
+          key: ${{ runner.os }}-cargo-${{ hashFiles('./Cargo.lock') }}
 
       - name: Run cargo clippy
         run: ./run_clippy.sh

COPYRIGHT

@@ -0,0 +1,20 @@
+This software is licensed under the Apache 2.0 License:
+
+----------------------------------------------------------------------------
+Copyright 2021 Zenith Labs, Inc
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+----------------------------------------------------------------------------
+
+The PostgreSQL submodule in vendor/postgres is licensed under the
+PostgreSQL license. See vendor/postgres/COPYRIGHT.

Cargo.lock

@@ -166,7 +166,7 @@ dependencies = [
  "cc",
  "cfg-if",
  "libc",
- "miniz_oxide 0.4.4",
+ "miniz_oxide",
  "object",
  "rustc-demangle",
 ]
@@ -363,16 +363,6 @@ dependencies = [
  "textwrap 0.14.2",
 ]
 
-[[package]]
-name = "close_fds"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3bc416f33de9d59e79e57560f450d21ff8393adcf1cdfc3e6d8fb93d5f88a2ed"
-dependencies = [
- "cfg-if",
- "libc",
-]
-
 [[package]]
 name = "cmake"
 version = "0.1.48"
@@ -811,7 +801,6 @@ name = "etcd_broker"
 version = "0.1.0"
 dependencies = [
  "etcd-client",
- "once_cell",
  "regex",
  "serde",
  "serde_json",
@@ -879,18 +868,6 @@ version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e"
 
-[[package]]
-name = "flate2"
-version = "1.0.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af"
-dependencies = [
- "cfg-if",
- "crc32fast",
- "libc",
- "miniz_oxide 0.5.1",
-]
-
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -1550,15 +1527,6 @@ dependencies = [
  "autocfg",
 ]
 
-[[package]]
-name = "miniz_oxide"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082"
-dependencies = [
- "adler",
-]
-
 [[package]]
 name = "mio"
 version = "0.8.2"
@@ -1733,9 +1701,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.10.0"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
+checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5"
 
 [[package]]
 name = "oorandom"
@@ -1800,12 +1768,10 @@ dependencies = [
  "bytes",
  "chrono",
  "clap 3.0.14",
- "close_fds",
  "const_format",
  "crc32c",
  "crossbeam-utils",
  "daemonize",
- "etcd_broker",
  "fail",
  "futures",
  "git-version",
@@ -2059,18 +2025,15 @@ dependencies = [
  "bytes",
  "chrono",
  "crc32c",
- "env_logger",
  "hex",
  "lazy_static",
  "log",
  "memoffset",
- "postgres",
  "rand",
  "regex",
  "serde",
  "thiserror",
  "utils",
- "wal_generate",
  "workspace_hack",
 ]
 
@@ -2124,20 +2087,6 @@ dependencies = [
  "unicode-xid",
 ]
 
-[[package]]
-name = "procfs"
-version = "0.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95e344cafeaeefe487300c361654bcfc85db3ac53619eeccced29f5ea18c4c70"
-dependencies = [
- "bitflags",
- "byteorder",
- "flate2",
- "hex",
- "lazy_static",
- "libc",
-]
-
 [[package]]
 name = "prometheus"
 version = "0.13.0"
@@ -2147,10 +2096,8 @@ dependencies = [
  "cfg-if",
  "fnv",
  "lazy_static",
- "libc",
  "memchr",
  "parking_lot 0.11.2",
- "procfs",
  "thiserror",
 ]
 
@@ -2406,8 +2353,6 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "async-trait",
- "metrics",
- "once_cell",
  "rusoto_core",
  "rusoto_s3",
  "serde",
@@ -2415,7 +2360,6 @@ dependencies = [
  "tempfile",
  "tokio",
  "tokio-util 0.7.0",
- "toml_edit",
  "tracing",
  "workspace_hack",
 ]
@@ -2667,7 +2611,6 @@ name = "safekeeper"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "async-trait",
  "byteorder",
  "bytes",
  "clap 3.0.14",
@@ -2676,14 +2619,12 @@ dependencies = [
  "daemonize",
  "etcd_broker",
  "fs2",
- "futures",
  "git-version",
  "hex",
  "humantime",
  "hyper",
  "lazy_static",
  "metrics",
- "once_cell",
  "postgres",
  "postgres-protocol",
  "postgres_ffi",
@@ -2697,7 +2638,6 @@ dependencies = [
  "tokio",
  "tokio-postgres",
  "tokio-util 0.7.0",
- "toml_edit",
  "tracing",
  "url",
  "utils",
@@ -3649,18 +3589,6 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
-[[package]]
-name = "wal_generate"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "clap 3.0.14",
- "env_logger",
- "log",
- "postgres",
- "tempfile",
-]
-
 [[package]]
 name = "walkdir"
 version = "2.3.2"

Dockerfile

@@ -25,7 +25,7 @@ COPY --from=pg-build /pg/tmp_install/include/postgresql/server tmp_install/inclu
 COPY . .
 
 # Show build caching stats to check if it was used in the end.
-# Has to be the part of the same RUN since cachepot daemon is killed in the end of this RUN, losing the compilation stats.
+# Has to be the part of the same RUN since cachepot daemon is killed in the end of this RUN, loosing the compilation stats.
 RUN set -e \
     && sudo -E "PATH=$PATH" mold -run cargo build --release \
     && cachepot -s

Dockerfile.alpine

@@ -0,0 +1,95 @@
+#
+# Docker image for console integration testing.
+#
+# We may also reuse it in CI to unify installation process and as a general binaries building
+# tool for production servers.
+#
+# Dynamic linking is used for librocksdb and libstdc++ bacause librocksdb-sys calls
+# bindgen with "dynamic" feature flag. This also prevents usage of dockerhub alpine-rust
+# images which are statically linked and have guards against any dlopen. I would rather
+# prefer all static binaries so we may change the way librocksdb-sys builds or wait until
+# we will have our own storage and drop rockdb dependency.
+#
+# Cargo-chef is used to separate dependencies building from main binaries building. This
+# way `docker build` will download and install dependencies only of there are changes to
+# out Cargo.toml files.
+#
+
+
+#
+# build postgres separately -- this layer will be rebuilt only if one of
+# mentioned paths will get any changes
+#
+FROM alpine:3.13 as pg-build
+RUN apk add --update clang llvm compiler-rt compiler-rt-static lld musl-dev binutils \
+                     make bison flex readline-dev zlib-dev perl linux-headers libseccomp-dev
+WORKDIR zenith
+COPY ./vendor/postgres vendor/postgres
+COPY ./Makefile Makefile
+# Build using clang and lld
+RUN CC='clang' LD='lld' CFLAGS='-fuse-ld=lld --rtlib=compiler-rt' make postgres -j4
+
+#
+# Calculate cargo dependencies.
+# This will always run, but only generate recipe.json with list of dependencies without
+# installing them.
+#
+FROM alpine:20210212 as cargo-deps-inspect
+RUN apk add --update rust cargo
+RUN cargo install cargo-chef
+WORKDIR zenith
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+#
+# Build cargo dependencies.
+# This temp cantainner would be build only if recipe.json was changed.
+#
+FROM alpine:20210212 as deps-build
+RUN apk add --update rust cargo openssl-dev clang build-base
+# rust-rocksdb can be built against system-wide rocksdb -- that saves about
+# 10 minutes during build. Rocksdb apk package is in testing now, but use it
+# anyway. In case of any troubles we can download and build rocksdb here manually
+# (to cache it as a docker layer).
+RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
+WORKDIR zenith
+COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
+COPY --from=cargo-deps-inspect /root/.cargo/bin/cargo-chef /root/.cargo/bin/
+COPY --from=cargo-deps-inspect /zenith/recipe.json recipe.json
+RUN ROCKSDB_LIB_DIR=/usr/lib/ cargo chef cook --release --recipe-path recipe.json
+
+#
+# Build zenith binaries
+#
+FROM alpine:20210212 as build
+RUN apk add --update rust cargo openssl-dev clang build-base
+RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev
+WORKDIR zenith
+COPY . .
+# Copy cached dependencies
+COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
+COPY --from=deps-build /zenith/target target
+COPY --from=deps-build /root/.cargo /root/.cargo
+RUN cargo build --release
+
+#
+# Copy binaries to resulting image.
+# build-base hare to provide libstdc++ (it will also bring gcc, but leave it this way until we figure
+# out how to statically link rocksdb or avoid it at all).
+#
+FROM alpine:3.13
+RUN apk add --update openssl build-base libseccomp-dev
+RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb
+COPY --from=build /zenith/target/release/pageserver /usr/local/bin
+COPY --from=build /zenith/target/release/safekeeper /usr/local/bin
+COPY --from=build /zenith/target/release/proxy /usr/local/bin
+COPY --from=pg-build /zenith/tmp_install /usr/local
+COPY docker-entrypoint.sh /docker-entrypoint.sh
+
+RUN addgroup zenith && adduser -h /data -D -G zenith zenith
+VOLUME ["/data"]
+WORKDIR /data
+USER zenith
+EXPOSE 6400
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["pageserver"]

Dockerfile.compute-tools

@@ -15,4 +15,4 @@ RUN set -e \
 # Final image that only has one binary
 FROM debian:buster-slim
 
-COPY --from=rust-build /home/circleci/project/target/release/compute_ctl /usr/local/bin/compute_ctl
+COPY --from=rust-build /home/circleci/project/target/release/zenith_ctl /usr/local/bin/zenith_ctl

Makefile

@@ -12,21 +12,15 @@ endif
 #
 BUILD_TYPE ?= debug
 ifeq ($(BUILD_TYPE),release)
-	PG_CONFIGURE_OPTS = --enable-debug --with-openssl
+	PG_CONFIGURE_OPTS = --enable-debug
 	PG_CFLAGS = -O2 -g3 $(CFLAGS)
 	# Unfortunately, `--profile=...` is a nightly feature
 	CARGO_BUILD_FLAGS += --release
 else ifeq ($(BUILD_TYPE),debug)
-	PG_CONFIGURE_OPTS = --enable-debug --with-openssl --enable-cassert --enable-depend
+	PG_CONFIGURE_OPTS = --enable-debug --enable-cassert --enable-depend
 	PG_CFLAGS = -O0 -g3 $(CFLAGS)
 else
-	$(error Bad build type '$(BUILD_TYPE)', see Makefile for options)
-endif
-
-# macOS with brew-installed openssl requires explicit paths
-UNAME_S := $(shell uname -s)
-ifeq ($(UNAME_S),Darwin)
-    PG_CONFIGURE_OPTS += --with-includes=$(HOMEBREW_PREFIX)/opt/openssl/include --with-libraries=$(HOMEBREW_PREFIX)/opt/openssl/lib
+$(error Bad build type `$(BUILD_TYPE)', see Makefile for options)
 endif
 
 # Choose whether we should be silent or verbose
@@ -74,16 +68,16 @@ postgres-headers: postgres-configure
 	+@echo "Installing PostgreSQL headers"
 	$(MAKE) -C tmp_install/build/src/include MAKELEVEL=0 install
 
-# Compile and install PostgreSQL and contrib/neon
+# Compile and install PostgreSQL and contrib/zenith
 .PHONY: postgres
 postgres: postgres-configure \
 		  postgres-headers # to prevent `make install` conflicts with zenith's `postgres-headers`
 	+@echo "Compiling PostgreSQL"
 	$(MAKE) -C tmp_install/build MAKELEVEL=0 install
-	+@echo "Compiling contrib/neon"
-	$(MAKE) -C tmp_install/build/contrib/neon install
-	+@echo "Compiling contrib/neon_test_utils"
-	$(MAKE) -C tmp_install/build/contrib/neon_test_utils install
+	+@echo "Compiling contrib/zenith"
+	$(MAKE) -C tmp_install/build/contrib/zenith install
+	+@echo "Compiling contrib/zenith_test_utils"
+	$(MAKE) -C tmp_install/build/contrib/zenith_test_utils install
 	+@echo "Compiling pg_buffercache"
 	$(MAKE) -C tmp_install/build/contrib/pg_buffercache install
 	+@echo "Compiling pageinspect"

NOTICE

@@ -1,5 +0,0 @@
-Neon
-Copyright 2022 Neon Inc.
-
-The PostgreSQL submodule in vendor/postgres is licensed under the
-PostgreSQL license. See vendor/postgres/COPYRIGHT.

README.md

@@ -5,11 +5,6 @@ Neon is a serverless open source alternative to AWS Aurora Postgres. It separate
 The project used to be called "Zenith". Many of the commands and code comments
 still refer to "zenith", but we are in the process of renaming things.
 
-## Quick start
-[Join the waitlist](https://neon.tech/) for our free tier to receive your serverless postgres instance. Then connect to it with your preferred postgres client (psql, dbeaver, etc) or use the online SQL editor.
-
-Alternatively, compile and run the project [locally](#running-local-installation).
-
 ## Architecture overview
 
 A Neon installation consists of compute nodes and Neon storage engine.
@@ -28,70 +23,29 @@ Pageserver consists of:
 
 ## Running local installation
 
-
-#### building on Linux
 1. Install build dependencies and other useful packages
 
-* On Ubuntu or Debian this set of packages should be sufficient to build the code:
-```bash
+On Ubuntu or Debian this set of packages should be sufficient to build the code:
+```text
 apt install build-essential libtool libreadline-dev zlib1g-dev flex bison libseccomp-dev \
-libssl-dev clang pkg-config libpq-dev etcd cmake postgresql-client
-```
-* On Fedora these packages are needed:
-```bash
-dnf install flex bison readline-devel zlib-devel openssl-devel \
-  libseccomp-devel perl clang cmake etcd postgresql postgresql-contrib
+libssl-dev clang pkg-config libpq-dev
 ```
 
-2. [Install Rust](https://www.rust-lang.org/tools/install)
-```
-# recommended approach from https://www.rust-lang.org/tools/install
-curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
-```
+[Rust] 1.58 or later is also required.
 
-3. Build neon and patched postgres
-```sh
-git clone --recursive https://github.com/neondatabase/neon.git
-cd neon
-make -j`nproc`
-```
+To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `tmp_install/bin` and `tmp_install/lib`, respectively.
 
-#### building on OSX (12.3.1)
-1. Install XCode and dependencies
-```
-xcode-select --install
-brew install protobuf etcd
-```
+To run the integration tests or Python scripts (not required to use the code), install
+Python (3.7 or higher), and install python3 packages using `./scripts/pysync` (requires poetry) in the project directory.
 
-2. [Install Rust](https://www.rust-lang.org/tools/install)
-```
-# recommended approach from https://www.rust-lang.org/tools/install
-curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
-```
-
-3. Install PostgreSQL Client
-```
-# from https://stackoverflow.com/questions/44654216/correct-way-to-install-psql-without-full-postgres-on-macos
-brew install libpq
-brew link --force libpq
-```
-
-4. Build neon and patched postgres
+2. Build neon and patched postgres
 ```sh
 git clone --recursive https://github.com/neondatabase/neon.git
 cd neon
 make -j5
 ```
 
-#### dependency installation notes
-To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `tmp_install/bin` and `tmp_install/lib`, respectively.
-
-To run the integration tests or Python scripts (not required to use the code), install
-Python (3.9 or higher), and install python3 packages using `./scripts/pysync` (requires poetry) in the project directory.
-
-
-#### running neon database
-1. Start pageserver and postgres on top of it (should be called from repo root):
+3. Start pageserver and postgres on top of it (should be called from repo root):
 ```sh
 # Create repository in .zenith with proper paths to binaries and data
 # Later that would be responsibility of a package install script
@@ -113,7 +67,7 @@ Safekeeper started
 > ./target/debug/neon_local pg start main
 Starting new postgres main on timeline de200bd42b49cc1814412c7e592dd6e9 ...
 Extracting base backup to create postgres instance: path=.zenith/pgdatadirs/tenants/9ef87a5bf0d92544f6fafeeb3239695c/main port=55432
-Starting postgres node at 'host=127.0.0.1 port=55432 user=cloud_admin dbname=postgres'
+Starting postgres node at 'host=127.0.0.1 port=55432 user=zenith_admin dbname=postgres'
 
 # check list of running postgres instances
 > ./target/debug/neon_local pg list
@@ -121,9 +75,9 @@ Starting postgres node at 'host=127.0.0.1 port=55432 user=cloud_admin dbname=pos
  main  127.0.0.1:55432  de200bd42b49cc1814412c7e592dd6e9  main         0/16B5BA8  running
 ```
 
-2. Now it is possible to connect to postgres and run some queries:
+4. Now it is possible to connect to postgres and run some queries:
 ```text
-> psql -p55432 -h 127.0.0.1 -U cloud_admin postgres
+> psql -p55432 -h 127.0.0.1 -U zenith_admin postgres
 postgres=# CREATE TABLE t(key int primary key, value text);
 CREATE TABLE
 postgres=# insert into t values(1,1);
@@ -135,7 +89,7 @@ postgres=# select * from t;
 (1 row)
 ```
 
-3. And create branches and run postgres on them:
+5. And create branches and run postgres on them:
 ```sh
 # create branch named migration_check
 > ./target/debug/neon_local timeline branch --branch-name migration_check
@@ -150,7 +104,7 @@ Created timeline 'b3b863fa45fa9e57e615f9f2d944e601' at Lsn 0/16F9A00 for tenant:
 > ./target/debug/neon_local pg start migration_check --branch-name migration_check
 Starting new postgres migration_check on timeline b3b863fa45fa9e57e615f9f2d944e601 ...
 Extracting base backup to create postgres instance: path=.zenith/pgdatadirs/tenants/9ef87a5bf0d92544f6fafeeb3239695c/migration_check port=55433
-Starting postgres node at 'host=127.0.0.1 port=55433 user=cloud_admin dbname=postgres'
+Starting postgres node at 'host=127.0.0.1 port=55433 user=zenith_admin dbname=postgres'
 
 # check the new list of running postgres instances
 > ./target/debug/neon_local pg list
@@ -160,7 +114,7 @@ Starting postgres node at 'host=127.0.0.1 port=55433 user=cloud_admin dbname=pos
 
 # this new postgres instance will have all the data from 'main' postgres,
 # but all modifications would not affect data in original postgres
-> psql -p55433 -h 127.0.0.1 -U cloud_admin postgres
+> psql -p55433 -h 127.0.0.1 -U zenith_admin postgres
 postgres=# select * from t;
  key | value
 -----+-------
@@ -171,7 +125,7 @@ postgres=# insert into t values(2,2);
 INSERT 0 1
 
 # check that the new change doesn't affect the 'main' postgres
-> psql -p55432 -h 127.0.0.1 -U cloud_admin postgres
+> psql -p55432 -h 127.0.0.1 -U zenith_admin postgres
 postgres=# select * from t;
  key | value
 -----+-------
@@ -179,7 +133,7 @@ postgres=# select * from t;
 (1 row)
 ```
 
-4. If you want to run tests afterwards (see below), you have to stop all the running the pageserver, safekeeper and postgres instances
+6. If you want to run tests afterwards (see below), you have to stop all the running the pageserver, safekeeper and postgres instances
    you have just started. You can stop them all with one command:
 ```sh
 > ./target/debug/neon_local stop

compute_tools/README.md

@@ -1,9 +1,9 @@
 # Compute node tools
 
-Postgres wrapper (`compute_ctl`) is intended to be run as a Docker entrypoint or as a `systemd`
-`ExecStart` option. It will handle all the `Neon` specifics during compute node
+Postgres wrapper (`zenith_ctl`) is intended to be run as a Docker entrypoint or as a `systemd`
+`ExecStart` option. It will handle all the `zenith` specifics during compute node
 initialization:
-- `compute_ctl` accepts cluster (compute node) specification as a JSON file.
+- `zenith_ctl` accepts cluster (compute node) specification as a JSON file.
 - Every start is a fresh start, so the data directory is removed and
   initialized again on each run.
 - Next it will put configuration files into the `PGDATA` directory.
@@ -13,18 +13,18 @@ initialization:
 - Check and alter/drop/create roles and databases.
 - Hang waiting on the `postmaster` process to exit.
 
-Also `compute_ctl` spawns two separate service threads:
+Also `zenith_ctl` spawns two separate service threads:
 - `compute-monitor` checks the last Postgres activity timestamp and saves it
-  into the shared `ComputeNode`;
+  into the shared `ComputeState`;
 - `http-endpoint` runs a Hyper HTTP API server, which serves readiness and the
   last activity requests.
 
 Usage example:
 ```sh
-compute_ctl -D /var/db/postgres/compute \
-            -C 'postgresql://cloud_admin@localhost/postgres' \
-            -S /var/db/postgres/specs/current.json \
-            -b /usr/local/bin/postgres
+zenith_ctl -D /var/db/postgres/compute \
+           -C 'postgresql://zenith_admin@localhost/postgres' \
+           -S /var/db/postgres/specs/current.json \
+           -b /usr/local/bin/postgres
 ```
 
 ## Tests

compute_tools/src/bin/compute_ctl.rs

@@ -1,174 +0,0 @@
-//!
-//! Postgres wrapper (`compute_ctl`) is intended to be run as a Docker entrypoint or as a `systemd`
-//! `ExecStart` option. It will handle all the `Neon` specifics during compute node
-//! initialization:
-//! - `compute_ctl` accepts cluster (compute node) specification as a JSON file.
-//! - Every start is a fresh start, so the data directory is removed and
-//!   initialized again on each run.
-//! - Next it will put configuration files into the `PGDATA` directory.
-//! - Sync safekeepers and get commit LSN.
-//! - Get `basebackup` from pageserver using the returned on the previous step LSN.
-//! - Try to start `postgres` and wait until it is ready to accept connections.
-//! - Check and alter/drop/create roles and databases.
-//! - Hang waiting on the `postmaster` process to exit.
-//!
-//! Also `compute_ctl` spawns two separate service threads:
-//! - `compute-monitor` checks the last Postgres activity timestamp and saves it
-//!   into the shared `ComputeNode`;
-//! - `http-endpoint` runs a Hyper HTTP API server, which serves readiness and the
-//!   last activity requests.
-//!
-//! Usage example:
-//! ```sh
-//! compute_ctl -D /var/db/postgres/compute \
-//!             -C 'postgresql://cloud_admin@localhost/postgres' \
-//!             -S /var/db/postgres/specs/current.json \
-//!             -b /usr/local/bin/postgres
-//! ```
-//!
-use std::fs::File;
-use std::panic;
-use std::path::Path;
-use std::process::exit;
-use std::sync::{Arc, RwLock};
-use std::{thread, time::Duration};
-
-use anyhow::Result;
-use chrono::Utc;
-use clap::Arg;
-use log::{error, info};
-
-use compute_tools::compute::{ComputeMetrics, ComputeNode, ComputeState, ComputeStatus};
-use compute_tools::http::api::launch_http_server;
-use compute_tools::logger::*;
-use compute_tools::monitor::launch_monitor;
-use compute_tools::params::*;
-use compute_tools::pg_helpers::*;
-use compute_tools::spec::*;
-
-fn main() -> Result<()> {
-    // TODO: re-use `utils::logging` later
-    init_logger(DEFAULT_LOG_LEVEL)?;
-
-    // Env variable is set by `cargo`
-    let version: Option<&str> = option_env!("CARGO_PKG_VERSION");
-    let matches = clap::App::new("compute_ctl")
-        .version(version.unwrap_or("unknown"))
-        .arg(
-            Arg::new("connstr")
-                .short('C')
-                .long("connstr")
-                .value_name("DATABASE_URL")
-                .required(true),
-        )
-        .arg(
-            Arg::new("pgdata")
-                .short('D')
-                .long("pgdata")
-                .value_name("DATADIR")
-                .required(true),
-        )
-        .arg(
-            Arg::new("pgbin")
-                .short('b')
-                .long("pgbin")
-                .value_name("POSTGRES_PATH"),
-        )
-        .arg(
-            Arg::new("spec")
-                .short('s')
-                .long("spec")
-                .value_name("SPEC_JSON"),
-        )
-        .arg(
-            Arg::new("spec-path")
-                .short('S')
-                .long("spec-path")
-                .value_name("SPEC_PATH"),
-        )
-        .get_matches();
-
-    let pgdata = matches.value_of("pgdata").expect("PGDATA path is required");
-    let connstr = matches
-        .value_of("connstr")
-        .expect("Postgres connection string is required");
-    let spec = matches.value_of("spec");
-    let spec_path = matches.value_of("spec-path");
-
-    // Try to use just 'postgres' if no path is provided
-    let pgbin = matches.value_of("pgbin").unwrap_or("postgres");
-
-    let spec: ComputeSpec = match spec {
-        // First, try to get cluster spec from the cli argument
-        Some(json) => serde_json::from_str(json)?,
-        None => {
-            // Second, try to read it from the file if path is provided
-            if let Some(sp) = spec_path {
-                let path = Path::new(sp);
-                let file = File::open(path)?;
-                serde_json::from_reader(file)?
-            } else {
-                panic!("cluster spec should be provided via --spec or --spec-path argument");
-            }
-        }
-    };
-
-    let pageserver_connstr = spec
-        .cluster
-        .settings
-        .find("neon.pageserver_connstring")
-        .expect("pageserver connstr should be provided");
-    let tenant = spec
-        .cluster
-        .settings
-        .find("neon.tenant_id")
-        .expect("tenant id should be provided");
-    let timeline = spec
-        .cluster
-        .settings
-        .find("neon.timeline_id")
-        .expect("tenant id should be provided");
-
-    let compute_state = ComputeNode {
-        start_time: Utc::now(),
-        connstr: connstr.to_string(),
-        pgdata: pgdata.to_string(),
-        pgbin: pgbin.to_string(),
-        spec,
-        tenant,
-        timeline,
-        pageserver_connstr,
-        metrics: ComputeMetrics::new(),
-        state: RwLock::new(ComputeState::new()),
-    };
-    let compute = Arc::new(compute_state);
-
-    // Launch service threads first, so we were able to serve availability
-    // requests, while configuration is still in progress.
-    let _http_handle = launch_http_server(&compute).expect("cannot launch http endpoint thread");
-    let _monitor_handle = launch_monitor(&compute).expect("cannot launch compute monitor thread");
-
-    // Run compute (Postgres) and hang waiting on it.
-    match compute.prepare_and_run() {
-        Ok(ec) => {
-            let code = ec.code().unwrap_or(1);
-            info!("Postgres exited with code {}, shutting down", code);
-            exit(code)
-        }
-        Err(error) => {
-            error!("could not start the compute node: {}", error);
-
-            let mut state = compute.state.write().unwrap();
-            state.error = Some(format!("{:?}", error));
-            state.status = ComputeStatus::Failed;
-            drop(state);
-
-            // Keep serving HTTP requests, so the cloud control plane was able to
-            // get the actual error.
-            info!("giving control plane 30s to collect the error before shutdown");
-            thread::sleep(Duration::from_secs(30));
-            info!("shutting down");
-            Err(error)
-        }
-    }
-}

compute_tools/src/bin/zenith_ctl.rs

@@ -0,0 +1,252 @@
+//!
+//! Postgres wrapper (`zenith_ctl`) is intended to be run as a Docker entrypoint or as a `systemd`
+//! `ExecStart` option. It will handle all the `zenith` specifics during compute node
+//! initialization:
+//! - `zenith_ctl` accepts cluster (compute node) specification as a JSON file.
+//! - Every start is a fresh start, so the data directory is removed and
+//!   initialized again on each run.
+//! - Next it will put configuration files into the `PGDATA` directory.
+//! - Sync safekeepers and get commit LSN.
+//! - Get `basebackup` from pageserver using the returned on the previous step LSN.
+//! - Try to start `postgres` and wait until it is ready to accept connections.
+//! - Check and alter/drop/create roles and databases.
+//! - Hang waiting on the `postmaster` process to exit.
+//!
+//! Also `zenith_ctl` spawns two separate service threads:
+//! - `compute-monitor` checks the last Postgres activity timestamp and saves it
+//!   into the shared `ComputeState`;
+//! - `http-endpoint` runs a Hyper HTTP API server, which serves readiness and the
+//!   last activity requests.
+//!
+//! Usage example:
+//! ```sh
+//! zenith_ctl -D /var/db/postgres/compute \
+//!            -C 'postgresql://zenith_admin@localhost/postgres' \
+//!            -S /var/db/postgres/specs/current.json \
+//!            -b /usr/local/bin/postgres
+//! ```
+//!
+use std::fs::File;
+use std::panic;
+use std::path::Path;
+use std::process::{exit, Command, ExitStatus};
+use std::sync::{Arc, RwLock};
+
+use anyhow::{Context, Result};
+use chrono::Utc;
+use clap::Arg;
+use log::info;
+use postgres::{Client, NoTls};
+
+use compute_tools::checker::create_writablity_check_data;
+use compute_tools::config;
+use compute_tools::http_api::launch_http_server;
+use compute_tools::logger::*;
+use compute_tools::monitor::launch_monitor;
+use compute_tools::params::*;
+use compute_tools::pg_helpers::*;
+use compute_tools::spec::*;
+use compute_tools::zenith::*;
+
+/// Do all the preparations like PGDATA directory creation, configuration,
+/// safekeepers sync, basebackup, etc.
+fn prepare_pgdata(state: &Arc<RwLock<ComputeState>>) -> Result<()> {
+    let state = state.read().unwrap();
+    let spec = &state.spec;
+    let pgdata_path = Path::new(&state.pgdata);
+    let pageserver_connstr = spec
+        .cluster
+        .settings
+        .find("zenith.page_server_connstring")
+        .expect("pageserver connstr should be provided");
+    let tenant = spec
+        .cluster
+        .settings
+        .find("zenith.zenith_tenant")
+        .expect("tenant id should be provided");
+    let timeline = spec
+        .cluster
+        .settings
+        .find("zenith.zenith_timeline")
+        .expect("tenant id should be provided");
+
+    info!(
+        "starting cluster #{}, operation #{}",
+        spec.cluster.cluster_id,
+        spec.operation_uuid.as_ref().unwrap()
+    );
+
+    // Remove/create an empty pgdata directory and put configuration there.
+    create_pgdata(&state.pgdata)?;
+    config::write_postgres_conf(&pgdata_path.join("postgresql.conf"), spec)?;
+
+    info!("starting safekeepers syncing");
+    let lsn = sync_safekeepers(&state.pgdata, &state.pgbin)
+        .with_context(|| "failed to sync safekeepers")?;
+    info!("safekeepers synced at LSN {}", lsn);
+
+    info!(
+        "getting basebackup@{} from pageserver {}",
+        lsn, pageserver_connstr
+    );
+    get_basebackup(&state.pgdata, &pageserver_connstr, &tenant, &timeline, &lsn).with_context(
+        || {
+            format!(
+                "failed to get basebackup@{} from pageserver {}",
+                lsn, pageserver_connstr
+            )
+        },
+    )?;
+
+    // Update pg_hba.conf received with basebackup.
+    update_pg_hba(pgdata_path)?;
+
+    Ok(())
+}
+
+/// Start Postgres as a child process and manage DBs/roles.
+/// After that this will hang waiting on the postmaster process to exit.
+fn run_compute(state: &Arc<RwLock<ComputeState>>) -> Result<ExitStatus> {
+    let read_state = state.read().unwrap();
+    let pgdata_path = Path::new(&read_state.pgdata);
+
+    // Run postgres as a child process.
+    let mut pg = Command::new(&read_state.pgbin)
+        .args(&["-D", &read_state.pgdata])
+        .spawn()
+        .expect("cannot start postgres process");
+
+    // Try default Postgres port if it is not provided
+    let port = read_state
+        .spec
+        .cluster
+        .settings
+        .find("port")
+        .unwrap_or_else(|| "5432".to_string());
+    wait_for_postgres(&port, pgdata_path)?;
+
+    let mut client = Client::connect(&read_state.connstr, NoTls)?;
+
+    handle_roles(&read_state.spec, &mut client)?;
+    handle_databases(&read_state.spec, &mut client)?;
+    handle_grants(&read_state.spec, &mut client)?;
+    create_writablity_check_data(&mut client)?;
+
+    // 'Close' connection
+    drop(client);
+
+    info!(
+        "finished configuration of cluster #{}",
+        read_state.spec.cluster.cluster_id
+    );
+
+    // Release the read lock.
+    drop(read_state);
+
+    // Get the write lock, update state and release the lock, so HTTP API
+    // was able to serve requests, while we are blocked waiting on
+    // Postgres.
+    let mut state = state.write().unwrap();
+    state.ready = true;
+    drop(state);
+
+    // Wait for child postgres process basically forever. In this state Ctrl+C
+    // will be propagated to postgres and it will be shut down as well.
+    let ecode = pg.wait().expect("failed to wait on postgres");
+
+    Ok(ecode)
+}
+
+fn main() -> Result<()> {
+    // TODO: re-use `utils::logging` later
+    init_logger(DEFAULT_LOG_LEVEL)?;
+
+    // Env variable is set by `cargo`
+    let version: Option<&str> = option_env!("CARGO_PKG_VERSION");
+    let matches = clap::App::new("zenith_ctl")
+        .version(version.unwrap_or("unknown"))
+        .arg(
+            Arg::new("connstr")
+                .short('C')
+                .long("connstr")
+                .value_name("DATABASE_URL")
+                .required(true),
+        )
+        .arg(
+            Arg::new("pgdata")
+                .short('D')
+                .long("pgdata")
+                .value_name("DATADIR")
+                .required(true),
+        )
+        .arg(
+            Arg::new("pgbin")
+                .short('b')
+                .long("pgbin")
+                .value_name("POSTGRES_PATH"),
+        )
+        .arg(
+            Arg::new("spec")
+                .short('s')
+                .long("spec")
+                .value_name("SPEC_JSON"),
+        )
+        .arg(
+            Arg::new("spec-path")
+                .short('S')
+                .long("spec-path")
+                .value_name("SPEC_PATH"),
+        )
+        .get_matches();
+
+    let pgdata = matches.value_of("pgdata").expect("PGDATA path is required");
+    let connstr = matches
+        .value_of("connstr")
+        .expect("Postgres connection string is required");
+    let spec = matches.value_of("spec");
+    let spec_path = matches.value_of("spec-path");
+
+    // Try to use just 'postgres' if no path is provided
+    let pgbin = matches.value_of("pgbin").unwrap_or("postgres");
+
+    let spec: ClusterSpec = match spec {
+        // First, try to get cluster spec from the cli argument
+        Some(json) => serde_json::from_str(json)?,
+        None => {
+            // Second, try to read it from the file if path is provided
+            if let Some(sp) = spec_path {
+                let path = Path::new(sp);
+                let file = File::open(path)?;
+                serde_json::from_reader(file)?
+            } else {
+                panic!("cluster spec should be provided via --spec or --spec-path argument");
+            }
+        }
+    };
+
+    let compute_state = ComputeState {
+        connstr: connstr.to_string(),
+        pgdata: pgdata.to_string(),
+        pgbin: pgbin.to_string(),
+        spec,
+        ready: false,
+        last_active: Utc::now(),
+    };
+    let compute_state = Arc::new(RwLock::new(compute_state));
+
+    // Launch service threads first, so we were able to serve availability
+    // requests, while configuration is still in progress.
+    let mut _threads = vec![
+        launch_http_server(&compute_state).expect("cannot launch compute monitor thread"),
+        launch_monitor(&compute_state).expect("cannot launch http endpoint thread"),
+    ];
+
+    prepare_pgdata(&compute_state)?;
+
+    // Run compute (Postgres) and hang waiting on it. Panic if any error happens,
+    // it will help us to trigger unwind and kill postmaster as well.
+    match run_compute(&compute_state) {
+        Ok(ec) => exit(ec.success() as i32),
+        Err(error) => panic!("cannot start compute node, error: {}", error),
+    }
+}

compute_tools/src/checker.rs

@@ -1,11 +1,11 @@
-use std::sync::Arc;
+use std::sync::{Arc, RwLock};
 
 use anyhow::{anyhow, Result};
 use log::error;
 use postgres::Client;
 use tokio_postgres::NoTls;
 
-use crate::compute::ComputeNode;
+use crate::zenith::ComputeState;
 
 pub fn create_writablity_check_data(client: &mut Client) -> Result<()> {
     let query = "
@@ -23,9 +23,9 @@ pub fn create_writablity_check_data(client: &mut Client) -> Result<()> {
     Ok(())
 }
 
-pub async fn check_writability(compute: &Arc<ComputeNode>) -> Result<()> {
-    let connstr = &compute.connstr;
-    let (client, connection) = tokio_postgres::connect(connstr, NoTls).await?;
+pub async fn check_writability(state: &Arc<RwLock<ComputeState>>) -> Result<()> {
+    let connstr = state.read().unwrap().connstr.clone();
+    let (client, connection) = tokio_postgres::connect(&connstr, NoTls).await?;
     if client.is_closed() {
         return Err(anyhow!("connection to postgres closed"));
     }

compute_tools/src/compute.rs

@@ -1,344 +0,0 @@
-//
-// XXX: This starts to be scarry similar to the `PostgresNode` from `control_plane`,
-// but there are several things that makes `PostgresNode` usage inconvenient in the
-// cloud:
-// - it inherits from `LocalEnv`, which contains **all-all** the information about
-//   a complete service running
-// - it uses `PageServerNode` with information about http endpoint, which we do not
-//   need in the cloud again
-// - many tiny pieces like, for example, we do not use `pg_ctl` in the cloud
-//
-// Thus, to use `PostgresNode` in the cloud, we need to 'mock' a bunch of required
-// attributes (not required for the cloud). Yet, it is still tempting to unify these
-// `PostgresNode` and `ComputeNode` and use one in both places.
-//
-// TODO: stabilize `ComputeNode` and think about using it in the `control_plane`.
-//
-use std::fs;
-use std::os::unix::fs::PermissionsExt;
-use std::path::Path;
-use std::process::{Command, ExitStatus, Stdio};
-use std::sync::atomic::{AtomicU64, Ordering};
-use std::sync::RwLock;
-
-use anyhow::{Context, Result};
-use chrono::{DateTime, Utc};
-use log::info;
-use postgres::{Client, NoTls};
-use serde::{Serialize, Serializer};
-
-use crate::checker::create_writablity_check_data;
-use crate::config;
-use crate::pg_helpers::*;
-use crate::spec::*;
-
-/// Compute node info shared across several `compute_ctl` threads.
-pub struct ComputeNode {
-    pub start_time: DateTime<Utc>,
-    pub connstr: String,
-    pub pgdata: String,
-    pub pgbin: String,
-    pub spec: ComputeSpec,
-    pub tenant: String,
-    pub timeline: String,
-    pub pageserver_connstr: String,
-    pub metrics: ComputeMetrics,
-    /// Volatile part of the `ComputeNode` so should be used under `RwLock`
-    /// to allow HTTP API server to serve status requests, while configuration
-    /// is in progress.
-    pub state: RwLock<ComputeState>,
-}
-
-fn rfc3339_serialize<S>(x: &DateTime<Utc>, s: S) -> Result<S::Ok, S::Error>
-where
-    S: Serializer,
-{
-    x.to_rfc3339().serialize(s)
-}
-
-#[derive(Serialize)]
-#[serde(rename_all = "snake_case")]
-pub struct ComputeState {
-    pub status: ComputeStatus,
-    /// Timestamp of the last Postgres activity
-    #[serde(serialize_with = "rfc3339_serialize")]
-    pub last_active: DateTime<Utc>,
-    pub error: Option<String>,
-}
-
-impl ComputeState {
-    pub fn new() -> Self {
-        Self {
-            status: ComputeStatus::Init,
-            last_active: Utc::now(),
-            error: None,
-        }
-    }
-}
-
-impl Default for ComputeState {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-#[derive(Serialize, Clone, Copy, PartialEq, Eq)]
-#[serde(rename_all = "snake_case")]
-pub enum ComputeStatus {
-    Init,
-    Running,
-    Failed,
-}
-
-#[derive(Serialize)]
-pub struct ComputeMetrics {
-    pub sync_safekeepers_ms: AtomicU64,
-    pub basebackup_ms: AtomicU64,
-    pub config_ms: AtomicU64,
-    pub total_startup_ms: AtomicU64,
-}
-
-impl ComputeMetrics {
-    pub fn new() -> Self {
-        Self {
-            sync_safekeepers_ms: AtomicU64::new(0),
-            basebackup_ms: AtomicU64::new(0),
-            config_ms: AtomicU64::new(0),
-            total_startup_ms: AtomicU64::new(0),
-        }
-    }
-}
-
-impl Default for ComputeMetrics {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl ComputeNode {
-    pub fn set_status(&self, status: ComputeStatus) {
-        self.state.write().unwrap().status = status;
-    }
-
-    pub fn get_status(&self) -> ComputeStatus {
-        self.state.read().unwrap().status
-    }
-
-    // Remove `pgdata` directory and create it again with right permissions.
-    fn create_pgdata(&self) -> Result<()> {
-        // Ignore removal error, likely it is a 'No such file or directory (os error 2)'.
-        // If it is something different then create_dir() will error out anyway.
-        let _ok = fs::remove_dir_all(&self.pgdata);
-        fs::create_dir(&self.pgdata)?;
-        fs::set_permissions(&self.pgdata, fs::Permissions::from_mode(0o700))?;
-
-        Ok(())
-    }
-
-    // Get basebackup from the libpq connection to pageserver using `connstr` and
-    // unarchive it to `pgdata` directory overriding all its previous content.
-    fn get_basebackup(&self, lsn: &str) -> Result<()> {
-        let start_time = Utc::now();
-
-        let mut client = Client::connect(&self.pageserver_connstr, NoTls)?;
-        let basebackup_cmd = match lsn {
-            "0/0" => format!("basebackup {} {}", &self.tenant, &self.timeline), // First start of the compute
-            _ => format!("basebackup {} {} {}", &self.tenant, &self.timeline, lsn),
-        };
-        let copyreader = client.copy_out(basebackup_cmd.as_str())?;
-
-        // Read the archive directly from the `CopyOutReader`
-        //
-        // Set `ignore_zeros` so that unpack() reads all the Copy data and
-        // doesn't stop at the end-of-archive marker. Otherwise, if the server
-        // sends an Error after finishing the tarball, we will not notice it.
-        let mut ar = tar::Archive::new(copyreader);
-        ar.set_ignore_zeros(true);
-        ar.unpack(&self.pgdata)?;
-
-        self.metrics.basebackup_ms.store(
-            Utc::now()
-                .signed_duration_since(start_time)
-                .to_std()
-                .unwrap()
-                .as_millis() as u64,
-            Ordering::Relaxed,
-        );
-
-        Ok(())
-    }
-
-    // Run `postgres` in a special mode with `--sync-safekeepers` argument
-    // and return the reported LSN back to the caller.
-    fn sync_safekeepers(&self) -> Result<String> {
-        let start_time = Utc::now();
-
-        let sync_handle = Command::new(&self.pgbin)
-            .args(&["--sync-safekeepers"])
-            .env("PGDATA", &self.pgdata) // we cannot use -D in this mode
-            .stdout(Stdio::piped())
-            .spawn()
-            .expect("postgres --sync-safekeepers failed to start");
-
-        // `postgres --sync-safekeepers` will print all log output to stderr and
-        // final LSN to stdout. So we pipe only stdout, while stderr will be automatically
-        // redirected to the caller output.
-        let sync_output = sync_handle
-            .wait_with_output()
-            .expect("postgres --sync-safekeepers failed");
-        if !sync_output.status.success() {
-            anyhow::bail!(
-                "postgres --sync-safekeepers exited with non-zero status: {}",
-                sync_output.status,
-            );
-        }
-
-        self.metrics.sync_safekeepers_ms.store(
-            Utc::now()
-                .signed_duration_since(start_time)
-                .to_std()
-                .unwrap()
-                .as_millis() as u64,
-            Ordering::Relaxed,
-        );
-
-        let lsn = String::from(String::from_utf8(sync_output.stdout)?.trim());
-
-        Ok(lsn)
-    }
-
-    /// Do all the preparations like PGDATA directory creation, configuration,
-    /// safekeepers sync, basebackup, etc.
-    pub fn prepare_pgdata(&self) -> Result<()> {
-        let spec = &self.spec;
-        let pgdata_path = Path::new(&self.pgdata);
-
-        // Remove/create an empty pgdata directory and put configuration there.
-        self.create_pgdata()?;
-        config::write_postgres_conf(&pgdata_path.join("postgresql.conf"), spec)?;
-
-        info!("starting safekeepers syncing");
-        let lsn = self
-            .sync_safekeepers()
-            .with_context(|| "failed to sync safekeepers")?;
-        info!("safekeepers synced at LSN {}", lsn);
-
-        info!(
-            "getting basebackup@{} from pageserver {}",
-            lsn, &self.pageserver_connstr
-        );
-        self.get_basebackup(&lsn).with_context(|| {
-            format!(
-                "failed to get basebackup@{} from pageserver {}",
-                lsn, &self.pageserver_connstr
-            )
-        })?;
-
-        // Update pg_hba.conf received with basebackup.
-        update_pg_hba(pgdata_path)?;
-
-        Ok(())
-    }
-
-    /// Start Postgres as a child process and manage DBs/roles.
-    /// After that this will hang waiting on the postmaster process to exit.
-    pub fn run(&self) -> Result<ExitStatus> {
-        let start_time = Utc::now();
-
-        let pgdata_path = Path::new(&self.pgdata);
-
-        // Run postgres as a child process.
-        let mut pg = Command::new(&self.pgbin)
-            .args(&["-D", &self.pgdata])
-            .spawn()
-            .expect("cannot start postgres process");
-
-        // Try default Postgres port if it is not provided
-        let port = self
-            .spec
-            .cluster
-            .settings
-            .find("port")
-            .unwrap_or_else(|| "5432".to_string());
-        wait_for_postgres(&mut pg, &port, pgdata_path)?;
-
-        // If connection fails,
-        // it may be the old node with `zenith_admin` superuser.
-        //
-        // In this case we need to connect with old `zenith_admin`name
-        // and create new user. We cannot simply rename connected user,
-        // but we can create a new one and grant it all privileges.
-        let mut client = match Client::connect(&self.connstr, NoTls) {
-            Err(e) => {
-                info!(
-                    "cannot connect to postgres: {}, retrying with `zenith_admin` username",
-                    e
-                );
-                let zenith_admin_connstr = self.connstr.replacen("cloud_admin", "zenith_admin", 1);
-
-                let mut client = Client::connect(&zenith_admin_connstr, NoTls)?;
-                client.simple_query("CREATE USER cloud_admin WITH SUPERUSER")?;
-                client.simple_query("GRANT zenith_admin TO cloud_admin")?;
-                drop(client);
-
-                // reconnect with connsting with expected name
-                Client::connect(&self.connstr, NoTls)?
-            }
-            Ok(client) => client,
-        };
-
-        handle_roles(&self.spec, &mut client)?;
-        handle_databases(&self.spec, &mut client)?;
-        handle_grants(&self.spec, &mut client)?;
-        create_writablity_check_data(&mut client)?;
-
-        // 'Close' connection
-        drop(client);
-        let startup_end_time = Utc::now();
-
-        self.metrics.config_ms.store(
-            startup_end_time
-                .signed_duration_since(start_time)
-                .to_std()
-                .unwrap()
-                .as_millis() as u64,
-            Ordering::Relaxed,
-        );
-        self.metrics.total_startup_ms.store(
-            startup_end_time
-                .signed_duration_since(self.start_time)
-                .to_std()
-                .unwrap()
-                .as_millis() as u64,
-            Ordering::Relaxed,
-        );
-
-        self.set_status(ComputeStatus::Running);
-
-        info!(
-            "finished configuration of compute for project {}",
-            self.spec.cluster.cluster_id
-        );
-
-        // Wait for child Postgres process basically forever. In this state Ctrl+C
-        // will propagate to Postgres and it will be shut down as well.
-        let ecode = pg
-            .wait()
-            .expect("failed to start waiting on Postgres process");
-
-        Ok(ecode)
-    }
-
-    pub fn prepare_and_run(&self) -> Result<ExitStatus> {
-        info!(
-            "starting compute for project {}, operation {}, tenant {}, timeline {}",
-            self.spec.cluster.cluster_id,
-            self.spec.operation_uuid.as_ref().unwrap(),
-            self.tenant,
-            self.timeline,
-        );
-
-        self.prepare_pgdata()?;
-        self.run()
-    }
-}

compute_tools/src/config.rs

@@ -6,7 +6,7 @@ use std::path::Path;
 use anyhow::Result;
 
 use crate::pg_helpers::PgOptionsSerialize;
-use crate::spec::ComputeSpec;
+use crate::zenith::ClusterSpec;
 
 /// Check that `line` is inside a text file and put it there if it is not.
 /// Create file if it doesn't exist.
@@ -32,20 +32,20 @@ pub fn line_in_file(path: &Path, line: &str) -> Result<bool> {
 }
 
 /// Create or completely rewrite configuration file specified by `path`
-pub fn write_postgres_conf(path: &Path, spec: &ComputeSpec) -> Result<()> {
+pub fn write_postgres_conf(path: &Path, spec: &ClusterSpec) -> Result<()> {
     // File::create() destroys the file content if it exists.
     let mut postgres_conf = File::create(path)?;
 
-    write_auto_managed_block(&mut postgres_conf, &spec.cluster.settings.as_pg_settings())?;
+    write_zenith_managed_block(&mut postgres_conf, &spec.cluster.settings.as_pg_settings())?;
 
     Ok(())
 }
 
 // Write Postgres config block wrapped with generated comment section
-fn write_auto_managed_block(file: &mut File, buf: &str) -> Result<()> {
-    writeln!(file, "# Managed by compute_ctl: begin")?;
+fn write_zenith_managed_block(file: &mut File, buf: &str) -> Result<()> {
+    writeln!(file, "# Managed by Zenith: begin")?;
     writeln!(file, "{}", buf)?;
-    writeln!(file, "# Managed by compute_ctl: end")?;
+    writeln!(file, "# Managed by Zenith: end")?;
 
     Ok(())
 }

compute_tools/src/http/mod.rs

@@ -1 +0,0 @@
-pub mod api;

compute_tools/src/http/openapi_spec.yaml

@@ -1,158 +0,0 @@
-openapi: "3.0.2"
-info:
-  title: Compute node control API
-  version: "1.0"
-
-servers:
-  - url: "http://localhost:3080"
-
-paths:
-  /status:
-    get:
-      tags:
-      - "info"
-      summary: Get compute node internal status
-      description: ""
-      operationId: getComputeStatus
-      responses:
-        "200":
-          description: ComputeState
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ComputeState"
-
-  /metrics.json:
-    get:
-      tags:
-      - "info"
-      summary: Get compute node startup metrics in JSON format
-      description: ""
-      operationId: getComputeMetricsJSON
-      responses:
-        "200":
-          description: ComputeMetrics
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ComputeMetrics"
-
-  /ready:
-    get:
-      deprecated: true
-      tags:
-      - "info"
-      summary: Check whether compute startup process finished successfully
-      description: ""
-      operationId: computeIsReady
-      responses:
-        "200":
-          description: Compute is ready ('true') or not ('false')
-          content:
-            text/plain:
-              schema:
-                type: string
-                example: "true"
-
-  /last_activity:
-    get:
-      deprecated: true
-      tags:
-      - "info"
-      summary: Get timestamp of the last compute activity
-      description: ""
-      operationId: getLastComputeActivityTS
-      responses:
-        "200":
-          description: Timestamp of the last compute activity
-          content:
-            text/plain:
-              schema:
-                type: string
-                example: "2022-10-12T07:20:50.52Z"
-
-  /check_writability:
-    get:
-      deprecated: true
-      tags:
-      - "check"
-      summary: Check that we can write new data on this compute
-      description: ""
-      operationId: checkComputeWritabilityDeprecated
-      responses:
-        "200":
-          description: Check result
-          content:
-            text/plain:
-              schema:
-                type: string
-                description: Error text or 'true' if check passed
-                example: "true"
-
-    post:
-      tags:
-      - "check"
-      summary: Check that we can write new data on this compute
-      description: ""
-      operationId: checkComputeWritability
-      responses:
-        "200":
-          description: Check result
-          content:
-            text/plain:
-              schema:
-                type: string
-                description: Error text or 'true' if check passed
-                example: "true"
-
-components:
-  securitySchemes:
-    JWT:
-      type: http
-      scheme: bearer
-      bearerFormat: JWT
-
-  schemas:
-    ComputeMetrics:
-      type: object
-      description: Compute startup metrics
-      required:
-        - sync_safekeepers_ms
-        - basebackup_ms
-        - config_ms
-        - total_startup_ms
-      properties:
-        sync_safekeepers_ms:
-          type: integer
-        basebackup_ms:
-          type: integer
-        config_ms:
-          type: integer
-        total_startup_ms:
-          type: integer
-
-    ComputeState:
-      type: object
-      required:
-        - status
-        - last_active
-      properties:
-        status:
-          $ref: '#/components/schemas/ComputeStatus'
-        last_active:
-          type: string
-          description: The last detected compute activity timestamp in UTC and RFC3339 format
-          example: "2022-10-12T07:20:50.52Z"
-        error:
-          type: string
-          description: Text of the error during compute startup, if any
-
-    ComputeStatus:
-      type: string
-      enum:
-        - init
-        - failed
-        - running
-
-security:
-  - JWT: []

compute_tools/src/http_api.rs

@@ -1,64 +1,37 @@
 use std::convert::Infallible;
 use std::net::SocketAddr;
-use std::sync::Arc;
+use std::sync::{Arc, RwLock};
 use std::thread;
 
 use anyhow::Result;
 use hyper::service::{make_service_fn, service_fn};
 use hyper::{Body, Method, Request, Response, Server, StatusCode};
 use log::{error, info};
-use serde_json;
 
-use crate::compute::{ComputeNode, ComputeStatus};
+use crate::zenith::*;
 
 // Service function to handle all available routes.
-async fn routes(req: Request<Body>, compute: Arc<ComputeNode>) -> Response<Body> {
+async fn routes(req: Request<Body>, state: Arc<RwLock<ComputeState>>) -> Response<Body> {
     match (req.method(), req.uri().path()) {
         // Timestamp of the last Postgres activity in the plain text.
-        // DEPRECATED in favour of /status
         (&Method::GET, "/last_activity") => {
             info!("serving /last_active GET request");
-            let state = compute.state.read().unwrap();
+            let state = state.read().unwrap();
 
             // Use RFC3339 format for consistency.
             Response::new(Body::from(state.last_active.to_rfc3339()))
         }
 
-        // Has compute setup process finished? -> true/false.
-        // DEPRECATED in favour of /status
+        // Has compute setup process finished? -> true/false
         (&Method::GET, "/ready") => {
             info!("serving /ready GET request");
-            let status = compute.get_status();
-            Response::new(Body::from(format!("{}", status == ComputeStatus::Running)))
+            let state = state.read().unwrap();
+            Response::new(Body::from(format!("{}", state.ready)))
         }
 
-        // Serialized compute state.
-        (&Method::GET, "/status") => {
-            info!("serving /status GET request");
-            let state = compute.state.read().unwrap();
-            Response::new(Body::from(serde_json::to_string(&*state).unwrap()))
-        }
-
-        // Startup metrics in JSON format. Keep /metrics reserved for a possible
-        // future use for Prometheus metrics format.
-        (&Method::GET, "/metrics.json") => {
-            info!("serving /metrics.json GET request");
-            Response::new(Body::from(serde_json::to_string(&compute.metrics).unwrap()))
-        }
-
-        // DEPRECATED, use POST instead
         (&Method::GET, "/check_writability") => {
             info!("serving /check_writability GET request");
-            let res = crate::checker::check_writability(&compute).await;
-            match res {
-                Ok(_) => Response::new(Body::from("true")),
-                Err(e) => Response::new(Body::from(e.to_string())),
-            }
-        }
-
-        (&Method::POST, "/check_writability") => {
-            info!("serving /check_writability POST request");
-            let res = crate::checker::check_writability(&compute).await;
+            let res = crate::checker::check_writability(&state).await;
             match res {
                 Ok(_) => Response::new(Body::from("true")),
                 Err(e) => Response::new(Body::from(e.to_string())),
@@ -76,7 +49,7 @@ async fn routes(req: Request<Body>, compute: Arc<ComputeNode>) -> Response<Body>
 
 // Main Hyper HTTP server function that runs it and blocks waiting on it forever.
 #[tokio::main]
-async fn serve(state: Arc<ComputeNode>) {
+async fn serve(state: Arc<RwLock<ComputeState>>) {
     let addr = SocketAddr::from(([0, 0, 0, 0], 3080));
 
     let make_service = make_service_fn(move |_conn| {
@@ -100,7 +73,7 @@ async fn serve(state: Arc<ComputeNode>) {
 }
 
 /// Launch a separate Hyper HTTP API server thread and return its `JoinHandle`.
-pub fn launch_http_server(state: &Arc<ComputeNode>) -> Result<thread::JoinHandle<()>> {
+pub fn launch_http_server(state: &Arc<RwLock<ComputeState>>) -> Result<thread::JoinHandle<()>> {
     let state = Arc::clone(state);
 
     Ok(thread::Builder::new()

compute_tools/src/lib.rs

@@ -4,11 +4,11 @@
 //!
 pub mod checker;
 pub mod config;
-pub mod http;
+pub mod http_api;
 #[macro_use]
 pub mod logger;
-pub mod compute;
 pub mod monitor;
 pub mod params;
 pub mod pg_helpers;
 pub mod spec;
+pub mod zenith;

compute_tools/src/monitor.rs

@@ -1,4 +1,4 @@
-use std::sync::Arc;
+use std::sync::{Arc, RwLock};
 use std::{thread, time};
 
 use anyhow::Result;
@@ -6,16 +6,16 @@ use chrono::{DateTime, Utc};
 use log::{debug, info};
 use postgres::{Client, NoTls};
 
-use crate::compute::ComputeNode;
+use crate::zenith::ComputeState;
 
 const MONITOR_CHECK_INTERVAL: u64 = 500; // milliseconds
 
 // Spin in a loop and figure out the last activity time in the Postgres.
 // Then update it in the shared state. This function never errors out.
 // XXX: the only expected panic is at `RwLock` unwrap().
-fn watch_compute_activity(compute: &Arc<ComputeNode>) {
+fn watch_compute_activity(state: &Arc<RwLock<ComputeState>>) {
     // Suppose that `connstr` doesn't change
-    let connstr = compute.connstr.clone();
+    let connstr = state.read().unwrap().connstr.clone();
     // Define `client` outside of the loop to reuse existing connection if it's active.
     let mut client = Client::connect(&connstr, NoTls);
     let timeout = time::Duration::from_millis(MONITOR_CHECK_INTERVAL);
@@ -43,10 +43,10 @@ fn watch_compute_activity(compute: &Arc<ComputeNode>) {
                          FROM pg_stat_activity
                          WHERE backend_type = 'client backend'
                             AND pid != pg_backend_pid()
-                            AND usename != 'cloud_admin';", // XXX: find a better way to filter other monitors?
+                            AND usename != 'zenith_admin';", // XXX: find a better way to filter other monitors?
                         &[],
                     );
-                let mut last_active = compute.state.read().unwrap().last_active;
+                let mut last_active = state.read().unwrap().last_active;
 
                 if let Ok(backs) = backends {
                     let mut idle_backs: Vec<DateTime<Utc>> = vec![];
@@ -83,14 +83,14 @@ fn watch_compute_activity(compute: &Arc<ComputeNode>) {
                 }
 
                 // Update the last activity in the shared state if we got a more recent one.
-                let mut state = compute.state.write().unwrap();
+                let mut state = state.write().unwrap();
                 if last_active > state.last_active {
                     state.last_active = last_active;
                     debug!("set the last compute activity time to: {}", last_active);
                 }
             }
             Err(e) => {
-                debug!("cannot connect to postgres: {}, retrying", e);
+                info!("cannot connect to postgres: {}, retrying", e);
 
                 // Establish a new connection and try again.
                 client = Client::connect(&connstr, NoTls);
@@ -100,7 +100,7 @@ fn watch_compute_activity(compute: &Arc<ComputeNode>) {
 }
 
 /// Launch a separate compute monitor thread and return its `JoinHandle`.
-pub fn launch_monitor(state: &Arc<ComputeNode>) -> Result<thread::JoinHandle<()>> {
+pub fn launch_monitor(state: &Arc<RwLock<ComputeState>>) -> Result<thread::JoinHandle<()>> {
     let state = Arc::clone(state);
 
     Ok(thread::Builder::new()

compute_tools/src/pg_helpers.rs

@@ -1,9 +1,7 @@
-use std::fs::File;
-use std::io::{BufRead, BufReader};
 use std::net::{SocketAddr, TcpStream};
 use std::os::unix::fs::PermissionsExt;
 use std::path::Path;
-use std::process::Child;
+use std::process::Command;
 use std::str::FromStr;
 use std::{fs, thread, time};
 
@@ -222,12 +220,12 @@ pub fn get_existing_dbs(client: &mut Client) -> Result<Vec<Database>> {
 /// Wait for Postgres to become ready to accept connections:
 /// - state should be `ready` in the `pgdata/postmaster.pid`
 /// - and we should be able to connect to 127.0.0.1:5432
-pub fn wait_for_postgres(pg: &mut Child, port: &str, pgdata: &Path) -> Result<()> {
+pub fn wait_for_postgres(port: &str, pgdata: &Path) -> Result<()> {
     let pid_path = pgdata.join("postmaster.pid");
     let mut slept: u64 = 0; // ms
     let pause = time::Duration::from_millis(100);
 
-    let timeout = time::Duration::from_millis(10);
+    let timeout = time::Duration::from_millis(200);
     let addr = SocketAddr::from_str(&format!("127.0.0.1:{}", port)).unwrap();
 
     loop {
@@ -238,19 +236,14 @@ pub fn wait_for_postgres(pg: &mut Child, port: &str, pgdata: &Path) -> Result<()
             bail!("timed out while waiting for Postgres to start");
         }
 
-        if let Ok(Some(status)) = pg.try_wait() {
-            // Postgres exited, that is not what we expected, bail out earlier.
-            let code = status.code().unwrap_or(-1);
-            bail!("Postgres exited unexpectedly with code {}", code);
-        }
-
         if pid_path.exists() {
-            let file = BufReader::new(File::open(&pid_path)?);
-            let status = file
-                .lines()
-                .last()
-                .unwrap()
-                .unwrap_or_else(|_| "unknown".to_string());
+            // XXX: dumb and the simplest way to get the last line in a text file
+            // TODO: better use `.lines().last()` later
+            let stdout = Command::new("tail")
+                .args(&["-n1", pid_path.to_str().unwrap()])
+                .output()?
+                .stdout;
+            let status = String::from_utf8(stdout)?;
             let can_connect = TcpStream::connect_timeout(&addr, timeout).is_ok();
 
             // Now Postgres is ready to accept connections

compute_tools/src/spec.rs

@@ -3,53 +3,16 @@ use std::path::Path;
 use anyhow::Result;
 use log::{info, log_enabled, warn, Level};
 use postgres::Client;
-use serde::Deserialize;
 
 use crate::config;
 use crate::params::PG_HBA_ALL_MD5;
 use crate::pg_helpers::*;
-
-/// Cluster spec or configuration represented as an optional number of
-/// delta operations + final cluster state description.
-#[derive(Clone, Deserialize)]
-pub struct ComputeSpec {
-    pub format_version: f32,
-    pub timestamp: String,
-    pub operation_uuid: Option<String>,
-    /// Expected cluster state at the end of transition process.
-    pub cluster: Cluster,
-    pub delta_operations: Option<Vec<DeltaOp>>,
-}
-
-/// Cluster state seen from the perspective of the external tools
-/// like Rails web console.
-#[derive(Clone, Deserialize)]
-pub struct Cluster {
-    pub cluster_id: String,
-    pub name: String,
-    pub state: Option<String>,
-    pub roles: Vec<Role>,
-    pub databases: Vec<Database>,
-    pub settings: GenericOptions,
-}
-
-/// Single cluster state changing operation that could not be represented as
-/// a static `Cluster` structure. For example:
-/// - DROP DATABASE
-/// - DROP ROLE
-/// - ALTER ROLE name RENAME TO new_name
-/// - ALTER DATABASE name RENAME TO new_name
-#[derive(Clone, Deserialize)]
-pub struct DeltaOp {
-    pub action: String,
-    pub name: PgIdent,
-    pub new_name: Option<PgIdent>,
-}
+use crate::zenith::ClusterSpec;
 
 /// It takes cluster specification and does the following:
 /// - Serialize cluster config and put it into `postgresql.conf` completely rewriting the file.
 /// - Update `pg_hba.conf` to allow external connections.
-pub fn handle_configuration(spec: &ComputeSpec, pgdata_path: &Path) -> Result<()> {
+pub fn handle_configuration(spec: &ClusterSpec, pgdata_path: &Path) -> Result<()> {
     // File `postgresql.conf` is no longer included into `basebackup`, so just
     // always write all config into it creating new file.
     config::write_postgres_conf(&pgdata_path.join("postgresql.conf"), spec)?;
@@ -76,7 +39,7 @@ pub fn update_pg_hba(pgdata_path: &Path) -> Result<()> {
 
 /// Given a cluster spec json and open transaction it handles roles creation,
 /// deletion and update.
-pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
+pub fn handle_roles(spec: &ClusterSpec, client: &mut Client) -> Result<()> {
     let mut xact = client.transaction()?;
     let existing_roles: Vec<Role> = get_existing_roles(&mut xact)?;
 
@@ -202,7 +165,7 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
 /// like `CREATE DATABASE` and `DROP DATABASE` do not support it. Statement-level
 /// atomicity should be enough here due to the order of operations and various checks,
 /// which together provide us idempotency.
-pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
+pub fn handle_databases(spec: &ClusterSpec, client: &mut Client) -> Result<()> {
     let existing_dbs: Vec<Database> = get_existing_dbs(client)?;
 
     // Print a list of existing Postgres databases (only in debug mode)
@@ -291,7 +254,7 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
 
 // Grant CREATE ON DATABASE to the database owner
 // to allow clients create trusted extensions.
-pub fn handle_grants(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
+pub fn handle_grants(spec: &ClusterSpec, client: &mut Client) -> Result<()> {
     info!("cluster spec grants:");
 
     for db in &spec.cluster.databases {

compute_tools/src/zenith.rs

@@ -0,0 +1,109 @@
+use std::process::{Command, Stdio};
+
+use anyhow::Result;
+use chrono::{DateTime, Utc};
+use postgres::{Client, NoTls};
+use serde::Deserialize;
+
+use crate::pg_helpers::*;
+
+/// Compute node state shared across several `zenith_ctl` threads.
+/// Should be used under `RwLock` to allow HTTP API server to serve
+/// status requests, while configuration is in progress.
+pub struct ComputeState {
+    pub connstr: String,
+    pub pgdata: String,
+    pub pgbin: String,
+    pub spec: ClusterSpec,
+    /// Compute setup process has finished
+    pub ready: bool,
+    /// Timestamp of the last Postgres activity
+    pub last_active: DateTime<Utc>,
+}
+
+/// Cluster spec or configuration represented as an optional number of
+/// delta operations + final cluster state description.
+#[derive(Clone, Deserialize)]
+pub struct ClusterSpec {
+    pub format_version: f32,
+    pub timestamp: String,
+    pub operation_uuid: Option<String>,
+    /// Expected cluster state at the end of transition process.
+    pub cluster: Cluster,
+    pub delta_operations: Option<Vec<DeltaOp>>,
+}
+
+/// Cluster state seen from the perspective of the external tools
+/// like Rails web console.
+#[derive(Clone, Deserialize)]
+pub struct Cluster {
+    pub cluster_id: String,
+    pub name: String,
+    pub state: Option<String>,
+    pub roles: Vec<Role>,
+    pub databases: Vec<Database>,
+    pub settings: GenericOptions,
+}
+
+/// Single cluster state changing operation that could not be represented as
+/// a static `Cluster` structure. For example:
+/// - DROP DATABASE
+/// - DROP ROLE
+/// - ALTER ROLE name RENAME TO new_name
+/// - ALTER DATABASE name RENAME TO new_name
+#[derive(Clone, Deserialize)]
+pub struct DeltaOp {
+    pub action: String,
+    pub name: PgIdent,
+    pub new_name: Option<PgIdent>,
+}
+
+/// Get basebackup from the libpq connection to pageserver using `connstr` and
+/// unarchive it to `pgdata` directory overriding all its previous content.
+pub fn get_basebackup(
+    pgdata: &str,
+    connstr: &str,
+    tenant: &str,
+    timeline: &str,
+    lsn: &str,
+) -> Result<()> {
+    let mut client = Client::connect(connstr, NoTls)?;
+    let basebackup_cmd = match lsn {
+        "0/0" => format!("basebackup {} {}", tenant, timeline), // First start of the compute
+        _ => format!("basebackup {} {} {}", tenant, timeline, lsn),
+    };
+    let copyreader = client.copy_out(basebackup_cmd.as_str())?;
+    let mut ar = tar::Archive::new(copyreader);
+
+    ar.unpack(&pgdata)?;
+
+    Ok(())
+}
+
+/// Run `postgres` in a special mode with `--sync-safekeepers` argument
+/// and return the reported LSN back to the caller.
+pub fn sync_safekeepers(pgdata: &str, pgbin: &str) -> Result<String> {
+    let sync_handle = Command::new(&pgbin)
+        .args(&["--sync-safekeepers"])
+        .env("PGDATA", &pgdata) // we cannot use -D in this mode
+        .stdout(Stdio::piped())
+        .spawn()
+        .expect("postgres --sync-safekeepers failed to start");
+
+    // `postgres --sync-safekeepers` will print all log output to stderr and
+    // final LSN to stdout. So we pipe only stdout, while stderr will be automatically
+    // redirected to the caller output.
+    let sync_output = sync_handle
+        .wait_with_output()
+        .expect("postgres --sync-safekeepers failed");
+    if !sync_output.status.success() {
+        anyhow::bail!(
+            "postgres --sync-safekeepers exited with non-zero status: {}",
+            sync_output.status,
+        );
+    }
+
+    let lsn = String::from(String::from_utf8(sync_output.stdout)?.trim());
+
+    Ok(lsn)
+}

compute_tools/tests/cluster_spec.json

@@ -85,7 +85,7 @@
                 "vartype": "bool"
             },
             {
-                "name": "safekeepers",
+                "name": "wal_acceptors",
                 "value": "127.0.0.1:6502,127.0.0.1:6503,127.0.0.1:6501",
                 "vartype": "string"
             },
@@ -150,7 +150,7 @@
                 "vartype": "integer"
             },
             {
-                "name": "neon.tenant_id",
+                "name": "zenith.zenith_tenant",
                 "value": "b0554b632bd4d547a63b86c3630317e8",
                 "vartype": "string"
             },
@@ -160,13 +160,13 @@
                 "vartype": "integer"
             },
             {
-                "name": "neon.timeline_id",
+                "name": "zenith.zenith_timeline",
                 "value": "2414a61ffc94e428f14b5758fe308e13",
                 "vartype": "string"
             },
             {
                 "name": "shared_preload_libraries",
-                "value": "neon",
+                "value": "zenith",
                 "vartype": "string"
             },
             {
@@ -175,7 +175,7 @@
                 "vartype": "string"
             },
             {
-                "name": "neon.pageserver_connstring",
+                "name": "zenith.page_server_connstring",
                 "value": "host=127.0.0.1 port=6400",
                 "vartype": "string"
             }

compute_tools/tests/pg_helpers_tests.rs

@@ -4,12 +4,12 @@ mod pg_helpers_tests {
     use std::fs::File;
 
     use compute_tools::pg_helpers::*;
-    use compute_tools::spec::ComputeSpec;
+    use compute_tools::zenith::ClusterSpec;
 
     #[test]
     fn params_serialize() {
         let file = File::open("tests/cluster_spec.json").unwrap();
-        let spec: ComputeSpec = serde_json::from_reader(file).unwrap();
+        let spec: ClusterSpec = serde_json::from_reader(file).unwrap();
 
         assert_eq!(
             spec.cluster.databases.first().unwrap().to_pg_options(),
@@ -24,11 +24,11 @@ mod pg_helpers_tests {
     #[test]
     fn settings_serialize() {
         let file = File::open("tests/cluster_spec.json").unwrap();
-        let spec: ComputeSpec = serde_json::from_reader(file).unwrap();
+        let spec: ClusterSpec = serde_json::from_reader(file).unwrap();
 
         assert_eq!(
             spec.cluster.settings.as_pg_settings(),
-            "fsync = off\nwal_level = replica\nhot_standby = on\nsafekeepers = '127.0.0.1:6502,127.0.0.1:6503,127.0.0.1:6501'\nwal_log_hints = on\nlog_connections = on\nshared_buffers = 32768\nport = 55432\nmax_connections = 100\nmax_wal_senders = 10\nlisten_addresses = '0.0.0.0'\nwal_sender_timeout = 0\npassword_encryption = md5\nmaintenance_work_mem = 65536\nmax_parallel_workers = 8\nmax_worker_processes = 8\nneon.tenant_id = 'b0554b632bd4d547a63b86c3630317e8'\nmax_replication_slots = 10\nneon.timeline_id = '2414a61ffc94e428f14b5758fe308e13'\nshared_preload_libraries = 'neon'\nsynchronous_standby_names = 'walproposer'\nneon.pageserver_connstring = 'host=127.0.0.1 port=6400'"
+            "fsync = off\nwal_level = replica\nhot_standby = on\nwal_acceptors = '127.0.0.1:6502,127.0.0.1:6503,127.0.0.1:6501'\nwal_log_hints = on\nlog_connections = on\nshared_buffers = 32768\nport = 55432\nmax_connections = 100\nmax_wal_senders = 10\nlisten_addresses = '0.0.0.0'\nwal_sender_timeout = 0\npassword_encryption = md5\nmaintenance_work_mem = 65536\nmax_parallel_workers = 8\nmax_worker_processes = 8\nzenith.zenith_tenant = 'b0554b632bd4d547a63b86c3630317e8'\nmax_replication_slots = 10\nzenith.zenith_timeline = '2414a61ffc94e428f14b5758fe308e13'\nshared_preload_libraries = 'zenith'\nsynchronous_standby_names = 'walproposer'\nzenith.page_server_connstring = 'host=127.0.0.1 port=6400'"
         );
     }
 

control_plane/Cargo.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-tar = "0.4.38"
+tar = "0.4.33"
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
 serde = { version = "1.0", features = ["derive"] }
 serde_with = "1.12.0"

control_plane/simple.conf

@@ -9,6 +9,3 @@ auth_type = 'Trust'
 id = 1
 pg_port = 5454
 http_port = 7676
-
-[etcd_broker]
-broker_endpoints = ['http://127.0.0.1:2379']

control_plane/src/compute.rs

@@ -148,9 +148,9 @@ impl PostgresNode {
         // Read a few options from the config file
         let context = format!("in config file {}", cfg_path_str);
         let port: u16 = conf.parse_field("port", &context)?;
-        let timeline_id: ZTimelineId = conf.parse_field("neon.timeline_id", &context)?;
-        let tenant_id: ZTenantId = conf.parse_field("neon.tenant_id", &context)?;
-        let uses_wal_proposer = conf.get("safekeepers").is_some();
+        let timeline_id: ZTimelineId = conf.parse_field("zenith.zenith_timeline", &context)?;
+        let tenant_id: ZTenantId = conf.parse_field("zenith.zenith_tenant", &context)?;
+        let uses_wal_proposer = conf.get("wal_acceptors").is_some();
 
         // parse recovery_target_lsn, if any
         let recovery_target_lsn: Option<Lsn> =
@@ -231,13 +231,8 @@ impl PostgresNode {
             .context("page server 'basebackup' command failed")?;
 
         // Read the archive directly from the `CopyOutReader`
-        //
-        // Set `ignore_zeros` so that unpack() reads all the Copy data and
-        // doesn't stop at the end-of-archive marker. Otherwise, if the server
-        // sends an Error after finishing the tarball, we will not notice it.
-        let mut ar = tar::Archive::new(copyreader);
-        ar.set_ignore_zeros(true);
-        ar.unpack(&self.pgdata())
+        tar::Archive::new(copyreader)
+            .unpack(&self.pgdata())
             .context("extracting base backup failed")?;
 
         Ok(())
@@ -279,8 +274,6 @@ impl PostgresNode {
         conf.append("listen_addresses", &self.address.ip().to_string());
         conf.append("port", &self.address.port().to_string());
         conf.append("wal_keep_size", "0");
-        // walproposer panics when basebackup is invalid, it is pointless to restart in this case.
-        conf.append("restart_after_crash", "off");
 
         // Configure the node to fetch pages from pageserver
         let pageserver_connstr = {
@@ -303,11 +296,11 @@ impl PostgresNode {
             // uses only needed variables namely host, port, user, password.
             format!("postgresql://no_user:{}@{}:{}", password, host, port)
         };
-        conf.append("shared_preload_libraries", "neon");
+        conf.append("shared_preload_libraries", "zenith");
         conf.append_line("");
-        conf.append("neon.pageserver_connstring", &pageserver_connstr);
-        conf.append("neon.tenant_id", &self.tenant_id.to_string());
-        conf.append("neon.timeline_id", &self.timeline_id.to_string());
+        conf.append("zenith.page_server_connstring", &pageserver_connstr);
+        conf.append("zenith.zenith_tenant", &self.tenant_id.to_string());
+        conf.append("zenith.zenith_timeline", &self.timeline_id.to_string());
         if let Some(lsn) = self.lsn {
             conf.append("recovery_target_lsn", &lsn.to_string());
         }
@@ -341,7 +334,7 @@ impl PostgresNode {
                 .map(|sk| format!("localhost:{}", sk.pg_port))
                 .collect::<Vec<String>>()
                 .join(",");
-            conf.append("safekeepers", &safekeepers);
+            conf.append("wal_acceptors", &safekeepers);
         } else {
             // We only use setup without safekeepers for tests,
             // and don't care about data durability on pageserver,
@@ -352,6 +345,7 @@ impl PostgresNode {
             // This isn't really a supported configuration, but can be useful for
             // testing.
             conf.append("synchronous_standby_names", "pageserver");
+            conf.append("zenith.callmemaybe_connstring", &self.connstr());
         }
 
         let mut file = File::create(self.pgdata().join("postgresql.conf"))?;
@@ -498,7 +492,7 @@ impl PostgresNode {
             "host={} port={} user={} dbname={}",
             self.address.ip(),
             self.address.port(),
-            "cloud_admin",
+            "zenith_admin",
             "postgres"
         )
     }

control_plane/src/etcd.rs

@@ -1,97 +0,0 @@
-use std::{
-    fs,
-    path::PathBuf,
-    process::{Command, Stdio},
-};
-
-use anyhow::Context;
-use nix::{
-    sys::signal::{kill, Signal},
-    unistd::Pid,
-};
-
-use crate::{local_env, read_pidfile};
-
-pub fn start_etcd_process(env: &local_env::LocalEnv) -> anyhow::Result<()> {
-    let etcd_broker = &env.etcd_broker;
-    println!(
-        "Starting etcd broker using {}",
-        etcd_broker.etcd_binary_path.display()
-    );
-
-    let etcd_data_dir = env.base_data_dir.join("etcd");
-    fs::create_dir_all(&etcd_data_dir).with_context(|| {
-        format!(
-            "Failed to create etcd data dir: {}",
-            etcd_data_dir.display()
-        )
-    })?;
-
-    let etcd_stdout_file =
-        fs::File::create(etcd_data_dir.join("etcd.stdout.log")).with_context(|| {
-            format!(
-                "Failed to create ectd stout file in directory {}",
-                etcd_data_dir.display()
-            )
-        })?;
-    let etcd_stderr_file =
-        fs::File::create(etcd_data_dir.join("etcd.stderr.log")).with_context(|| {
-            format!(
-                "Failed to create ectd stderr file in directory {}",
-                etcd_data_dir.display()
-            )
-        })?;
-    let client_urls = etcd_broker.comma_separated_endpoints();
-
-    let etcd_process = Command::new(&etcd_broker.etcd_binary_path)
-        .args(&[
-            format!("--data-dir={}", etcd_data_dir.display()),
-            format!("--listen-client-urls={client_urls}"),
-            format!("--advertise-client-urls={client_urls}"),
-            // Set --quota-backend-bytes to keep the etcd virtual memory
-            // size smaller. Our test etcd clusters are very small.
-            // See https://github.com/etcd-io/etcd/issues/7910
-            "--quota-backend-bytes=100000000".to_string(),
-        ])
-        .stdout(Stdio::from(etcd_stdout_file))
-        .stderr(Stdio::from(etcd_stderr_file))
-        .spawn()
-        .context("Failed to spawn etcd subprocess")?;
-    let pid = etcd_process.id();
-
-    let etcd_pid_file_path = etcd_pid_file_path(env);
-    fs::write(&etcd_pid_file_path, pid.to_string()).with_context(|| {
-        format!(
-            "Failed to create etcd pid file at {}",
-            etcd_pid_file_path.display()
-        )
-    })?;
-
-    Ok(())
-}
-
-pub fn stop_etcd_process(env: &local_env::LocalEnv) -> anyhow::Result<()> {
-    let etcd_path = &env.etcd_broker.etcd_binary_path;
-    println!("Stopping etcd broker at {}", etcd_path.display());
-
-    let etcd_pid_file_path = etcd_pid_file_path(env);
-    let pid = Pid::from_raw(read_pidfile(&etcd_pid_file_path).with_context(|| {
-        format!(
-            "Failed to read etcd pid file at {}",
-            etcd_pid_file_path.display()
-        )
-    })?);
-
-    kill(pid, Signal::SIGTERM).with_context(|| {
-        format!(
-            "Failed to stop etcd with pid {pid} at {}",
-            etcd_pid_file_path.display()
-        )
-    })?;
-
-    Ok(())
-}
-
-fn etcd_pid_file_path(env: &local_env::LocalEnv) -> PathBuf {
-    env.base_data_dir.join("etcd.pid")
-}

control_plane/src/lib.rs

@@ -12,7 +12,6 @@ use std::path::Path;
 use std::process::Command;
 
 pub mod compute;
-pub mod etcd;
 pub mod local_env;
 pub mod postgresql_conf;
 pub mod safekeeper;
@@ -49,12 +48,3 @@ fn fill_rust_env_vars(cmd: &mut Command) -> &mut Command {
         cmd
     }
 }
-
-fn fill_aws_secrets_vars(mut cmd: &mut Command) -> &mut Command {
-    for env_key in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] {
-        if let Ok(value) = std::env::var(env_key) {
-            cmd = cmd.env(env_key, value);
-        }
-    }
-    cmd
-}

control_plane/src/local_env.rs

@@ -4,7 +4,6 @@
 //! script which will use local paths.
 
 use anyhow::{bail, ensure, Context};
-use reqwest::Url;
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
 use std::collections::HashMap;
@@ -15,7 +14,7 @@ use std::process::{Command, Stdio};
 use utils::{
     auth::{encode_from_key_file, Claims, Scope},
     postgres_backend::AuthType,
-    zid::{NodeId, ZTenantId, ZTenantTimelineId, ZTimelineId},
+    zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId},
 };
 
 use crate::safekeeper::SafekeeperNode;
@@ -60,7 +59,13 @@ pub struct LocalEnv {
     #[serde(default)]
     pub private_key_path: PathBuf,
 
-    pub etcd_broker: EtcdBroker,
+    // A comma separated broker (etcd) endpoints for storage nodes coordination, e.g. 'http://127.0.0.1:2379'.
+    #[serde(default)]
+    pub broker_endpoints: Option<String>,
+
+    /// A prefix to all to any key when pushing/polling etcd from a node.
+    #[serde(default)]
+    pub broker_etcd_prefix: Option<String>,
 
     pub pageserver: PageServerConf,
 
@@ -76,75 +81,11 @@ pub struct LocalEnv {
     branch_name_mappings: HashMap<String, Vec<(ZTenantId, ZTimelineId)>>,
 }
 
-/// Etcd broker config for cluster internal communication.
-#[serde_as]
-#[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)]
-pub struct EtcdBroker {
-    /// A prefix to all to any key when pushing/polling etcd from a node.
-    #[serde(default)]
-    pub broker_etcd_prefix: Option<String>,
-
-    /// Broker (etcd) endpoints for storage nodes coordination, e.g. 'http://127.0.0.1:2379'.
-    #[serde(default)]
-    #[serde_as(as = "Vec<DisplayFromStr>")]
-    pub broker_endpoints: Vec<Url>,
-
-    /// Etcd binary path to use.
-    #[serde(default)]
-    pub etcd_binary_path: PathBuf,
-}
-
-impl EtcdBroker {
-    pub fn locate_etcd() -> anyhow::Result<PathBuf> {
-        let which_output = Command::new("which")
-            .arg("etcd")
-            .output()
-            .context("Failed to run 'which etcd' command")?;
-        let stdout = String::from_utf8_lossy(&which_output.stdout);
-        ensure!(
-            which_output.status.success(),
-            "'which etcd' invocation failed. Status: {}, stdout: {stdout}, stderr: {}",
-            which_output.status,
-            String::from_utf8_lossy(&which_output.stderr)
-        );
-
-        let etcd_path = PathBuf::from(stdout.trim());
-        ensure!(
-            etcd_path.is_file(),
-            "'which etcd' invocation was successful, but the path it returned is not a file or does not exist: {}",
-            etcd_path.display()
-        );
-
-        Ok(etcd_path)
-    }
-
-    pub fn comma_separated_endpoints(&self) -> String {
-        self.broker_endpoints
-            .iter()
-            .map(|url| {
-                // URL by default adds a '/' path at the end, which is not what etcd CLI wants.
-                let url_string = url.as_str();
-                if url_string.ends_with('/') {
-                    &url_string[0..url_string.len() - 1]
-                } else {
-                    url_string
-                }
-            })
-            .fold(String::new(), |mut comma_separated_urls, url| {
-                if !comma_separated_urls.is_empty() {
-                    comma_separated_urls.push(',');
-                }
-                comma_separated_urls.push_str(url);
-                comma_separated_urls
-            })
-    }
-}
-
 #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)]
 #[serde(default)]
 pub struct PageServerConf {
     // node id
-    pub id: NodeId,
+    pub id: ZNodeId,
     // Pageserver connection settings
     pub listen_pg_addr: String,
     pub listen_http_addr: String,
@@ -159,7 +100,7 @@ pub struct PageServerConf {
 impl Default for PageServerConf {
     fn default() -> Self {
         Self {
-            id: NodeId(0),
+            id: ZNodeId(0),
             listen_pg_addr: String::new(),
             listen_http_addr: String::new(),
             auth_type: AuthType::Trust,
@@ -171,23 +112,19 @@ impl Default for PageServerConf {
 #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)]
 #[serde(default)]
 pub struct SafekeeperConf {
-    pub id: NodeId,
+    pub id: ZNodeId,
     pub pg_port: u16,
     pub http_port: u16,
     pub sync: bool,
-    pub remote_storage: Option<String>,
-    pub backup_threads: Option<u32>,
 }
 
 impl Default for SafekeeperConf {
     fn default() -> Self {
         Self {
-            id: NodeId(0),
+            id: ZNodeId(0),
             pg_port: 0,
             http_port: 0,
             sync: true,
-            remote_storage: None,
-            backup_threads: None,
         }
     }
 }
@@ -247,7 +184,12 @@ impl LocalEnv {
             if old_timeline_id == &timeline_id {
                 Ok(())
             } else {
-                bail!("branch '{branch_name}' is already mapped to timeline {old_timeline_id}, cannot map to another timeline {timeline_id}");
+                bail!(
+                    "branch '{}' is already mapped to timeline {}, cannot map to another timeline {}",
+                    branch_name,
+                    old_timeline_id,
+                    timeline_id
+                );
             }
         } else {
             existing_values.push((tenant_id, timeline_id));
@@ -283,7 +225,7 @@ impl LocalEnv {
     ///
     /// Unlike 'load_config', this function fills in any defaults that are missing
     /// from the config file.
-    pub fn parse_config(toml: &str) -> anyhow::Result<Self> {
+    pub fn create_config(toml: &str) -> anyhow::Result<Self> {
         let mut env: LocalEnv = toml::from_str(toml)?;
 
         // Find postgres binaries.
@@ -296,11 +238,26 @@ impl LocalEnv {
                 env.pg_distrib_dir = cwd.join("tmp_install")
             }
         }
+        if !env.pg_distrib_dir.join("bin/postgres").exists() {
+            bail!(
+                "Can't find postgres binary at {}",
+                env.pg_distrib_dir.display()
+            );
+        }
 
         // Find zenith binaries.
         if env.zenith_distrib_dir == Path::new("") {
             env.zenith_distrib_dir = env::current_exe()?.parent().unwrap().to_owned();
         }
+        for binary in ["pageserver", "safekeeper"] {
+            if !env.zenith_distrib_dir.join(binary).exists() {
+                bail!(
+                    "Can't find binary '{}' in zenith distrib dir '{}'",
+                    binary,
+                    env.zenith_distrib_dir.display()
+                );
+            }
+        }
 
         // If no initial tenant ID was given, generate it.
         if env.default_tenant_id.is_none() {
@@ -389,42 +346,11 @@ impl LocalEnv {
             base_path != Path::new(""),
             "repository base path is missing"
         );
-
         ensure!(
             !base_path.exists(),
             "directory '{}' already exists. Perhaps already initialized?",
             base_path.display()
         );
-        if !self.pg_distrib_dir.join("bin/postgres").exists() {
-            bail!(
-                "Can't find postgres binary at {}",
-                self.pg_distrib_dir.display()
-            );
-        }
-        for binary in ["pageserver", "safekeeper"] {
-            if !self.zenith_distrib_dir.join(binary).exists() {
-                bail!(
-                    "Can't find binary '{}' in zenith distrib dir '{}'",
-                    binary,
-                    self.zenith_distrib_dir.display()
-                );
-            }
-        }
-
-        for binary in ["pageserver", "safekeeper"] {
-            if !self.zenith_distrib_dir.join(binary).exists() {
-                bail!(
-                    "Can't find binary '{binary}' in zenith distrib dir '{}'",
-                    self.zenith_distrib_dir.display()
-                );
-            }
-        }
-        if !self.pg_distrib_dir.join("bin/postgres").exists() {
-            bail!(
-                "Can't find postgres binary at {}",
-                self.pg_distrib_dir.display()
-            );
-        }
 
         fs::create_dir(&base_path)?;
 
@@ -482,35 +408,7 @@ impl LocalEnv {
 
 fn base_path() -> PathBuf {
     match std::env::var_os("ZENITH_REPO_DIR") {
-        Some(val) => PathBuf::from(val),
-        None => PathBuf::from(".zenith"),
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn simple_conf_parsing() {
-        let simple_conf_toml = include_str!("../simple.conf");
-        let simple_conf_parse_result = LocalEnv::parse_config(simple_conf_toml);
-        assert!(
-            simple_conf_parse_result.is_ok(),
-            "failed to parse simple config {simple_conf_toml}, reason: {simple_conf_parse_result:?}"
-        );
-
-        let string_to_replace = "broker_endpoints = ['http://127.0.0.1:2379']";
-        let spoiled_url_str = "broker_endpoints = ['!@$XOXO%^&']";
-        let spoiled_url_toml = simple_conf_toml.replace(string_to_replace, spoiled_url_str);
-        assert!(
-            spoiled_url_toml.contains(spoiled_url_str),
-            "Failed to replace string {string_to_replace} in the toml file {simple_conf_toml}"
-        );
-        let spoiled_url_parse_result = LocalEnv::parse_config(&spoiled_url_toml);
-        assert!(
-            spoiled_url_parse_result.is_err(),
-            "expected toml with invalid Url {spoiled_url_toml} to fail the parsing, but got {spoiled_url_parse_result:?}"
-        );
+        Some(val) => PathBuf::from(val.to_str().unwrap()),
+        None => ".zenith".into(),
     }
 }

control_plane/src/safekeeper.rs

@@ -18,12 +18,12 @@ use thiserror::Error;
 use utils::{
     connstring::connection_address,
     http::error::HttpErrorBody,
-    zid::{NodeId, ZTenantId, ZTimelineId},
+    zid::{ZNodeId, ZTenantId, ZTimelineId},
 };
 
 use crate::local_env::{LocalEnv, SafekeeperConf};
 use crate::storage::PageServerNode;
-use crate::{fill_aws_secrets_vars, fill_rust_env_vars, read_pidfile};
+use crate::{fill_rust_env_vars, read_pidfile};
 
 #[derive(Error, Debug)]
 pub enum SafekeeperHttpError {
@@ -52,7 +52,7 @@ impl ResponseErrorMessageExt for Response {
         Err(SafekeeperHttpError::Response(
             match self.json::<HttpErrorBody>() {
                 Ok(err_body) => format!("Error: {}", err_body.msg),
-                Err(_) => format!("Http error ({}) at {url}.", status.as_u16()),
+                Err(_) => format!("Http error ({}) at {}.", status.as_u16(), url),
             },
         ))
     }
@@ -65,7 +65,7 @@ impl ResponseErrorMessageExt for Response {
 //
 #[derive(Debug)]
 pub struct SafekeeperNode {
-    pub id: NodeId,
+    pub id: ZNodeId,
 
     pub conf: SafekeeperConf,
 
@@ -75,12 +75,17 @@ pub struct SafekeeperNode {
     pub http_base_url: String,
 
     pub pageserver: Arc<PageServerNode>,
+
+    broker_endpoints: Option<String>,
+    broker_etcd_prefix: Option<String>,
 }
 
 impl SafekeeperNode {
     pub fn from_env(env: &LocalEnv, conf: &SafekeeperConf) -> SafekeeperNode {
         let pageserver = Arc::new(PageServerNode::from_env(env));
 
+        println!("initializing for sk {} for {}", conf.id, conf.http_port);
+
         SafekeeperNode {
             id: conf.id,
             conf: conf.clone(),
@@ -89,6 +94,8 @@ impl SafekeeperNode {
             http_client: Client::new(),
             http_base_url: format!("http://127.0.0.1:{}/v1", conf.http_port),
             pageserver,
+            broker_endpoints: env.broker_endpoints.clone(),
+            broker_etcd_prefix: env.broker_etcd_prefix.clone(),
         }
     }
 
@@ -100,7 +107,7 @@ impl SafekeeperNode {
             .unwrap()
     }
 
-    pub fn datadir_path_by_id(env: &LocalEnv, sk_id: NodeId) -> PathBuf {
+    pub fn datadir_path_by_id(env: &LocalEnv, sk_id: ZNodeId) -> PathBuf {
         env.safekeeper_data_dir(format!("sk{}", sk_id).as_ref())
     }
 
@@ -135,22 +142,12 @@ impl SafekeeperNode {
         if !self.conf.sync {
             cmd.arg("--no-sync");
         }
-
-        let comma_separated_endpoints = self.env.etcd_broker.comma_separated_endpoints();
-        if !comma_separated_endpoints.is_empty() {
-            cmd.args(&["--broker-endpoints", &comma_separated_endpoints]);
+        if let Some(ref ep) = self.broker_endpoints {
+            cmd.args(&["--broker-endpoints", ep]);
         }
-        if let Some(prefix) = self.env.etcd_broker.broker_etcd_prefix.as_deref() {
+        if let Some(prefix) = self.broker_etcd_prefix.as_deref() {
             cmd.args(&["--broker-etcd-prefix", prefix]);
         }
-        if let Some(threads) = self.conf.backup_threads {
-            cmd.args(&["--backup-threads", threads.to_string().as_ref()]);
-        }
-        if let Some(ref remote_storage) = self.conf.remote_storage {
-            cmd.args(&["--remote-storage", remote_storage]);
-        }
-
-        fill_aws_secrets_vars(&mut cmd);
 
         if !cmd.status()?.success() {
             bail!(
@@ -213,13 +210,12 @@ impl SafekeeperNode {
         let pid = Pid::from_raw(pid);
 
         let sig = if immediate {
-            print!("Stopping safekeeper {} immediately..", self.id);
+            println!("Stop safekeeper immediately");
             Signal::SIGQUIT
         } else {
-            print!("Stopping safekeeper {} gracefully..", self.id);
+            println!("Stop safekeeper gracefully");
             Signal::SIGTERM
         };
-        io::stdout().flush().unwrap();
         match kill(pid, sig) {
             Ok(_) => (),
             Err(Errno::ESRCH) => {
@@ -241,35 +237,25 @@ impl SafekeeperNode {
         // TODO Remove this "timeout" and handle it on caller side instead.
         // Shutting down may take a long time,
         // if safekeeper flushes a lot of data
-        let mut tcp_stopped = false;
         for _ in 0..100 {
-            if !tcp_stopped {
-                if let Err(err) = TcpStream::connect(&address) {
-                    tcp_stopped = true;
-                    if err.kind() != io::ErrorKind::ConnectionRefused {
-                        eprintln!("\nSafekeeper connection failed with error: {err}");
-                    }
-                }
-            }
-            if tcp_stopped {
-                // Also check status on the HTTP port
+            if let Err(_e) = TcpStream::connect(&address) {
+                println!("Safekeeper stopped receiving connections");
+
+                //Now check status
                 match self.check_status() {
-                    Err(SafekeeperHttpError::Transport(err)) if err.is_connect() => {
-                        println!("done!");
-                        return Ok(());
+                    Ok(_) => {
+                        println!("Safekeeper status is OK. Wait a bit.");
+                        thread::sleep(Duration::from_secs(1));
                     }
                     Err(err) => {
-                        eprintln!("\nSafekeeper status check failed with error: {err}");
+                        println!("Safekeeper status is: {}", err);
                         return Ok(());
                     }
-                    Ok(()) => {
-                        // keep waiting
-                    }
                 }
+            } else {
+                println!("Safekeeper still receives connections");
+                thread::sleep(Duration::from_secs(1));
             }
-            print!(".");
-            io::stdout().flush().unwrap();
-            thread::sleep(Duration::from_secs(1));
         }
 
         bail!("Failed to stop safekeeper with pid {}", pid);
@@ -294,7 +280,7 @@ impl SafekeeperNode {
         &self,
         tenant_id: ZTenantId,
         timeline_id: ZTimelineId,
-        peer_ids: Vec<NodeId>,
+        peer_ids: Vec<ZNodeId>,
     ) -> Result<()> {
         Ok(self
             .http_request(

control_plane/src/storage.rs

@@ -1,7 +1,6 @@
 use std::collections::HashMap;
 use std::io::Write;
 use std::net::TcpStream;
-use std::num::NonZeroU64;
 use std::path::PathBuf;
 use std::process::Command;
 use std::time::Duration;
@@ -12,7 +11,6 @@ use nix::errno::Errno;
 use nix::sys::signal::{kill, Signal};
 use nix::unistd::Pid;
 use pageserver::http::models::{TenantConfigRequest, TenantCreateRequest, TimelineCreateRequest};
-use pageserver::tenant_mgr::TenantInfo;
 use pageserver::timelines::TimelineInfo;
 use postgres::{Config, NoTls};
 use reqwest::blocking::{Client, RequestBuilder, Response};
@@ -27,7 +25,8 @@ use utils::{
 };
 
 use crate::local_env::LocalEnv;
-use crate::{fill_aws_secrets_vars, fill_rust_env_vars, read_pidfile};
+use crate::{fill_rust_env_vars, read_pidfile};
+use pageserver::tenant_mgr::TenantInfo;
 
 #[derive(Error, Debug)]
 pub enum PageserverHttpError {
@@ -38,12 +37,6 @@ pub enum PageserverHttpError {
     Response(String),
 }
 
-impl From<anyhow::Error> for PageserverHttpError {
-    fn from(e: anyhow::Error) -> Self {
-        Self::Response(e.to_string())
-    }
-}
-
 type Result<T> = result::Result<T, PageserverHttpError>;
 
 pub trait ResponseErrorMessageExt: Sized {
@@ -128,16 +121,6 @@ impl PageServerNode {
         );
         let listen_pg_addr_param =
             format!("listen_pg_addr='{}'", self.env.pageserver.listen_pg_addr);
-        let broker_endpoints_param = format!(
-            "broker_endpoints=[{}]",
-            self.env
-                .etcd_broker
-                .broker_endpoints
-                .iter()
-                .map(|url| format!("'{url}'"))
-                .collect::<Vec<_>>()
-                .join(",")
-        );
         let mut args = Vec::with_capacity(20);
 
         args.push("--init");
@@ -146,19 +129,8 @@ impl PageServerNode {
         args.extend(["-c", &authg_type_param]);
         args.extend(["-c", &listen_http_addr_param]);
         args.extend(["-c", &listen_pg_addr_param]);
-        args.extend(["-c", &broker_endpoints_param]);
         args.extend(["-c", &id]);
 
-        let broker_etcd_prefix_param = self
-            .env
-            .etcd_broker
-            .broker_etcd_prefix
-            .as_ref()
-            .map(|prefix| format!("broker_etcd_prefix='{prefix}'"));
-        if let Some(broker_etcd_prefix_param) = broker_etcd_prefix_param.as_deref() {
-            args.extend(["-c", broker_etcd_prefix_param]);
-        }
-
         for config_override in config_overrides {
             args.extend(["-c", config_override]);
         }
@@ -288,13 +260,12 @@ impl PageServerNode {
         let pid = Pid::from_raw(read_pidfile(&pid_file)?);
 
         let sig = if immediate {
-            print!("Stopping pageserver immediately..");
+            println!("Stop pageserver immediately");
             Signal::SIGQUIT
         } else {
-            print!("Stopping pageserver gracefully..");
+            println!("Stop pageserver gracefully");
             Signal::SIGTERM
         };
-        io::stdout().flush().unwrap();
         match kill(pid, sig) {
             Ok(_) => (),
             Err(Errno::ESRCH) => {
@@ -316,36 +287,25 @@ impl PageServerNode {
         // TODO Remove this "timeout" and handle it on caller side instead.
         // Shutting down may take a long time,
         // if pageserver checkpoints a lot of data
-        let mut tcp_stopped = false;
         for _ in 0..100 {
-            if !tcp_stopped {
-                if let Err(err) = TcpStream::connect(&address) {
-                    tcp_stopped = true;
-                    if err.kind() != io::ErrorKind::ConnectionRefused {
-                        eprintln!("\nPageserver connection failed with error: {err}");
-                    }
-                }
-            }
-            if tcp_stopped {
-                // Also check status on the HTTP port
+            if let Err(_e) = TcpStream::connect(&address) {
+                println!("Pageserver stopped receiving connections");
 
+                //Now check status
                 match self.check_status() {
-                    Err(PageserverHttpError::Transport(err)) if err.is_connect() => {
-                        println!("done!");
-                        return Ok(());
+                    Ok(_) => {
+                        println!("Pageserver status is OK. Wait a bit.");
+                        thread::sleep(Duration::from_secs(1));
                     }
                     Err(err) => {
-                        eprintln!("\nPageserver status check failed with error: {err}");
+                        println!("Pageserver status is: {}", err);
                         return Ok(());
                     }
-                    Ok(()) => {
-                        // keep waiting
-                    }
                 }
+            } else {
+                println!("Pageserver still receives connections");
+                thread::sleep(Duration::from_secs(1));
             }
-            print!(".");
-            io::stdout().flush().unwrap();
-            thread::sleep(Duration::from_secs(1));
         }
 
         bail!("Failed to stop pageserver with pid {}", pid);
@@ -417,15 +377,6 @@ impl PageServerNode {
                     .map(|x| x.parse::<usize>())
                     .transpose()?,
                 pitr_interval: settings.get("pitr_interval").map(|x| x.to_string()),
-                walreceiver_connect_timeout: settings
-                    .get("walreceiver_connect_timeout")
-                    .map(|x| x.to_string()),
-                lagging_wal_timeout: settings.get("lagging_wal_timeout").map(|x| x.to_string()),
-                max_lsn_wal_lag: settings
-                    .get("max_lsn_wal_lag")
-                    .map(|x| x.parse::<NonZeroU64>())
-                    .transpose()
-                    .context("Failed to parse 'max_lsn_wal_lag' as non zero integer")?,
             })
             .send()?
             .error_from_body()?
@@ -449,41 +400,22 @@ impl PageServerNode {
                 tenant_id,
                 checkpoint_distance: settings
                     .get("checkpoint_distance")
-                    .map(|x| x.parse::<u64>())
-                    .transpose()
-                    .context("Failed to parse 'checkpoint_distance' as an integer")?,
+                    .map(|x| x.parse::<u64>().unwrap()),
                 compaction_target_size: settings
                     .get("compaction_target_size")
-                    .map(|x| x.parse::<u64>())
-                    .transpose()
-                    .context("Failed to parse 'compaction_target_size' as an integer")?,
+                    .map(|x| x.parse::<u64>().unwrap()),
                 compaction_period: settings.get("compaction_period").map(|x| x.to_string()),
                 compaction_threshold: settings
                     .get("compaction_threshold")
-                    .map(|x| x.parse::<usize>())
-                    .transpose()
-                    .context("Failed to parse 'compaction_threshold' as an integer")?,
+                    .map(|x| x.parse::<usize>().unwrap()),
                 gc_horizon: settings
                     .get("gc_horizon")
-                    .map(|x| x.parse::<u64>())
-                    .transpose()
-                    .context("Failed to parse 'gc_horizon' as an integer")?,
+                    .map(|x| x.parse::<u64>().unwrap()),
                 gc_period: settings.get("gc_period").map(|x| x.to_string()),
                 image_creation_threshold: settings
                     .get("image_creation_threshold")
-                    .map(|x| x.parse::<usize>())
-                    .transpose()
-                    .context("Failed to parse 'image_creation_threshold' as non zero integer")?,
+                    .map(|x| x.parse::<usize>().unwrap()),
                 pitr_interval: settings.get("pitr_interval").map(|x| x.to_string()),
-                walreceiver_connect_timeout: settings
-                    .get("walreceiver_connect_timeout")
-                    .map(|x| x.to_string()),
-                lagging_wal_timeout: settings.get("lagging_wal_timeout").map(|x| x.to_string()),
-                max_lsn_wal_lag: settings
-                    .get("max_lsn_wal_lag")
-                    .map(|x| x.parse::<NonZeroU64>())
-                    .transpose()
-                    .context("Failed to parse 'max_lsn_wal_lag' as non zero integer")?,
             })
             .send()?
             .error_from_body()?;
@@ -528,3 +460,12 @@ impl PageServerNode {
         Ok(timeline_info_response)
     }
 }
+
+fn fill_aws_secrets_vars(mut cmd: &mut Command) -> &mut Command {
+    for env_key in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] {
+        if let Ok(value) = std::env::var(env_key) {
+            cmd = cmd.env(env_key, value);
+        }
+    }
+    cmd
+}

docker-entrypoint.sh

@@ -1,20 +1,13 @@
 #!/bin/sh
 set -eux
 
-broker_endpoints_param="${BROKER_ENDPOINT:-absent}"
-if [ "$broker_endpoints_param" != "absent" ]; then
-    broker_endpoints_param="-c broker_endpoints=['$broker_endpoints_param']"
-else
-    broker_endpoints_param=''
-fi
-
 if [ "$1" = 'pageserver' ]; then
     if [ ! -d "/data/tenants" ]; then
         echo "Initializing pageserver data directory"
-        pageserver --init -D /data -c "pg_distrib_dir='/usr/local'" -c "id=10" $broker_endpoints_param
+        pageserver --init -D /data -c "pg_distrib_dir='/usr/local'" -c "id=10"
     fi
     echo "Staring pageserver at 0.0.0.0:6400"
-    pageserver -c "listen_pg_addr='0.0.0.0:6400'" -c "listen_http_addr='0.0.0.0:9898'" $broker_endpoints_param -D /data
+    pageserver -c "listen_pg_addr='0.0.0.0:6400'" -c "listen_http_addr='0.0.0.0:9898'" -D /data
 else
     "$@"
 fi

docs/README.md

@@ -6,7 +6,7 @@
 - [docker.md](docker.md) — Docker images and building pipeline.
 - [glossary.md](glossary.md) — Glossary of all the terms used in codebase.
 - [multitenancy.md](multitenancy.md) — how multitenancy is organized in the pageserver and Zenith CLI.
-- [sourcetree.md](sourcetree.md) — Overview of the source tree layout.
+- [sourcetree.md](sourcetree.md) — Overview of the source tree layeout.
 - [pageserver/README.md](/pageserver/README.md) — pageserver overview.
 - [postgres_ffi/README.md](/libs/postgres_ffi/README.md) — Postgres FFI overview.
 - [test_runner/README.md](/test_runner/README.md) — tests infrastructure overview.

docs/core_changes.md

@@ -188,7 +188,7 @@ Not currently committed but proposed:
 3. Prefetching
 - Why?
   As far as pages in Zenith are loaded on demand, to reduce node startup time
-  and also speedup some massive queries we need some mechanism for bulk loading to
+  and also sppedup some massive queries we need some mechanism for bulk loading to
   reduce page request round-trip overhead.
 
   Currently Postgres is supporting prefetching only for bitmap scan.

docs/docker.md

@@ -1,20 +1,20 @@
-# Docker images of Neon
+# Docker images of Zenith
 
 ## Images
 
 Currently we build two main images:
 
-- [neondatabase/neon](https://hub.docker.com/repository/docker/zenithdb/zenith) — image with pre-built `pageserver`, `safekeeper` and `proxy` binaries and all the required runtime dependencies. Built from [/Dockerfile](/Dockerfile).
-- [neondatabase/compute-node](https://hub.docker.com/repository/docker/zenithdb/compute-node) — compute node image with pre-built Postgres binaries from [neondatabase/postgres](https://github.com/neondatabase/postgres).
+- [zenithdb/zenith](https://hub.docker.com/repository/docker/zenithdb/zenith) — image with pre-built `pageserver`, `safekeeper` and `proxy` binaries and all the required runtime dependencies. Built from [/Dockerfile](/Dockerfile).
+- [zenithdb/compute-node](https://hub.docker.com/repository/docker/zenithdb/compute-node) — compute node image with pre-built Postgres binaries from [zenithdb/postgres](https://github.com/zenithdb/postgres).
 
-And additional intermediate image:
+And additional intermediate images:
 
-- [neondatabase/compute-tools](https://hub.docker.com/repository/docker/neondatabase/compute-tools) — compute node configuration management tools.
+- [zenithdb/compute-tools](https://hub.docker.com/repository/docker/zenithdb/compute-tools) — compute node configuration management tools.
 
 ## Building pipeline
 
-We build all images after a successful `release` tests run and push automatically to Docker Hub with two parallel CI jobs
+1. Image `zenithdb/compute-tools` is re-built automatically.
 
-1. `neondatabase/compute-tools` and `neondatabase/compute-node`
+2. Image `zenithdb/compute-node` is built independently in the [zenithdb/postgres](https://github.com/zenithdb/postgres) repo.
 
-2. `neondatabase/neon`
+3. Image `zenithdb/zenith` is built in this repo after a successful `release` tests run and pushed to Docker Hub automatically.

docs/glossary.md

@@ -2,7 +2,7 @@
 
 ### Authentication
 
-### Backpressure
+### Backpresssure
 
 Backpressure is used to limit the lag between pageserver and compute node or WAL service.
 
@@ -21,7 +21,7 @@ NOTE:It has nothing to do with PostgreSQL pg_basebackup.
 
 ### Branch
 
-We can create branch at certain LSN using `neon_local timeline branch` command.
+We can create branch at certain LSN using `zenith timeline branch` command.
 Each Branch lives in a corresponding timeline[] and has an ancestor[].
 
 
@@ -91,7 +91,7 @@ The layer map tracks what layers exist in a timeline.
 
 ### Layered repository
 
-Neon repository implementation that keeps data in layers.
+Zenith repository implementation that keeps data in layers.
 ### LSN
 
 The Log Sequence Number (LSN) is a unique identifier of the WAL record[] in the WAL log.
@@ -101,7 +101,7 @@ It is printed as two hexadecimal numbers of up to 8 digits each, separated by a
 Check also [PostgreSQL doc about pg_lsn type](https://www.postgresql.org/docs/devel/datatype-pg-lsn.html)
 Values can be compared to calculate the volume of WAL data that separates them, so they are used to measure the progress of replication and recovery.
 
-In Postgres and Neon LSNs are used to describe certain points in WAL handling.
+In postgres and Zenith lsns are used to describe certain points in WAL handling.
 
 PostgreSQL LSNs and functions to monitor them:
 * `pg_current_wal_insert_lsn()` - Returns the current write-ahead log insert location.
@@ -111,13 +111,13 @@ PostgreSQL LSNs and functions to monitor them:
 * `pg_last_wal_replay_lsn ()` - Returns the last write-ahead log location that has been replayed during recovery. If recovery is still in progress this will increase monotonically.
 [source PostgreSQL documentation](https://www.postgresql.org/docs/devel/functions-admin.html):
 
-Neon safekeeper LSNs. For more check [safekeeper/README_PROTO.md](/safekeeper/README_PROTO.md)
+Zenith safekeeper LSNs. For more check [safekeeper/README_PROTO.md](/safekeeper/README_PROTO.md)
 * `CommitLSN`: position in WAL confirmed by quorum safekeepers.
 * `RestartLSN`: position in WAL confirmed by all safekeepers.
 * `FlushLSN`: part of WAL persisted to the disk by safekeeper.
-* `VCL`: the largest LSN for which we can guarantee availability of all prior records.
+* `VCL`: the largerst LSN for which we can guarantee availablity of all prior records.
 
-Neon pageserver LSNs:
+Zenith pageserver LSNs:
 * `last_record_lsn` - the end of last processed WAL record.
 * `disk_consistent_lsn` - data is known to be fully flushed and fsync'd to local disk on pageserver up to this LSN.
 * `remote_consistent_lsn` - The last LSN that is synced to remote storage and is guaranteed to survive pageserver crash.
@@ -132,7 +132,7 @@ This is the unit of data exchange between compute node and pageserver.
 
 ### Pageserver
 
-Neon storage engine: repositories + wal receiver + page service + wal redo.
+Zenith storage engine: repositories + wal receiver + page service + wal redo.
 
 ### Page service
 
@@ -184,10 +184,10 @@ relation exceeds that size, it is split into multiple segments.
 SLRUs include pg_clog, pg_multixact/members, and
 pg_multixact/offsets. There are other SLRUs in PostgreSQL, but
 they don't need to be stored permanently (e.g. pg_subtrans),
-or we do not support them in neon yet (pg_commit_ts).
+or we do not support them in zenith yet (pg_commit_ts).
 
 ### Tenant (Multitenancy)
-Tenant represents a single customer, interacting with Neon.
+Tenant represents a single customer, interacting with Zenith.
 Wal redo[] activity, timelines[], layers[] are managed for each tenant independently.
 One pageserver[] can serve multiple tenants at once.
 One safekeeper

docs/multitenancy.md

@@ -6,7 +6,7 @@ Zenith supports multitenancy. One pageserver can serve multiple tenants at once.
 
 ### Tenants in other commands
 
-By default during `zenith init` new tenant is created on the pageserver. Newly created tenant's id is saved to cli config, so other commands can use it automatically if no direct argument `--tenantid=<tenantid>` is provided. So generally tenantid more frequently appears in internal pageserver interface. Its commands take tenantid argument to distinguish to which tenant operation should be applied. CLI support creation of new tenants.
+By default during `zenith init` new tenant is created on the pageserver. Newly created tenant's id is saved to cli config, so other commands can use it automatically if no direct arugment `--tenantid=<tenantid>` is provided. So generally tenantid more frequently appears in internal pageserver interface. Its commands take tenantid argument to distinguish to which tenant operation should be applied. CLI support creation of new tenants.
 
 Examples for cli:
 

docs/rfcs/002-storage.md

@@ -77,7 +77,7 @@ Upon storage node restart recent WAL files are applied to appropriate pages and
 
 ### **Checkpointing**
 
-No such mechanism is needed. Or we may look at the storage node as at kind of continuous checkpointer.
+No such mechanism is needed. Or we may look at the storage node as at kind of continuous chekpointer.
 
 ### **Full page writes (torn page protection)**
 
@@ -111,13 +111,13 @@ Since we are storing page diffs of variable sizes there is no structural depende
 
 ### **Chunk metadata**
 
-Chunk metadata is a file lies in chunk directory that stores info about current snapshots and PITR regions. Chunk should always consult this data when merging SSTables and applying delete markers.
+Chunk metadata is a file lies in chunk directory that stores info about current snapshots and PITR regions. Chunck should always consult this data when merging SSTables and applying delete markers.
 
 ### **Chunk splitting**
 
 *(NB: following paragraph is about how to avoid page splitting)*
 
-When chunks hits some soft storage limit (let's say 100Gb) it should be split in half and global metadata about chunk boundaries should be updated. Here i assume that chunk split is a local operation happening on single node. Process of chink splitting should look like following:
+When chunks hits some soft storage limit (let's say 100Gb) it should be split in half and global matadata about chunk boundaries should be updated. Here i assume that chunk split is a local operation happening on single node. Process of chink splitting should look like following:
 
 1. Find separation key and spawn two new chunks with [lo, mid) [mid, hi) boundaries.
 
@@ -166,7 +166,7 @@ Multi-tenant storage makes sense even on a laptop, when you work with different
 
 Few databases are stored in one chunk, replicated three times
 
-- When database can't fit into one storage node it can occupy lots of chunks that were split while database was growing. Chunk placement on nodes is controlled by us with some automatization, but we always may manually move chunks around the cluster.
+- When database can't fit into one storage node it can occupy lots of chunks that were split while database was growing. Chunk placement on nodes is controlled by us with some automatization, but we alway may manually move chunks around the cluster.
 
 <img width="940" alt="Screenshot_2021-02-22_at_16 49 10" src="https://user-images.githubusercontent.com/284219/108729815-fb071e00-753b-11eb-86e0-be6703e47d82.png">
 

docs/rfcs/003-laptop-cli.md

@@ -123,7 +123,7 @@ Show currently attached storages. For example:
 > zenith storage list
 NAME            USED    TYPE                OPTIONS          PATH
 local           5.1G    zenith-local                         /opt/zenith/store/local
-local.compr     20.4G   zenith-local        compression=on    /opt/zenith/store/local.compr
+local.compr     20.4G   zenith-local        comression=on    /opt/zenith/store/local.compr
 zcloud          60G     zenith-remote                        zenith.tech/stas/mystore
 s3tank          80G     S3
 ```
@@ -136,9 +136,9 @@ s3tank          80G     S3
 
 ## pg
 
-Manages postgres data directories and can start postgres instances with proper configuration. An experienced user may avoid using that (except pg create) and configure/run postgres by themselves.
+Manages postgres data directories and can start postgreses with proper configuration. An experienced user may avoid using that (except pg create) and configure/run postgres by themself.
 
-Pg is a term for a single postgres running on some data. I'm trying to avoid separation of datadir management and postgres instance management -- both that concepts bundled here together.
+Pg is a term for a single postgres running on some data. I'm trying to avoid here separation of datadir management and postgres instance management -- both that concepts bundled here together.
 
 **zenith pg create** [--no-start --snapshot --cow] -s storage-name -n pgdata
 

docs/rfcs/005-zenith_local.md

@@ -31,7 +31,7 @@ Ideally, just one binary that incorporates all elements we need.
 
 #### Components:
 
-- **zenith-CLI** - interface for end-users.  Turns commands to REST requests and handles responses to show them in a user-friendly way.  
+- **zenith-CLI** - interface for end-users.  Turns commands to REST requests and handles responces to show them in a user-friendly way.  
 CLI proposal is here https://github.com/libzenith/rfcs/blob/003-laptop-cli.md/003-laptop-cli.md
 WIP code is here: https://github.com/libzenith/postgres/tree/main/pageserver/src/bin/cli
 

docs/rfcs/006-laptop-cli-v2-CLI.md

@@ -25,9 +25,9 @@ To make changes in the catalog you need to run compute nodes
 zenith start /home/pipedpiper/northwind:main -- starts a compute instance
 zenith start zenith://zenith.tech/northwind:main -- starts a compute instance in the cloud
 -- you can start a compute node against any hash or branch
-zenith start /home/pipedpiper/northwind:experimental --port 8008 -- start another compute instance (on different port)
+zenith start /home/pipedpiper/northwind:experimental --port 8008 -- start anothe compute instance (on different port)
 -- you can start a compute node against any hash or branch
-zenith start /home/pipedpiper/northwind:<hash> --port 8009 -- start another compute instance (on different port)
+zenith start /home/pipedpiper/northwind:<hash> --port 8009 -- start anothe compute instance (on different port)
 
 -- After running some DML you can run 
 -- zenith status and see how there are two WAL streams one on top of 

docs/rfcs/006-laptop-cli-v2-repository-structure.md

@@ -121,7 +121,7 @@ repository, launch an instance on the same branch in both clones, and
 later try to push/pull between them? Perhaps create a new timeline
 every time you start up an instance? Then you would detect that the
 timelines have diverged. That would match with the "epoch" concept
-that we have in the WAL safekeeper
+that we have in the WAL safekeepr
 
 ### zenith checkout/commit
 

docs/rfcs/009-snapshot-first-storage-cli.md

@@ -2,9 +2,9 @@ While working on export/import commands, I understood that they fit really well
 
 We may think about backups as snapshots in a different format (i.e plain pgdata format, basebackup tar format, WAL-G format (if they want to support it) and so on). They use same storage API, the only difference is the code that packs/unpacks files.
 
-Even if zenith aims to maintains durability using it's own snapshots, backups will be useful for uploading data from postgres to zenith.
+Even if zenith aims to maintains durability using it's own snapshots, backups will be useful for uploading data from postges to zenith.
 
-So here is an attempt to design consistent CLI for different usage scenarios:
+So here is an attemt to design consistent CLI for diferent usage scenarios:
 
 #### 1. Start empty pageserver.
 That is what we have now.

docs/rfcs/009-snapshot-first-storage-pitr.md

@@ -3,7 +3,7 @@
 GetPage@LSN can be called with older LSNs, and the page server needs
 to be able to reconstruct older page versions. That's needed for
 having read-only replicas that lag behind the primary, or that are
-"anchored" at an older LSN, and internally in the page server when you
+"anchored" at an older LSN, and internally in the page server whne you
 branch at an older point in time. How do you do that?
 
 For now, I'm not considering incremental snapshots at all. I don't
@@ -192,7 +192,7 @@ for a particular relation readily available alongside the snapshot
 files, and you don't need to track what snapshot LSNs exist
 separately.
 
-(If we wanted to minimize the number of files, you could include the
+(If we wanted to minize the number of files, you could include the
 snapshot @300 and the WAL between 200 and 300 in the same file, but I
 feel it's probably better to keep them separate)
 

docs/rfcs/009-snapshot-first-storage.md

@@ -121,7 +121,7 @@ The properties of s3 that we depend on are:
 list objects
 streaming read of entire object
 read byte range from object
-streaming write new object (may use multipart upload for better reliability)
+streaming write new object (may use multipart upload for better relialibity)
 delete object (that should not disrupt an already-started read).
 
 Uploaded files, restored backups, or s3 buckets controlled by users could contain malicious content. We should always validate that objects contain the content they’re supposed to. Incorrect, Corrupt or malicious-looking contents should cause software (cloud tools, pageserver) to fail gracefully.

docs/rfcs/010-storage_details.md

@@ -40,7 +40,7 @@ b) overwrite older pages with the newer pages -- if there is no replica we proba
 
 I imagine that newly created pages would just be added to the back of PageStore (again in queue-like fashion) and this way there wouldn't be any meaningful ordering inside of that queue. When we are forming a new incremental snapshot we may prohibit any updates to the current set of pages in PageStore (giving up on single page version rule) and cut off that whole set when snapshot creation is complete.
 
-With option b) we can also treat PageStor as an uncompleted incremental snapshot.
+With option b) we can also treat PageStor as an uncompleted increamental snapshot.
 
 ### LocalStore
 
@@ -123,7 +123,7 @@ As far as I understand Bookfile/Aversion addresses versioning and serialization
 As for exact data that should go to snapshots I think it is the following for each snapshot:
 
 * format version number
-* set of key/values to interpret content (e.g. is page compression enabled, is that a full or incremental snapshot, previous snapshot id, is there WAL at the end on file, etc) -- it is up to a reader to decide what to do if some keys are missing or some unknown key are present. If we add something backward compatible to the file we can keep the version number.
+* set of key/values to interpret content (e.g. is page compression enabled, is that a full or incremental snapshot, previous snapshot id, is there WAL at the end on file, etc) -- it is up to a reader to decide what to do if some keys are missing or some unknow key are present. If we add something backward compatible to the file we can keep the version number.
 * array of [BuffTag, corresponding offset in file] for pages -- IIUC that is analogous to ToC in Bookfile
 * array of [(BuffTag, LSN), corresponding offset in file] for the WAL records
 * pages, one by one
@@ -131,7 +131,7 @@ As for exact data that should go to snapshots I think it is the following for ea
 
 It is also important to be able to load metadata quickly since it would be one of the main factors impacting the time of page server start. E.g. if would store/cache about 10TB of data per page server, the size of uncompressed page references would be about 30GB (10TB / ( 8192 bytes page size / ( ~18 bytes per ObjectTag + 8 bytes offset in the file))).
 
-1) Since our ToC/array of entries can be sorted by ObjectTag we can store the whole BufferTag only when relation_id is changed and store only delta-encoded offsets for a given relation. That would reduce the average per-page metadata size to something less than 4 bytes instead of 26 (assuming that pages would follow the same order and offset deltas would be small).
+1) Since our ToC/array of entries can be sorted by ObjectTag we can store the whole BufferTag only when realtion_id is changed and store only delta-encoded offsets for a given relation. That would reduce the average per-page metadata size to something less than 4 bytes instead of 26 (assuming that pages would follow the same order and offset delatas would be small).
 2) It makes sense to keep ToC at the beginning of the file to avoid extra seeks to locate it. Doesn't matter too much with the local files but matters on S3 -- if we are accessing a lot of ~1Gb files with the size of metadata ~ 1Mb then the time to transfer this metadata would be comparable with access latency itself (which is about a half of a second). So by slurping metadata with one read of file header instead of N reads we can improve the speed of page server start by this N factor.
 
 I think both of that optimizations can be done later, but that is something to keep in mind when we are designing our storage serialization routines.

docs/rfcs/013-term-history.md

@@ -7,13 +7,13 @@ and e.g. prevents electing two proposers with the same term -- it is actually
 called `term` in the code. The second, called `epoch`, reflects progress of log
 receival and this might lag behind `term`; safekeeper switches to epoch `n` when
 it has received all committed log records from all `< n` terms. This roughly
-corresponds to proposed in
+correspones to proposed in
 
 https://github.com/zenithdb/rfcs/pull/3/files
 
 
 This makes our biggest our difference from Raft. In Raft, every log record is
-stamped with term in which it was generated; while we essentially store in
+stamped with term in which it was generated; while we essentialy store in
 `epoch` only the term of the highest record on this safekeeper -- when we know
 it -- because during recovery generally we don't, and `epoch` is bumped directly
 to the term of the proposer who performs the recovery when it is finished. It is

docs/rfcs/015-storage-messaging.md

@@ -124,7 +124,7 @@ Each storage node can subscribe to the relevant sets of keys and maintain a loca
 
 ### Safekeeper address discovery
 
-During the startup safekeeper should publish the address he is listening on as the part of `{"sk_#{sk_id}" => ip_address}`. Then the pageserver can resolve `sk_#{sk_id}` to the actual address. This way it would work both locally and in the cloud setup. Safekeeper should have `--advertised-address` CLI option so that we can listen on e.g. 0.0.0.0 but advertise something more useful.
+During the startup safekeeper should publish the address he is listening on as the part of `{"sk_#{sk_id}" => ip_address}`. Then the pageserver can resolve `sk_#{sk_id}` to the actual address. This way it would work both locally and in the cloud setup. Safekeeper should have `--advertised-address` CLI option so that we can listen on e.g. 0.0.0.0 but advertize something more useful.
 
 ### Safekeeper behavior
 
@@ -195,7 +195,7 @@ sequenceDiagram
     PS1->>SK1: start replication
 ```
 
-#### Behaviour of services during typical operations
+#### Behavour of services during typical operations
 
 ```mermaid
 sequenceDiagram
@@ -250,7 +250,7 @@ sequenceDiagram
     PS2->>M: Register downloaded timeline
     PS2->>M: Get safekeepers for timeline, subscribe to changes
     PS2->>SK1: Start replication to catch up
-    note over O: PS2 caught up, time to switch compute
+    note over O: PS2 catched up, time to switch compute
     O->>C: Restart compute with new pageserver url in config
     note over C: Wal push is restarted
     loop request pages

docs/rfcs/README.md

@@ -49,7 +49,7 @@ topics.
 
 RFC lifecycle:
 
-- Should be submitted in a pull request with and full RFC text in a committed markdown file and copy of the Summary and Motivation sections also included in the PR body.
+- Should be submitted in a pull request with and full RFC text in a commited markdown file and copy of the Summary and Motivation sections also included in the PR body.
 - RFC should be published for review before most of the actual code is written. This isn’t a strict rule, don’t hesitate to experiment and build a POC in parallel with writing an RFC.
 - Add labels to the PR in the same manner as you do Issues. Example TBD
 - Request the review from your peers. Reviewing the RFCs from your peers is a priority, same as reviewing the actual code.

docs/rfcs/cluster-size-limits.md

@@ -22,8 +22,8 @@ so we don't want to give users access to the functionality that we don't think i
 
 * pageserver - calculate the size consumed by a timeline and add it to the feedback message.
 * safekeeper - pass feedback message from pageserver to compute.
-* compute - receive feedback message, enforce size limit based on GUC `neon.max_cluster_size`.
-* console - set and update `neon.max_cluster_size` setting
+* compute - receive feedback message, enforce size limit based on GUC `zenith.max_cluster_size`.
+* console - set and update `zenith.max_cluster_size` setting
 
 ## Proposed implementation
 
@@ -49,7 +49,7 @@ This message is received by the safekeeper and propagated to compute node as a p
 
 Finally, when compute node receives the `current_timeline_size` from safekeeper (or from pageserver directly), it updates the global variable.
 
-And then every zenith_extend() operation checks if limit is reached `(current_timeline_size > neon.max_cluster_size)` and throws `ERRCODE_DISK_FULL` error if so.
+And then every zenith_extend() operation checks if limit is reached `(current_timeline_size > zenith.max_cluster_size)` and throws `ERRCODE_DISK_FULL` error if so.
 (see Postgres error codes [https://www.postgresql.org/docs/devel/errcodes-appendix.html](https://www.postgresql.org/docs/devel/errcodes-appendix.html))
 
 TODO:
@@ -75,5 +75,5 @@ We should warn users if the limit is soon to be reached.
 ### **Security implications**
 
 We treat compute as an untrusted component. That's why we try to isolate it with secure container runtime or a VM.
-Malicious users may change the `neon.max_cluster_size`, so we need an extra size limit check.
+Malicious users may change the `zenith.max_cluster_size`, so we need an extra size limit check.
 To cover this case, we also monitor the compute node size in the console.

docs/settings.md

@@ -23,22 +23,18 @@ gc_horizon = '67108864'
 max_file_descriptors = '100'
 
 # initial superuser role name to use when creating a new tenant
-initial_superuser_name = 'cloud_admin'
-
-broker_etcd_prefix = 'neon'
-broker_endpoints = ['some://etcd']
+initial_superuser_name = 'zenith_admin'
 
 # [remote_storage]
 ```
 
-The config above shows default values for all basic pageserver settings, besides `broker_endpoints`: that one has to be set by the user,
-see the corresponding section below.
+The config above shows default values for all basic pageserver settings.
 Pageserver uses default values for all files that are missing in the config, so it's not a hard error to leave the config blank.
 Yet, it validates the config values it can (e.g. postgres install dir) and errors if the validation fails, refusing to start.
 
 Note the `[remote_storage]` section: it's a [table](https://toml.io/en/v1.0.0#table) in TOML specification and
 
-- either has to be placed in the config after the table-less values such as `initial_superuser_name = 'cloud_admin'`
+- either has to be placed in the config after the table-less values such as `initial_superuser_name = 'zenith_admin'`
 
 - or can be placed anywhere if rewritten in identical form as [inline table](https://toml.io/en/v1.0.0#inline-table): `remote_storage = {foo = 2}`
 
@@ -50,17 +46,6 @@ Example: `${PAGESERVER_BIN} -c "checkpoint_period = '100 s'" -c "remote_storage=
 
 Note that TOML distinguishes between strings and integers, the former require single or double quotes around them.
 
-#### broker_endpoints
-
-A list of endpoints (etcd currently) to connect and pull the information from.
-Mandatory, does not have a default, since requires etcd to be started as a separate process,
-and its connection url should be specified separately.
-
-#### broker_etcd_prefix
-
-A prefix to add for every etcd key used, to separate one group of related instances from another, in the same cluster.
-Default is `neon`.
-
 #### checkpoint_distance
 
 `checkpoint_distance` is the amount of incoming WAL that is held in
@@ -105,31 +90,17 @@ Interval at which garbage collection is triggered. Default is 100 s.
 
 #### image_creation_threshold
 
-L0 delta layer threshold for L1 image layer creation. Default is 3.
+L0 delta layer threshold for L1 iamge layer creation. Default is 3.
 
 #### pitr_interval
 
 WAL retention duration for PITR branching. Default is 30 days.
 
-#### walreceiver_connect_timeout
-
-Time to wait to establish the wal receiver connection before failing
-
-#### lagging_wal_timeout
-
-Time the pageserver did not get any WAL updates from safekeeper (if any).
-Avoids lagging pageserver preemptively by forcing to switch it from stalled connections.
-
-#### max_lsn_wal_lag
-
-Difference between Lsn values of the latest available WAL on safekeepers: if currently connected safekeeper starts to lag too long and too much,
-it gets swapped to the different one.
-
 #### initial_superuser_name
 
 Name of the initial superuser role, passed to initdb when a new tenant
 is initialized. It doesn't affect anything after initialization. The
-default is Note: The default is 'cloud_admin', and the console
+default is Note: The default is 'zenith_admin', and the console
 depends on that, so if you change it, bad things will happen.
 
 #### page_cache_size
@@ -199,7 +170,7 @@ If no IAM bucket access is used during the remote storage usage, use the `AWS_AC
 
 ###### General remote storage configuration
 
-Pageserver allows only one remote storage configured concurrently and errors if parameters from multiple different remote configurations are used.
+Pagesever allows only one remote storage configured concurrently and errors if parameters from multiple different remote configurations are used.
 No default values are used for the remote storage configuration parameters.
 
 Besides, there are parameters common for all types of remote storage that can be configured, those have defaults:

docs/sourcetree.md

@@ -10,7 +10,7 @@ Intended to be used in integration tests and in CLI tools for local installation
 
 `/docs`:
 
-Documentation of the Zenith features and concepts.
+Documentaion of the Zenith features and concepts.
 Now it is mostly dev documentation.
 
 `/monitoring`:
@@ -42,13 +42,13 @@ Integration tests, written in Python using the `pytest` framework.
 
 `/vendor/postgres`:
 
-PostgreSQL source tree, with the modifications needed for Neon.
+PostgreSQL source tree, with the modifications needed for Zenith.
 
-`/vendor/postgres/contrib/neon`:
+`/vendor/postgres/contrib/zenith`:
 
 PostgreSQL extension that implements storage manager API and network communications with remote page server.
 
-`/vendor/postgres/contrib/neon_test_utils`:
+`/vendor/postgres/contrib/zenith_test_utils`:
 
 PostgreSQL extension that contains functions needed for testing and debugging.
 
@@ -91,22 +91,18 @@ so manual installation of dependencies is not recommended.
 A single virtual environment with all dependencies is described in the single `Pipfile`.
 
 ### Prerequisites
-- Install Python 3.9 (the minimal supported version) or greater.
-    - Our setup with poetry should work with newer python versions too. So feel free to open an issue with a `c/test-runner` label if something doesn't work as expected.
-    - If you have some trouble with other version you can resolve it by installing Python 3.9 separately, via [pyenv](https://github.com/pyenv/pyenv) or via system package manager e.g.:
+- Install Python 3.7 (the minimal supported version) or greater.
+    - Our setup with poetry should work with newer python versions too. So feel free to open an issue with a `c/test-runner` label if something doesnt work as expected.
+    - If you have some trouble with other version you can resolve it by installing Python 3.7 separately, via pyenv or via system package manager e.g.:
       ```bash
       # In Ubuntu
       sudo add-apt-repository ppa:deadsnakes/ppa
       sudo apt update
-      sudo apt install python3.9
+      sudo apt install python3.7
       ```
 - Install `poetry`
     - Exact version of `poetry` is not important, see installation instructions available at poetry's [website](https://python-poetry.org/docs/#installation)`.
-- Install dependencies via `./scripts/pysync`.
-    - Note that CI uses specific Python version (look for `PYTHON_VERSION` [here](https://github.com/neondatabase/docker-images/blob/main/rust/Dockerfile))
-      so if you have different version some linting tools can yield different result locally vs in the CI.
-    - You can explicitly specify which Python to use by running `poetry env use /path/to/python`, e.g. `poetry env use python3.9`.
-      This may also disable the `The currently activated Python version X.Y.Z is not supported by the project` warning.
+- Install dependencies via `./scripts/pysync`. Note that CI uses Python 3.7 so if you have different version some linting tools can yield different result locally vs in the CI.
 
 Run `poetry shell` to activate the virtual environment.
 Alternatively, use `poetry run` to run a single command in the venv, e.g. `poetry run pytest`.

libs/etcd_broker/Cargo.toml

@@ -9,7 +9,6 @@
  serde = { version = "1.0", features = ["derive"] }
  serde_json = "1"
  serde_with = "1.12.0"
- once_cell = "1.8.0"
 
  utils = { path = "../utils" }
  workspace_hack = { version = "0.1", path = "../../workspace_hack" }

libs/etcd_broker/src/lib.rs

@@ -6,7 +6,6 @@ use std::{
     str::FromStr,
 };
 
-use once_cell::sync::Lazy;
 use regex::{Captures, Regex};
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
@@ -17,22 +16,18 @@ use tokio::{sync::mpsc, task::JoinHandle};
 use tracing::*;
 use utils::{
     lsn::Lsn,
-    zid::{NodeId, ZTenantId, ZTenantTimelineId},
+    zid::{ZNodeId, ZTenantId, ZTenantTimelineId},
 };
 
-/// Default value to use for prefixing to all etcd keys with.
-/// This way allows isolating safekeeper/pageserver groups in the same etcd cluster.
-pub const DEFAULT_NEON_BROKER_ETCD_PREFIX: &str = "neon";
-
 #[derive(Debug, Deserialize, Serialize)]
 struct SafekeeperTimeline {
-    safekeeper_id: NodeId,
+    safekeeper_id: ZNodeId,
     info: SkTimelineInfo,
 }
 
 /// Published data about safekeeper's timeline. Fields made optional for easy migrations.
 #[serde_as]
-#[derive(Debug, Clone, Deserialize, Serialize)]
+#[derive(Debug, Deserialize, Serialize)]
 pub struct SkTimelineInfo {
     /// Term of the last entry.
     pub last_log_term: Option<u64>,
@@ -44,10 +39,10 @@ pub struct SkTimelineInfo {
     #[serde_as(as = "Option<DisplayFromStr>")]
     #[serde(default)]
     pub commit_lsn: Option<Lsn>,
-    /// LSN up to which safekeeper has backed WAL.
+    /// LSN up to which safekeeper offloaded WAL to s3.
     #[serde_as(as = "Option<DisplayFromStr>")]
     #[serde(default)]
-    pub backup_lsn: Option<Lsn>,
+    pub s3_wal_lsn: Option<Lsn>,
     /// LSN of last checkpoint uploaded by pageserver.
     #[serde_as(as = "Option<DisplayFromStr>")]
     #[serde(default)]
@@ -56,9 +51,7 @@ pub struct SkTimelineInfo {
     #[serde(default)]
     pub peer_horizon_lsn: Option<Lsn>,
     #[serde(default)]
-    pub safekeeper_connstr: Option<String>,
-    #[serde(default)]
-    pub pageserver_connstr: Option<String>,
+    pub safekeeper_connection_string: Option<String>,
 }
 
 #[derive(Debug, thiserror::Error)]
@@ -74,7 +67,7 @@ pub enum BrokerError {
 /// A way to control the data retrieval from a certain subscription.
 pub struct SkTimelineSubscription {
     safekeeper_timeline_updates:
-        mpsc::UnboundedReceiver<HashMap<ZTenantTimelineId, HashMap<NodeId, SkTimelineInfo>>>,
+        mpsc::UnboundedReceiver<HashMap<ZTenantTimelineId, HashMap<ZNodeId, SkTimelineInfo>>>,
     kind: SkTimelineSubscriptionKind,
     watcher_handle: JoinHandle<Result<(), BrokerError>>,
     watcher: Watcher,
@@ -84,7 +77,7 @@ impl SkTimelineSubscription {
     /// Asynchronously polls for more data from the subscription, suspending the current future if there's no data sent yet.
     pub async fn fetch_data(
         &mut self,
-    ) -> Option<HashMap<ZTenantTimelineId, HashMap<NodeId, SkTimelineInfo>>> {
+    ) -> Option<HashMap<ZTenantTimelineId, HashMap<ZNodeId, SkTimelineInfo>>> {
         self.safekeeper_timeline_updates.recv().await
     }
 
@@ -111,45 +104,68 @@ impl SkTimelineSubscription {
 /// The subscription kind to the timeline updates from safekeeper.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct SkTimelineSubscriptionKind {
-    broker_etcd_prefix: String,
+    broker_prefix: String,
     kind: SubscriptionKind,
 }
 
 impl SkTimelineSubscriptionKind {
-    pub fn all(broker_etcd_prefix: String) -> Self {
+    pub fn all(broker_prefix: String) -> Self {
         Self {
-            broker_etcd_prefix,
+            broker_prefix,
             kind: SubscriptionKind::All,
         }
     }
 
-    pub fn tenant(broker_etcd_prefix: String, tenant: ZTenantId) -> Self {
+    pub fn tenant(broker_prefix: String, tenant: ZTenantId) -> Self {
         Self {
-            broker_etcd_prefix,
+            broker_prefix,
             kind: SubscriptionKind::Tenant(tenant),
         }
     }
 
-    pub fn timeline(broker_etcd_prefix: String, timeline: ZTenantTimelineId) -> Self {
+    pub fn timeline(broker_prefix: String, timeline: ZTenantTimelineId) -> Self {
         Self {
-            broker_etcd_prefix,
+            broker_prefix,
             kind: SubscriptionKind::Timeline(timeline),
         }
     }
 
+    fn watch_regex(&self) -> Regex {
+        match self.kind {
+            SubscriptionKind::All => Regex::new(&format!(
+                r"^{}/([[:xdigit:]]+)/([[:xdigit:]]+)/safekeeper/([[:digit:]])$",
+                self.broker_prefix
+            ))
+            .expect("wrong regex for 'everything' subscription"),
+            SubscriptionKind::Tenant(tenant_id) => Regex::new(&format!(
+                r"^{}/{tenant_id}/([[:xdigit:]]+)/safekeeper/([[:digit:]])$",
+                self.broker_prefix
+            ))
+            .expect("wrong regex for 'tenant' subscription"),
+            SubscriptionKind::Timeline(ZTenantTimelineId {
+                tenant_id,
+                timeline_id,
+            }) => Regex::new(&format!(
+                r"^{}/{tenant_id}/{timeline_id}/safekeeper/([[:digit:]])$",
+                self.broker_prefix
+            ))
+            .expect("wrong regex for 'timeline' subscription"),
+        }
+    }
+
     /// Etcd key to use for watching a certain timeline updates from safekeepers.
     pub fn watch_key(&self) -> String {
         match self.kind {
-            SubscriptionKind::All => self.broker_etcd_prefix.to_string(),
+            SubscriptionKind::All => self.broker_prefix.to_string(),
             SubscriptionKind::Tenant(tenant_id) => {
-                format!("{}/{tenant_id}/safekeeper", self.broker_etcd_prefix)
+                format!("{}/{tenant_id}/safekeeper", self.broker_prefix)
             }
             SubscriptionKind::Timeline(ZTenantTimelineId {
                 tenant_id,
                 timeline_id,
             }) => format!(
                 "{}/{tenant_id}/{timeline_id}/safekeeper",
-                self.broker_etcd_prefix
+                self.broker_prefix
             ),
         }
     }
@@ -174,7 +190,6 @@ pub async fn subscribe_to_safekeeper_timeline_updates(
     subscription: SkTimelineSubscriptionKind,
 ) -> Result<SkTimelineSubscription, BrokerError> {
     info!("Subscribing to timeline updates, subscription kind: {subscription:?}");
-    let kind = subscription.clone();
 
     let (watcher, mut stream) = client
         .watch(
@@ -190,16 +205,19 @@ pub async fn subscribe_to_safekeeper_timeline_updates(
         })?;
 
     let (timeline_updates_sender, safekeeper_timeline_updates) = mpsc::unbounded_channel();
+
+    let subscription_kind = subscription.kind;
+    let regex = subscription.watch_regex();
     let watcher_handle = tokio::spawn(async move {
         while let Some(resp) = stream.message().await.map_err(|e| BrokerError::InternalError(format!(
-            "Failed to get messages from the subscription stream, kind: {:?}, error: {e}", subscription.kind
+            "Failed to get messages from the subscription stream, kind: {subscription_kind:?}, error: {e}"
         )))? {
             if resp.canceled() {
                 info!("Watch for timeline updates subscription was canceled, exiting");
                 break;
             }
 
-            let mut timeline_updates: HashMap<ZTenantTimelineId, HashMap<NodeId, SkTimelineInfo>> = HashMap::new();
+            let mut timeline_updates: HashMap<ZTenantTimelineId, HashMap<ZNodeId, SkTimelineInfo>> = HashMap::new();
             // Keep track that the timeline data updates from etcd arrive in the right order.
             // https://etcd.io/docs/v3.5/learning/api_guarantees/#isolation-level-and-consistency-of-replicas
             // > etcd does not ensure linearizability for watch operations. Users are expected to verify the revision of watch responses to ensure correct ordering.
@@ -213,16 +231,9 @@ pub async fn subscribe_to_safekeeper_timeline_updates(
                 if EventType::Put == event.event_type() {
                     if let Some(new_etcd_kv) = event.kv() {
                         let new_kv_version = new_etcd_kv.version();
-                        let (key_str, value_str) = match extract_key_value_str(new_etcd_kv) {
-                            Ok(strs) => strs,
-                            Err(e) => {
-                                error!("Failed to represent etcd KV {new_etcd_kv:?} as pair of str: {e}");
-                                continue;
-                            },
-                        };
 
-                        match parse_etcd_key_value(&subscription,  key_str, value_str) {
-                            Ok((zttid, timeline)) => {
+                        match parse_etcd_key_value(subscription_kind, &regex, new_etcd_kv) {
+                            Ok(Some((zttid, timeline))) => {
                                 match timeline_updates
                                     .entry(zttid)
                                     .or_default()
@@ -233,8 +244,6 @@ pub async fn subscribe_to_safekeeper_timeline_updates(
                                         if old_etcd_kv_version < new_kv_version {
                                             o.insert(timeline.info);
                                             timeline_etcd_versions.insert(zttid,new_kv_version);
-                                        } else {
-                                            debug!("Skipping etcd timeline update due to older version compared to one that's already stored");
                                         }
                                     }
                                     hash_map::Entry::Vacant(v) => {
@@ -243,6 +252,7 @@ pub async fn subscribe_to_safekeeper_timeline_updates(
                                     }
                                 }
                             }
+                            Ok(None) => {}
                             Err(e) => error!("Failed to parse timeline update: {e}"),
                         };
                     }
@@ -256,72 +266,64 @@ pub async fn subscribe_to_safekeeper_timeline_updates(
         }
 
         Ok(())
-    }.instrument(info_span!("etcd_broker")));
+    });
 
     Ok(SkTimelineSubscription {
-        kind,
+        kind: subscription,
         safekeeper_timeline_updates,
         watcher_handle,
         watcher,
     })
 }
 
-fn extract_key_value_str(kv: &KeyValue) -> Result<(&str, &str), BrokerError> {
-    let key = kv.key_str().map_err(|e| {
-        BrokerError::EtcdClient(e, "Failed to extract key str out of etcd KV".to_string())
-    })?;
-    let value = kv.value_str().map_err(|e| {
-        BrokerError::EtcdClient(e, "Failed to extract value str out of etcd KV".to_string())
-    })?;
-    Ok((key, value))
-}
-
-static SK_TIMELINE_KEY_REGEX: Lazy<Regex> = Lazy::new(|| {
-    Regex::new("/([[:xdigit:]]+)/([[:xdigit:]]+)/safekeeper/([[:digit:]]+)$")
-        .expect("wrong regex for safekeeper timeline etcd key")
-});
-
 fn parse_etcd_key_value(
-    subscription: &SkTimelineSubscriptionKind,
-    key_str: &str,
-    value_str: &str,
-) -> Result<(ZTenantTimelineId, SafekeeperTimeline), BrokerError> {
-    let broker_prefix = subscription.broker_etcd_prefix.as_str();
-    if !key_str.starts_with(broker_prefix) {
-        return Err(BrokerError::ParsingError(format!(
-            "KV has unexpected key '{key_str}' that does not start with broker prefix {broker_prefix}"
-        )));
-    }
-
-    let key_part = &key_str[broker_prefix.len()..];
-    let key_captures = match SK_TIMELINE_KEY_REGEX.captures(key_part) {
-        Some(captures) => captures,
-        None => {
-            return Err(BrokerError::ParsingError(format!(
-                "KV has unexpected key part '{key_part}' that does not match required regex {}",
-                SK_TIMELINE_KEY_REGEX.as_str()
-            )));
-        }
+    subscription_kind: SubscriptionKind,
+    regex: &Regex,
+    kv: &KeyValue,
+) -> Result<Option<(ZTenantTimelineId, SafekeeperTimeline)>, BrokerError> {
+    let caps = if let Some(caps) = regex.captures(kv.key_str().map_err(|e| {
+        BrokerError::EtcdClient(e, format!("Failed to represent kv {kv:?} as key str"))
+    })?) {
+        caps
+    } else {
+        return Ok(None);
     };
-    let info = serde_json::from_str(value_str).map_err(|e| {
-        BrokerError::ParsingError(format!(
-            "Failed to parse '{value_str}' as safekeeper timeline info: {e}"
-        ))
+
+    let (zttid, safekeeper_id) = match subscription_kind {
+        SubscriptionKind::All => (
+            ZTenantTimelineId::new(
+                parse_capture(&caps, 1).map_err(BrokerError::ParsingError)?,
+                parse_capture(&caps, 2).map_err(BrokerError::ParsingError)?,
+            ),
+            ZNodeId(parse_capture(&caps, 3).map_err(BrokerError::ParsingError)?),
+        ),
+        SubscriptionKind::Tenant(tenant_id) => (
+            ZTenantTimelineId::new(
+                tenant_id,
+                parse_capture(&caps, 1).map_err(BrokerError::ParsingError)?,
+            ),
+            ZNodeId(parse_capture(&caps, 2).map_err(BrokerError::ParsingError)?),
+        ),
+        SubscriptionKind::Timeline(zttid) => (
+            zttid,
+            ZNodeId(parse_capture(&caps, 1).map_err(BrokerError::ParsingError)?),
+        ),
+    };
+
+    let info_str = kv.value_str().map_err(|e| {
+        BrokerError::EtcdClient(e, format!("Failed to represent kv {kv:?} as value str"))
     })?;
-
-    let zttid = ZTenantTimelineId::new(
-        parse_capture(&key_captures, 1).map_err(BrokerError::ParsingError)?,
-        parse_capture(&key_captures, 2).map_err(BrokerError::ParsingError)?,
-    );
-    let safekeeper_id = NodeId(parse_capture(&key_captures, 3).map_err(BrokerError::ParsingError)?);
-
-    Ok((
+    Ok(Some((
         zttid,
         SafekeeperTimeline {
             safekeeper_id,
-            info,
+            info: serde_json::from_str(info_str).map_err(|e| {
+                BrokerError::ParsingError(format!(
+                    "Failed to parse '{info_str}' as safekeeper timeline info: {e}"
+                ))
+            })?,
         },
-    ))
+    )))
 }
 
 fn parse_capture<T>(caps: &Captures, index: usize) -> Result<T, String>
@@ -340,53 +342,3 @@ where
         )
     })
 }
-
-#[cfg(test)]
-mod tests {
-    use utils::zid::ZTimelineId;
-
-    use super::*;
-
-    #[test]
-    fn typical_etcd_prefix_should_be_parsed() {
-        let prefix = "neon";
-        let tenant_id = ZTenantId::generate();
-        let timeline_id = ZTimelineId::generate();
-        let all_subscription = SkTimelineSubscriptionKind {
-            broker_etcd_prefix: prefix.to_string(),
-            kind: SubscriptionKind::All,
-        };
-        let tenant_subscription = SkTimelineSubscriptionKind {
-            broker_etcd_prefix: prefix.to_string(),
-            kind: SubscriptionKind::Tenant(tenant_id),
-        };
-        let timeline_subscription = SkTimelineSubscriptionKind {
-            broker_etcd_prefix: prefix.to_string(),
-            kind: SubscriptionKind::Timeline(ZTenantTimelineId::new(tenant_id, timeline_id)),
-        };
-
-        let typical_etcd_kv_strs = [
-            (
-                format!("{prefix}/{tenant_id}/{timeline_id}/safekeeper/1"),
-                r#"{"last_log_term":231,"flush_lsn":"0/241BB70","commit_lsn":"0/241BB70","backup_lsn":"0/2000000","remote_consistent_lsn":"0/0","peer_horizon_lsn":"0/16960E8","safekeeper_connstr":"something.local:1234","pageserver_connstr":"postgresql://(null):@somethine.else.local:3456"}"#,
-            ),
-            (
-                format!("{prefix}/{tenant_id}/{timeline_id}/safekeeper/13"),
-                r#"{"last_log_term":231,"flush_lsn":"0/241BB70","commit_lsn":"0/241BB70","backup_lsn":"0/2000000","remote_consistent_lsn":"0/0","peer_horizon_lsn":"0/16960E8","safekeeper_connstr":"something.local:1234","pageserver_connstr":"postgresql://(null):@somethine.else.local:3456"}"#,
-            ),
-        ];
-
-        for (key_string, value_str) in typical_etcd_kv_strs {
-            for subscription in [
-                &all_subscription,
-                &tenant_subscription,
-                &timeline_subscription,
-            ] {
-                let (id, _timeline) =
-                    parse_etcd_key_value(subscription, &key_string, value_str)
-                        .unwrap_or_else(|e| panic!("Should be able to parse etcd key string '{key_string}' and etcd value string '{value_str}' for subscription {subscription:?}, but got: {e}"));
-                assert_eq!(id, ZTenantTimelineId::new(tenant_id, timeline_id));
-            }
-        }
-    }
-}

libs/metrics/Cargo.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-prometheus = {version = "0.13", default_features=false, features = ["process"]} # removes protobuf dependency
+prometheus = {version = "0.13", default_features=false} # removes protobuf dependency
 libc = "0.2"
 lazy_static = "1.4"
 once_cell = "1.8.0"

libs/metrics/src/lib.rs

@@ -3,7 +3,7 @@
 //! Otherwise, we might not see all metrics registered via
 //! a default registry.
 use lazy_static::lazy_static;
-pub use prometheus::{core, default_registry, proto};
+use once_cell::race::OnceBox;
 pub use prometheus::{exponential_buckets, linear_buckets};
 pub use prometheus::{register_gauge, Gauge};
 pub use prometheus::{register_gauge_vec, GaugeVec};
@@ -27,15 +27,48 @@ pub fn gather() -> Vec<prometheus::proto::MetricFamily> {
     prometheus::gather()
 }
 
+static COMMON_METRICS_PREFIX: OnceBox<&str> = OnceBox::new();
+
+/// Sets a prefix which will be used for all common metrics, typically a service
+/// name like 'pageserver'. Should be executed exactly once in the beginning of
+/// any executable which uses common metrics.
+pub fn set_common_metrics_prefix(prefix: &'static str) {
+    // Not unwrap() because metrics may be initialized after multiple threads have been started.
+    COMMON_METRICS_PREFIX
+        .set(prefix.into())
+        .unwrap_or_else(|_| {
+            eprintln!(
+                "set_common_metrics_prefix() was called second time with '{}', exiting",
+                prefix
+            );
+            std::process::exit(1);
+        });
+}
+
+/// Prepends a prefix to a common metric name so they are distinguished between
+/// different services, see <https://github.com/zenithdb/zenith/pull/681>
+/// A call to set_common_metrics_prefix() is necessary prior to calling this.
+pub fn new_common_metric_name(unprefixed_metric_name: &str) -> String {
+    // Not unwrap() because metrics may be initialized after multiple threads have been started.
+    format!(
+        "{}_{}",
+        COMMON_METRICS_PREFIX.get().unwrap_or_else(|| {
+            eprintln!("set_common_metrics_prefix() was not called, but metrics are used, exiting");
+            std::process::exit(1);
+        }),
+        unprefixed_metric_name
+    )
+}
+
 lazy_static! {
     static ref DISK_IO_BYTES: IntGaugeVec = register_int_gauge_vec!(
-        "libmetrics_disk_io_bytes_total",
+        new_common_metric_name("disk_io_bytes"),
         "Bytes written and read from disk, grouped by the operation (read|write)",
         &["io_operation"]
     )
     .expect("Failed to register disk i/o bytes int gauge vec");
     static ref MAXRSS_KB: IntGauge = register_int_gauge!(
-        "libmetrics_maxrss_kb",
+        new_common_metric_name("maxrss_kb"),
         "Memory usage (Maximum Resident Set Size)"
     )
     .expect("Failed to register maxrss_kb int gauge");

libs/postgres_ffi/Cargo.toml

@@ -20,10 +20,5 @@ serde = { version = "1.0", features = ["derive"] }
 utils = { path = "../utils" }
 workspace_hack = { version = "0.1", path = "../../workspace_hack" }
 
-[dev-dependencies]
-env_logger = "0.9"
-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-wal_generate = { path = "wal_generate" }
-
 [build-dependencies]
 bindgen = "0.59.1"

libs/postgres_ffi/src/waldecoder.rs

@@ -73,7 +73,7 @@ impl WalStreamDecoder {
     /// Returns one of the following:
     ///     Ok((Lsn, Bytes)): a tuple containing the LSN of next record, and the record itself
     ///     Ok(None): there is not enough data in the input buffer. Feed more by calling the `feed_bytes` function
-    ///     Err(WalDecodeError): an error occurred while decoding, meaning the input was invalid.
+    ///     Err(WalDecodeError): an error occured while decoding, meaning the input was invalid.
     ///
     pub fn poll_decode(&mut self) -> Result<Option<(Lsn, Bytes)>, WalDecodeError> {
         let recordbuf;

libs/postgres_ffi/src/xlog_utils.rs

@@ -15,7 +15,7 @@ use crate::XLogPageHeaderData;
 use crate::XLogRecord;
 use crate::XLOG_PAGE_MAGIC;
 
-use anyhow::{bail, ensure};
+use anyhow::bail;
 use byteorder::{ByteOrder, LittleEndian};
 use bytes::BytesMut;
 use bytes::{Buf, Bytes};
@@ -30,7 +30,6 @@ use std::path::{Path, PathBuf};
 use std::time::SystemTime;
 use utils::bin_ser::DeserializeError;
 use utils::bin_ser::SerializeError;
-use utils::const_assert;
 use utils::lsn::Lsn;
 
 pub const XLOG_FNAME_LEN: usize = 24;
@@ -150,9 +149,8 @@ fn find_end_of_wal_segment(
 ) -> anyhow::Result<u32> {
     // step back to the beginning of the page to read it in...
     let mut offs: usize = start_offset - start_offset % XLOG_BLCKSZ;
-    let mut skipping_first_contrecord: bool = false;
     let mut contlen: usize = 0;
-    let mut xl_crc: u32 = 0;
+    let mut wal_crc: u32 = 0;
     let mut crc: u32 = 0;
     let mut rec_offs: usize = 0;
     let mut buf = [0u8; XLOG_BLCKSZ];
@@ -160,15 +158,11 @@ fn find_end_of_wal_segment(
     let mut last_valid_rec_pos: usize = start_offset; // assume at given start_offset begins new record
     let mut file = File::open(data_dir.join(file_name.clone() + ".partial")).unwrap();
     file.seek(SeekFrom::Start(offs as u64))?;
-    // xl_crc is the last field in XLogRecord, will not be read into rec_hdr
-    const_assert!(XLOG_RECORD_CRC_OFFS + 4 == XLOG_SIZE_OF_XLOG_RECORD);
     let mut rec_hdr = [0u8; XLOG_RECORD_CRC_OFFS];
 
-    trace!("find_end_of_wal_segment(data_dir={}, segno={}, tli={}, wal_seg_size={}, start_offset=0x{:x})", data_dir.display(), segno, tli, wal_seg_size, start_offset);
     while offs < wal_seg_size {
         // we are at the beginning of the page; read it in
         if offs % XLOG_BLCKSZ == 0 {
-            trace!("offs=0x{:x}: new page", offs);
             let bytes_read = file.read(&mut buf)?;
             if bytes_read != buf.len() {
                 bail!(
@@ -182,49 +176,30 @@ fn find_end_of_wal_segment(
             let xlp_magic = LittleEndian::read_u16(&buf[0..2]);
             let xlp_info = LittleEndian::read_u16(&buf[2..4]);
             let xlp_rem_len = LittleEndian::read_u32(&buf[XLP_REM_LEN_OFFS..XLP_REM_LEN_OFFS + 4]);
-            trace!(
-                "  xlp_magic=0x{:x}, xlp_info=0x{:x}, xlp_rem_len={}",
-                xlp_magic,
-                xlp_info,
-                xlp_rem_len
-            );
             // this is expected in current usage when valid WAL starts after page header
             if xlp_magic != XLOG_PAGE_MAGIC as u16 {
                 trace!(
-                    "  invalid WAL file {}.partial magic {} at {:?}",
+                    "invalid WAL file {}.partial magic {} at {:?}",
                     file_name,
                     xlp_magic,
                     Lsn(XLogSegNoOffsetToRecPtr(segno, offs as u32, wal_seg_size)),
                 );
             }
             if offs == 0 {
-                offs += XLOG_SIZE_OF_XLOG_LONG_PHD;
+                offs = XLOG_SIZE_OF_XLOG_LONG_PHD;
                 if (xlp_info & XLP_FIRST_IS_CONTRECORD) != 0 {
-                    trace!("  first record is contrecord");
-                    skipping_first_contrecord = true;
-                    contlen = xlp_rem_len as usize;
-                    if offs < start_offset {
-                        // Pre-condition failed: the beginning of the segment is unexpectedly corrupted.
-                        ensure!(start_offset - offs >= contlen,
-                            "start_offset is in the middle of the first record (which happens to be a contrecord), \
-                             expected to be on a record boundary. Is beginning of the segment corrupted?");
-                        contlen = 0;
-                        // keep skipping_first_contrecord to avoid counting the contrecord as valid, we did not check it.
-                    }
-                } else {
-                    trace!("  first record is not contrecord");
+                    offs += ((xlp_rem_len + 7) & !7) as usize;
                 }
             } else {
                 offs += XLOG_SIZE_OF_XLOG_SHORT_PHD;
             }
             // ... and step forward again if asked
-            trace!("  skipped header to 0x{:x}", offs);
             offs = max(offs, start_offset);
+
         // beginning of the next record
         } else if contlen == 0 {
             let page_offs = offs % XLOG_BLCKSZ;
             let xl_tot_len = LittleEndian::read_u32(&buf[page_offs..page_offs + 4]) as usize;
-            trace!("offs=0x{:x}: new record, xl_tot_len={}", offs, xl_tot_len);
             if xl_tot_len == 0 {
                 info!(
                     "find_end_of_wal_segment reached zeros at {:?}, last records ends at {:?}",
@@ -237,25 +212,10 @@ fn find_end_of_wal_segment(
                 );
                 break; // zeros, reached the end
             }
-            if skipping_first_contrecord {
-                skipping_first_contrecord = false;
-                trace!("  first contrecord has been just completed");
-            } else {
-                trace!(
-                    "  updating last_valid_rec_pos: 0x{:x} --> 0x{:x}",
-                    last_valid_rec_pos,
-                    offs
-                );
-                last_valid_rec_pos = offs;
-            }
+            last_valid_rec_pos = offs;
             offs += 4;
             rec_offs = 4;
             contlen = xl_tot_len - 4;
-            trace!(
-                "  reading rec_hdr[0..4] <-- [0x{:x}; 0x{:x})",
-                page_offs,
-                page_offs + 4
-            );
             rec_hdr[0..4].copy_from_slice(&buf[page_offs..page_offs + 4]);
         } else {
             // we're continuing a record, possibly from previous page.
@@ -264,118 +224,42 @@ fn find_end_of_wal_segment(
 
             // read the rest of the record, or as much as fits on this page.
             let n = min(contlen, pageleft);
-            trace!(
-                "offs=0x{:x}, record continuation, pageleft={}, contlen={}",
-                offs,
-                pageleft,
-                contlen
-            );
-            // fill rec_hdr header up to (but not including) xl_crc field
-            trace!(
-                "  rec_offs={}, XLOG_RECORD_CRC_OFFS={}, XLOG_SIZE_OF_XLOG_RECORD={}",
-                rec_offs,
-                XLOG_RECORD_CRC_OFFS,
-                XLOG_SIZE_OF_XLOG_RECORD
-            );
+            // fill rec_hdr (header up to (but not including) xl_crc field)
             if rec_offs < XLOG_RECORD_CRC_OFFS {
                 let len = min(XLOG_RECORD_CRC_OFFS - rec_offs, n);
-                trace!(
-                    "  reading rec_hdr[{}..{}] <-- [0x{:x}; 0x{:x})",
-                    rec_offs,
-                    rec_offs + len,
-                    page_offs,
-                    page_offs + len
-                );
                 rec_hdr[rec_offs..rec_offs + len].copy_from_slice(&buf[page_offs..page_offs + len]);
             }
             if rec_offs <= XLOG_RECORD_CRC_OFFS && rec_offs + n >= XLOG_SIZE_OF_XLOG_RECORD {
                 let crc_offs = page_offs - rec_offs + XLOG_RECORD_CRC_OFFS;
-                // All records are aligned on 8-byte boundary, so their 8-byte frames
-                // cannot be split between pages. As xl_crc is the last field,
-                // its content is always on the same page.
-                const_assert!(XLOG_RECORD_CRC_OFFS % 8 == 4);
-                // We should always start reading aligned records even in incorrect WALs so if
-                // the condition is false it is likely a bug. However, it is localized somewhere
-                // in this function, hence we do not crash and just report failure instead.
-                ensure!(crc_offs % 8 == 4, "Record is not aligned properly (bug?)");
-                xl_crc = LittleEndian::read_u32(&buf[crc_offs..crc_offs + 4]);
-                trace!(
-                    "  reading xl_crc: [0x{:x}; 0x{:x}) = 0x{:x}",
-                    crc_offs,
-                    crc_offs + 4,
-                    xl_crc
-                );
+                wal_crc = LittleEndian::read_u32(&buf[crc_offs..crc_offs + 4]);
                 crc = crc32c_append(0, &buf[crc_offs + 4..page_offs + n]);
-                trace!(
-                    "  initializing crc: [0x{:x}; 0x{:x}); crc = 0x{:x}",
-                    crc_offs + 4,
-                    page_offs + n,
-                    crc
-                );
-            } else if rec_offs > XLOG_RECORD_CRC_OFFS {
-                // As all records are 8-byte aligned, the header is already fully read and `crc` is initialized in the branch above.
-                ensure!(rec_offs >= XLOG_SIZE_OF_XLOG_RECORD);
-                let old_crc = crc;
-                crc = crc32c_append(crc, &buf[page_offs..page_offs + n]);
-                trace!(
-                    "  appending to crc: [0x{:x}; 0x{:x}); 0x{:x} --> 0x{:x}",
-                    page_offs,
-                    page_offs + n,
-                    old_crc,
-                    crc
-                );
             } else {
-                // Correct because of the way conditions are written above.
-                assert!(rec_offs + n < XLOG_SIZE_OF_XLOG_RECORD);
-                // If `skipping_first_contrecord == true`, we may be reading from a middle of a record
-                // which started in the previous segment. Hence there is no point in validating the header.
-                if !skipping_first_contrecord && rec_offs + n > XLOG_RECORD_CRC_OFFS {
-                    info!(
-                        "Curiously corrupted WAL: a record stops inside the header; \
-                             offs=0x{:x}, record continuation, pageleft={}, contlen={}",
-                        offs, pageleft, contlen
-                    );
-                    break;
-                }
-                // Do nothing: we are still reading the header. It's accounted in CRC in the end of the record.
+                crc ^= 0xFFFFFFFFu32;
+                crc = crc32c_append(crc, &buf[page_offs..page_offs + n]);
             }
+            crc = !crc;
             rec_offs += n;
             offs += n;
             contlen -= n;
 
             if contlen == 0 {
-                trace!("  record completed at 0x{:x}", offs);
+                crc = !crc;
                 crc = crc32c_append(crc, &rec_hdr);
                 offs = (offs + 7) & !7; // pad on 8 bytes boundary */
-                trace!(
-                    "  padded offs to 0x{:x}, crc is {:x}, expected crc is {:x}",
-                    offs,
-                    crc,
-                    xl_crc
-                );
-                if skipping_first_contrecord {
-                    // do nothing, the flag will go down on next iteration when we're reading new record
-                    trace!("  first conrecord has been just completed");
-                } else if crc == xl_crc {
+                if crc == wal_crc {
                     // record is valid, advance the result to its end (with
                     // alignment to the next record taken into account)
-                    trace!(
-                        "  updating last_valid_rec_pos: 0x{:x} --> 0x{:x}",
-                        last_valid_rec_pos,
-                        offs
-                    );
                     last_valid_rec_pos = offs;
                 } else {
                     info!(
                         "CRC mismatch {} vs {} at {}",
-                        crc, xl_crc, last_valid_rec_pos
+                        crc, wal_crc, last_valid_rec_pos
                     );
                     break;
                 }
             }
         }
     }
-    trace!("last_valid_rec_pos=0x{:x}", last_valid_rec_pos);
     Ok(last_valid_rec_pos as u32)
 }
 
@@ -531,7 +415,7 @@ impl CheckPoint {
     ///
     /// Returns 'true' if the XID was updated.
     pub fn update_next_xid(&mut self, xid: u32) -> bool {
-        // nextXid should nw greater than any XID in WAL, so increment provided XID and check for wraparround.
+        // nextXid should nw greate than any XID in WAL, so increment provided XID and check for wraparround.
         let mut new_xid = std::cmp::max(xid + 1, pg_constants::FIRST_NORMAL_TRANSACTION_ID);
         // To reduce number of metadata checkpoints, we forward align XID on XID_CHECKPOINT_INTERVAL.
         // XID_CHECKPOINT_INTERVAL should not be larger than BLCKSZ*CLOG_XACTS_PER_BYTE
@@ -592,126 +476,78 @@ pub fn generate_wal_segment(segno: u64, system_id: u64) -> Result<Bytes, Seriali
 mod tests {
     use super::*;
     use regex::Regex;
-    use std::{env, str::FromStr};
+    use std::{env, process::Command, str::FromStr};
 
-    fn init_logging() {
-        let _ = env_logger::Builder::from_env(
-            env_logger::Env::default()
-                .default_filter_or("wal_generate=info,postgres_ffi::xlog_utils=trace"),
-        )
-        .is_test(true)
-        .try_init();
-    }
-
-    fn test_end_of_wal(
-        test_name: &str,
-        generate_wal: impl Fn(&mut postgres::Client) -> anyhow::Result<postgres::types::PgLsn>,
-        expected_end_of_wal_non_partial: Lsn,
-        last_segment: &str,
-    ) {
-        use wal_generate::*;
-        // 1. Generate some WAL
+    // Run find_end_of_wal against file in test_wal dir
+    // Ensure that it finds last record correctly
+    #[test]
+    pub fn test_find_end_of_wal() {
+        // 1. Run initdb to generate some WAL
         let top_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
             .join("..")
             .join("..");
-        let cfg = Conf {
-            pg_distrib_dir: top_path.join("tmp_install"),
-            datadir: top_path.join(format!("test_output/{}", test_name)),
-        };
-        if cfg.datadir.exists() {
-            fs::remove_dir_all(&cfg.datadir).unwrap();
+        let data_dir = top_path.join("test_output/test_find_end_of_wal");
+        let initdb_path = top_path.join("tmp_install/bin/initdb");
+        let lib_path = top_path.join("tmp_install/lib");
+        if data_dir.exists() {
+            fs::remove_dir_all(&data_dir).unwrap();
         }
-        cfg.initdb().unwrap();
-        let mut srv = cfg.start_server().unwrap();
-        let expected_wal_end: Lsn =
-            u64::from(generate_wal(&mut srv.connect_with_timeout().unwrap()).unwrap()).into();
-        srv.kill();
+        println!("Using initdb from '{}'", initdb_path.display());
+        println!("Data directory '{}'", data_dir.display());
+        let initdb_output = Command::new(initdb_path)
+            .args(&["-D", data_dir.to_str().unwrap()])
+            .arg("--no-instructions")
+            .arg("--no-sync")
+            .env_clear()
+            .env("LD_LIBRARY_PATH", &lib_path)
+            .env("DYLD_LIBRARY_PATH", &lib_path)
+            .output()
+            .unwrap();
+        assert!(
+            initdb_output.status.success(),
+            "initdb failed. Status: '{}', stdout: '{}', stderr: '{}'",
+            initdb_output.status,
+            String::from_utf8_lossy(&initdb_output.stdout),
+            String::from_utf8_lossy(&initdb_output.stderr),
+        );
 
         // 2. Pick WAL generated by initdb
-        let wal_dir = cfg.datadir.join("pg_wal");
+        let wal_dir = data_dir.join("pg_wal");
         let wal_seg_size = 16 * 1024 * 1024;
 
         // 3. Check end_of_wal on non-partial WAL segment (we treat it as fully populated)
         let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true, Lsn(0)).unwrap();
         let wal_end = Lsn(wal_end);
-        info!(
-            "find_end_of_wal returned (wal_end={}, tli={})",
-            wal_end, tli
-        );
-        assert_eq!(wal_end, expected_end_of_wal_non_partial);
+        println!("wal_end={}, tli={}", wal_end, tli);
+        assert_eq!(wal_end, "0/2000000".parse::<Lsn>().unwrap());
 
         // 4. Get the actual end of WAL by pg_waldump
-        let waldump_output = cfg
-            .pg_waldump("000000010000000000000001", last_segment)
-            .unwrap()
-            .stderr;
-        let waldump_output = std::str::from_utf8(&waldump_output).unwrap();
-        let caps = match Regex::new(r"invalid record length at (.+):")
-            .unwrap()
-            .captures(waldump_output)
-        {
-            Some(caps) => caps,
-            None => {
-                error!("Unable to parse pg_waldump's stderr:\n{}", waldump_output);
-                panic!();
-            }
-        };
+        let waldump_path = top_path.join("tmp_install/bin/pg_waldump");
+        let waldump_output = Command::new(waldump_path)
+            .arg(wal_dir.join("000000010000000000000001"))
+            .env_clear()
+            .env("LD_LIBRARY_PATH", &lib_path)
+            .env("DYLD_LIBRARY_PATH", &lib_path)
+            .output()
+            .unwrap();
+        let waldump_output = std::str::from_utf8(&waldump_output.stderr).unwrap();
+        println!("waldump_output = '{}'", &waldump_output);
+        let re = Regex::new(r"invalid record length at (.+):").unwrap();
+        let caps = re.captures(waldump_output).unwrap();
         let waldump_wal_end = Lsn::from_str(caps.get(1).unwrap().as_str()).unwrap();
-        info!(
-            "waldump erred on {}, expected wal end at {}",
-            waldump_wal_end, expected_wal_end
-        );
-        assert_eq!(waldump_wal_end, expected_wal_end);
 
         // 5. Rename file to partial to actually find last valid lsn
         fs::rename(
-            wal_dir.join(last_segment),
-            wal_dir.join(format!("{}.partial", last_segment)),
+            wal_dir.join("000000010000000000000001"),
+            wal_dir.join("000000010000000000000001.partial"),
         )
         .unwrap();
         let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true, Lsn(0)).unwrap();
         let wal_end = Lsn(wal_end);
-        info!(
-            "find_end_of_wal returned (wal_end={}, tli={})",
-            wal_end, tli
-        );
+        println!("wal_end={}, tli={}", wal_end, tli);
         assert_eq!(wal_end, waldump_wal_end);
     }
 
-    #[test]
-    pub fn test_find_end_of_wal_simple() {
-        init_logging();
-        test_end_of_wal(
-            "test_find_end_of_wal_simple",
-            wal_generate::generate_simple,
-            "0/2000000".parse::<Lsn>().unwrap(),
-            "000000010000000000000001",
-        );
-    }
-
-    #[test]
-    pub fn test_find_end_of_wal_crossing_segment_followed_by_small_one() {
-        init_logging();
-        test_end_of_wal(
-            "test_find_end_of_wal_crossing_segment_followed_by_small_one",
-            wal_generate::generate_wal_record_crossing_segment_followed_by_small_one,
-            "0/3000000".parse::<Lsn>().unwrap(),
-            "000000010000000000000002",
-        );
-    }
-
-    #[test]
-    #[ignore = "not yet fixed, needs correct parsing of pre-last segments"] // TODO
-    pub fn test_find_end_of_wal_last_crossing_segment() {
-        init_logging();
-        test_end_of_wal(
-            "test_find_end_of_wal_last_crossing_segment",
-            wal_generate::generate_last_wal_record_crossing_segment,
-            "0/3000000".parse::<Lsn>().unwrap(),
-            "000000010000000000000002",
-        );
-    }
-
     /// Check the math in update_next_xid
     ///
     /// NOTE: These checks are sensitive to the value of XID_CHECKPOINT_INTERVAL,

libs/postgres_ffi/wal_generate/Cargo.toml

@@ -1,14 +0,0 @@
-[package]
-name = "wal_generate"
-version = "0.1.0"
-edition = "2021"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-anyhow = "1.0"
-clap = "3.0"
-env_logger = "0.9"
-log = "0.4"
-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-tempfile = "3.2"

libs/postgres_ffi/wal_generate/src/bin/wal_generate.rs

@@ -1,58 +0,0 @@
-use anyhow::*;
-use clap::{App, Arg};
-use wal_generate::*;
-
-fn main() -> Result<()> {
-    env_logger::Builder::from_env(
-        env_logger::Env::default().default_filter_or("wal_generate=info"),
-    )
-    .init();
-    let arg_matches = App::new("Postgres WAL generator")
-        .about("Generates Postgres databases with specific WAL properties")
-        .arg(
-            Arg::new("datadir")
-                .short('D')
-                .long("datadir")
-                .takes_value(true)
-                .help("Data directory for the Postgres server")
-                .required(true)
-        )
-        .arg(
-            Arg::new("pg-distrib-dir")
-                .long("pg-distrib-dir")
-                .takes_value(true)
-                .help("Directory with Postgres distribution (bin and lib directories, e.g. tmp_install)")
-                .default_value("/usr/local")
-        )
-        .arg(
-            Arg::new("type")
-                .long("type")
-                .takes_value(true)
-                .help("Type of WAL to generate")
-                .possible_values(["simple", "last_wal_record_crossing_segment", "wal_record_crossing_segment_followed_by_small_one"])
-                .required(true)
-        )
-        .get_matches();
-
-    let cfg = Conf {
-        pg_distrib_dir: arg_matches.value_of("pg-distrib-dir").unwrap().into(),
-        datadir: arg_matches.value_of("datadir").unwrap().into(),
-    };
-    cfg.initdb()?;
-    let mut srv = cfg.start_server()?;
-    let lsn = match arg_matches.value_of("type").unwrap() {
-        "simple" => generate_simple(&mut srv.connect_with_timeout()?)?,
-        "last_wal_record_crossing_segment" => {
-            generate_last_wal_record_crossing_segment(&mut srv.connect_with_timeout()?)?
-        }
-        "wal_record_crossing_segment_followed_by_small_one" => {
-            generate_wal_record_crossing_segment_followed_by_small_one(
-                &mut srv.connect_with_timeout()?,
-            )?
-        }
-        a => panic!("Unknown --type argument: {}", a),
-    };
-    println!("end_of_wal = {}", lsn);
-    srv.kill();
-    Ok(())
-}

libs/postgres_ffi/wal_generate/src/lib.rs

@@ -1,278 +0,0 @@
-use anyhow::*;
-use core::time::Duration;
-use log::*;
-use postgres::types::PgLsn;
-use postgres::Client;
-use std::cmp::Ordering;
-use std::path::{Path, PathBuf};
-use std::process::{Command, Stdio};
-use std::time::Instant;
-use tempfile::{tempdir, TempDir};
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct Conf {
-    pub pg_distrib_dir: PathBuf,
-    pub datadir: PathBuf,
-}
-
-pub struct PostgresServer {
-    process: std::process::Child,
-    _unix_socket_dir: TempDir,
-    client_config: postgres::Config,
-}
-
-impl Conf {
-    fn pg_bin_dir(&self) -> PathBuf {
-        self.pg_distrib_dir.join("bin")
-    }
-
-    fn pg_lib_dir(&self) -> PathBuf {
-        self.pg_distrib_dir.join("lib")
-    }
-
-    fn new_pg_command(&self, command: impl AsRef<Path>) -> Result<Command> {
-        let path = self.pg_bin_dir().join(command);
-        ensure!(path.exists(), "Command {:?} does not exist", path);
-        let mut cmd = Command::new(path);
-        cmd.env_clear()
-            .env("LD_LIBRARY_PATH", self.pg_lib_dir())
-            .env("DYLD_LIBRARY_PATH", self.pg_lib_dir());
-        Ok(cmd)
-    }
-
-    pub fn initdb(&self) -> Result<()> {
-        if let Some(parent) = self.datadir.parent() {
-            info!("Pre-creating parent directory {:?}", parent);
-            // Tests may be run concurrently and there may be a race to create `test_output/`.
-            // std::fs::create_dir_all is guaranteed to have no races with another thread creating directories.
-            std::fs::create_dir_all(parent)?;
-        }
-        info!(
-            "Running initdb in {:?} with user \"postgres\"",
-            self.datadir
-        );
-        let output = self
-            .new_pg_command("initdb")?
-            .arg("-D")
-            .arg(self.datadir.as_os_str())
-            .args(&["-U", "postgres", "--no-instructions", "--no-sync"])
-            .output()?;
-        debug!("initdb output: {:?}", output);
-        ensure!(
-            output.status.success(),
-            "initdb failed, stdout and stderr follow:\n{}{}",
-            String::from_utf8_lossy(&output.stdout),
-            String::from_utf8_lossy(&output.stderr),
-        );
-        Ok(())
-    }
-
-    pub fn start_server(&self) -> Result<PostgresServer> {
-        info!("Starting Postgres server in {:?}", self.datadir);
-        let unix_socket_dir = tempdir()?; // We need a directory with a short name for Unix socket (up to 108 symbols)
-        let unix_socket_dir_path = unix_socket_dir.path().to_owned();
-        let server_process = self
-            .new_pg_command("postgres")?
-            .args(&["-c", "listen_addresses="])
-            .arg("-k")
-            .arg(unix_socket_dir_path.as_os_str())
-            .arg("-D")
-            .arg(self.datadir.as_os_str())
-            .args(&["-c", "wal_keep_size=50MB"]) // Ensure old WAL is not removed
-            .args(&["-c", "logging_collector=on"]) // stderr will mess up with tests output
-            .args(&["-c", "shared_preload_libraries=neon"]) // can only be loaded at startup
-            // Disable background processes as much as possible
-            .args(&["-c", "wal_writer_delay=10s"])
-            .args(&["-c", "autovacuum=off"])
-            .stderr(Stdio::null())
-            .spawn()?;
-        let server = PostgresServer {
-            process: server_process,
-            _unix_socket_dir: unix_socket_dir,
-            client_config: {
-                let mut c = postgres::Config::new();
-                c.host_path(&unix_socket_dir_path);
-                c.user("postgres");
-                c.connect_timeout(Duration::from_millis(1000));
-                c
-            },
-        };
-        Ok(server)
-    }
-
-    pub fn pg_waldump(
-        &self,
-        first_segment_name: &str,
-        last_segment_name: &str,
-    ) -> Result<std::process::Output> {
-        let first_segment_file = self.datadir.join(first_segment_name);
-        let last_segment_file = self.datadir.join(last_segment_name);
-        info!(
-            "Running pg_waldump for {} .. {}",
-            first_segment_file.display(),
-            last_segment_file.display()
-        );
-        let output = self
-            .new_pg_command("pg_waldump")?
-            .args(&[
-                &first_segment_file.as_os_str(),
-                &last_segment_file.as_os_str(),
-            ])
-            .output()?;
-        debug!("waldump output: {:?}", output);
-        Ok(output)
-    }
-}
-
-impl PostgresServer {
-    pub fn connect_with_timeout(&self) -> Result<Client> {
-        let retry_until = Instant::now() + *self.client_config.get_connect_timeout().unwrap();
-        while Instant::now() < retry_until {
-            use std::result::Result::Ok;
-            if let Ok(client) = self.client_config.connect(postgres::NoTls) {
-                return Ok(client);
-            }
-            std::thread::sleep(Duration::from_millis(100));
-        }
-        bail!("Connection timed out");
-    }
-
-    pub fn kill(&mut self) {
-        self.process.kill().unwrap();
-        self.process.wait().unwrap();
-    }
-}
-
-impl Drop for PostgresServer {
-    fn drop(&mut self) {
-        use std::result::Result::Ok;
-        match self.process.try_wait() {
-            Ok(Some(_)) => return,
-            Ok(None) => {
-                warn!("Server was not terminated, will be killed");
-            }
-            Err(e) => {
-                error!("Unable to get status of the server: {}, will be killed", e);
-            }
-        }
-        let _ = self.process.kill();
-    }
-}
-
-pub trait PostgresClientExt: postgres::GenericClient {
-    fn pg_current_wal_insert_lsn(&mut self) -> Result<PgLsn> {
-        Ok(self
-            .query_one("SELECT pg_current_wal_insert_lsn()", &[])?
-            .get(0))
-    }
-    fn pg_current_wal_flush_lsn(&mut self) -> Result<PgLsn> {
-        Ok(self
-            .query_one("SELECT pg_current_wal_flush_lsn()", &[])?
-            .get(0))
-    }
-}
-
-impl<C: postgres::GenericClient> PostgresClientExt for C {}
-
-fn generate_internal<C: postgres::GenericClient>(
-    client: &mut C,
-    f: impl Fn(&mut C, PgLsn) -> Result<Option<PgLsn>>,
-) -> Result<PgLsn> {
-    client.execute("create extension if not exists neon_test_utils", &[])?;
-
-    let wal_segment_size = client.query_one(
-        "select cast(setting as bigint) as setting, unit \
-         from pg_settings where name = 'wal_segment_size'",
-        &[],
-    )?;
-    ensure!(
-        wal_segment_size.get::<_, String>("unit") == "B",
-        "Unexpected wal_segment_size unit"
-    );
-    ensure!(
-        wal_segment_size.get::<_, i64>("setting") == 16 * 1024 * 1024,
-        "Unexpected wal_segment_size in bytes"
-    );
-
-    let initial_lsn = client.pg_current_wal_insert_lsn()?;
-    info!("LSN initial = {}", initial_lsn);
-
-    let last_lsn = match f(client, initial_lsn)? {
-        None => client.pg_current_wal_insert_lsn()?,
-        Some(last_lsn) => match last_lsn.cmp(&client.pg_current_wal_insert_lsn()?) {
-            Ordering::Less => bail!("Some records were inserted after the generated WAL"),
-            Ordering::Equal => last_lsn,
-            Ordering::Greater => bail!("Reported LSN is greater than insert_lsn"),
-        },
-    };
-
-    // Some records may be not flushed, e.g. non-transactional logical messages.
-    client.execute("select neon_xlogflush(pg_current_wal_insert_lsn())", &[])?;
-    match last_lsn.cmp(&client.pg_current_wal_flush_lsn()?) {
-        Ordering::Less => bail!("Some records were flushed after the generated WAL"),
-        Ordering::Equal => {}
-        Ordering::Greater => bail!("Reported LSN is greater than flush_lsn"),
-    }
-    Ok(last_lsn)
-}
-
-pub fn generate_simple(client: &mut impl postgres::GenericClient) -> Result<PgLsn> {
-    generate_internal(client, |client, _| {
-        client.execute("CREATE table t(x int)", &[])?;
-        Ok(None)
-    })
-}
-
-fn generate_single_logical_message(
-    client: &mut impl postgres::GenericClient,
-    transactional: bool,
-) -> Result<PgLsn> {
-    generate_internal(client, |client, initial_lsn| {
-        ensure!(
-            initial_lsn < PgLsn::from(0x0200_0000 - 1024 * 1024),
-            "Initial LSN is too far in the future"
-        );
-
-        let message_lsn: PgLsn = client
-            .query_one(
-                "select pg_logical_emit_message($1, 'big-16mb-msg', \
-                 concat(repeat('abcd', 16 * 256 * 1024), 'end')) as message_lsn",
-                &[&transactional],
-            )?
-            .get("message_lsn");
-        ensure!(
-            message_lsn > PgLsn::from(0x0200_0000 + 4 * 8192),
-            "Logical message did not cross the segment boundary"
-        );
-        ensure!(
-            message_lsn < PgLsn::from(0x0400_0000),
-            "Logical message crossed two segments"
-        );
-
-        if transactional {
-            // Transactional logical messages are part of a transaction, so the one above is
-            // followed by a small COMMIT record.
-
-            let after_message_lsn = client.pg_current_wal_insert_lsn()?;
-            ensure!(
-                message_lsn < after_message_lsn,
-                "No record found after the emitted message"
-            );
-            Ok(Some(after_message_lsn))
-        } else {
-            Ok(Some(message_lsn))
-        }
-    })
-}
-
-pub fn generate_wal_record_crossing_segment_followed_by_small_one(
-    client: &mut impl postgres::GenericClient,
-) -> Result<PgLsn> {
-    generate_single_logical_message(client, true)
-}
-
-pub fn generate_last_wal_record_crossing_segment<C: postgres::GenericClient>(
-    client: &mut C,
-) -> Result<PgLsn> {
-    generate_single_logical_message(client, false)
-}

libs/remote_storage/Cargo.toml

@@ -5,17 +5,14 @@ edition = "2021"
 
 [dependencies]
 anyhow = { version = "1.0", features = ["backtrace"] }
-async-trait = "0.1"
-metrics = { version = "0.1", path = "../metrics" }
-once_cell = "1.8.0"
+tokio = { version = "1.17", features = ["sync", "macros", "fs", "io-util"] }
+tokio-util = { version = "0.7", features = ["io"] }
+tracing = "0.1.27"
 rusoto_core = "0.48"
 rusoto_s3 = "0.48"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
-tokio = { version = "1.17", features = ["sync", "macros", "fs", "io-util"] }
-tokio-util = { version = "0.7", features = ["io"] }
-toml_edit = { version = "0.13", features = ["easy"] }
-tracing = "0.1.27"
+async-trait = "0.1"
 
 workspace_hack = { version = "0.1", path = "../../workspace_hack" }
 

libs/remote_storage/src/lib.rs

@@ -16,10 +16,8 @@ use std::{
     path::{Path, PathBuf},
 };
 
-use anyhow::{bail, Context};
-
+use anyhow::Context;
 use tokio::io;
-use toml_edit::Item;
 use tracing::info;
 
 pub use self::{
@@ -89,8 +87,7 @@ pub trait RemoteStorage: Send + Sync {
     async fn delete(&self, path: &Self::RemoteObjectId) -> anyhow::Result<()>;
 }
 
-/// Every storage, currently supported.
-/// Serves as a simple way to pass around the [`RemoteStorage`] without dealing with generics.
+/// TODO kb
 pub enum GenericRemoteStorage {
     Local(LocalFs),
     S3(S3Bucket),
@@ -205,90 +202,6 @@ pub fn path_with_suffix_extension(original_path: impl AsRef<Path>, suffix: &str)
         .with_extension(new_extension.as_ref())
 }
 
-impl RemoteStorageConfig {
-    pub fn from_toml(toml: &toml_edit::Item) -> anyhow::Result<RemoteStorageConfig> {
-        let local_path = toml.get("local_path");
-        let bucket_name = toml.get("bucket_name");
-        let bucket_region = toml.get("bucket_region");
-
-        let max_concurrent_syncs = NonZeroUsize::new(
-            parse_optional_integer("max_concurrent_syncs", toml)?
-                .unwrap_or(DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNCS),
-        )
-        .context("Failed to parse 'max_concurrent_syncs' as a positive integer")?;
-
-        let max_sync_errors = NonZeroU32::new(
-            parse_optional_integer("max_sync_errors", toml)?
-                .unwrap_or(DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS),
-        )
-        .context("Failed to parse 'max_sync_errors' as a positive integer")?;
-
-        let concurrency_limit = NonZeroUsize::new(
-            parse_optional_integer("concurrency_limit", toml)?
-                .unwrap_or(DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT),
-        )
-        .context("Failed to parse 'concurrency_limit' as a positive integer")?;
-
-        let storage = match (local_path, bucket_name, bucket_region) {
-            (None, None, None) => bail!("no 'local_path' nor 'bucket_name' option"),
-            (_, Some(_), None) => {
-                bail!("'bucket_region' option is mandatory if 'bucket_name' is given ")
-            }
-            (_, None, Some(_)) => {
-                bail!("'bucket_name' option is mandatory if 'bucket_region' is given ")
-            }
-            (None, Some(bucket_name), Some(bucket_region)) => RemoteStorageKind::AwsS3(S3Config {
-                bucket_name: parse_toml_string("bucket_name", bucket_name)?,
-                bucket_region: parse_toml_string("bucket_region", bucket_region)?,
-                prefix_in_bucket: toml
-                    .get("prefix_in_bucket")
-                    .map(|prefix_in_bucket| parse_toml_string("prefix_in_bucket", prefix_in_bucket))
-                    .transpose()?,
-                endpoint: toml
-                    .get("endpoint")
-                    .map(|endpoint| parse_toml_string("endpoint", endpoint))
-                    .transpose()?,
-                concurrency_limit,
-            }),
-            (Some(local_path), None, None) => RemoteStorageKind::LocalFs(PathBuf::from(
-                parse_toml_string("local_path", local_path)?,
-            )),
-            (Some(_), Some(_), _) => bail!("local_path and bucket_name are mutually exclusive"),
-        };
-
-        Ok(RemoteStorageConfig {
-            max_concurrent_syncs,
-            max_sync_errors,
-            storage,
-        })
-    }
-}
-
-// Helper functions to parse a toml Item
-fn parse_optional_integer<I, E>(name: &str, item: &toml_edit::Item) -> anyhow::Result<Option<I>>
-where
-    I: TryFrom<i64, Error = E>,
-    E: std::error::Error + Send + Sync + 'static,
-{
-    let toml_integer = match item.get(name) {
-        Some(item) => item
-            .as_integer()
-            .with_context(|| format!("configure option {name} is not an integer"))?,
-        None => return Ok(None),
-    };
-
-    I::try_from(toml_integer)
-        .map(Some)
-        .with_context(|| format!("configure option {name} is too large"))
-}
-
-fn parse_toml_string(name: &str, item: &Item) -> anyhow::Result<String> {
-    let s = item
-        .as_str()
-        .with_context(|| format!("configure option {name} is not a string"))?;
-    Ok(s.to_string())
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

libs/remote_storage/src/s3_bucket.rs

@@ -23,71 +23,6 @@ use crate::{strip_path_prefix, RemoteStorage, S3Config};
 
 use super::StorageMetadata;
 
-pub(super) mod metrics {
-    use metrics::{register_int_counter_vec, IntCounterVec};
-    use once_cell::sync::Lazy;
-
-    static S3_REQUESTS_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
-        register_int_counter_vec!(
-            "remote_storage_s3_requests_count",
-            "Number of s3 requests of particular type",
-            &["request_type"],
-        )
-        .expect("failed to define a metric")
-    });
-
-    static S3_REQUESTS_FAIL_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
-        register_int_counter_vec!(
-            "remote_storage_s3_failures_count",
-            "Number of failed s3 requests of particular type",
-            &["request_type"],
-        )
-        .expect("failed to define a metric")
-    });
-
-    pub fn inc_get_object() {
-        S3_REQUESTS_COUNT.with_label_values(&["get_object"]).inc();
-    }
-
-    pub fn inc_get_object_fail() {
-        S3_REQUESTS_FAIL_COUNT
-            .with_label_values(&["get_object"])
-            .inc();
-    }
-
-    pub fn inc_put_object() {
-        S3_REQUESTS_COUNT.with_label_values(&["put_object"]).inc();
-    }
-
-    pub fn inc_put_object_fail() {
-        S3_REQUESTS_FAIL_COUNT
-            .with_label_values(&["put_object"])
-            .inc();
-    }
-
-    pub fn inc_delete_object() {
-        S3_REQUESTS_COUNT
-            .with_label_values(&["delete_object"])
-            .inc();
-    }
-
-    pub fn inc_delete_object_fail() {
-        S3_REQUESTS_FAIL_COUNT
-            .with_label_values(&["delete_object"])
-            .inc();
-    }
-
-    pub fn inc_list_objects() {
-        S3_REQUESTS_COUNT.with_label_values(&["list_objects"]).inc();
-    }
-
-    pub fn inc_list_objects_fail() {
-        S3_REQUESTS_FAIL_COUNT
-            .with_label_values(&["list_objects"])
-            .inc();
-    }
-}
-
 const S3_PREFIX_SEPARATOR: char = '/';
 
 #[derive(Debug, Eq, PartialEq, PartialOrd, Ord, Hash)]
@@ -217,9 +152,6 @@ impl RemoteStorage for S3Bucket {
                 .acquire()
                 .await
                 .context("Concurrency limiter semaphore got closed during S3 list")?;
-
-            metrics::inc_list_objects();
-
             let fetch_response = self
                 .client
                 .list_objects_v2(ListObjectsV2Request {
@@ -228,11 +160,7 @@ impl RemoteStorage for S3Bucket {
                     continuation_token,
                     ..ListObjectsV2Request::default()
                 })
-                .await
-                .map_err(|e| {
-                    metrics::inc_list_objects_fail();
-                    e
-                })?;
+                .await?;
             document_keys.extend(
                 fetch_response
                     .contents
@@ -262,8 +190,6 @@ impl RemoteStorage for S3Bucket {
             .acquire()
             .await
             .context("Concurrency limiter semaphore got closed during S3 upload")?;
-
-        metrics::inc_put_object();
         self.client
             .put_object(PutObjectRequest {
                 body: Some(StreamingBody::new_with_size(
@@ -275,11 +201,7 @@ impl RemoteStorage for S3Bucket {
                 metadata: metadata.map(|m| m.0),
                 ..PutObjectRequest::default()
             })
-            .await
-            .map_err(|e| {
-                metrics::inc_put_object_fail();
-                e
-            })?;
+            .await?;
         Ok(())
     }
 
@@ -293,9 +215,6 @@ impl RemoteStorage for S3Bucket {
             .acquire()
             .await
             .context("Concurrency limiter semaphore got closed during S3 download")?;
-
-        metrics::inc_get_object();
-
         let object_output = self
             .client
             .get_object(GetObjectRequest {
@@ -303,11 +222,7 @@ impl RemoteStorage for S3Bucket {
                 key: from.key().to_owned(),
                 ..GetObjectRequest::default()
             })
-            .await
-            .map_err(|e| {
-                metrics::inc_get_object_fail();
-                e
-            })?;
+            .await?;
 
         if let Some(body) = object_output.body {
             let mut from = io::BufReader::new(body.into_async_read());
@@ -336,9 +251,6 @@ impl RemoteStorage for S3Bucket {
             .acquire()
             .await
             .context("Concurrency limiter semaphore got closed during S3 range download")?;
-
-        metrics::inc_get_object();
-
         let object_output = self
             .client
             .get_object(GetObjectRequest {
@@ -347,11 +259,7 @@ impl RemoteStorage for S3Bucket {
                 range,
                 ..GetObjectRequest::default()
             })
-            .await
-            .map_err(|e| {
-                metrics::inc_get_object_fail();
-                e
-            })?;
+            .await?;
 
         if let Some(body) = object_output.body {
             let mut from = io::BufReader::new(body.into_async_read());
@@ -367,20 +275,13 @@ impl RemoteStorage for S3Bucket {
             .acquire()
             .await
             .context("Concurrency limiter semaphore got closed during S3 delete")?;
-
-        metrics::inc_delete_object();
-
         self.client
             .delete_object(DeleteObjectRequest {
                 bucket: self.bucket_name.clone(),
                 key: path.key().to_owned(),
                 ..DeleteObjectRequest::default()
             })
-            .await
-            .map_err(|e| {
-                metrics::inc_delete_object_fail();
-                e
-            })?;
+            .await?;
         Ok(())
     }
 }

libs/utils/scripts/restore_from_wal.sh

@@ -5,7 +5,7 @@ DATA_DIR=$3
 PORT=$4
 SYSID=`od -A n -j 24 -N 8 -t d8 $WAL_PATH/000000010000000000000002* | cut -c 3-`
 rm -fr $DATA_DIR
-env -i LD_LIBRARY_PATH=$PG_BIN/../lib $PG_BIN/initdb -E utf8 -U cloud_admin -D $DATA_DIR --sysid=$SYSID
+env -i LD_LIBRARY_PATH=$PG_BIN/../lib $PG_BIN/initdb -E utf8 -U zenith_admin -D $DATA_DIR --sysid=$SYSID
 echo port=$PORT >> $DATA_DIR/postgresql.conf
 REDO_POS=0x`$PG_BIN/pg_controldata -D $DATA_DIR | fgrep "REDO location"| cut -c 42-`
 declare -i WAL_SIZE=$REDO_POS+114

libs/utils/scripts/restore_from_wal_archive.sh

@@ -5,7 +5,7 @@ PORT=$4
 SYSID=`od -A n -j 24 -N 8 -t d8 $WAL_PATH/000000010000000000000002* | cut -c 3-`
 rm -fr $DATA_DIR /tmp/pg_wals
 mkdir /tmp/pg_wals
-env -i LD_LIBRARY_PATH=$PG_BIN/../lib $PG_BIN/initdb -E utf8 -U cloud_admin -D $DATA_DIR --sysid=$SYSID
+env -i LD_LIBRARY_PATH=$PG_BIN/../lib $PG_BIN/initdb -E utf8 -U zenith_admin -D $DATA_DIR --sysid=$SYSID
 echo port=$PORT >> $DATA_DIR/postgresql.conf
 REDO_POS=0x`$PG_BIN/pg_controldata -D $DATA_DIR | fgrep "REDO location"| cut -c 42-`
 declare -i WAL_SIZE=$REDO_POS+114

libs/utils/src/bin_ser.rs

@@ -71,7 +71,7 @@ impl From<bincode::Error> for SerializeError {
 /// - Fixed integer encoding (i.e. 1u32 is 00000001 not 01)
 ///
 /// Does not allow trailing bytes in deserialization. If this is desired, you
-/// may set [`Options::allow_trailing_bytes`] to explicitly accommodate this.
+/// may set [`Options::allow_trailing_bytes`] to explicitly accomodate this.
 pub fn be_coder() -> impl Options {
     bincode::DefaultOptions::new()
         .with_big_endian()
@@ -85,7 +85,7 @@ pub fn be_coder() -> impl Options {
 /// - Fixed integer encoding (i.e. 1u32 is 00000001 not 01)
 ///
 /// Does not allow trailing bytes in deserialization. If this is desired, you
-/// may set [`Options::allow_trailing_bytes`] to explicitly accommodate this.
+/// may set [`Options::allow_trailing_bytes`] to explicitly accomodate this.
 pub fn le_coder() -> impl Options {
     bincode::DefaultOptions::new()
         .with_little_endian()

libs/utils/src/http/endpoint.rs

@@ -5,7 +5,7 @@ use anyhow::anyhow;
 use hyper::header::AUTHORIZATION;
 use hyper::{header::CONTENT_TYPE, Body, Request, Response, Server};
 use lazy_static::lazy_static;
-use metrics::{register_int_counter, Encoder, IntCounter, TextEncoder};
+use metrics::{new_common_metric_name, register_int_counter, Encoder, IntCounter, TextEncoder};
 use routerify::ext::RequestExt;
 use routerify::RequestInfo;
 use routerify::{Middleware, Router, RouterBuilder, RouterService};
@@ -18,7 +18,7 @@ use super::error::ApiError;
 
 lazy_static! {
     static ref SERVE_METRICS_COUNT: IntCounter = register_int_counter!(
-        "libmetrics_metric_handler_requests_total",
+        new_common_metric_name("serve_metrics_count"),
         "Number of metric requests made"
     )
     .expect("failed to define a metric");

libs/utils/src/lib.rs

@@ -64,7 +64,7 @@ pub mod signals;
 /// One thing to note is that .git is not available in docker (and it is bad to include it there).
 /// So everything becides docker build is covered by git_version crate, and docker uses a `GIT_VERSION` argument to get the value required.
 /// It takes variable from build process env and puts it to the rustc env. And then we can retrieve it here by using env! macro.
-/// Git version received from environment variable used as a fallback in git_version invocation.
+/// Git version received from environment variable used as a fallback in git_version invokation.
 /// And to avoid running buildscript every recompilation, we use rerun-if-env-changed option.
 /// So the build script will be run only when GIT_VERSION envvar has changed.
 ///
@@ -95,11 +95,3 @@ macro_rules! project_git_version {
         );
     };
 }
-
-/// Same as `assert!`, but evaluated during compilation and gets optimized out in runtime.
-#[macro_export]
-macro_rules! const_assert {
-    ($($args:tt)*) => {
-        const _: () = assert!($($args)*);
-    };
-}

libs/utils/src/lsn.rs

@@ -26,9 +26,6 @@ impl Lsn {
     /// Maximum possible value for an LSN
     pub const MAX: Lsn = Lsn(u64::MAX);
 
-    /// Invalid value for InvalidXLogRecPtr, as defined in xlogdefs.h
-    pub const INVALID: Lsn = Lsn(0);
-
     /// Subtract a number, returning None on overflow.
     pub fn checked_sub<T: Into<u64>>(self, other: T) -> Option<Lsn> {
         let other: u64 = other.into();
@@ -106,12 +103,6 @@ impl Lsn {
     pub fn is_aligned(&self) -> bool {
         *self == self.align()
     }
-
-    /// Return if the LSN is valid
-    /// mimics postgres XLogRecPtrIsInvalid macro
-    pub fn is_valid(self) -> bool {
-        self != Lsn::INVALID
-    }
 }
 
 impl From<u64> for Lsn {

libs/utils/src/postgres_backend.rs

@@ -336,11 +336,11 @@ impl PostgresBackend {
         let have_tls = self.tls_config.is_some();
         match msg {
             FeMessage::StartupPacket(m) => {
-                trace!("got startup message {m:?}");
+                trace!("got startup message {:?}", m);
 
                 match m {
                     FeStartupPacket::SslRequest => {
-                        debug!("SSL requested");
+                        info!("SSL requested");
 
                         self.write_message(&BeMessage::EncryptionResponse(have_tls))?;
                         if have_tls {
@@ -349,7 +349,7 @@ impl PostgresBackend {
                         }
                     }
                     FeStartupPacket::GssEncRequest => {
-                        debug!("GSS requested");
+                        info!("GSS requested");
                         self.write_message(&BeMessage::EncryptionResponse(false))?;
                     }
                     FeStartupPacket::StartupMessage { .. } => {
@@ -433,7 +433,12 @@ impl PostgresBackend {
                     // full cause of the error, not just the top-level context + its trace.
                     // We don't want to send that in the ErrorResponse though,
                     // because it's not relevant to the compute node logs.
-                    error!("query handler for '{}' failed: {:?}", query_string, e);
+                    if query_string.starts_with("callmemaybe") {
+                        // FIXME avoid printing a backtrace for tenant x not found errors until this is properly fixed
+                        error!("query handler for '{}' failed: {}", query_string, e);
+                    } else {
+                        error!("query handler for '{}' failed: {:?}", query_string, e);
+                    }
                     self.write_message_noflush(&BeMessage::ErrorResponse(&e.to_string()))?;
                     // TODO: untangle convoluted control flow
                     if e.to_string().contains("failed to run") {
@@ -470,7 +475,7 @@ impl PostgresBackend {
                     self.write_message(&BeMessage::ErrorResponse(&e.to_string()))?;
                 }
                 // NOTE there is no ReadyForQuery message. This handler is used
-                // for basebackup and it uses CopyOut which doesn't require
+                // for basebackup and it uses CopyOut which doesnt require
                 // ReadyForQuery message and backend just switches back to
                 // processing mode after sending CopyDone or ErrorResponse.
             }

libs/utils/src/pq_proto.rs

@@ -464,7 +464,7 @@ impl BeParameterStatusMessage<'static> {
     }
 }
 
-// One row description in RowDescription packet.
+// One row desciption in RowDescription packet.
 #[derive(Debug)]
 pub struct RowDescriptor<'a> {
     pub name: &'a [u8],
@@ -613,7 +613,7 @@ fn cstr_to_str(b: &Bytes) -> Result<&str> {
 impl<'a> BeMessage<'a> {
     /// Write message to the given buf.
     // Unlike the reading side, we use BytesMut
-    // here as msg len precedes its body and it is handy to write it down first
+    // here as msg len preceeds its body and it is handy to write it down first
     // and then fill the length. With Write we would have to either calc it
     // manually or have one more buffer.
     pub fn write(buf: &mut BytesMut, message: &BeMessage) -> io::Result<()> {
@@ -1047,7 +1047,7 @@ mod tests {
     #[test]
     fn test_zenithfeedback_serialization() {
         let mut zf = ZenithFeedback::empty();
-        // Fill zf with some values
+        // Fill zf wih some values
         zf.current_timeline_size = 12345678;
         // Set rounded time to be able to compare it with deserialized value,
         // because it is rounded up to microseconds during serialization.
@@ -1062,7 +1062,7 @@ mod tests {
     #[test]
     fn test_zenithfeedback_unknown_key() {
         let mut zf = ZenithFeedback::empty();
-        // Fill zf with some values
+        // Fill zf wih some values
         zf.current_timeline_size = 12345678;
         // Set rounded time to be able to compare it with deserialized value,
         // because it is rounded up to microseconds during serialization.

libs/utils/src/zid.rs

@@ -193,7 +193,7 @@ pub struct ZTenantId(ZId);
 zid_newtype!(ZTenantId);
 
 // A pair uniquely identifying Zenith instance.
-#[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash)]
 pub struct ZTenantTimelineId {
     pub tenant_id: ZTenantId,
     pub timeline_id: ZTimelineId,
@@ -218,7 +218,7 @@ impl ZTenantTimelineId {
 
 impl fmt::Display for ZTenantTimelineId {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}/{}", self.tenant_id, self.timeline_id)
+        write!(f, "{}-{}", self.tenant_id, self.timeline_id)
     }
 }
 
@@ -226,9 +226,9 @@ impl fmt::Display for ZTenantTimelineId {
 // by the console.
 #[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Hash, Debug, Serialize, Deserialize)]
 #[serde(transparent)]
-pub struct NodeId(pub u64);
+pub struct ZNodeId(pub u64);
 
-impl fmt::Display for NodeId {
+impl fmt::Display for ZNodeId {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "{}", self.0)
     }

monitoring/docker-compose.yml

@@ -0,0 +1,25 @@
+version: "3"
+services:
+
+  prometheus:
+    container_name: prometheus
+    image: prom/prometheus:latest
+    volumes:
+      - ./prometheus.yaml:/etc/prometheus/prometheus.yml
+    # ports:
+    #   - "9090:9090"
+    # TODO: find a proper portable solution
+    network_mode: "host"
+
+  grafana:
+    image: grafana/grafana:latest
+    volumes:
+      - ./grafana.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+    # ports:
+    #   - "3000:3000"
+    # TODO: find a proper portable solution
+    network_mode: "host"

monitoring/grafana.yaml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+datasources:
+- name: Prometheus
+  type: prometheus
+  access: proxy
+  orgId: 1
+  url: http://localhost:9090
+  basicAuth: false
+  isDefault: false
+  version: 1
+  editable: false

monitoring/prometheus.yaml

@@ -0,0 +1,5 @@
+scrape_configs:
+  - job_name: 'default'
+    scrape_interval: 10s
+    static_configs:
+      - targets: ['localhost:9898']

neon_local/src/main.rs

@@ -1,10 +1,10 @@
 use anyhow::{anyhow, bail, Context, Result};
 use clap::{App, AppSettings, Arg, ArgMatches};
 use control_plane::compute::ComputeControlPlane;
-use control_plane::local_env::{EtcdBroker, LocalEnv};
+use control_plane::local_env;
+use control_plane::local_env::LocalEnv;
 use control_plane::safekeeper::SafekeeperNode;
 use control_plane::storage::PageServerNode;
-use control_plane::{etcd, local_env};
 use pageserver::config::defaults::{
     DEFAULT_HTTP_LISTEN_ADDR as DEFAULT_PAGESERVER_HTTP_ADDR,
     DEFAULT_PG_LISTEN_ADDR as DEFAULT_PAGESERVER_PG_ADDR,
@@ -14,7 +14,6 @@ use safekeeper::defaults::{
     DEFAULT_PG_LISTEN_PORT as DEFAULT_SAFEKEEPER_PG_PORT,
 };
 use std::collections::{BTreeSet, HashMap};
-use std::path::Path;
 use std::process::exit;
 use std::str::FromStr;
 use utils::{
@@ -22,38 +21,39 @@ use utils::{
     lsn::Lsn,
     postgres_backend::AuthType,
     project_git_version,
-    zid::{NodeId, ZTenantId, ZTenantTimelineId, ZTimelineId},
+    zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId},
 };
 
 use pageserver::timelines::TimelineInfo;
 
 // Default id of a safekeeper node, if not specified on the command line.
-const DEFAULT_SAFEKEEPER_ID: NodeId = NodeId(1);
-const DEFAULT_PAGESERVER_ID: NodeId = NodeId(1);
+const DEFAULT_SAFEKEEPER_ID: ZNodeId = ZNodeId(1);
+const DEFAULT_PAGESERVER_ID: ZNodeId = ZNodeId(1);
 const DEFAULT_BRANCH_NAME: &str = "main";
 project_git_version!(GIT_VERSION);
 
-fn default_conf(etcd_binary_path: &Path) -> String {
+fn default_conf() -> String {
     format!(
         r#"
 # Default built-in configuration, defined in main.rs
-[etcd_broker]
-broker_endpoints = ['http://localhost:2379']
-etcd_binary_path = '{etcd_binary_path}'
-
 [pageserver]
-id = {DEFAULT_PAGESERVER_ID}
-listen_pg_addr = '{DEFAULT_PAGESERVER_PG_ADDR}'
-listen_http_addr = '{DEFAULT_PAGESERVER_HTTP_ADDR}'
+id = {pageserver_id}
+listen_pg_addr = '{pageserver_pg_addr}'
+listen_http_addr = '{pageserver_http_addr}'
 auth_type = '{pageserver_auth_type}'
 
 [[safekeepers]]
-id = {DEFAULT_SAFEKEEPER_ID}
-pg_port = {DEFAULT_SAFEKEEPER_PG_PORT}
-http_port = {DEFAULT_SAFEKEEPER_HTTP_PORT}
+id = {safekeeper_id}
+pg_port = {safekeeper_pg_port}
+http_port = {safekeeper_http_port}
 "#,
-        etcd_binary_path = etcd_binary_path.display(),
+        pageserver_id = DEFAULT_PAGESERVER_ID,
+        pageserver_pg_addr = DEFAULT_PAGESERVER_PG_ADDR,
+        pageserver_http_addr = DEFAULT_PAGESERVER_HTTP_ADDR,
         pageserver_auth_type = AuthType::Trust,
+        safekeeper_id = DEFAULT_SAFEKEEPER_ID,
+        safekeeper_pg_port = DEFAULT_SAFEKEEPER_PG_PORT,
+        safekeeper_http_port = DEFAULT_SAFEKEEPER_HTTP_PORT,
     )
 }
 
@@ -167,12 +167,12 @@ fn main() -> Result<()> {
             .subcommand(App::new("create")
                 .arg(tenant_id_arg.clone())
                 .arg(timeline_id_arg.clone().help("Use a specific timeline id when creating a tenant and its initial timeline"))
-                .arg(Arg::new("config").short('c').takes_value(true).multiple_occurrences(true).required(false))
-                )
+				.arg(Arg::new("config").short('c').takes_value(true).multiple_occurrences(true).required(false))
+				)
             .subcommand(App::new("config")
                 .arg(tenant_id_arg.clone())
-                .arg(Arg::new("config").short('c').takes_value(true).multiple_occurrences(true).required(false))
-                )
+				.arg(Arg::new("config").short('c').takes_value(true).multiple_occurrences(true).required(false))
+				)
         )
         .subcommand(
             App::new("pageserver")
@@ -275,7 +275,7 @@ fn main() -> Result<()> {
             "pageserver" => handle_pageserver(sub_args, &env),
             "pg" => handle_pg(sub_args, &env),
             "safekeeper" => handle_safekeeper(sub_args, &env),
-            _ => bail!("unexpected subcommand {sub_name}"),
+            _ => bail!("unexpected subcommand {}", sub_name),
         };
 
         if original_env != env {
@@ -289,7 +289,7 @@ fn main() -> Result<()> {
         Ok(Some(updated_env)) => updated_env.persist_config(&updated_env.base_data_dir)?,
         Ok(None) => (),
         Err(e) => {
-            eprintln!("command failed: {e:?}");
+            eprintln!("command failed: {:?}", e);
             exit(1);
         }
     }
@@ -468,21 +468,21 @@ fn parse_timeline_id(sub_match: &ArgMatches) -> anyhow::Result<Option<ZTimelineI
         .context("Failed to parse timeline id from the argument string")
 }
 
-fn handle_init(init_match: &ArgMatches) -> anyhow::Result<LocalEnv> {
+fn handle_init(init_match: &ArgMatches) -> Result<LocalEnv> {
     let initial_timeline_id_arg = parse_timeline_id(init_match)?;
 
     // Create config file
     let toml_file: String = if let Some(config_path) = init_match.value_of("config") {
         // load and parse the file
         std::fs::read_to_string(std::path::Path::new(config_path))
-            .with_context(|| format!("Could not read configuration file '{config_path}'"))?
+            .with_context(|| format!("Could not read configuration file \"{}\"", config_path))?
     } else {
         // Built-in default config
-        default_conf(&EtcdBroker::locate_etcd()?)
+        default_conf()
     };
 
     let mut env =
-        LocalEnv::parse_config(&toml_file).context("Failed to create neon configuration")?;
+        LocalEnv::create_config(&toml_file).context("Failed to create neon configuration")?;
     env.init().context("Failed to initialize neon repository")?;
 
     // default_tenantid was generated by the `env.init()` call above
@@ -497,7 +497,7 @@ fn handle_init(init_match: &ArgMatches) -> anyhow::Result<LocalEnv> {
             &pageserver_config_overrides(init_match),
         )
         .unwrap_or_else(|e| {
-            eprintln!("pageserver init failed: {e}");
+            eprintln!("pageserver init failed: {}", e);
             exit(1);
         });
 
@@ -860,7 +860,7 @@ fn handle_pageserver(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul
     Ok(())
 }
 
-fn get_safekeeper(env: &local_env::LocalEnv, id: NodeId) -> Result<SafekeeperNode> {
+fn get_safekeeper(env: &local_env::LocalEnv, id: ZNodeId) -> Result<SafekeeperNode> {
     if let Some(node) = env.safekeepers.iter().find(|node| node.id == id) {
         Ok(SafekeeperNode::from_env(env, node))
     } else {
@@ -876,7 +876,7 @@ fn handle_safekeeper(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul
 
     // All the commands take an optional safekeeper name argument
     let sk_id = if let Some(id_str) = sub_args.value_of("id") {
-        NodeId(id_str.parse().context("while parsing safekeeper id")?)
+        ZNodeId(id_str.parse().context("while parsing safekeeper id")?)
     } else {
         DEFAULT_SAFEKEEPER_ID
     };
@@ -920,23 +920,20 @@ fn handle_safekeeper(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul
     Ok(())
 }
 
-fn handle_start_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow::Result<()> {
-    etcd::start_etcd_process(env)?;
+fn handle_start_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> {
     let pageserver = PageServerNode::from_env(env);
 
     // Postgres nodes are not started automatically
 
     if let Err(e) = pageserver.start(&pageserver_config_overrides(sub_match)) {
-        eprintln!("pageserver start failed: {e}");
-        try_stop_etcd_process(env);
+        eprintln!("pageserver start failed: {}", e);
         exit(1);
     }
 
     for node in env.safekeepers.iter() {
         let safekeeper = SafekeeperNode::from_env(env, node);
         if let Err(e) = safekeeper.start() {
-            eprintln!("safekeeper '{}' start failed: {e}", safekeeper.id);
-            try_stop_etcd_process(env);
+            eprintln!("safekeeper '{}' start failed: {}", safekeeper.id, e);
             exit(1);
         }
     }
@@ -966,14 +963,5 @@ fn handle_stop_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<
             eprintln!("safekeeper '{}' stop failed: {}", safekeeper.id, e);
         }
     }
-
-    try_stop_etcd_process(env);
-
     Ok(())
 }
-
-fn try_stop_etcd_process(env: &local_env::LocalEnv) {
-    if let Err(e) = etcd::stop_etcd_process(env) {
-        eprintln!("etcd stop failed: {e}");
-    }
-}

pageserver/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 
 [features]
 # It is simpler infra-wise to have failpoints enabled by default
-# It shouldn't affect performance in any way because failpoints
+# It shouldn't affect perf in any way because failpoints
 # are not placed in hot code paths
 default = ["failpoints"]
 profiling = ["pprof"]
@@ -55,12 +55,10 @@ fail = "0.5.0"
 git-version = "0.3.5"
 
 postgres_ffi = { path = "../libs/postgres_ffi" }
-etcd_broker = { path = "../libs/etcd_broker" }
 metrics = { path = "../libs/metrics" }
 utils = { path = "../libs/utils" }
 remote_storage = { path = "../libs/remote_storage" }
 workspace_hack = { version = "0.1", path = "../workspace_hack" }
-close_fds = "0.3.2"
 
 [dev-dependencies]
 hex-literal = "0.3"

pageserver/src/basebackup.rs

@@ -10,9 +10,8 @@
 //! This module is responsible for creation of such tarball
 //! from data stored in object storage.
 //!
-use anyhow::{anyhow, bail, ensure, Context, Result};
+use anyhow::{anyhow, ensure, Context, Result};
 use bytes::{BufMut, BytesMut};
-use fail::fail_point;
 use std::fmt::Write as FmtWrite;
 use std::io;
 use std::io::Write;
@@ -31,16 +30,11 @@ use utils::lsn::Lsn;
 /// This is short-living object only for the time of tarball creation,
 /// created mostly to avoid passing a lot of parameters between various functions
 /// used for constructing tarball.
-pub struct Basebackup<'a, W>
-where
-    W: Write,
-{
-    ar: Builder<AbortableWrite<W>>,
+pub struct Basebackup<'a> {
+    ar: Builder<&'a mut dyn Write>,
     timeline: &'a Arc<DatadirTimelineImpl>,
     pub lsn: Lsn,
     prev_record_lsn: Lsn,
-
-    finished: bool,
 }
 
 // Create basebackup with non-rel data in it. Omit relational data.
@@ -50,15 +44,12 @@ where
 //  * When working without safekeepers. In this situation it is important to match the lsn
 //    we are taking basebackup on with the lsn that is used in pageserver's walreceiver
 //    to start the replication.
-impl<'a, W> Basebackup<'a, W>
-where
-    W: Write,
-{
+impl<'a> Basebackup<'a> {
     pub fn new(
-        write: W,
+        write: &'a mut dyn Write,
         timeline: &'a Arc<DatadirTimelineImpl>,
         req_lsn: Option<Lsn>,
-    ) -> Result<Basebackup<'a, W>> {
+    ) -> Result<Basebackup<'a>> {
         // Compute postgres doesn't have any previous WAL files, but the first
         // record that it's going to write needs to include the LSN of the
         // previous record (xl_prev). We include prev_record_lsn in the
@@ -99,15 +90,14 @@ where
         );
 
         Ok(Basebackup {
-            ar: Builder::new(AbortableWrite::new(write)),
+            ar: Builder::new(write),
             timeline,
             lsn: backup_lsn,
             prev_record_lsn: backup_prev,
-            finished: false,
         })
     }
 
-    pub fn send_tarball(mut self) -> anyhow::Result<()> {
+    pub fn send_tarball(&mut self) -> anyhow::Result<()> {
         // Create pgdata subdirs structure
         for dir in pg_constants::PGDATA_SUBDIRS.iter() {
             let header = new_tar_header_dir(*dir)?;
@@ -145,14 +135,9 @@ where
             self.add_twophase_file(xid)?;
         }
 
-        fail_point!("basebackup-before-control-file", |_| {
-            bail!("failpoint basebackup-before-control-file")
-        });
-
         // Generate pg_control and bootstrap WAL segment.
         self.add_pgcontrol_file()?;
         self.ar.finish()?;
-        self.finished = true;
         debug!("all tarred up!");
         Ok(())
     }
@@ -346,19 +331,6 @@ where
     }
 }
 
-impl<'a, W> Drop for Basebackup<'a, W>
-where
-    W: Write,
-{
-    /// If the basebackup was not finished, prevent the Archive::drop() from
-    /// writing the end-of-archive marker.
-    fn drop(&mut self) {
-        if !self.finished {
-            self.ar.get_mut().abort();
-        }
-    }
-}
-
 //
 // Create new tarball entry header
 //
@@ -394,49 +366,3 @@ fn new_tar_header_dir(path: &str) -> anyhow::Result<Header> {
     header.set_cksum();
     Ok(header)
 }
-
-/// A wrapper that passes through all data to the underlying Write,
-/// until abort() is called.
-///
-/// tar::Builder has an annoying habit of finishing the archive with
-/// a valid tar end-of-archive marker (two 512-byte sectors of zeros),
-/// even if an error occurs and we don't finish building the archive.
-/// We'd rather abort writing the tarball immediately than construct
-/// a seemingly valid but incomplete archive. This wrapper allows us
-/// to swallow the end-of-archive marker that Builder::drop() emits,
-/// without writing it to the underlying sink.
-///
-struct AbortableWrite<W> {
-    w: W,
-    aborted: bool,
-}
-
-impl<W> AbortableWrite<W> {
-    pub fn new(w: W) -> Self {
-        AbortableWrite { w, aborted: false }
-    }
-
-    pub fn abort(&mut self) {
-        self.aborted = true;
-    }
-}
-
-impl<W> Write for AbortableWrite<W>
-where
-    W: Write,
-{
-    fn write(&mut self, data: &[u8]) -> io::Result<usize> {
-        if self.aborted {
-            Ok(data.len())
-        } else {
-            self.w.write(data)
-        }
-    }
-    fn flush(&mut self) -> io::Result<()> {
-        if self.aborted {
-            Ok(())
-        } else {
-            self.w.flush()
-        }
-    }
-}

pageserver/src/bin/pageserver.rs

@@ -38,6 +38,7 @@ fn version() -> String {
 }
 
 fn main() -> anyhow::Result<()> {
+    metrics::set_common_metrics_prefix("pageserver");
     let arg_matches = App::new("Zenith page server")
         .about("Materializes WAL stream to pages and serves them to the postgres")
         .version(&*version())
@@ -97,8 +98,6 @@ fn main() -> anyhow::Result<()> {
         let features: &[&str] = &[
             #[cfg(feature = "failpoints")]
             "failpoints",
-            #[cfg(feature = "profiling")]
-            "profiling",
         ];
         println!("{{\"features\": {features:?} }}");
         return Ok(());
@@ -184,8 +183,13 @@ fn main() -> anyhow::Result<()> {
     // as a ref.
     let conf: &'static PageServerConf = Box::leak(Box::new(conf));
 
-    // Initialize up failpoints support
+    // If failpoints are used, terminate the whole pageserver process if they are hit.
     let scenario = FailScenario::setup();
+    if fail::has_failpoints() {
+        std::panic::set_hook(Box::new(|_| {
+            std::process::exit(1);
+        }));
+    }
 
     // Basic initialization of things that don't change after startup
     virtual_file::init(conf.max_file_descriptors);
@@ -254,7 +258,7 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
         // Otherwise, the coverage data will be damaged.
         match daemonize.exit_action(|| exit_now(0)).start() {
             Ok(_) => info!("Success, daemonized"),
-            Err(err) => bail!("{err}. could not daemonize. bailing."),
+            Err(err) => error!(%err, "could not daemonize"),
         }
     }
 

pageserver/src/config.rs

@@ -5,18 +5,17 @@
 //! See also `settings.md` for better description on every parameter.
 
 use anyhow::{anyhow, bail, ensure, Context, Result};
-use remote_storage::RemoteStorageConfig;
+use remote_storage::{RemoteStorageConfig, RemoteStorageKind, S3Config};
 use std::env;
-
+use std::num::{NonZeroU32, NonZeroUsize};
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use std::time::Duration;
 use toml_edit;
 use toml_edit::{Document, Item};
-use url::Url;
 use utils::{
     postgres_backend::AuthType,
-    zid::{NodeId, ZTenantId, ZTimelineId},
+    zid::{ZNodeId, ZTenantId, ZTimelineId},
 };
 
 use crate::layered_repository::TIMELINES_SEGMENT_NAME;
@@ -34,7 +33,7 @@ pub mod defaults {
     pub const DEFAULT_WAIT_LSN_TIMEOUT: &str = "60 s";
     pub const DEFAULT_WAL_REDO_TIMEOUT: &str = "60 s";
 
-    pub const DEFAULT_SUPERUSER: &str = "cloud_admin";
+    pub const DEFAULT_SUPERUSER: &str = "zenith_admin";
 
     pub const DEFAULT_PAGE_CACHE_SIZE: usize = 8192;
     pub const DEFAULT_MAX_FILE_DESCRIPTORS: usize = 100;
@@ -78,7 +77,7 @@ pub mod defaults {
 pub struct PageServerConf {
     // Identifier of that particular pageserver so e g safekeepers
     // can safely distinguish different pageservers
-    pub id: NodeId,
+    pub id: ZNodeId,
 
     /// Example (default): 127.0.0.1:64000
     pub listen_pg_addr: String,
@@ -112,13 +111,6 @@ pub struct PageServerConf {
 
     pub profiling: ProfilingConfig,
     pub default_tenant_conf: TenantConf,
-
-    /// A prefix to add in etcd brokers before every key.
-    /// Can be used for isolating different pageserver groups within the same etcd cluster.
-    pub broker_etcd_prefix: String,
-
-    /// Etcd broker endpoints to connect to.
-    pub broker_endpoints: Vec<Url>,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -180,11 +172,9 @@ struct PageServerConfigBuilder {
     auth_validation_public_key_path: BuilderValue<Option<PathBuf>>,
     remote_storage_config: BuilderValue<Option<RemoteStorageConfig>>,
 
-    id: BuilderValue<NodeId>,
+    id: BuilderValue<ZNodeId>,
 
     profiling: BuilderValue<ProfilingConfig>,
-    broker_etcd_prefix: BuilderValue<String>,
-    broker_endpoints: BuilderValue<Vec<Url>>,
 }
 
 impl Default for PageServerConfigBuilder {
@@ -210,8 +200,6 @@ impl Default for PageServerConfigBuilder {
             remote_storage_config: Set(None),
             id: NotSet,
             profiling: Set(ProfilingConfig::Disabled),
-            broker_etcd_prefix: Set(etcd_broker::DEFAULT_NEON_BROKER_ETCD_PREFIX.to_string()),
-            broker_endpoints: Set(Vec::new()),
         }
     }
 }
@@ -268,15 +256,7 @@ impl PageServerConfigBuilder {
         self.remote_storage_config = BuilderValue::Set(remote_storage_config)
     }
 
-    pub fn broker_endpoints(&mut self, broker_endpoints: Vec<Url>) {
-        self.broker_endpoints = BuilderValue::Set(broker_endpoints)
-    }
-
-    pub fn broker_etcd_prefix(&mut self, broker_etcd_prefix: String) {
-        self.broker_etcd_prefix = BuilderValue::Set(broker_etcd_prefix)
-    }
-
-    pub fn id(&mut self, node_id: NodeId) {
+    pub fn id(&mut self, node_id: ZNodeId) {
         self.id = BuilderValue::Set(node_id)
     }
 
@@ -284,11 +264,7 @@ impl PageServerConfigBuilder {
         self.profiling = BuilderValue::Set(profiling)
     }
 
-    pub fn build(self) -> anyhow::Result<PageServerConf> {
-        let broker_endpoints = self
-            .broker_endpoints
-            .ok_or(anyhow!("No broker endpoints provided"))?;
-
+    pub fn build(self) -> Result<PageServerConf> {
         Ok(PageServerConf {
             listen_pg_addr: self
                 .listen_pg_addr
@@ -324,10 +300,6 @@ impl PageServerConfigBuilder {
             profiling: self.profiling.ok_or(anyhow!("missing profiling"))?,
             // TenantConf is handled separately
             default_tenant_conf: TenantConf::default(),
-            broker_endpoints,
-            broker_etcd_prefix: self
-                .broker_etcd_prefix
-                .ok_or(anyhow!("missing broker_etcd_prefix"))?,
         })
     }
 }
@@ -369,7 +341,7 @@ impl PageServerConf {
     /// validating the input and failing on errors.
     ///
     /// This leaves any options not present in the file in the built-in defaults.
-    pub fn parse_and_validate(toml: &Document, workdir: &Path) -> anyhow::Result<Self> {
+    pub fn parse_and_validate(toml: &Document, workdir: &Path) -> Result<Self> {
         let mut builder = PageServerConfigBuilder::default();
         builder.workdir(workdir.to_owned());
 
@@ -394,24 +366,13 @@ impl PageServerConf {
                 )),
                 "auth_type" => builder.auth_type(parse_toml_from_str(key, item)?),
                 "remote_storage" => {
-                    builder.remote_storage_config(Some(RemoteStorageConfig::from_toml(item)?))
+                    builder.remote_storage_config(Some(Self::parse_remote_storage_config(item)?))
                 }
                 "tenant_config" => {
                     t_conf = Self::parse_toml_tenant_conf(item)?;
                 }
-                "id" => builder.id(NodeId(parse_toml_u64(key, item)?)),
+                "id" => builder.id(ZNodeId(parse_toml_u64(key, item)?)),
                 "profiling" => builder.profiling(parse_toml_from_str(key, item)?),
-                "broker_etcd_prefix" => builder.broker_etcd_prefix(parse_toml_string(key, item)?),
-                "broker_endpoints" => builder.broker_endpoints(
-                    parse_toml_array(key, item)?
-                        .into_iter()
-                        .map(|endpoint_str| {
-                            endpoint_str.parse::<Url>().with_context(|| {
-                                format!("Array item {endpoint_str} for key {key} is not a valid url endpoint")
-                            })
-                        })
-                        .collect::<anyhow::Result<_>>()?,
-                ),
                 _ => bail!("unrecognized pageserver option '{key}'"),
             }
         }
@@ -480,25 +441,68 @@ impl PageServerConf {
         if let Some(pitr_interval) = item.get("pitr_interval") {
             t_conf.pitr_interval = Some(parse_toml_duration("pitr_interval", pitr_interval)?);
         }
-        if let Some(walreceiver_connect_timeout) = item.get("walreceiver_connect_timeout") {
-            t_conf.walreceiver_connect_timeout = Some(parse_toml_duration(
-                "walreceiver_connect_timeout",
-                walreceiver_connect_timeout,
-            )?);
-        }
-        if let Some(lagging_wal_timeout) = item.get("lagging_wal_timeout") {
-            t_conf.lagging_wal_timeout = Some(parse_toml_duration(
-                "lagging_wal_timeout",
-                lagging_wal_timeout,
-            )?);
-        }
-        if let Some(max_lsn_wal_lag) = item.get("max_lsn_wal_lag") {
-            t_conf.max_lsn_wal_lag = Some(parse_toml_from_str("max_lsn_wal_lag", max_lsn_wal_lag)?);
-        }
 
         Ok(t_conf)
     }
 
+    /// subroutine of parse_config(), to parse the `[remote_storage]` table.
+    fn parse_remote_storage_config(toml: &toml_edit::Item) -> anyhow::Result<RemoteStorageConfig> {
+        let local_path = toml.get("local_path");
+        let bucket_name = toml.get("bucket_name");
+        let bucket_region = toml.get("bucket_region");
+
+        let max_concurrent_syncs = NonZeroUsize::new(
+            parse_optional_integer("max_concurrent_syncs", toml)?
+                .unwrap_or(remote_storage::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNCS),
+        )
+        .context("Failed to parse 'max_concurrent_syncs' as a positive integer")?;
+
+        let max_sync_errors = NonZeroU32::new(
+            parse_optional_integer("max_sync_errors", toml)?
+                .unwrap_or(remote_storage::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS),
+        )
+        .context("Failed to parse 'max_sync_errors' as a positive integer")?;
+
+        let concurrency_limit = NonZeroUsize::new(
+            parse_optional_integer("concurrency_limit", toml)?
+                .unwrap_or(remote_storage::DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT),
+        )
+        .context("Failed to parse 'concurrency_limit' as a positive integer")?;
+
+        let storage = match (local_path, bucket_name, bucket_region) {
+            (None, None, None) => bail!("no 'local_path' nor 'bucket_name' option"),
+            (_, Some(_), None) => {
+                bail!("'bucket_region' option is mandatory if 'bucket_name' is given ")
+            }
+            (_, None, Some(_)) => {
+                bail!("'bucket_name' option is mandatory if 'bucket_region' is given ")
+            }
+            (None, Some(bucket_name), Some(bucket_region)) => RemoteStorageKind::AwsS3(S3Config {
+                bucket_name: parse_toml_string("bucket_name", bucket_name)?,
+                bucket_region: parse_toml_string("bucket_region", bucket_region)?,
+                prefix_in_bucket: toml
+                    .get("prefix_in_bucket")
+                    .map(|prefix_in_bucket| parse_toml_string("prefix_in_bucket", prefix_in_bucket))
+                    .transpose()?,
+                endpoint: toml
+                    .get("endpoint")
+                    .map(|endpoint| parse_toml_string("endpoint", endpoint))
+                    .transpose()?,
+                concurrency_limit,
+            }),
+            (Some(local_path), None, None) => RemoteStorageKind::LocalFs(PathBuf::from(
+                parse_toml_string("local_path", local_path)?,
+            )),
+            (Some(_), Some(_), _) => bail!("local_path and bucket_name are mutually exclusive"),
+        };
+
+        Ok(RemoteStorageConfig {
+            max_concurrent_syncs,
+            max_sync_errors,
+            storage,
+        })
+    }
+
     #[cfg(test)]
     pub fn test_repo_dir(test_name: &str) -> PathBuf {
         PathBuf::from(format!("../tmp_check/test_{test_name}"))
@@ -507,14 +511,14 @@ impl PageServerConf {
     #[cfg(test)]
     pub fn dummy_conf(repo_dir: PathBuf) -> Self {
         PageServerConf {
-            id: NodeId(0),
+            id: ZNodeId(0),
             wait_lsn_timeout: Duration::from_secs(60),
             wal_redo_timeout: Duration::from_secs(60),
             page_cache_size: defaults::DEFAULT_PAGE_CACHE_SIZE,
             max_file_descriptors: defaults::DEFAULT_MAX_FILE_DESCRIPTORS,
             listen_pg_addr: defaults::DEFAULT_PG_LISTEN_ADDR.to_string(),
             listen_http_addr: defaults::DEFAULT_HTTP_LISTEN_ADDR.to_string(),
-            superuser: "cloud_admin".to_string(),
+            superuser: "zenith_admin".to_string(),
             workdir: repo_dir,
             pg_distrib_dir: PathBuf::new(),
             auth_type: AuthType::Trust,
@@ -522,8 +526,6 @@ impl PageServerConf {
             remote_storage_config: None,
             profiling: ProfilingConfig::Disabled,
             default_tenant_conf: TenantConf::dummy_conf(),
-            broker_endpoints: Vec::new(),
-            broker_etcd_prefix: etcd_broker::DEFAULT_NEON_BROKER_ETCD_PREFIX.to_string(),
         }
     }
 }
@@ -549,6 +551,23 @@ fn parse_toml_u64(name: &str, item: &Item) -> Result<u64> {
     Ok(i as u64)
 }
 
+fn parse_optional_integer<I, E>(name: &str, item: &toml_edit::Item) -> anyhow::Result<Option<I>>
+where
+    I: TryFrom<i64, Error = E>,
+    E: std::error::Error + Send + Sync + 'static,
+{
+    let toml_integer = match item.get(name) {
+        Some(item) => item
+            .as_integer()
+            .with_context(|| format!("configure option {name} is not an integer"))?,
+        None => return Ok(None),
+    };
+
+    I::try_from(toml_integer)
+        .map(Some)
+        .with_context(|| format!("configure option {name} is too large"))
+}
+
 fn parse_toml_duration(name: &str, item: &Item) -> Result<Duration> {
     let s = item
         .as_str()
@@ -557,46 +576,20 @@ fn parse_toml_duration(name: &str, item: &Item) -> Result<Duration> {
     Ok(humantime::parse_duration(s)?)
 }
 
-fn parse_toml_from_str<T>(name: &str, item: &Item) -> anyhow::Result<T>
+fn parse_toml_from_str<T>(name: &str, item: &Item) -> Result<T>
 where
-    T: FromStr,
-    <T as FromStr>::Err: std::fmt::Display,
+    T: FromStr<Err = anyhow::Error>,
 {
     let v = item
         .as_str()
         .with_context(|| format!("configure option {name} is not a string"))?;
-    T::from_str(v).map_err(|e| {
-        anyhow!(
-            "Failed to parse string as {parse_type} for configure option {name}: {e}",
-            parse_type = stringify!(T)
-        )
-    })
-}
-
-fn parse_toml_array(name: &str, item: &Item) -> anyhow::Result<Vec<String>> {
-    let array = item
-        .as_array()
-        .with_context(|| format!("configure option {name} is not an array"))?;
-
-    array
-        .iter()
-        .map(|value| {
-            value
-                .as_str()
-                .map(str::to_string)
-                .with_context(|| format!("Array item {value:?} for key {name} is not a string"))
-        })
-        .collect()
+    T::from_str(v)
 }
 
 #[cfg(test)]
 mod tests {
-    use std::{
-        fs,
-        num::{NonZeroU32, NonZeroUsize},
-    };
+    use std::fs;
 
-    use remote_storage::{RemoteStorageKind, S3Config};
     use tempfile::{tempdir, TempDir};
 
     use super::*;
@@ -623,21 +616,17 @@ id = 10
     fn parse_defaults() -> anyhow::Result<()> {
         let tempdir = tempdir()?;
         let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?;
-        let broker_endpoint = "http://127.0.0.1:7777";
-        // we have to create dummy values to overcome the validation errors
-        let config_string = format!(
-            "pg_distrib_dir='{}'\nid=10\nbroker_endpoints = ['{broker_endpoint}']",
-            pg_distrib_dir.display()
-        );
+        // we have to create dummy pathes to overcome the validation errors
+        let config_string = format!("pg_distrib_dir='{}'\nid=10", pg_distrib_dir.display());
         let toml = config_string.parse()?;
 
         let parsed_config = PageServerConf::parse_and_validate(&toml, &workdir)
-            .unwrap_or_else(|e| panic!("Failed to parse config '{config_string}', reason: {e:?}"));
+            .unwrap_or_else(|e| panic!("Failed to parse config '{config_string}', reason: {e}"));
 
         assert_eq!(
             parsed_config,
             PageServerConf {
-                id: NodeId(10),
+                id: ZNodeId(10),
                 listen_pg_addr: defaults::DEFAULT_PG_LISTEN_ADDR.to_string(),
                 listen_http_addr: defaults::DEFAULT_HTTP_LISTEN_ADDR.to_string(),
                 wait_lsn_timeout: humantime::parse_duration(defaults::DEFAULT_WAIT_LSN_TIMEOUT)?,
@@ -652,10 +641,6 @@ id = 10
                 remote_storage_config: None,
                 profiling: ProfilingConfig::Disabled,
                 default_tenant_conf: TenantConf::default(),
-                broker_endpoints: vec![broker_endpoint
-                    .parse()
-                    .expect("Failed to parse a valid broker endpoint URL")],
-                broker_etcd_prefix: etcd_broker::DEFAULT_NEON_BROKER_ETCD_PREFIX.to_string(),
             },
             "Correct defaults should be used when no config values are provided"
         );
@@ -667,21 +652,20 @@ id = 10
     fn parse_basic_config() -> anyhow::Result<()> {
         let tempdir = tempdir()?;
         let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?;
-        let broker_endpoint = "http://127.0.0.1:7777";
 
         let config_string = format!(
-            "{ALL_BASE_VALUES_TOML}pg_distrib_dir='{}'\nbroker_endpoints = ['{broker_endpoint}']",
+            "{ALL_BASE_VALUES_TOML}pg_distrib_dir='{}'",
             pg_distrib_dir.display()
         );
         let toml = config_string.parse()?;
 
         let parsed_config = PageServerConf::parse_and_validate(&toml, &workdir)
-            .unwrap_or_else(|e| panic!("Failed to parse config '{config_string}', reason: {e:?}"));
+            .unwrap_or_else(|e| panic!("Failed to parse config '{config_string}', reason: {e}"));
 
         assert_eq!(
             parsed_config,
             PageServerConf {
-                id: NodeId(10),
+                id: ZNodeId(10),
                 listen_pg_addr: "127.0.0.1:64000".to_string(),
                 listen_http_addr: "127.0.0.1:9898".to_string(),
                 wait_lsn_timeout: Duration::from_secs(111),
@@ -696,10 +680,6 @@ id = 10
                 remote_storage_config: None,
                 profiling: ProfilingConfig::Disabled,
                 default_tenant_conf: TenantConf::default(),
-                broker_endpoints: vec![broker_endpoint
-                    .parse()
-                    .expect("Failed to parse a valid broker endpoint URL")],
-                broker_etcd_prefix: etcd_broker::DEFAULT_NEON_BROKER_ETCD_PREFIX.to_string(),
             },
             "Should be able to parse all basic config values correctly"
         );
@@ -711,7 +691,6 @@ id = 10
     fn parse_remote_fs_storage_config() -> anyhow::Result<()> {
         let tempdir = tempdir()?;
         let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?;
-        let broker_endpoint = "http://127.0.0.1:7777";
 
         let local_storage_path = tempdir.path().join("local_remote_storage");
 
@@ -731,7 +710,6 @@ local_path = '{}'"#,
             let config_string = format!(
                 r#"{ALL_BASE_VALUES_TOML}
 pg_distrib_dir='{}'
-broker_endpoints = ['{broker_endpoint}']
 
 {remote_storage_config_str}"#,
                 pg_distrib_dir.display(),
@@ -740,9 +718,7 @@ broker_endpoints = ['{broker_endpoint}']
             let toml = config_string.parse()?;
 
             let parsed_remote_storage_config = PageServerConf::parse_and_validate(&toml, &workdir)
-                .unwrap_or_else(|e| {
-                    panic!("Failed to parse config '{config_string}', reason: {e:?}")
-                })
+                .unwrap_or_else(|e| panic!("Failed to parse config '{config_string}', reason: {e}"))
                 .remote_storage_config
                 .expect("Should have remote storage config for the local FS");
 
@@ -752,7 +728,7 @@ broker_endpoints = ['{broker_endpoint}']
                     max_concurrent_syncs: NonZeroUsize::new(
                         remote_storage::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNCS
                     )
-                        .unwrap(),
+                    .unwrap(),
                     max_sync_errors: NonZeroU32::new(remote_storage::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS)
                         .unwrap(),
                     storage: RemoteStorageKind::LocalFs(local_storage_path.clone()),
@@ -775,7 +751,6 @@ broker_endpoints = ['{broker_endpoint}']
         let max_concurrent_syncs = NonZeroUsize::new(111).unwrap();
         let max_sync_errors = NonZeroU32::new(222).unwrap();
         let s3_concurrency_limit = NonZeroUsize::new(333).unwrap();
-        let broker_endpoint = "http://127.0.0.1:7777";
 
         let identical_toml_declarations = &[
             format!(
@@ -798,7 +773,6 @@ concurrency_limit = {s3_concurrency_limit}"#
             let config_string = format!(
                 r#"{ALL_BASE_VALUES_TOML}
 pg_distrib_dir='{}'
-broker_endpoints = ['{broker_endpoint}']
 
 {remote_storage_config_str}"#,
                 pg_distrib_dir.display(),
@@ -807,9 +781,7 @@ broker_endpoints = ['{broker_endpoint}']
             let toml = config_string.parse()?;
 
             let parsed_remote_storage_config = PageServerConf::parse_and_validate(&toml, &workdir)
-                .unwrap_or_else(|e| {
-                    panic!("Failed to parse config '{config_string}', reason: {e:?}")
-                })
+                .unwrap_or_else(|e| panic!("Failed to parse config '{config_string}', reason: {e}"))
                 .remote_storage_config
                 .expect("Should have remote storage config for S3");
 

pageserver/src/http/models.rs

@@ -1,10 +1,8 @@
-use std::num::NonZeroU64;
-
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
 use utils::{
     lsn::Lsn,
-    zid::{NodeId, ZTenantId, ZTimelineId},
+    zid::{ZNodeId, ZTenantId, ZTimelineId},
 };
 
 #[serde_as]
@@ -35,9 +33,6 @@ pub struct TenantCreateRequest {
     pub gc_period: Option<String>,
     pub image_creation_threshold: Option<usize>,
     pub pitr_interval: Option<String>,
-    pub walreceiver_connect_timeout: Option<String>,
-    pub lagging_wal_timeout: Option<String>,
-    pub max_lsn_wal_lag: Option<NonZeroU64>,
 }
 
 #[serde_as]
@@ -47,7 +42,7 @@ pub struct TenantCreateResponse(#[serde_as(as = "DisplayFromStr")] pub ZTenantId
 
 #[derive(Serialize)]
 pub struct StatusResponse {
-    pub id: NodeId,
+    pub id: ZNodeId,
 }
 
 impl TenantCreateRequest {
@@ -73,9 +68,6 @@ pub struct TenantConfigRequest {
     pub gc_period: Option<String>,
     pub image_creation_threshold: Option<usize>,
     pub pitr_interval: Option<String>,
-    pub walreceiver_connect_timeout: Option<String>,
-    pub lagging_wal_timeout: Option<String>,
-    pub max_lsn_wal_lag: Option<NonZeroU64>,
 }
 
 impl TenantConfigRequest {
@@ -90,21 +82,6 @@ impl TenantConfigRequest {
             gc_period: None,
             image_creation_threshold: None,
             pitr_interval: None,
-            walreceiver_connect_timeout: None,
-            lagging_wal_timeout: None,
-            max_lsn_wal_lag: None,
         }
     }
 }
-
-/// A WAL receiver's data stored inside the global `WAL_RECEIVERS`.
-/// We keep one WAL receiver active per timeline.
-#[serde_as]
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct WalReceiverEntry {
-    pub wal_producer_connstr: Option<String>,
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub last_received_msg_lsn: Option<Lsn>,
-    /// the timestamp (in microseconds) of the last received message
-    pub last_received_msg_ts: Option<u128>,
-}

pageserver/src/http/openapi_spec.yml

@@ -123,53 +123,6 @@ paths:
               schema:
                 $ref: "#/components/schemas/Error"
 
-  /v1/tenant/{tenant_id}/timeline/{timeline_id}/wal_receiver:
-    parameters:
-      - name: tenant_id
-        in: path
-        required: true
-        schema:
-          type: string
-          format: hex
-      - name: timeline_id
-        in: path
-        required: true
-        schema:
-          type: string
-          format: hex
-    get:
-      description: Get wal receiver's data attached to the timeline
-      responses:
-        "200":
-          description: WalReceiverEntry
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/WalReceiverEntry"
-        "401":
-          description: Unauthorized Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/UnauthorizedError"
-        "403":
-          description: Forbidden Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ForbiddenError"
-        "404":
-          description: Error when no wal receiver is running or found
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/NotFoundError"
-        "500":
-          description: Generic operation error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
 
   /v1/tenant/{tenant_id}/timeline/{timeline_id}/attach:
     parameters:
@@ -567,21 +520,6 @@ components:
           type: integer
         current_logical_size_non_incremental:
           type: integer
-    WalReceiverEntry:
-      type: object
-      required:
-        - thread_id
-        - wal_producer_connstr
-      properties:
-        thread_id:
-          type: integer
-        wal_producer_connstr:
-          type: string
-        last_received_msg_lsn:
-          type: string
-          format: hex
-        last_received_msg_ts:
-          type: integer
 
     Error:
       type: object

pageserver/src/http/routes.rs

@@ -224,23 +224,6 @@ async fn timeline_detail_handler(request: Request<Body>) -> Result<Response<Body
     json_response(StatusCode::OK, timeline_info)
 }
 
-async fn wal_receiver_get_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    let tenant_id: ZTenantId = parse_request_param(&request, "tenant_id")?;
-    check_permission(&request, Some(tenant_id))?;
-
-    let timeline_id: ZTimelineId = parse_request_param(&request, "timeline_id")?;
-    let wal_receiver_entry = crate::walreceiver::get_wal_receiver_entry(tenant_id, timeline_id)
-        .instrument(info_span!("wal_receiver_get", tenant = %tenant_id, timeline = %timeline_id))
-        .await
-        .ok_or_else(|| {
-            ApiError::NotFound(format!(
-                "WAL receiver data not found for tenant {tenant_id} and timeline {timeline_id}"
-            ))
-        })?;
-
-    json_response(StatusCode::OK, &wal_receiver_entry)
-}
-
 async fn timeline_attach_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
     let tenant_id: ZTenantId = parse_request_param(&request, "tenant_id")?;
     check_permission(&request, Some(tenant_id))?;
@@ -395,19 +378,6 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
             Some(humantime::parse_duration(&pitr_interval).map_err(ApiError::from_err)?);
     }
 
-    if let Some(walreceiver_connect_timeout) = request_data.walreceiver_connect_timeout {
-        tenant_conf.walreceiver_connect_timeout = Some(
-            humantime::parse_duration(&walreceiver_connect_timeout).map_err(ApiError::from_err)?,
-        );
-    }
-    if let Some(lagging_wal_timeout) = request_data.lagging_wal_timeout {
-        tenant_conf.lagging_wal_timeout =
-            Some(humantime::parse_duration(&lagging_wal_timeout).map_err(ApiError::from_err)?);
-    }
-    if let Some(max_lsn_wal_lag) = request_data.max_lsn_wal_lag {
-        tenant_conf.max_lsn_wal_lag = Some(max_lsn_wal_lag);
-    }
-
     tenant_conf.checkpoint_distance = request_data.checkpoint_distance;
     tenant_conf.compaction_target_size = request_data.compaction_target_size;
     tenant_conf.compaction_threshold = request_data.compaction_threshold;
@@ -455,18 +425,6 @@ async fn tenant_config_handler(mut request: Request<Body>) -> Result<Response<Bo
         tenant_conf.pitr_interval =
             Some(humantime::parse_duration(&pitr_interval).map_err(ApiError::from_err)?);
     }
-    if let Some(walreceiver_connect_timeout) = request_data.walreceiver_connect_timeout {
-        tenant_conf.walreceiver_connect_timeout = Some(
-            humantime::parse_duration(&walreceiver_connect_timeout).map_err(ApiError::from_err)?,
-        );
-    }
-    if let Some(lagging_wal_timeout) = request_data.lagging_wal_timeout {
-        tenant_conf.lagging_wal_timeout =
-            Some(humantime::parse_duration(&lagging_wal_timeout).map_err(ApiError::from_err)?);
-    }
-    if let Some(max_lsn_wal_lag) = request_data.max_lsn_wal_lag {
-        tenant_conf.max_lsn_wal_lag = Some(max_lsn_wal_lag);
-    }
 
     tenant_conf.checkpoint_distance = request_data.checkpoint_distance;
     tenant_conf.compaction_target_size = request_data.compaction_target_size;
@@ -527,10 +485,6 @@ pub fn make_router(
             "/v1/tenant/:tenant_id/timeline/:timeline_id",
             timeline_detail_handler,
         )
-        .get(
-            "/v1/tenant/:tenant_id/timeline/:timeline_id/wal_receiver",
-            wal_receiver_get_handler,
-        )
         .post(
             "/v1/tenant/:tenant_id/timeline/:timeline_id/attach",
             timeline_attach_handler,

pageserver/src/keyspace.rs

@@ -15,7 +15,7 @@ pub struct KeySpace {
 impl KeySpace {
     ///
     /// Partition a key space into roughly chunks of roughly 'target_size' bytes
-    /// in each partition.
+    /// in each patition.
     ///
     pub fn partition(&self, target_size: u64) -> KeyPartitioning {
         // Assume that each value is 8k in size.