Tonic based broker for SkTimelineInfo.

This API is rather pointless, as sane choice anyway requires knowledge of peers
status and leaders lifetime in any case can intersect, which is fine for us --
so manual elections are straightforward. Here, we deterministically choose among
the reasonably caught up safekeepers, shifting by timeline id to spread the
load.

A step towards custom broker https://github.com/neondatabase/neon/issues/2394

.github/ansible/deploy.yaml

@@ -1,7 +1,7 @@
 - name: Upload Neon binaries
   hosts: storage
   gather_facts: False
-  remote_user: "{{ remote_user }}"
+  remote_user: admin
 
   tasks:
 
@@ -36,7 +36,7 @@
 - name: Deploy pageserver
   hosts: pageservers
   gather_facts: False
-  remote_user: "{{ remote_user }}"
+  remote_user: admin
 
   tasks:
 
@@ -124,7 +124,7 @@
 - name: Deploy safekeeper
   hosts: safekeepers
   gather_facts: False
-  remote_user: "{{ remote_user }}"
+  remote_user: admin
 
   tasks:
 

.github/ansible/neon-stress.hosts.yaml

@@ -12,20 +12,19 @@ storage:
         bucket_name: "{{ bucket_name }}"
         bucket_region: "{{ bucket_region }}"
         prefix_in_bucket: "{{ inventory_hostname }}"
-    hostname_suffix: ".local"
-    remote_user: admin
+
   children:
     pageservers:
       hosts:
         neon-stress-ps-1:
-          console_region_id: aws-eu-west-1
+          console_region_id: 1
         neon-stress-ps-2:
-          console_region_id: aws-eu-west-1
+          console_region_id: 1
     safekeepers:
       hosts:
         neon-stress-sk-1:
-          console_region_id: aws-eu-west-1
+          console_region_id: 1
         neon-stress-sk-2:
-          console_region_id: aws-eu-west-1
+          console_region_id: 1
         neon-stress-sk-3:
-          console_region_id: aws-eu-west-1
+          console_region_id: 1

.github/ansible/production.hosts.yaml

@@ -12,22 +12,20 @@ storage:
         bucket_name: "{{ bucket_name }}"
         bucket_region: "{{ bucket_region }}"
         prefix_in_bucket: "{{ inventory_hostname }}"
-    hostname_suffix: ".local"
-    remote_user: admin
 
   children:
     pageservers:
       hosts:
         zenith-1-ps-2:
-          console_region_id: aws-us-west-2
+          console_region_id: 1
         zenith-1-ps-3:
-          console_region_id: aws-us-west-2
+          console_region_id: 1
 
     safekeepers:
       hosts:
         zenith-1-sk-1:
-          console_region_id: aws-us-west-2
+          console_region_id: 1
         zenith-1-sk-2:
-          console_region_id: aws-us-west-2
+          console_region_id: 1
         zenith-1-sk-3:
-          console_region_id: aws-us-west-2
+          console_region_id: 1

.github/ansible/scripts/init_pageserver.sh

@@ -12,19 +12,18 @@ cat <<EOF | tee /tmp/payload
   "version": 1,
   "host": "${HOST}",
   "port": 6400,
-  "region_id": "{{ console_region_id }}",
+  "region_id": {{ console_region_id }},
   "instance_id": "${INSTANCE_ID}",
   "http_host": "${HOST}",
-  "http_port": 9898,
-  "active": false
+  "http_port": 9898
 }
 EOF
 
 # check if pageserver already registered or not
-if ! curl -sf -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/pageservers/${INSTANCE_ID} -o /dev/null; then
+if ! curl -sf -X PATCH -d '{}' {{ console_mgmt_base_url }}/api/v1/pageservers/${INSTANCE_ID} -o /dev/null; then
 
     # not registered, so register it now
-    ID=$(curl -sf -X POST -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/pageservers -d@/tmp/payload | jq -r '.id')
+    ID=$(curl -sf -X POST {{ console_mgmt_base_url }}/api/v1/pageservers -d@/tmp/payload | jq -r '.ID')
 
     # init pageserver
     sudo -u pageserver /usr/local/bin/pageserver -c "id=${ID}" -c "pg_distrib_dir='/usr/local'" --init -D /storage/pageserver/data

.github/ansible/scripts/init_safekeeper.sh

@@ -14,18 +14,18 @@ cat <<EOF | tee /tmp/payload
   "host": "${HOST}",
   "port": 6500,
   "http_port": 7676,
-  "region_id": "{{ console_region_id }}",
+  "region_id": {{ console_region_id }},
   "instance_id": "${INSTANCE_ID}",
-  "availability_zone_id": "${AZ_ID}",
-  "active": false
+  "availability_zone_id": "${AZ_ID}"
 }
 EOF
 
 # check if safekeeper already registered or not
-if ! curl -sf -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/safekeepers/${INSTANCE_ID} -o /dev/null; then
+if ! curl -sf -X PATCH -d '{}' {{ console_mgmt_base_url }}/api/v1/safekeepers/${INSTANCE_ID} -o /dev/null; then
 
     # not registered, so register it now
-    ID=$(curl -sf -X POST -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/safekeepers -d@/tmp/payload | jq -r '.id')
+    ID=$(curl -sf -X POST {{ console_mgmt_base_url }}/api/v1/safekeepers -d@/tmp/payload | jq -r '.ID')
+
     # init safekeeper
     sudo -u safekeeper /usr/local/bin/safekeeper --id ${ID} --init -D /storage/safekeeper/data
 fi

.github/ansible/ssm_config

@@ -1,3 +0,0 @@
-ansible_connection: aws_ssm
-ansible_aws_ssm_bucket_name: neon-dev-bucket
-ansible_python_interpreter: /usr/bin/python3

.github/ansible/staging.hosts.yaml

@@ -11,24 +11,30 @@ storage:
         bucket_name: "{{ bucket_name }}"
         bucket_region: "{{ bucket_region }}"
         prefix_in_bucket: "{{ inventory_hostname }}"
-    hostname_suffix: ".local"
-    remote_user: admin
 
   children:
     pageservers:
       hosts:
         zenith-us-stage-ps-2:
-          console_region_id: aws-us-east-1
+          console_region_id: 27
         zenith-us-stage-ps-3:
-          console_region_id: aws-us-east-1
+          console_region_id: 27
         zenith-us-stage-ps-4:
-          console_region_id: aws-us-east-1
+          console_region_id: 27
+        zenith-us-stage-test-ps-1:
+          console_region_id: 28
 
     safekeepers:
       hosts:
         zenith-us-stage-sk-4:
-          console_region_id: aws-us-east-1
+          console_region_id: 27
         zenith-us-stage-sk-5:
-          console_region_id: aws-us-east-1
+          console_region_id: 27
         zenith-us-stage-sk-6:
-          console_region_id: aws-us-east-1
+          console_region_id: 27
+        zenith-us-stage-test-sk-1:
+          console_region_id: 28
+        zenith-us-stage-test-sk-2:
+          console_region_id: 28
+        zenith-us-stage-test-sk-3:
+          console_region_id: 28

.github/ansible/staging.us-east-2.hosts.yaml

@@ -1,32 +0,0 @@
-storage:
-  vars:
-    bucket_name: neon-staging-storage-us-east-2
-    bucket_region: us-east-2
-    console_mgmt_base_url: http://console-staging.local
-    env_name: us-stage
-    etcd_endpoints: etcd-0.us-east-2.aws.neon.build:2379
-    pageserver_config_stub:
-      pg_distrib_dir: /usr/local
-      remote_storage:
-        bucket_name: "{{ bucket_name }}"
-        bucket_region: "{{ bucket_region }}"
-        prefix_in_bucket: "pageserver/v1"
-    hostname_suffix: ""
-    remote_user: ssm-user
-    ansible_aws_ssm_region: us-east-2
-    console_region_id: aws-us-east-2
-
-  children:
-    pageservers:
-      hosts:
-        pageserver-0.us-east-2.aws.neon.build:
-          ansible_host: i-0c3e70929edb5d691
-
-    safekeepers:
-      hosts:
-        safekeeper-0.us-east-2.aws.neon.build:
-          ansible_host: i-027662bd552bf5db0
-        safekeeper-1.us-east-2.aws.neon.build:
-          ansible_host: i-0171efc3604a7b907
-        safekeeper-2.us-east-2.aws.neon.build:
-          ansible_host: i-0de0b03a51676a6ce

.github/ansible/systemd/pageserver.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=Neon pageserver
+Description=Zenith pageserver
 After=network.target auditd.service
 
 [Service]

.github/ansible/systemd/safekeeper.service

@@ -1,12 +1,12 @@
 [Unit]
-Description=Neon safekeeper
+Description=Zenith safekeeper
 After=network.target auditd.service
 
 [Service]
 Type=simple
 User=safekeeper
 Environment=RUST_BACKTRACE=1 NEON_REPO_DIR=/storage/safekeeper/data LD_LIBRARY_PATH=/usr/local/v14/lib
-ExecStart=/usr/local/bin/safekeeper -l {{ inventory_hostname }}{{ hostname_suffix }}:6500 --listen-http {{ inventory_hostname }}{{ hostname_suffix }}:7676 -D /storage/safekeeper/data --broker-endpoints={{ etcd_endpoints }} --remote-storage='{bucket_name="{{bucket_name}}", bucket_region="{{bucket_region}}", prefix_in_bucket="{{ env_name }}/wal"}'
+ExecStart=/usr/local/bin/safekeeper -l {{ inventory_hostname }}.local:6500 --listen-http {{ inventory_hostname }}.local:7676 -D /storage/safekeeper/data --broker-endpoints={{ etcd_endpoints }} --remote-storage='{bucket_name="{{bucket_name}}", bucket_region="{{bucket_region}}", prefix_in_bucket="{{ env_name }}/wal"}'
 ExecReload=/bin/kill -HUP $MAINPID
 KillMode=mixed
 KillSignal=SIGINT

.github/helm-values/dev-us-east-2-beta.neon-proxy-scram.yaml

@@ -1,31 +0,0 @@
-# Helm chart values for neon-proxy-scram.
-# This is a YAML-formatted file.
-
-image:
-  repository: neondatabase/neon
-
-settings:
-  authBackend: "console"
-  authEndpoint: "http://console-staging.local/management/api/v2"
-  domain: "*.us-east-2.aws.neon.build"
-
-# -- Additional labels for neon-proxy pods
-podLabels:
-  zenith_service: proxy-scram
-  zenith_env: dev
-  zenith_region: us-east-2
-  zenith_region_slug: us-east-2
-
-exposedService:
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: us-east-2.aws.neon.build
-
-#metrics:
-#  enabled: true
-#  serviceMonitor:
-#    enabled: true
-#    selector:
-#      release: kube-prometheus-stack

.github/workflows/build_and_test.yml

@@ -481,7 +481,6 @@ jobs:
 
   neon-image:
     runs-on: dev
-    needs: [ tag ]
     container: gcr.io/kaniko-project/executor:v1.9.0-debug
 
     steps:
@@ -495,11 +494,10 @@ jobs:
         run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json
 
       - name: Kaniko build neon
-        run: /kaniko/executor --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --build-arg GIT_VERSION=${{ github.sha }} --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}}
+        run: /kaniko/executor --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --build-arg GIT_VERSION=${{ github.sha }} --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:$GITHUB_RUN_ID
 
   compute-tools-image:
     runs-on: dev
-    needs: [ tag ]
     container: gcr.io/kaniko-project/executor:v1.9.0-debug
 
     steps:
@@ -510,12 +508,11 @@ jobs:
         run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json
 
       - name: Kaniko build compute tools
-        run: /kaniko/executor --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-tools --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}}
+        run: /kaniko/executor --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-tools --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:$GITHUB_RUN_ID
 
   compute-node-image:
     runs-on: dev
     container: gcr.io/kaniko-project/executor:v1.9.0-debug
-    needs: [ tag ]
     steps:
       - name: Checkout
         uses: actions/checkout@v1 # v3 won't work with kaniko
@@ -530,12 +527,11 @@ jobs:
         # cloud repo depends on this image name, thus duplicating it
         # remove compute-node when cloud repo is updated
       - name: Kaniko build compute node with extensions v14 (compatibility)
-        run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-node-v14 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:${{needs.tag.outputs.build-tag}}
+        run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-node-v14 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:$GITHUB_RUN_ID
 
   compute-node-image-v14:
     runs-on: dev
     container: gcr.io/kaniko-project/executor:v1.9.0-debug
-    needs: [ tag ]
     steps:
       - name: Checkout
         uses: actions/checkout@v1 # v3 won't work with kaniko
@@ -547,13 +543,12 @@ jobs:
         run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json
 
       - name: Kaniko build compute node with extensions v14
-        run: /kaniko/executor --skip-unused-stages  --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache  --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-node-v14 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}}
+        run: /kaniko/executor --skip-unused-stages  --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache  --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-node-v14 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:$GITHUB_RUN_ID
 
 
   compute-node-image-v15:
     runs-on: dev
     container: gcr.io/kaniko-project/executor:v1.9.0-debug
-    needs: [ tag ]
     steps:
       - name: Checkout
         uses: actions/checkout@v1 # v3 won't work with kaniko
@@ -565,11 +560,11 @@ jobs:
         run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json
 
       - name: Kaniko build compute node with extensions v15
-        run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-node-v15 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:${{needs.tag.outputs.build-tag}}
+        run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-node-v15 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:$GITHUB_RUN_ID
 
   promote-images:
     runs-on: dev
-    needs: [ tag, neon-image, compute-node-image, compute-node-image-v14, compute-node-image-v15, compute-tools-image ]
+    needs: [ neon-image, compute-node-image, compute-node-image-v14, compute-node-image-v15, compute-tools-image ]
     if: github.event_name != 'workflow_dispatch'
     container: amazon/aws-cli
     strategy:
@@ -582,9 +577,8 @@ jobs:
 
     steps:
       - name: Promote image to latest
-        run: |
-          export MANIFEST=$(aws ecr batch-get-image --repository-name ${{ matrix.name }} --image-ids imageTag=${{needs.tag.outputs.build-tag}} --query 'images[].imageManifest' --output text)
-          aws ecr put-image --repository-name ${{ matrix.name }} --image-tag latest --image-manifest "$MANIFEST"
+        run:
+          MANIFEST=$(aws ecr batch-get-image --repository-name ${{ matrix.name }} --image-ids imageTag=$GITHUB_RUN_ID --query 'images[].imageManifest' --output text) && aws ecr put-image --repository-name ${{ matrix.name }} --image-tag latest --image-manifest "$MANIFEST"
 
   push-docker-hub:
     runs-on: dev
@@ -603,19 +597,19 @@ jobs:
           echo "{\"credsStore\":\"ecr-login\"}" > /github/home/.docker/config.json
 
       - name: Pull neon image from ECR
-        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}} neon
+        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:latest neon
 
       - name: Pull compute tools image from ECR
-        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} compute-tools
+        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:latest compute-tools
 
       - name: Pull compute node image from ECR
-        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:${{needs.tag.outputs.build-tag}} compute-node
+        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:latest compute-node
 
       - name: Pull compute node v14 image from ECR
-        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}} compute-node-v14
+        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:latest compute-node-v14
 
       - name: Pull compute node v15 image from ECR
-        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:${{needs.tag.outputs.build-tag}} compute-node-v15
+        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:latest compute-node-v15
 
       - name: Pull rust image from ECR
         run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned rust
@@ -625,11 +619,11 @@ jobs:
           (github.ref_name == 'main' || github.ref_name == 'release') &&
           github.event_name != 'workflow_dispatch'
         run: |
-          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.us-east-2.amazonaws.com/neon:latest
-          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.us-east-2.amazonaws.com/compute-tools:latest
-          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.us-east-2.amazonaws.com/compute-node:latest
-          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.us-east-2.amazonaws.com/compute-node-v14:latest
-          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.us-east-2.amazonaws.com/compute-node-v15:latest
+          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:$GITHUB_RUN_ID 093970136003.dkr.ecr.us-east-2.amazonaws.com/neon:latest
+          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:$GITHUB_RUN_ID 093970136003.dkr.ecr.us-east-2.amazonaws.com/compute-tools:latest
+          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:$GITHUB_RUN_ID 093970136003.dkr.ecr.us-east-2.amazonaws.com/compute-node:latest
+          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:$GITHUB_RUN_ID 093970136003.dkr.ecr.us-east-2.amazonaws.com/compute-node-v14:latest
+          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:$GITHUB_RUN_ID 093970136003.dkr.ecr.us-east-2.amazonaws.com/compute-node-v15:latest
 
       - name: Configure Docker Hub login
         run: |
@@ -677,11 +671,11 @@ jobs:
       - id: set-matrix
         run: |
           if [[ "$GITHUB_REF_NAME" == "main" ]]; then
-            STAGING='{"env_name": "staging", "proxy_job": "neon-proxy", "proxy_config": "staging.proxy", "kubeconfig_secret": "STAGING_KUBECONFIG_DATA", "console_api_key_secret": "NEON_STAGING_API_KEY"}'
-            NEON_STRESS='{"env_name": "neon-stress", "proxy_job": "neon-stress-proxy", "proxy_config": "neon-stress.proxy", "kubeconfig_secret": "NEON_STRESS_KUBECONFIG_DATA", "console_api_key_secret": "NEON_CAPTEST_API_KEY"}'
+            STAGING='{"env_name": "staging", "proxy_job": "neon-proxy", "proxy_config": "staging.proxy", "kubeconfig_secret": "STAGING_KUBECONFIG_DATA"}'
+            NEON_STRESS='{"env_name": "neon-stress", "proxy_job": "neon-stress-proxy", "proxy_config": "neon-stress.proxy", "kubeconfig_secret": "NEON_STRESS_KUBECONFIG_DATA"}'
             echo "include=[$STAGING, $NEON_STRESS]" >> $GITHUB_OUTPUT
           elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
-            PRODUCTION='{"env_name": "production", "proxy_job": "neon-proxy", "proxy_config": "production.proxy", "kubeconfig_secret": "PRODUCTION_KUBECONFIG_DATA", "console_api_key_secret": "NEON_PRODUCTION_API_KEY"}'
+            PRODUCTION='{"env_name": "production", "proxy_job": "neon-proxy", "proxy_config": "production.proxy", "kubeconfig_secret": "PRODUCTION_KUBECONFIG_DATA"}'
             echo "include=[$PRODUCTION]" >> $GITHUB_OUTPUT
           else
             echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
@@ -741,47 +735,7 @@ jobs:
           ssh-add ssh-key
           rm -f ssh-key ssh-key-cert.pub
           ansible-galaxy collection install sivel.toiletwater
-          ansible-playbook deploy.yaml -i ${{ matrix.env_name }}.hosts.yaml -e CONSOLE_API_TOKEN=${{ secrets[matrix.console_api_key_secret] }}
-          rm -f neon_install.tar.gz .neon_current_version
-
-  deploy-new:
-    runs-on: dev
-    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
-    # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version.
-    # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly
-    needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
-    if: |
-      (github.ref_name == 'main') &&
-      github.event_name != 'workflow_dispatch'
-    defaults:
-      run:
-        shell: bash
-    env:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          fetch-depth: 0
-
-      - name: Redeploy
-        run: |
-          export DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          cd "$(pwd)/.github/ansible"
-
-          if [[ "$GITHUB_REF_NAME" == "main" ]]; then
-            ./get_binaries.sh
-          elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
-            RELEASE=true ./get_binaries.sh
-          else
-            echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
-            exit 1
-          fi
-
-          ansible-galaxy collection install sivel.toiletwater
-          ansible-playbook deploy.yaml -i staging.us-east-2.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_STAGING_API_KEY}}
+          ansible-playbook deploy.yaml -i ${{ matrix.env_name }}.hosts.yaml
           rm -f neon_install.tar.gz .neon_current_version
 
   deploy-proxy:
@@ -825,31 +779,3 @@ jobs:
           DOCKER_TAG=${{needs.tag.outputs.build-tag}}
           helm upgrade ${{ matrix.proxy_job }}       neondatabase/neon-proxy --namespace neon-proxy --install -f .github/helm-values/${{ matrix.proxy_config }}.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
           helm upgrade ${{ matrix.proxy_job }}-scram neondatabase/neon-proxy --namespace neon-proxy --install -f .github/helm-values/${{ matrix.proxy_config }}-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
-
-  deploy-proxy-new:
-    runs-on: dev
-    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
-    # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
-    needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
-    if: |
-      (github.ref_name == 'main' || github.ref_name == 'release') &&
-      github.event_name != 'workflow_dispatch'
-    defaults:
-      run:
-        shell: bash
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          fetch-depth: 0
-
-      - name: Configure environment 
-        run: |
-          helm repo add neondatabase https://neondatabase.github.io/helm-charts
-          aws --region us-east-2 eks update-kubeconfig --name dev-us-east-2-beta --role-arn arn:aws:iam::369495373322:role/github-runner
-
-      - name: Re-deploy proxy
-        run: |
-          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/dev-us-east-2-beta.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s

.github/workflows/codestyle.yml

@@ -36,7 +36,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
         with:
           submodules: true
           fetch-depth: 2

Cargo.lock

@@ -40,7 +40,8 @@ dependencies = [
 [[package]]
 name = "amplify_num"
 version = "0.4.1"
-source = "git+https://github.com/hlinnaka/rust-amplify.git?branch=unsigned-int-perf#bd49b737c2e6e623ab8e9ba5ceaed5712d3a3940"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f27d3d00d3d115395a7a8a4dc045feb7aa82b641e485f7e15f4e67ac16f4f56d"
 
 [[package]]
 name = "android_system_properties"
@@ -74,9 +75,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
 
 [[package]]
 name = "asn1-rs"
-version = "0.5.1"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf6690c370453db30743b373a60ba498fc0d6d83b11f4abfd87a84a075db5dd4"
+checksum = "30ff05a702273012438132f449575dbc804e27b2f3cbe3069aa237d26c98fa33"
 dependencies = [
  "asn1-rs-derive",
  "asn1-rs-impl",
@@ -90,9 +91,9 @@ dependencies = [
 
 [[package]]
 name = "asn1-rs-derive"
-version = "0.4.0"
+version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c"
+checksum = "db8b7511298d5b7784b40b092d9e9dcd3a627a5707e4b5e507931ab0d44eeebf"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -261,13 +262,15 @@ dependencies = [
 
 [[package]]
 name = "bindgen"
-version = "0.61.0"
+version = "0.60.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a022e58a142a46fea340d68012b9201c094e93ec3d033a944a24f8fd4a4f09a"
+checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6"
 dependencies = [
  "bitflags",
  "cexpr",
  "clang-sys",
+ "clap",
+ "env_logger",
  "lazy_static",
  "lazycell",
  "log",
@@ -277,7 +280,6 @@ dependencies = [
  "regex",
  "rustc-hash",
  "shlex",
- "syn",
  "which",
 ]
 
@@ -325,14 +327,13 @@ checksum = "5988cb1d626264ac94100be357308f29ff7cbdd3b36bda27f450a4ee3f713426"
 
 [[package]]
 name = "bstr"
-version = "1.0.1"
+version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fca0852af221f458706eb0725c03e4ed6c46af9ac98e6a689d5e634215d594dd"
+checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
 dependencies = [
+ "lazy_static",
  "memchr",
- "once_cell",
  "regex-automata",
- "serde",
 ]
 
 [[package]]
@@ -449,23 +450,28 @@ version = "3.2.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "86447ad904c7fb335a790c9d7fe3d0d971dc523b8ccd1561a520de9a85302750"
 dependencies = [
+ "atty",
  "bitflags",
- "clap_lex 0.2.4",
+ "clap_derive",
+ "clap_lex",
  "indexmap",
+ "once_cell",
+ "strsim",
+ "termcolor",
  "textwrap",
 ]
 
 [[package]]
-name = "clap"
-version = "4.0.15"
+name = "clap_derive"
+version = "3.2.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bf8832993da70a4c6d13c581f4463c2bdda27b9bf1c5498dc4365543abe6d6f"
+checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65"
 dependencies = [
- "atty",
- "bitflags",
- "clap_lex 0.3.0",
- "strsim",
- "termcolor",
+ "heck",
+ "proc-macro-error",
+ "proc-macro2",
+ "quote",
+ "syn",
 ]
 
 [[package]]
@@ -477,15 +483,6 @@ dependencies = [
  "os_str_bytes",
 ]
 
-[[package]]
-name = "clap_lex"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8"
-dependencies = [
- "os_str_bytes",
-]
-
 [[package]]
 name = "close_fds"
 version = "0.3.2"
@@ -543,7 +540,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "chrono",
- "clap 4.0.15",
+ "clap",
  "env_logger",
  "futures",
  "hyper",
@@ -585,7 +582,7 @@ name = "control_plane"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "clap 4.0.15",
+ "clap",
  "comfy-table",
  "git-version",
  "nix 0.25.0",
@@ -678,7 +675,7 @@ dependencies = [
  "atty",
  "cast",
  "ciborium",
- "clap 3.2.22",
+ "clap",
  "criterion-plot",
  "itertools",
  "lazy_static",
@@ -746,7 +743,7 @@ dependencies = [
  "autocfg",
  "cfg-if",
  "crossbeam-utils",
- "memoffset 0.6.5",
+ "memoffset",
  "scopeguard",
 ]
 
@@ -906,14 +903,14 @@ version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6ee87af31d84ef885378aebca32be3d682b0e0dc119d5b4860a2c5bb5046730"
 dependencies = [
- "uuid 0.8.2",
+ "uuid",
 ]
 
 [[package]]
 name = "der-parser"
-version = "8.1.0"
+version = "7.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42d4bc9b0db0a0df9ae64634ac5bdefb7afcb534e182275ca0beadbe486701c1"
+checksum = "fe398ac75057914d7d07307bf67dc7f3f574a26783b4fc7805a20ffa9f506e82"
 dependencies = [
  "asn1-rs",
  "displaydoc",
@@ -1020,11 +1017,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9fb8664f6ea68aba5503d42dd1be786b0f1bd9b7972e7f40208c83ef74db91bf"
 dependencies = [
  "http",
- "prost",
+ "prost 0.10.4",
  "tokio",
  "tokio-stream",
- "tonic",
- "tonic-build",
+ "tonic 0.7.2",
+ "tonic-build 0.7.2",
  "tower",
  "tower-service",
 ]
@@ -1222,12 +1219,6 @@ version = "0.3.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1"
 
-[[package]]
-name = "futures-timer"
-version = "3.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c"
-
 [[package]]
 name = "futures-util"
 version = "0.3.24"
@@ -1814,15 +1805,6 @@ dependencies = [
  "autocfg",
 ]
 
-[[package]]
-name = "memoffset"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
-dependencies = [
- "autocfg",
-]
-
 [[package]]
 name = "metrics"
 version = "0.1.0"
@@ -1905,6 +1887,22 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "546c37ac5d9e56f55e73b677106873d9d9f5190605e41a856503623648488cae"
 
+[[package]]
+name = "neon_broker"
+version = "0.1.0"
+dependencies = [
+ "async-stream",
+ "clap",
+ "futures-core",
+ "futures-util",
+ "prost 0.11.0",
+ "tokio",
+ "tokio-stream",
+ "tonic 0.8.2",
+ "tonic-build 0.8.2",
+ "utils",
+]
+
 [[package]]
 name = "nix"
 version = "0.23.1"
@@ -1915,7 +1913,7 @@ dependencies = [
  "cc",
  "cfg-if",
  "libc",
- "memoffset 0.6.5",
+ "memoffset",
 ]
 
 [[package]]
@@ -1928,7 +1926,7 @@ dependencies = [
  "bitflags",
  "cfg-if",
  "libc",
- "memoffset 0.6.5",
+ "memoffset",
  "pin-utils",
 ]
 
@@ -2041,9 +2039,9 @@ dependencies = [
 
 [[package]]
 name = "oid-registry"
-version = "0.6.0"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d4bda43fd1b844cbc6e6e54b5444e2b1bc7838bce59ad205902cccbb26d6761"
+checksum = "38e20717fa0541f39bd146692035c37bedfa532b3e5071b35761082407546b2a"
 dependencies = [
  "asn1-rs",
 ]
@@ -2134,11 +2132,10 @@ dependencies = [
  "byteorder",
  "bytes",
  "chrono",
- "clap 4.0.15",
+ "clap",
  "close_fds",
  "const_format",
  "crc32c",
- "criterion",
  "crossbeam-utils",
  "daemonize",
  "etcd_broker",
@@ -2411,7 +2408,7 @@ dependencies = [
  "env_logger",
  "hex",
  "log",
- "memoffset 0.7.1",
+ "memoffset",
  "once_cell",
  "postgres",
  "rand",
@@ -2458,6 +2455,30 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "proc-macro-error"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
+dependencies = [
+ "proc-macro-error-attr",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro-error-attr"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "version_check",
+]
+
 [[package]]
 name = "proc-macro-hack"
 version = "0.5.19"
@@ -2509,7 +2530,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71adf41db68aa0daaefc69bb30bcd68ded9b9abaad5d1fbb6304c4fb390e083e"
 dependencies = [
  "bytes",
- "prost-derive",
+ "prost-derive 0.10.1",
+]
+
+[[package]]
+name = "prost"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "399c3c31cdec40583bb68f0b18403400d01ec4289c383aa047560439952c4dd7"
+dependencies = [
+ "bytes",
+ "prost-derive 0.11.0",
 ]
 
 [[package]]
@@ -2527,8 +2558,28 @@ dependencies = [
  "log",
  "multimap",
  "petgraph",
- "prost",
- "prost-types",
+ "prost 0.10.4",
+ "prost-types 0.10.1",
+ "regex",
+ "tempfile",
+ "which",
+]
+
+[[package]]
+name = "prost-build"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f835c582e6bd972ba8347313300219fed5bfa52caf175298d860b61ff6069bb"
+dependencies = [
+ "bytes",
+ "heck",
+ "itertools",
+ "lazy_static",
+ "log",
+ "multimap",
+ "petgraph",
+ "prost 0.11.0",
+ "prost-types 0.11.1",
  "regex",
  "tempfile",
  "which",
@@ -2547,6 +2598,19 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "prost-derive"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7345d5f0e08c0536d7ac7229952590239e77abf0a0100a1b1d890add6ea96364"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "prost-types"
 version = "0.10.1"
@@ -2554,7 +2618,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68"
 dependencies = [
  "bytes",
- "prost",
+ "prost 0.10.4",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4dfaa718ad76a44b3415e6c4d53b17c8f99160dcb3a99b10470fce8ad43f6e3e"
+dependencies = [
+ "bytes",
+ "prost 0.11.0",
 ]
 
 [[package]]
@@ -2567,7 +2641,7 @@ dependencies = [
  "base64",
  "bstr",
  "bytes",
- "clap 4.0.15",
+ "clap",
  "futures",
  "git-version",
  "hashbrown",
@@ -2601,7 +2675,7 @@ dependencies = [
  "tracing-subscriber",
  "url",
  "utils",
- "uuid 1.2.1",
+ "uuid",
  "workspace_hack",
  "x509-parser",
 ]
@@ -2690,13 +2764,13 @@ dependencies = [
 
 [[package]]
 name = "rcgen"
-version = "0.10.0"
+version = "0.8.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b"
+checksum = "5911d1403f4143c9d56a702069d593e8d0f3fab880a85e103604d0893ea31ba7"
 dependencies = [
+ "chrono",
  "pem",
  "ring",
- "time 0.3.15",
  "yasna",
 ]
 
@@ -2886,21 +2960,9 @@ dependencies = [
 
 [[package]]
 name = "rstest"
-version = "0.15.0"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9c9dc66cc29792b663ffb5269be669f1613664e69ad56441fdb895c2347b930"
-dependencies = [
- "futures",
- "futures-timer",
- "rstest_macros",
- "rustc_version 0.4.0",
-]
-
-[[package]]
-name = "rstest_macros"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5015e68a0685a95ade3eee617ff7101ab6a3fc689203101ca16ebc16f2b89c66"
+checksum = "d912f35156a3f99a66ee3e11ac2e0b3f34ac85a07e05263d05a7e2c8810d616f"
 dependencies = [
  "cfg-if",
  "proc-macro2",
@@ -3080,7 +3142,7 @@ dependencies = [
  "async-trait",
  "byteorder",
  "bytes",
- "clap 4.0.15",
+ "clap",
  "const_format",
  "crc32c",
  "daemonize",
@@ -3470,7 +3532,7 @@ dependencies = [
  "debugid",
  "memmap2",
  "stable_deref_trait",
- "uuid 0.8.2",
+ "uuid",
 ]
 
 [[package]]
@@ -3801,8 +3863,40 @@ dependencies = [
  "hyper-timeout",
  "percent-encoding",
  "pin-project",
- "prost",
- "prost-derive",
+ "prost 0.10.4",
+ "prost-derive 0.10.1",
+ "tokio",
+ "tokio-stream",
+ "tokio-util",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+ "tracing-futures",
+]
+
+[[package]]
+name = "tonic"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55b9af819e54b8f33d453655bef9b9acc171568fb49523078d0cc4e7484200ec"
+dependencies = [
+ "async-stream",
+ "async-trait",
+ "axum",
+ "base64",
+ "bytes",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-timeout",
+ "percent-encoding",
+ "pin-project",
+ "prost 0.11.0",
+ "prost-derive 0.11.0",
  "tokio",
  "tokio-stream",
  "tokio-util",
@@ -3821,7 +3915,20 @@ checksum = "d9263bf4c9bfaae7317c1c2faf7f18491d2fe476f70c414b73bf5d445b00ffa1"
 dependencies = [
  "prettyplease",
  "proc-macro2",
- "prost-build",
+ "prost-build 0.10.4",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tonic-build"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48c6fd7c2581e36d63388a9e04c350c21beb7a8b059580b2e93993c526899ddc"
+dependencies = [
+ "prettyplease",
+ "proc-macro2",
+ "prost-build 0.11.1",
  "quote",
  "syn",
 ]
@@ -4056,12 +4163,6 @@ name = "uuid"
 version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
-
-[[package]]
-name = "uuid"
-version = "1.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "feb41e78f93363bb2df8b0e86a2ca30eed7806ea16ea0c790d757cf93f79be83"
 dependencies = [
  "getrandom",
  "serde",
@@ -4111,7 +4212,7 @@ name = "wal_craft"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "clap 4.0.15",
+ "clap",
  "env_logger",
  "log",
  "once_cell",
@@ -4351,7 +4452,7 @@ dependencies = [
  "anyhow",
  "bytes",
  "chrono",
- "clap 4.0.15",
+ "clap",
  "crossbeam-utils",
  "either",
  "fail",
@@ -4359,12 +4460,11 @@ dependencies = [
  "indexmap",
  "libc",
  "log",
- "memchr",
  "nom",
  "num-bigint",
  "num-integer",
  "num-traits",
- "prost",
+ "prost 0.10.4",
  "rand",
  "regex",
  "regex-syntax",
@@ -4377,13 +4477,14 @@ dependencies = [
  "tokio-util",
  "tracing",
  "tracing-core",
+ "uuid",
 ]
 
 [[package]]
 name = "x509-parser"
-version = "0.14.0"
+version = "0.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0ecbeb7b67ce215e40e3cc7f2ff902f94a223acf44995934763467e7b1febc8"
+checksum = "9fb9bace5b5589ffead1afb76e43e34cff39cd0f3ce7e170ae0c29e53b88eb1c"
 dependencies = [
  "asn1-rs",
  "base64",
@@ -4414,11 +4515,11 @@ checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3"
 
 [[package]]
 name = "yasna"
-version = "0.5.0"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "346d34a236c9d3e5f3b9b74563f238f955bbd05fa0b8b4efa53c130c43982f4c"
+checksum = "e262a29d0e61ccf2b6190d7050d4b237535fc76ce4c1210d9caa316f71dffa75"
 dependencies = [
- "time 0.3.15",
+ "chrono",
 ]
 
 [[package]]

Cargo.toml

@@ -11,6 +11,7 @@ cargo-features = ["named-profiles"]
 
 [workspace]
 members = [
+    "broker",
     "compute_tools",
     "control_plane",
     "pageserver",

Dockerfile.compute-node-v14

@@ -71,12 +71,10 @@ RUN apt update && \
 RUN apt update && \
     apt install -y --no-install-recommends -t testing binutils
 
-# Sed is used to patch for https://github.com/plv8/plv8/issues/503
 RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.4.tar.gz && \
     tar xvzf v3.1.4.tar.gz && \
     cd plv8-3.1.4 && \
     export PATH="/usr/local/pgsql/bin:$PATH" && \
-    sed -i 's/MemoryContextAlloc(/MemoryContextAllocZero(/' plv8.cc && \
     make -j $(getconf _NPROCESSORS_ONLN) && \
     make -j $(getconf _NPROCESSORS_ONLN) install && \
     rm -rf /plv8-* && \
@@ -118,7 +116,8 @@ RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.0.1.tar.gz -O h3
 #
 FROM build-deps AS neon-pg-ext-build
 COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/
-COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/
+# plv8 still sometimes crashes during the creation
+# COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=h3-pg-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=h3-pg-build /h3/usr /
 COPY pgxn/ pgxn/

Dockerfile.compute-node-v15

@@ -76,12 +76,10 @@ RUN apt update && \
 RUN apt update && \
     apt install -y --no-install-recommends -t testing binutils
 
-# Sed is used to patch for https://github.com/plv8/plv8/issues/503
 RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.4.tar.gz && \
     tar xvzf v3.1.4.tar.gz && \
     cd plv8-3.1.4 && \
     export PATH="/usr/local/pgsql/bin:$PATH" && \
-    sed -i 's/MemoryContextAlloc(/MemoryContextAllocZero(/' plv8.cc && \
     make -j $(getconf _NPROCESSORS_ONLN) && \
     make -j $(getconf _NPROCESSORS_ONLN) install && \
     rm -rf /plv8-* && \
@@ -123,7 +121,8 @@ RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.0.1.tar.gz -O h3
 #
 FROM build-deps AS neon-pg-ext-build
 COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/
-COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/
+# plv8 still sometimes crashes during the creation
+# COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=h3-pg-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=h3-pg-build /h3/usr /
 COPY pgxn/ pgxn/

broker/Cargo.toml

@@ -0,0 +1,35 @@
+[package]
+name = "neon_broker"
+version = "0.1.0"
+edition = "2021"
+
+[features]
+bench = []
+
+[[bin]]
+name = "neon_broker"
+path = "src/broker.rs"
+
+[[bin]]
+name = "neon_broker_bench"
+path = "src/bench.rs"
+# build benchmarking binary only if explicitly requested with '--feature bench'
+# required-features = ["bench"]
+
+[dependencies]
+async-stream = "0.3"
+futures-core = "0.3"
+futures-util = "0.3"
+tonic = "0.8"
+prost = "0.11"
+tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] }
+# for exploring with tokio-console
+# tokio = { version = "1", features = ["full", "tracing"] }
+# console-subscriber = "0.1.8"
+tokio-stream = "0.1"
+clap = { version = "3.2.17", features = ["derive"] }
+
+utils = { path = "../libs/utils" }
+
+[build-dependencies]
+tonic-build = "0.8"

broker/build.rs

@@ -0,0 +1,4 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    tonic_build::compile_protos("proto/broker.proto")?;
+    Ok(())
+}

broker/proto/broker.proto

@@ -0,0 +1,38 @@
+syntax = "proto3";
+
+package neon_broker;
+
+service NeonBroker {
+    // Subscribe to safekeeper updates.
+    rpc SubscribeSafekeeperInfo(SubscribeSafekeeperInfoRequest) returns (stream SafekeeperTimelineInfo) {};
+
+    // Publish safekeeper updates.
+    rpc PublishSafekeeperInfo(stream SafekeeperTimelineInfo) returns (Empty) {};
+}
+
+message SubscribeSafekeeperInfoRequest {
+    oneof subscription_key {
+        Empty all = 1; // subscribe to everything
+        TenantTimelineId tenant_timeline_id = 2; // subscribe to specific timeline
+    }
+}
+
+message SafekeeperTimelineInfo {
+    uint64 safekeeper_id = 1;
+    TenantTimelineId tenant_timeline_id = 2;
+    uint64 last_log_term = 3;
+    uint64 flush_lsn = 4;
+    uint64 commit_lsn = 5;
+    uint64 backup_lsn = 6;
+    uint64 remote_consistent_lsn = 7;
+    uint64 peer_horizon_lsn = 8;
+    string safekeeper_connstr = 9;
+}
+
+message TenantTimelineId {
+    bytes tenant_id = 1;
+    bytes timeline_id = 2;
+}
+
+message Empty {
+}

broker/readme.md

@@ -0,0 +1,4 @@
+```
+cargo build -r -p neon_broker --features bench && target/release/neon_broker
+target/release/neon_broker_bench -s 1 -p 1
+```

broker/src/bench.rs

@@ -0,0 +1,179 @@
+pub mod neon_broker {
+    tonic::include_proto!("neon_broker");
+}
+
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use clap::Parser;
+use neon_broker::neon_broker_client::NeonBrokerClient;
+use neon_broker::subscribe_safekeeper_info_request::SubscriptionKey;
+use neon_broker::TenantTimelineId as ProtoTenantTimelineId;
+use neon_broker::{SafekeeperTimelineInfo, SubscribeSafekeeperInfoRequest};
+use tokio::time::{self, sleep};
+
+use tonic::transport::Channel;
+use tonic::Request;
+
+#[derive(Parser, Debug)]
+#[clap(author, version, about, long_about = None)]
+struct Args {
+    /// Number of publishers
+    #[clap(short = 'p', long, value_parser, default_value_t = 1)]
+    num_pubs: u64,
+    /// Number of subscribers
+    #[clap(short = 's', long, value_parser, default_value_t = 1)]
+    num_subs: u64,
+}
+
+async fn progress_reporter(counters: Vec<Arc<AtomicU64>>) {
+    let mut interval = time::interval(Duration::from_millis(1000));
+    let mut c_old = counters.iter().map(|c| c.load(Ordering::Relaxed)).sum();
+    let mut c_min_old = counters
+        .iter()
+        .map(|c| c.load(Ordering::Relaxed))
+        .min()
+        .unwrap_or(0);
+    let mut started_at = None;
+    let mut skipped: u64 = 0;
+    loop {
+        interval.tick().await;
+        // print!(
+        //     "cnts are {:?}",
+        //     counters
+        //         .iter()
+        //         .map(|c| c.load(Ordering::Relaxed))
+        //         .collect::<Vec<_>>()
+        // );
+        let c_new = counters.iter().map(|c| c.load(Ordering::Relaxed)).sum();
+        let c_min_new = counters
+            .iter()
+            .map(|c| c.load(Ordering::Relaxed))
+            .min()
+            .unwrap_or(0);
+        if c_new > 0 && started_at.is_none() {
+            started_at = Some(Instant::now());
+            skipped = c_new;
+        }
+        let avg_rps = started_at.map(|s| {
+            let dur = s.elapsed();
+            let dur_secs = dur.as_secs() as f64 + (dur.subsec_millis() as f64) / 1000.0;
+            let avg_rps = (c_new - skipped) as f64 / dur_secs;
+            (dur, avg_rps)
+        });
+        println!(
+            "sum rps {}, min rps {} total {}, total min {}, duration, avg sum rps {:?}",
+            c_new - c_old,
+            c_min_new - c_min_old,
+            c_new,
+            c_min_new,
+            avg_rps
+        );
+        c_old = c_new;
+        c_min_old = c_min_new;
+    }
+}
+
+fn tli_from_u64(i: u64) -> Vec<u8> {
+    let mut timeline_id = vec![0xFF; 8];
+    timeline_id.extend_from_slice(&i.to_be_bytes());
+    timeline_id
+}
+
+async fn subscribe(client: Option<NeonBrokerClient<Channel>>, counter: Arc<AtomicU64>, i: u64) {
+    let mut client = match client {
+        Some(c) => c,
+        None => NeonBrokerClient::connect("http://[::1]:50051")
+            .await
+            .unwrap(),
+    };
+
+    // let key = SubscriptionKey::All(Empty {});
+    let key = SubscriptionKey::TenantTimelineId(ProtoTenantTimelineId {
+        tenant_id: vec![0xFF; 16],
+        timeline_id: tli_from_u64(i),
+    });
+    let request = SubscribeSafekeeperInfoRequest {
+        subscription_key: Some(key),
+    };
+    let mut stream = client
+        .subscribe_safekeeper_info(request)
+        .await
+        .unwrap()
+        .into_inner();
+
+    while let Some(_feature) = stream.message().await.unwrap() {
+        counter.fetch_add(1, Ordering::Relaxed);
+        // println!("info = {:?}, client {}", _feature, i);
+    }
+}
+
+async fn publish(client: Option<NeonBrokerClient<Channel>>, n_keys: u64) {
+    let mut client = match client {
+        Some(c) => c,
+        None => NeonBrokerClient::connect("http://[::1]:50051")
+            .await
+            .unwrap(),
+    };
+    let mut counter: u64 = 0;
+
+    // create stream producing new values
+    let outbound = async_stream::stream! {
+        loop {
+            let info = SafekeeperTimelineInfo {
+                safekeeper_id: 1,
+                tenant_timeline_id: Some(ProtoTenantTimelineId {
+                    tenant_id: vec![0xFF; 16],
+                    timeline_id: tli_from_u64(counter % n_keys),
+                }),
+                last_log_term: 0,
+                flush_lsn: counter,
+                commit_lsn: 2,
+                backup_lsn: 3,
+                remote_consistent_lsn: 4,
+                peer_horizon_lsn: 5,
+                safekeeper_connstr: "zenith-1-sk-1.local:7676".to_owned(),
+            };
+            counter += 1;
+            // println!("sending info = {:?}", info);
+            // if counter >= 1000 {
+                // break;
+            // }
+            yield info;
+            // sleep(Duration::from_millis(100)).await;
+        }
+    };
+    let _response = client
+        .publish_safekeeper_info(Request::new(outbound))
+        .await
+        .unwrap();
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let args = Args::parse();
+
+    let mut counters = Vec::with_capacity(args.num_subs as usize);
+    for _ in 0..args.num_subs {
+        counters.push(Arc::new(AtomicU64::new(0)));
+    }
+    let h = tokio::spawn(progress_reporter(counters.clone()));
+
+    let c = NeonBrokerClient::connect("http://[::1]:50051")
+        .await
+        .unwrap();
+
+    for i in 0..args.num_subs {
+        let c = Some(c.clone());
+        // let c = None;
+        tokio::spawn(subscribe(c, counters[i as usize].clone(), i));
+    }
+    for _i in 0..args.num_pubs {
+        // let c = Some(c.clone());
+        tokio::spawn(publish(None, args.num_subs as u64));
+    }
+
+    h.await?;
+    Ok(())
+}

broker/src/broker.rs

@@ -0,0 +1,526 @@
+//! Simple pub-sub based on grpc (tonic) and Tokio mpsc for storage nodes
+//! messaging. The main design goal is to avoid central synchronization during
+//! normal flow, resorting to it only when pub/sub change happens. Each
+//! subscriber holds mpsc for messages it sits on; tx end is sent to existing
+//! publishers and saved in shared state for new ones. Publishers maintain
+//! locally set of subscribers they stream messages to.
+//!
+//! Subscriptions to 1) single timeline 2) everything are possible. We could add
+//! subscription to set of timelines to save grpc streams, but testing shows
+//! many individual streams is also ok.
+//!
+//! Message is dropped if subscriber can't consume it, not affecting other
+//! subscribers.
+//!
+//! Only safekeeper message is supported, but it is not hard to add something
+//! else with templating.
+use std::collections::hash_map::Entry;
+use std::collections::HashMap;
+use std::fmt;
+use std::pin::Pin;
+use std::sync::{Arc, Mutex};
+use std::task::{Context, Poll};
+use std::time::Duration;
+
+use futures_core::Stream;
+use futures_util::StreamExt;
+use tokio::sync::mpsc::error::TrySendError;
+use tokio::sync::mpsc::{self, Receiver, Sender};
+use tokio::{select, time};
+use tokio_stream::wrappers::ReceiverStream;
+use tonic::Code;
+use tonic::{transport::Server, Request, Response, Status};
+
+use neon_broker_proto::neon_broker_server::{NeonBroker, NeonBrokerServer};
+use neon_broker_proto::subscribe_safekeeper_info_request::SubscriptionKey as ProtoSubscriptionKey;
+use neon_broker_proto::TenantTimelineId as ProtoTenantTimelineId;
+use neon_broker_proto::{Empty, SafekeeperTimelineInfo, SubscribeSafekeeperInfoRequest};
+use utils::id::{TenantId, TenantTimelineId, TimelineId};
+
+pub mod neon_broker_proto {
+    // The string specified here must match the proto package name.
+    // If you want to have a look at the generated code, it is at path similar to
+    // target/debug/build/neon_broker-0fde81d03bedc3b2/out/neon_broker.rs
+    tonic::include_proto!("neon_broker");
+}
+
+// Max size of the queue to the subscriber.
+const CHAN_SIZE: usize = 256;
+
+type PubId = u64; // id of publisher for registering in maps
+type SubId = u64; // id of subscriber for registering in maps
+
+#[derive(Copy, Clone)]
+enum SubscriptionKey {
+    All,
+    Timeline(TenantTimelineId),
+}
+
+impl SubscriptionKey {
+    // Parse protobuf subkey (protobuf doesn't have fixed size bytes, we get vectors).
+    pub fn from_proto_subscription_key(key: ProtoSubscriptionKey) -> Result<Self, Status> {
+        match key {
+            ProtoSubscriptionKey::All(_) => Ok(SubscriptionKey::All),
+            ProtoSubscriptionKey::TenantTimelineId(proto_ttid) => {
+                Ok(SubscriptionKey::Timeline(parse_proto_ttid(&proto_ttid)?))
+            }
+        }
+    }
+}
+
+// Subscriber id + tx end of the channel for messages to it.
+#[derive(Clone)]
+struct SubSender(SubId, Sender<SafekeeperTimelineInfo>);
+
+impl fmt::Debug for SubSender {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "Subscription id {}", self.0)
+    }
+}
+
+// Announcements subscriber sends to publisher(s) asking it to stream to the
+// provided channel, or forget about it, releasing memory.
+#[derive(Clone)]
+enum SubAnnounce {
+    AddAll(Sender<SafekeeperTimelineInfo>), // add subscription to all timelines
+    AddTimeline(TenantTimelineId, SubSender), // add subsciption to the specific timeline
+    RemoveTimeline(TenantTimelineId, SubId), // remove subscription to the specific timeline
+                                            // RemoveAll is not needed as publisher will notice closed channel while
+                                            // trying to send the next message.
+}
+
+struct SharedState {
+    // Registered publishers. They sit on the rx end of these channels and
+    // receive through it tx handles of chans to subscribers.
+    //
+    // Note: publishers don't identify which keys they publish, so each
+    // publisher will receive channels to all subs and filter them before sending.
+    pub_txs: HashMap<PubId, Sender<SubAnnounce>>,
+    next_pub_id: PubId,
+    // Registered subscribers -- when publisher joins it walks over them,
+    // collecting txs to send messages.
+    subs_to_all: HashMap<SubId, Sender<SafekeeperTimelineInfo>>,
+    subs_to_timelines: HashMap<TenantTimelineId, Vec<SubSender>>,
+    next_sub_id: SubId,
+}
+
+// Utility func to remove subscription from the map
+fn remove_sub(
+    subs_to_timelines: &mut HashMap<TenantTimelineId, Vec<SubSender>>,
+    ttid: &TenantTimelineId,
+    sub_id: SubId,
+) {
+    if let Some(subsenders) = subs_to_timelines.get_mut(&ttid) {
+        subsenders.retain(|ss| ss.0 != sub_id);
+        if subsenders.len() == 0 {
+            subs_to_timelines.remove(&ttid);
+        }
+    }
+    // Note that subscription might be not here if subscriber task was aborted
+    // earlier than it managed to notify publisher about itself.
+}
+
+impl SharedState {
+    // Register new publisher.
+    pub fn register_publisher(&mut self, announce_tx: Sender<SubAnnounce>) -> PubId {
+        let pub_id = self.next_pub_id;
+        self.next_pub_id += 1;
+        assert!(!self.pub_txs.contains_key(&pub_id));
+        self.pub_txs.insert(pub_id, announce_tx);
+        pub_id
+    }
+
+    pub fn unregister_publisher(&mut self, pub_id: PubId) {
+        assert!(self.pub_txs.contains_key(&pub_id));
+        self.pub_txs.remove(&pub_id);
+    }
+
+    // Register new subscriber.
+    // Returns list of channels through which existing publishers must be notified
+    // about new subscriber; we can't do it here due to risk of deadlock.
+    pub fn register_subscriber(
+        &mut self,
+        sub_key: SubscriptionKey,
+        sub_tx: Sender<SafekeeperTimelineInfo>,
+    ) -> (SubId, Vec<Sender<SubAnnounce>>, SubAnnounce) {
+        let sub_id = self.next_sub_id;
+        self.next_sub_id += 1;
+        let announce = match sub_key {
+            SubscriptionKey::All => {
+                assert!(!self.subs_to_all.contains_key(&sub_id));
+                self.subs_to_all.insert(sub_id, sub_tx.clone());
+                SubAnnounce::AddAll(sub_tx)
+            }
+            SubscriptionKey::Timeline(ttid) => {
+                match self.subs_to_timelines.entry(ttid) {
+                    Entry::Occupied(mut o) => {
+                        let subsenders = o.get_mut();
+                        subsenders.push(SubSender(sub_id, sub_tx.clone()));
+                    }
+                    Entry::Vacant(v) => {
+                        v.insert(vec![SubSender(sub_id, sub_tx.clone())]);
+                    }
+                }
+                SubAnnounce::AddTimeline(ttid, SubSender(sub_id, sub_tx))
+            }
+        };
+        // Collect existing publishers to notify them after lock is released;
+        // TODO: the probability of channels being full here is tiny (publisher
+        // always blocks listening chan), we can try sending first and resort to
+        // cloning if needed.
+        //
+        // Deadlock is possible only if publisher tries to access shared state
+        // during its lifetime, i.e. we add maintenance of set of published
+        // tlis. Otherwise we can just await here (but lock must be replaced
+        // with Tokio one).
+        //
+        // We could also just error out if some chan is full, but that needs
+        // cleanup of incompleted job, and notifying publishers when unregistering
+        // is mandatory anyway.
+        (sub_id, self.pub_txs.values().cloned().collect(), announce)
+    }
+
+    // Unregister the subscriber. Similar to register_subscriber, returns list
+    // of channels through which publishers must be notified about the removal.
+    pub fn unregister_subscriber(
+        &mut self,
+        sub_id: SubId,
+        sub_key: SubscriptionKey,
+    ) -> Option<(Vec<Sender<SubAnnounce>>, SubAnnounce)> {
+        // We need to notify existing publishers only about per timeline
+        // subscriptions, 'all' kind is detected on its own through closed
+        // channels.
+        let announce = match sub_key {
+            SubscriptionKey::All => {
+                assert!(self.subs_to_all.contains_key(&sub_id));
+                self.subs_to_all.remove(&sub_id);
+                None
+            }
+            SubscriptionKey::Timeline(ref ttid) => {
+                remove_sub(&mut self.subs_to_timelines, ttid, sub_id);
+                Some(SubAnnounce::RemoveTimeline(*ttid, sub_id))
+            }
+        };
+        announce.map(|a| (self.pub_txs.values().cloned().collect(), a))
+    }
+
+    pub fn report(&mut self) {
+        println!(
+            "registered {} publishers, {} subs to all, {} subs to timelines",
+            self.pub_txs.len(),
+            self.subs_to_all.len(),
+            self.subs_to_timelines.len(),
+        );
+    }
+}
+
+// SharedState wrapper for post-locking operations (sending to pub_tx chans).
+#[derive(Clone)]
+struct Registry {
+    shared_state: Arc<Mutex<SharedState>>,
+}
+
+impl Registry {
+    // Register new publisher in shared state.
+    pub fn register_publisher(&self) -> Publisher {
+        let (announce_tx, announce_rx) = mpsc::channel(128);
+        let mut ss = self.shared_state.lock().unwrap();
+        let id = ss.register_publisher(announce_tx);
+        let (subs_to_all, subs_to_timelines) = (
+            ss.subs_to_all.values().cloned().collect(),
+            ss.subs_to_timelines.clone(),
+        );
+        drop(ss);
+        // println!("registered publisher {}", id);
+        Publisher {
+            id,
+            announce_rx: announce_rx.into(),
+            subs_to_all,
+            subs_to_timelines,
+            registry: self.clone(),
+        }
+    }
+
+    pub fn unregister_publisher(&self, publisher: &Publisher) {
+        self.shared_state
+            .lock()
+            .unwrap()
+            .unregister_publisher(publisher.id);
+        // println!("unregistered publisher {}", publisher.id);
+    }
+
+    // Register new subscriber in shared state.
+    pub async fn register_subscriber(&self, sub_key: SubscriptionKey) -> Subscriber {
+        let (tx, rx) = mpsc::channel(CHAN_SIZE);
+        let id;
+        let mut pub_txs;
+        let announce;
+        {
+            let mut ss = self.shared_state.lock().unwrap();
+            (id, pub_txs, announce) = ss.register_subscriber(sub_key, tx);
+        }
+        // Note: it is important to create Subscriber before .await. If client
+        // disconnects during await, which would terminate the Future we still
+        // need to run Subscriber's drop() which will unregister it from the
+        // shared state.
+        let subscriber = Subscriber {
+            id,
+            key: sub_key,
+            sub_rx: rx,
+            registry: self.clone(),
+        };
+        // Notify existing publishers about new subscriber.
+        for pub_tx in pub_txs.iter_mut() {
+            // Closed channel is fine; it means publisher has gone.
+            pub_tx.send(announce.clone()).await.ok();
+        }
+        // println!("registered subscriber {}", id);
+        subscriber
+    }
+
+    // Unregister the subscriber
+    pub fn unregister_subscriber(&self, sub: &Subscriber) {
+        let mut ss = self.shared_state.lock().unwrap();
+        let announce_pack = ss.unregister_subscriber(sub.id, sub.key);
+        drop(ss);
+        // Notify publishers about the removal. Apart from wanting to do it
+        // outside lock, here we also spin a task as Drop impl can't be async.
+        if let Some((mut pub_txs, announce)) = announce_pack {
+            tokio::spawn(async move {
+                for pub_tx in pub_txs.iter_mut() {
+                    // Closed channel is fine; it means publisher has gone.
+                    pub_tx.send(announce.clone()).await.ok();
+                }
+            });
+        }
+        // println!("unregistered subscriber {}", sub.id);
+    }
+
+    pub async fn report(&self) {
+        let mut interval = time::interval(Duration::from_millis(1000));
+        loop {
+            interval.tick().await;
+            self.shared_state.lock().unwrap().report();
+        }
+    }
+}
+
+// Private subscriber state.
+struct Subscriber {
+    id: SubId,
+    key: SubscriptionKey,
+    // Subscriber receives messages from publishers here.
+    sub_rx: Receiver<SafekeeperTimelineInfo>,
+    // to unregister itself from shared state in Drop
+    registry: Registry,
+}
+
+impl Drop for Subscriber {
+    fn drop(&mut self) {
+        self.registry.unregister_subscriber(self);
+    }
+}
+
+// Private publisher state
+struct Publisher {
+    id: PubId,
+    // new subscribers request to send (or stop sending) msgs them here.
+    // It could be just Receiver, but weirdly it doesn't implement futures_core Stream directly.
+    announce_rx: ReceiverStream<SubAnnounce>,
+    subs_to_all: Vec<Sender<SafekeeperTimelineInfo>>,
+    subs_to_timelines: HashMap<TenantTimelineId, Vec<SubSender>>,
+    // to unregister itself from shared state in Drop
+    registry: Registry,
+}
+
+impl Publisher {
+    // Send msg to relevant subscribers.
+    pub fn send_msg(&mut self, msg: &SafekeeperTimelineInfo) -> Result<(), Status> {
+        // send message to subscribers for everything
+        let mut cleanup_subs_to_all = false;
+        for sub in self.subs_to_all.iter() {
+            match sub.try_send(msg.clone()) {
+                Err(TrySendError::Full(_)) => {
+                    // println!("dropping message, channel is full");
+                }
+                Err(TrySendError::Closed(_)) => {
+                    cleanup_subs_to_all = true;
+                }
+                _ => (),
+            }
+        }
+        // some channels got closed (subscriber gone), remove them
+        if cleanup_subs_to_all {
+            self.subs_to_all.retain(|tx| !tx.is_closed());
+        }
+
+        // send message to per timeline subscribers
+        let ttid = parse_proto_ttid(msg.tenant_timeline_id.as_ref().ok_or(Status::new(
+            Code::InvalidArgument,
+            "missing tenant_timeline_id",
+        ))?)?;
+        if let Some(subs) = self.subs_to_timelines.get(&ttid) {
+            for tx in subs.iter().map(|sub_sender| &sub_sender.1) {
+                if let Err(TrySendError::Full(_)) = tx.try_send(msg.clone()) {
+                    // println!("dropping message, channel is full");
+                }
+                // closed channel is ignored here; we will be notified and remove it soon
+            }
+        }
+        Ok(())
+    }
+
+    // Add/remove subscriber according to sub_announce.
+    pub fn update_sub(&mut self, sub_announce: SubAnnounce) {
+        match sub_announce {
+            SubAnnounce::AddAll(tx) => self.subs_to_all.push(tx),
+            SubAnnounce::AddTimeline(ttid, sub_sender) => {
+                match self.subs_to_timelines.entry(ttid) {
+                    Entry::Occupied(mut o) => {
+                        let subsenders = o.get_mut();
+                        subsenders.push(sub_sender);
+                    }
+                    Entry::Vacant(v) => {
+                        v.insert(vec![sub_sender]);
+                    }
+                }
+            }
+            SubAnnounce::RemoveTimeline(ref ttid, sub_id) => {
+                remove_sub(&mut self.subs_to_timelines, ttid, sub_id);
+            }
+        }
+    }
+}
+
+impl Drop for Publisher {
+    fn drop(&mut self) {
+        self.registry.unregister_publisher(self);
+    }
+}
+
+struct NeonBrokerService {
+    registry: Registry,
+}
+
+#[tonic::async_trait]
+impl NeonBroker for NeonBrokerService {
+    async fn publish_safekeeper_info(
+        &self,
+        request: Request<tonic::Streaming<SafekeeperTimelineInfo>>,
+    ) -> Result<Response<Empty>, Status> {
+        let mut publisher = self.registry.register_publisher();
+
+        let mut stream = request.into_inner();
+
+        loop {
+            select! {
+                msg = stream.next() => {
+                    match msg {
+                        Some(Ok(msg)) => {publisher.send_msg(&msg)?;},
+                        Some(Err(e)) => {return Err(e);}, // grpc error from the stream
+                        None => {break;} // closed stream
+                    }
+                }
+                Some(announce) = publisher.announce_rx.next() => {
+                    publisher.update_sub(announce);
+                }
+            }
+        }
+
+        Ok(Response::new(Empty {}))
+    }
+
+    type SubscribeSafekeeperInfoStream =
+        Pin<Box<dyn Stream<Item = Result<SafekeeperTimelineInfo, Status>> + Send + 'static>>;
+
+    async fn subscribe_safekeeper_info(
+        &self,
+        request: Request<SubscribeSafekeeperInfoRequest>,
+    ) -> Result<Response<Self::SubscribeSafekeeperInfoStream>, Status> {
+        let proto_key = request.into_inner().subscription_key.ok_or(Status::new(
+            Code::InvalidArgument,
+            "missing subscription key",
+        ))?;
+        let sub_key = SubscriptionKey::from_proto_subscription_key(proto_key)?;
+        let mut subscriber = self.registry.register_subscriber(sub_key).await;
+
+        // transform rx into stream with item = Result, as method result demands
+        let output = async_stream::try_stream! {
+            while let Some(info) = subscriber.sub_rx.recv().await {
+                    yield info
+                }
+
+            // internal generator
+            // let _ = subscriber.sub_rx.try_recv().ok();
+            // let mut counter = 0;
+            // loop {
+            // let info = SafekeeperTimelineInfo {
+            //     safekeeper_id: 1,
+            //     tenant_timeline_id: Some(ProtoTenantTimelineId {
+            //         tenant_id: vec![0xFF; 16],
+            //         timeline_id: vec![0xFF; 16],
+            //         // timeline_id: tli_from_u64(counter),
+            //     }),
+            //     last_log_term: 0,
+            //     flush_lsn: counter,
+            //     commit_lsn: 2,
+            //     backup_lsn: 3,
+            //     remote_consistent_lsn: 4,
+            //     peer_horizon_lsn: 5,
+            //     safekeeper_connstr: "zenith-1-sk-1.local:7676".to_owned(),
+            // };
+            // counter += 1;
+            // yield info;
+            // }
+        };
+
+        Ok(Response::new(
+            Box::pin(output) as Self::SubscribeSafekeeperInfoStream
+        ))
+    }
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // console_subscriber::init();
+
+    let addr = "[::1]:50051".parse()?;
+    let registry = Registry {
+        shared_state: Arc::new(Mutex::new(SharedState {
+            pub_txs: HashMap::new(),
+            next_pub_id: 0,
+            subs_to_all: HashMap::new(),
+            subs_to_timelines: HashMap::new(),
+            next_sub_id: 0,
+        })),
+    };
+    let neon_broker_service = NeonBrokerService {
+        registry: registry.clone(),
+    };
+
+    tokio::spawn(async move { registry.report().await });
+
+    Server::builder()
+        .http2_keepalive_interval(Some(Duration::from_millis(5000)))
+        .add_service(NeonBrokerServer::new(neon_broker_service))
+        .serve(addr)
+        .await?;
+
+    Ok(())
+}
+
+// parse variable length bytes from protobuf
+fn parse_proto_ttid(proto_ttid: &ProtoTenantTimelineId) -> Result<TenantTimelineId, Status> {
+    let tenant_id = TenantId::from_vec(&proto_ttid.tenant_id)
+        .map_err(|e| Status::new(Code::InvalidArgument, format!("malformed tenant_id: {}", e)))?;
+    let timeline_id = TimelineId::from_vec(&proto_ttid.timeline_id).map_err(|e| {
+        Status::new(
+            Code::InvalidArgument,
+            format!("malformed timeline_id: {}", e),
+        )
+    })?;
+    Ok(TenantTimelineId {
+        tenant_id,
+        timeline_id,
+    })
+}

compute_tools/Cargo.toml

@@ -6,7 +6,7 @@ edition = "2021"
 [dependencies]
 anyhow = "1.0"
 chrono = "0.4"
-clap = "4.0"
+clap = "3.0"
 env_logger = "0.9"
 futures = "0.3.13"
 hyper = { version = "0.14", features = ["full"] }

compute_tools/src/bin/compute_ctl.rs

@@ -51,19 +51,53 @@ fn main() -> Result<()> {
     // TODO: re-use `utils::logging` later
     init_logger(DEFAULT_LOG_LEVEL)?;
 
-    let matches = cli().get_matches();
+    // Env variable is set by `cargo`
+    let version: Option<&str> = option_env!("CARGO_PKG_VERSION");
+    let matches = clap::App::new("compute_ctl")
+        .version(version.unwrap_or("unknown"))
+        .arg(
+            Arg::new("connstr")
+                .short('C')
+                .long("connstr")
+                .value_name("DATABASE_URL")
+                .required(true),
+        )
+        .arg(
+            Arg::new("pgdata")
+                .short('D')
+                .long("pgdata")
+                .value_name("DATADIR")
+                .required(true),
+        )
+        .arg(
+            Arg::new("pgbin")
+                .short('b')
+                .long("pgbin")
+                .value_name("POSTGRES_PATH"),
+        )
+        .arg(
+            Arg::new("spec")
+                .short('s')
+                .long("spec")
+                .value_name("SPEC_JSON"),
+        )
+        .arg(
+            Arg::new("spec-path")
+                .short('S')
+                .long("spec-path")
+                .value_name("SPEC_PATH"),
+        )
+        .get_matches();
 
-    let pgdata = matches
-        .get_one::<String>("pgdata")
-        .expect("PGDATA path is required");
+    let pgdata = matches.value_of("pgdata").expect("PGDATA path is required");
     let connstr = matches
-        .get_one::<String>("connstr")
+        .value_of("connstr")
         .expect("Postgres connection string is required");
-    let spec = matches.get_one::<String>("spec");
-    let spec_path = matches.get_one::<String>("spec-path");
+    let spec = matches.value_of("spec");
+    let spec_path = matches.value_of("spec-path");
 
     // Try to use just 'postgres' if no path is provided
-    let pgbin = matches.get_one::<String>("pgbin").unwrap();
+    let pgbin = matches.value_of("pgbin").unwrap_or("postgres");
 
     let spec: ComputeSpec = match spec {
         // First, try to get cluster spec from the cli argument
@@ -139,48 +173,3 @@ fn main() -> Result<()> {
         }
     }
 }
-
-fn cli() -> clap::Command {
-    // Env variable is set by `cargo`
-    let version = option_env!("CARGO_PKG_VERSION").unwrap_or("unknown");
-    clap::Command::new("compute_ctl")
-        .version(version)
-        .arg(
-            Arg::new("connstr")
-                .short('C')
-                .long("connstr")
-                .value_name("DATABASE_URL")
-                .required(true),
-        )
-        .arg(
-            Arg::new("pgdata")
-                .short('D')
-                .long("pgdata")
-                .value_name("DATADIR")
-                .required(true),
-        )
-        .arg(
-            Arg::new("pgbin")
-                .short('b')
-                .long("pgbin")
-                .default_value("postgres")
-                .value_name("POSTGRES_PATH"),
-        )
-        .arg(
-            Arg::new("spec")
-                .short('s')
-                .long("spec")
-                .value_name("SPEC_JSON"),
-        )
-        .arg(
-            Arg::new("spec-path")
-                .short('S')
-                .long("spec-path")
-                .value_name("SPEC_PATH"),
-        )
-}
-
-#[test]
-fn verify_cli() {
-    cli().debug_assert()
-}

compute_tools/src/pg_helpers.rs

@@ -8,10 +8,11 @@ use std::process::Child;
 use std::time::{Duration, Instant};
 
 use anyhow::{bail, Result};
-use notify::{RecursiveMode, Watcher};
 use postgres::{Client, Transaction};
 use serde::Deserialize;
 
+use notify::{RecursiveMode, Watcher};
+
 const POSTGRES_WAIT_TIMEOUT: Duration = Duration::from_millis(60 * 1000); // milliseconds
 
 /// Rust representation of Postgres role info with only those fields
@@ -168,7 +169,7 @@ impl Database {
     /// it may require a proper quoting too.
     pub fn to_pg_options(&self) -> String {
         let mut params: String = self.options.as_pg_options();
-        write!(params, " OWNER {}", &self.owner.pg_quote())
+        write!(params, " OWNER {}", &self.owner.quote())
             .expect("String is documented to not to error during write operations");
 
         params
@@ -179,17 +180,18 @@ impl Database {
 /// intended to be used for DB / role names.
 pub type PgIdent = String;
 
-/// Generic trait used to provide quoting / encoding for strings used in the
-/// Postgres SQL queries and DATABASE_URL.
-pub trait Escaping {
-    fn pg_quote(&self) -> String;
+/// Generic trait used to provide quoting for strings used in the
+/// Postgres SQL queries. Currently used only to implement quoting
+/// of identifiers, but could be used for literals in the future.
+pub trait PgQuote {
+    fn quote(&self) -> String;
 }
 
-impl Escaping for PgIdent {
+impl PgQuote for PgIdent {
     /// This is intended to mimic Postgres quote_ident(), but for simplicity it
-    /// always quotes provided string with `""` and escapes every `"`.
-    /// **Not idempotent**, i.e. if string is already escaped it will be escaped again.
-    fn pg_quote(&self) -> String {
+    /// always quotes provided string with `""` and escapes every `"`. Not idempotent,
+    /// i.e. if string is already escaped it will be escaped again.
+    fn quote(&self) -> String {
         let result = format!("\"{}\"", self.replace('"', "\"\""));
         result
     }

compute_tools/src/spec.rs

@@ -1,9 +1,7 @@
 use std::path::Path;
-use std::str::FromStr;
 
 use anyhow::Result;
 use log::{info, log_enabled, warn, Level};
-use postgres::config::Config;
 use postgres::{Client, NoTls};
 use serde::Deserialize;
 
@@ -117,8 +115,8 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
                     if existing_roles.iter().any(|r| r.name == op.name) {
                         let query: String = format!(
                             "ALTER ROLE {} RENAME TO {}",
-                            op.name.pg_quote(),
-                            new_name.pg_quote()
+                            op.name.quote(),
+                            new_name.quote()
                         );
 
                         warn!("renaming role '{}' to '{}'", op.name, new_name);
@@ -164,7 +162,7 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
             }
 
             if update_role {
-                let mut query: String = format!("ALTER ROLE {} ", name.pg_quote());
+                let mut query: String = format!("ALTER ROLE {} ", name.quote());
                 info_print!(" -> update");
 
                 query.push_str(&role.to_pg_options());
@@ -172,7 +170,7 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
             }
         } else {
             info!("role name: '{}'", &name);
-            let mut query: String = format!("CREATE ROLE {} ", name.pg_quote());
+            let mut query: String = format!("CREATE ROLE {} ", name.quote());
             info!("role create query: '{}'", &query);
             info_print!(" -> create");
 
@@ -181,7 +179,7 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
 
             let grant_query = format!(
                 "GRANT pg_read_all_data, pg_write_all_data TO {}",
-                name.pg_quote()
+                name.quote()
             );
             xact.execute(grant_query.as_str(), &[])?;
             info!("role grant query: '{}'", &grant_query);
@@ -217,7 +215,7 @@ pub fn handle_role_deletions(node: &ComputeNode, client: &mut Client) -> Result<
             // We do not check either role exists or not,
             // Postgres will take care of it for us
             if op.action == "delete_role" {
-                let query: String = format!("DROP ROLE IF EXISTS {}", &op.name.pg_quote());
+                let query: String = format!("DROP ROLE IF EXISTS {}", &op.name.quote());
 
                 warn!("deleting role '{}'", &op.name);
                 xact.execute(query.as_str(), &[])?;
@@ -232,16 +230,17 @@ pub fn handle_role_deletions(node: &ComputeNode, client: &mut Client) -> Result<
 fn reassign_owned_objects(node: &ComputeNode, role_name: &PgIdent) -> Result<()> {
     for db in &node.spec.cluster.databases {
         if db.owner != *role_name {
-            let mut conf = Config::from_str(node.connstr.as_str())?;
-            conf.dbname(&db.name);
+            let mut connstr = node.connstr.clone();
+            // database name is always the last and the only component of the path
+            connstr.set_path(&db.name);
 
-            let mut client = conf.connect(NoTls)?;
+            let mut client = Client::connect(connstr.as_str(), NoTls)?;
 
             // This will reassign all dependent objects to the db owner
             let reassign_query = format!(
                 "REASSIGN OWNED BY {} TO {}",
-                role_name.pg_quote(),
-                db.owner.pg_quote()
+                role_name.quote(),
+                db.owner.quote()
             );
             info!(
                 "reassigning objects owned by '{}' in db '{}' to '{}'",
@@ -250,7 +249,7 @@ fn reassign_owned_objects(node: &ComputeNode, role_name: &PgIdent) -> Result<()>
             client.simple_query(&reassign_query)?;
 
             // This now will only drop privileges of the role
-            let drop_query = format!("DROP OWNED BY {}", role_name.pg_quote());
+            let drop_query = format!("DROP OWNED BY {}", role_name.quote());
             client.simple_query(&drop_query)?;
         }
     }
@@ -280,7 +279,7 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
                 // We do not check either DB exists or not,
                 // Postgres will take care of it for us
                 "delete_db" => {
-                    let query: String = format!("DROP DATABASE IF EXISTS {}", &op.name.pg_quote());
+                    let query: String = format!("DROP DATABASE IF EXISTS {}", &op.name.quote());
 
                     warn!("deleting database '{}'", &op.name);
                     client.execute(query.as_str(), &[])?;
@@ -292,8 +291,8 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
                     if existing_dbs.iter().any(|r| r.name == op.name) {
                         let query: String = format!(
                             "ALTER DATABASE {} RENAME TO {}",
-                            op.name.pg_quote(),
-                            new_name.pg_quote()
+                            op.name.quote(),
+                            new_name.quote()
                         );
 
                         warn!("renaming database '{}' to '{}'", op.name, new_name);
@@ -321,7 +320,7 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
             // XXX: db owner name is returned as quoted string from Postgres,
             // when quoting is needed.
             let new_owner = if r.owner.starts_with('"') {
-                db.owner.pg_quote()
+                db.owner.quote()
             } else {
                 db.owner.clone()
             };
@@ -329,15 +328,15 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
             if new_owner != r.owner {
                 let query: String = format!(
                     "ALTER DATABASE {} OWNER TO {}",
-                    name.pg_quote(),
-                    db.owner.pg_quote()
+                    name.quote(),
+                    db.owner.quote()
                 );
                 info_print!(" -> update");
 
                 client.execute(query.as_str(), &[])?;
             }
         } else {
-            let mut query: String = format!("CREATE DATABASE {} ", name.pg_quote());
+            let mut query: String = format!("CREATE DATABASE {} ", name.quote());
             info_print!(" -> create");
 
             query.push_str(&db.to_pg_options());
@@ -367,7 +366,7 @@ pub fn handle_grants(node: &ComputeNode, client: &mut Client) -> Result<()> {
         .cluster
         .roles
         .iter()
-        .map(|r| r.name.pg_quote())
+        .map(|r| r.name.quote())
         .collect::<Vec<_>>();
 
     for db in &spec.cluster.databases {
@@ -375,7 +374,7 @@ pub fn handle_grants(node: &ComputeNode, client: &mut Client) -> Result<()> {
 
         let query: String = format!(
             "GRANT CREATE ON DATABASE {} TO {}",
-            dbname.pg_quote(),
+            dbname.quote(),
             roles.join(", ")
         );
         info!("grant query {}", &query);
@@ -386,11 +385,12 @@ pub fn handle_grants(node: &ComputeNode, client: &mut Client) -> Result<()> {
     // Do some per-database access adjustments. We'd better do this at db creation time,
     // but CREATE DATABASE isn't transactional. So we cannot create db + do some grants
     // atomically.
+    let mut db_connstr = node.connstr.clone();
     for db in &node.spec.cluster.databases {
-        let mut conf = Config::from_str(node.connstr.as_str())?;
-        conf.dbname(&db.name);
+        // database name is always the last and the only component of the path
+        db_connstr.set_path(&db.name);
 
-        let mut db_client = conf.connect(NoTls)?;
+        let mut db_client = Client::connect(db_connstr.as_str(), NoTls)?;
 
         // This will only change ownership on the schema itself, not the objects
         // inside it. Without it owner of the `public` schema will be `cloud_admin`
@@ -419,36 +419,9 @@ pub fn handle_grants(node: &ComputeNode, client: &mut Client) -> Result<()> {
                     END IF;\n\
                 END\n\
             $$;",
-            db.owner.pg_quote()
+            db.owner.quote()
         );
         db_client.simple_query(&alter_query)?;
-
-        // Explicitly grant CREATE ON SCHEMA PUBLIC to the web_access user.
-        // This is needed since postgres 15, where this privilege is removed by default.
-        let grant_query = "DO $$\n\
-                BEGIN\n\
-                    IF EXISTS(\n\
-                        SELECT nspname\n\
-                        FROM pg_catalog.pg_namespace\n\
-                        WHERE nspname = 'public'\n\
-                    ) AND\n\
-                    version() LIKE 'PostgreSQL 15%'\n\
-                    THEN\n\
-                        IF EXISTS(\n\
-                            SELECT rolname\n\
-                            FROM pg_catalog.pg_roles\n\
-                            WHERE rolname = 'web_access'\n\
-                        )\n\
-                        THEN\n\
-                            GRANT CREATE ON SCHEMA public TO web_access;\n\
-                        END IF;\n\
-                    END IF;\n\
-                END\n\
-            $$;"
-        .to_string();
-
-        info!("grant query for db {} : {}", &db.name, &grant_query);
-        db_client.simple_query(&grant_query)?;
     }
 
     Ok(())

compute_tools/tests/pg_helpers_tests.rs

@@ -33,9 +33,9 @@ mod pg_helpers_tests {
     }
 
     #[test]
-    fn ident_pg_quote() {
+    fn quote_ident() {
         let ident: PgIdent = PgIdent::from("\"name\";\\n select 1;");
 
-        assert_eq!(ident.pg_quote(), "\"\"\"name\"\";\\n select 1;\"");
+        assert_eq!(ident.quote(), "\"\"\"name\"\";\\n select 1;\"");
     }
 }

control_plane/Cargo.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-clap = "4.0"
+clap = "3.0"
 comfy-table = "6.1"
 git-version = "0.3.5"
 tar = "0.4.38"

control_plane/src/bin/neon_local.rs

@@ -6,7 +6,7 @@
 //! rely on `neon_local` to set up the environment for each test.
 //!
 use anyhow::{anyhow, bail, Context, Result};
-use clap::{value_parser, Arg, ArgAction, ArgMatches, Command};
+use clap::{App, AppSettings, Arg, ArgMatches};
 use control_plane::compute::ComputeControlPlane;
 use control_plane::local_env::{EtcdBroker, LocalEnv};
 use control_plane::safekeeper::SafekeeperNode;
@@ -85,7 +85,212 @@ struct TimelineTreeEl {
 //   * Providing CLI api to the pageserver
 //   * TODO: export/import to/from usual postgres
 fn main() -> Result<()> {
-    let matches = cli().get_matches();
+    let branch_name_arg = Arg::new("branch-name")
+        .long("branch-name")
+        .takes_value(true)
+        .help("Name of the branch to be created or used as an alias for other services")
+        .required(false);
+
+    let pg_node_arg = Arg::new("node").help("Postgres node name").required(false);
+
+    let safekeeper_id_arg = Arg::new("id").help("safekeeper id").required(false);
+
+    let tenant_id_arg = Arg::new("tenant-id")
+        .long("tenant-id")
+        .help("Tenant id. Represented as a hexadecimal string 32 symbols length")
+        .takes_value(true)
+        .required(false);
+
+    let timeline_id_arg = Arg::new("timeline-id")
+        .long("timeline-id")
+        .help("Timeline id. Represented as a hexadecimal string 32 symbols length")
+        .takes_value(true)
+        .required(false);
+
+    let pg_version_arg = Arg::new("pg-version")
+        .long("pg-version")
+        .help("Postgres version to use for the initial tenant")
+        .required(false)
+        .takes_value(true)
+        .default_value(DEFAULT_PG_VERSION);
+
+    let port_arg = Arg::new("port")
+        .long("port")
+        .required(false)
+        .value_name("port");
+
+    let stop_mode_arg = Arg::new("stop-mode")
+        .short('m')
+        .takes_value(true)
+        .possible_values(&["fast", "immediate"])
+        .help("If 'immediate', don't flush repository data at shutdown")
+        .required(false)
+        .value_name("stop-mode");
+
+    let pageserver_config_args = Arg::new("pageserver-config-override")
+        .long("pageserver-config-override")
+        .takes_value(true)
+        .number_of_values(1)
+        .multiple_occurrences(true)
+        .help("Additional pageserver's configuration options or overrides, refer to pageserver's 'config-override' CLI parameter docs for more")
+        .required(false);
+
+    let lsn_arg = Arg::new("lsn")
+        .long("lsn")
+        .help("Specify Lsn on the timeline to start from. By default, end of the timeline would be used.")
+        .takes_value(true)
+        .required(false);
+
+    let matches = App::new("Neon CLI")
+        .setting(AppSettings::ArgRequiredElseHelp)
+        .version(GIT_VERSION)
+        .subcommand(
+            App::new("init")
+                .about("Initialize a new Neon repository")
+                .arg(pageserver_config_args.clone())
+                .arg(timeline_id_arg.clone().help("Use a specific timeline id when creating a tenant and its initial timeline"))
+                .arg(
+                    Arg::new("config")
+                        .long("config")
+                        .required(false)
+                        .value_name("config"),
+                )
+                .arg(pg_version_arg.clone())
+        )
+        .subcommand(
+            App::new("timeline")
+            .about("Manage timelines")
+            .subcommand(App::new("list")
+                .about("List all timelines, available to this pageserver")
+                .arg(tenant_id_arg.clone()))
+            .subcommand(App::new("branch")
+                .about("Create a new timeline, using another timeline as a base, copying its data")
+                .arg(tenant_id_arg.clone())
+                .arg(branch_name_arg.clone())
+                .arg(Arg::new("ancestor-branch-name").long("ancestor-branch-name").takes_value(true)
+                    .help("Use last Lsn of another timeline (and its data) as base when creating the new timeline. The timeline gets resolved by its branch name.").required(false))
+                .arg(Arg::new("ancestor-start-lsn").long("ancestor-start-lsn").takes_value(true)
+                    .help("When using another timeline as base, use a specific Lsn in it instead of the latest one").required(false)))
+            .subcommand(App::new("create")
+                .about("Create a new blank timeline")
+                .arg(tenant_id_arg.clone())
+                .arg(branch_name_arg.clone())
+                .arg(pg_version_arg.clone())
+            )
+            .subcommand(App::new("import")
+                .about("Import timeline from basebackup directory")
+                .arg(tenant_id_arg.clone())
+                .arg(timeline_id_arg.clone())
+                .arg(Arg::new("node-name").long("node-name").takes_value(true)
+                    .help("Name to assign to the imported timeline"))
+                .arg(Arg::new("base-tarfile").long("base-tarfile").takes_value(true)
+                    .help("Basebackup tarfile to import"))
+                .arg(Arg::new("base-lsn").long("base-lsn").takes_value(true)
+                    .help("Lsn the basebackup starts at"))
+                .arg(Arg::new("wal-tarfile").long("wal-tarfile").takes_value(true)
+                    .help("Wal to add after base"))
+                .arg(Arg::new("end-lsn").long("end-lsn").takes_value(true)
+                    .help("Lsn the basebackup ends at"))
+                .arg(pg_version_arg.clone())
+            )
+        ).subcommand(
+            App::new("tenant")
+            .setting(AppSettings::ArgRequiredElseHelp)
+            .about("Manage tenants")
+            .subcommand(App::new("list"))
+            .subcommand(App::new("create")
+                .arg(tenant_id_arg.clone())
+                .arg(timeline_id_arg.clone().help("Use a specific timeline id when creating a tenant and its initial timeline"))
+                .arg(Arg::new("config").short('c').takes_value(true).multiple_occurrences(true).required(false))
+                .arg(pg_version_arg.clone())
+                )
+            .subcommand(App::new("config")
+                .arg(tenant_id_arg.clone())
+                .arg(Arg::new("config").short('c').takes_value(true).multiple_occurrences(true).required(false))
+                )
+        )
+        .subcommand(
+            App::new("pageserver")
+                .setting(AppSettings::ArgRequiredElseHelp)
+                .about("Manage pageserver")
+                .subcommand(App::new("status"))
+                .subcommand(App::new("start").about("Start local pageserver").arg(pageserver_config_args.clone()))
+                .subcommand(App::new("stop").about("Stop local pageserver")
+                            .arg(stop_mode_arg.clone()))
+                .subcommand(App::new("restart").about("Restart local pageserver").arg(pageserver_config_args.clone()))
+        )
+        .subcommand(
+            App::new("safekeeper")
+                .setting(AppSettings::ArgRequiredElseHelp)
+                .about("Manage safekeepers")
+                .subcommand(App::new("start")
+                            .about("Start local safekeeper")
+                            .arg(safekeeper_id_arg.clone())
+                )
+                .subcommand(App::new("stop")
+                            .about("Stop local safekeeper")
+                            .arg(safekeeper_id_arg.clone())
+                            .arg(stop_mode_arg.clone())
+                )
+                .subcommand(App::new("restart")
+                            .about("Restart local safekeeper")
+                            .arg(safekeeper_id_arg.clone())
+                            .arg(stop_mode_arg.clone())
+                )
+        )
+        .subcommand(
+            App::new("pg")
+                .setting(AppSettings::ArgRequiredElseHelp)
+                .about("Manage postgres instances")
+                .subcommand(App::new("list").arg(tenant_id_arg.clone()))
+                .subcommand(App::new("create")
+                    .about("Create a postgres compute node")
+                    .arg(pg_node_arg.clone())
+                    .arg(branch_name_arg.clone())
+                    .arg(tenant_id_arg.clone())
+                    .arg(lsn_arg.clone())
+                    .arg(port_arg.clone())
+                    .arg(
+                        Arg::new("config-only")
+                            .help("Don't do basebackup, create compute node with only config files")
+                            .long("config-only")
+                            .required(false))
+                    .arg(pg_version_arg.clone())
+                )
+                .subcommand(App::new("start")
+                    .about("Start a postgres compute node.\n This command actually creates new node from scratch, but preserves existing config files")
+                    .arg(pg_node_arg.clone())
+                    .arg(tenant_id_arg.clone())
+                    .arg(branch_name_arg.clone())
+                    .arg(timeline_id_arg.clone())
+                    .arg(lsn_arg.clone())
+                    .arg(port_arg.clone())
+                    .arg(pg_version_arg.clone())
+                )
+                .subcommand(
+                    App::new("stop")
+                    .arg(pg_node_arg.clone())
+                    .arg(tenant_id_arg.clone())
+                    .arg(
+                        Arg::new("destroy")
+                            .help("Also delete data directory (now optional, should be default in future)")
+                            .long("destroy")
+                            .required(false)
+                    )
+                    )
+
+        )
+        .subcommand(
+            App::new("start")
+                .about("Start page server and safekeepers")
+                .arg(pageserver_config_args)
+        )
+        .subcommand(
+            App::new("stop")
+                .about("Stop page server and safekeepers")
+                .arg(stop_mode_arg.clone())
+        )
+        .get_matches();
 
     let (sub_name, sub_args) = match matches.subcommand() {
         Some(subcommand_data) => subcommand_data,
@@ -270,16 +475,16 @@ fn get_tenant_id(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow::R
 
 fn parse_tenant_id(sub_match: &ArgMatches) -> anyhow::Result<Option<TenantId>> {
     sub_match
-        .get_one::<String>("tenant-id")
-        .map(|tenant_id| TenantId::from_str(tenant_id))
+        .value_of("tenant-id")
+        .map(TenantId::from_str)
         .transpose()
         .context("Failed to parse tenant id from the argument string")
 }
 
 fn parse_timeline_id(sub_match: &ArgMatches) -> anyhow::Result<Option<TimelineId>> {
     sub_match
-        .get_one::<String>("timeline-id")
-        .map(|timeline_id| TimelineId::from_str(timeline_id))
+        .value_of("timeline-id")
+        .map(TimelineId::from_str)
         .transpose()
         .context("Failed to parse timeline id from the argument string")
 }
@@ -288,22 +493,19 @@ fn handle_init(init_match: &ArgMatches) -> anyhow::Result<LocalEnv> {
     let initial_timeline_id_arg = parse_timeline_id(init_match)?;
 
     // Create config file
-    let toml_file: String = if let Some(config_path) = init_match.get_one::<PathBuf>("config") {
+    let toml_file: String = if let Some(config_path) = init_match.value_of("config") {
         // load and parse the file
-        std::fs::read_to_string(config_path).with_context(|| {
-            format!(
-                "Could not read configuration file '{}'",
-                config_path.display()
-            )
-        })?
+        std::fs::read_to_string(std::path::Path::new(config_path))
+            .with_context(|| format!("Could not read configuration file '{config_path}'"))?
     } else {
         // Built-in default config
         default_conf(&EtcdBroker::locate_etcd()?)
     };
 
     let pg_version = init_match
-        .get_one::<u32>("pg-version")
-        .copied()
+        .value_of("pg-version")
+        .unwrap()
+        .parse::<u32>()
         .context("Failed to parse postgres version from the argument string")?;
 
     let mut env =
@@ -339,10 +541,9 @@ fn handle_init(init_match: &ArgMatches) -> anyhow::Result<LocalEnv> {
 
 fn pageserver_config_overrides(init_match: &ArgMatches) -> Vec<&str> {
     init_match
-        .get_many::<String>("pageserver-config-override")
+        .values_of("pageserver-config-override")
         .into_iter()
         .flatten()
-        .map(|s| s.as_str())
         .collect()
 }
 
@@ -357,7 +558,7 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an
         Some(("create", create_match)) => {
             let initial_tenant_id = parse_tenant_id(create_match)?;
             let tenant_conf: HashMap<_, _> = create_match
-                .get_many::<String>("config")
+                .values_of("config")
                 .map(|vals| vals.flat_map(|c| c.split_once(':')).collect())
                 .unwrap_or_default();
             let new_tenant_id = pageserver.tenant_create(initial_tenant_id, tenant_conf)?;
@@ -366,8 +567,9 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an
             // Create an initial timeline for the new tenant
             let new_timeline_id = parse_timeline_id(create_match)?;
             let pg_version = create_match
-                .get_one::<u32>("pg-version")
-                .copied()
+                .value_of("pg-version")
+                .unwrap()
+                .parse::<u32>()
                 .context("Failed to parse postgres version from the argument string")?;
 
             let timeline_info = pageserver.timeline_create(
@@ -393,7 +595,7 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an
         Some(("config", create_match)) => {
             let tenant_id = get_tenant_id(create_match, env)?;
             let tenant_conf: HashMap<_, _> = create_match
-                .get_many::<String>("config")
+                .values_of("config")
                 .map(|vals| vals.flat_map(|c| c.split_once(':')).collect())
                 .unwrap_or_default();
 
@@ -420,12 +622,13 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) -
         Some(("create", create_match)) => {
             let tenant_id = get_tenant_id(create_match, env)?;
             let new_branch_name = create_match
-                .get_one::<String>("branch-name")
+                .value_of("branch-name")
                 .ok_or_else(|| anyhow!("No branch name provided"))?;
 
             let pg_version = create_match
-                .get_one::<u32>("pg-version")
-                .copied()
+                .value_of("pg-version")
+                .unwrap()
+                .parse::<u32>()
                 .context("Failed to parse postgres version from the argument string")?;
 
             let timeline_info =
@@ -444,32 +647,35 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) -
             let tenant_id = get_tenant_id(import_match, env)?;
             let timeline_id = parse_timeline_id(import_match)?.expect("No timeline id provided");
             let name = import_match
-                .get_one::<String>("node-name")
+                .value_of("node-name")
                 .ok_or_else(|| anyhow!("No node name provided"))?;
 
             // Parse base inputs
             let base_tarfile = import_match
-                .get_one::<PathBuf>("base-tarfile")
-                .ok_or_else(|| anyhow!("No base-tarfile provided"))?
-                .to_owned();
+                .value_of("base-tarfile")
+                .map(|s| PathBuf::from_str(s).unwrap())
+                .ok_or_else(|| anyhow!("No base-tarfile provided"))?;
             let base_lsn = Lsn::from_str(
                 import_match
-                    .get_one::<String>("base-lsn")
+                    .value_of("base-lsn")
                     .ok_or_else(|| anyhow!("No base-lsn provided"))?,
             )?;
             let base = (base_lsn, base_tarfile);
 
             // Parse pg_wal inputs
-            let wal_tarfile = import_match.get_one::<PathBuf>("wal-tarfile").cloned();
+            let wal_tarfile = import_match
+                .value_of("wal-tarfile")
+                .map(|s| PathBuf::from_str(s).unwrap());
             let end_lsn = import_match
-                .get_one::<String>("end-lsn")
+                .value_of("end-lsn")
                 .map(|s| Lsn::from_str(s).unwrap());
             // TODO validate both or none are provided
             let pg_wal = end_lsn.zip(wal_tarfile);
 
             let pg_version = import_match
-                .get_one::<u32>("pg-version")
-                .copied()
+                .value_of("pg-version")
+                .unwrap()
+                .parse::<u32>()
                 .context("Failed to parse postgres version from the argument string")?;
 
             let mut cplane = ComputeControlPlane::load(env.clone())?;
@@ -484,11 +690,10 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) -
         Some(("branch", branch_match)) => {
             let tenant_id = get_tenant_id(branch_match, env)?;
             let new_branch_name = branch_match
-                .get_one::<String>("branch-name")
+                .value_of("branch-name")
                 .ok_or_else(|| anyhow!("No branch name provided"))?;
             let ancestor_branch_name = branch_match
-                .get_one::<String>("ancestor-branch-name")
-                .map(|s| s.as_str())
+                .value_of("ancestor-branch-name")
                 .unwrap_or(DEFAULT_BRANCH_NAME);
             let ancestor_timeline_id = env
                 .get_branch_timeline_id(ancestor_branch_name, tenant_id)
@@ -497,8 +702,8 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) -
                 })?;
 
             let start_lsn = branch_match
-                .get_one::<String>("ancestor-start-lsn")
-                .map(|lsn_str| Lsn::from_str(lsn_str))
+                .value_of("ancestor-start-lsn")
+                .map(Lsn::from_str)
                 .transpose()
                 .context("Failed to parse ancestor start Lsn from the request")?;
             let timeline_info = pageserver.timeline_create(
@@ -599,39 +804,45 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> {
         }
         "create" => {
             let branch_name = sub_args
-                .get_one::<String>("branch-name")
-                .map(|s| s.as_str())
+                .value_of("branch-name")
                 .unwrap_or(DEFAULT_BRANCH_NAME);
             let node_name = sub_args
-                .get_one::<String>("node")
-                .map(|node_name| node_name.to_string())
-                .unwrap_or_else(|| format!("{branch_name}_node"));
+                .value_of("node")
+                .map(ToString::to_string)
+                .unwrap_or_else(|| format!("{}_node", branch_name));
 
             let lsn = sub_args
-                .get_one::<String>("lsn")
-                .map(|lsn_str| Lsn::from_str(lsn_str))
+                .value_of("lsn")
+                .map(Lsn::from_str)
                 .transpose()
                 .context("Failed to parse Lsn from the request")?;
             let timeline_id = env
                 .get_branch_timeline_id(branch_name, tenant_id)
-                .ok_or_else(|| anyhow!("Found no timeline id for branch name '{branch_name}'"))?;
+                .ok_or_else(|| anyhow!("Found no timeline id for branch name '{}'", branch_name))?;
 
-            let port: Option<u16> = sub_args.get_one::<u16>("port").copied();
+            let port: Option<u16> = match sub_args.value_of("port") {
+                Some(p) => Some(p.parse()?),
+                None => None,
+            };
 
             let pg_version = sub_args
-                .get_one::<u32>("pg-version")
-                .copied()
+                .value_of("pg-version")
+                .unwrap()
+                .parse::<u32>()
                 .context("Failed to parse postgres version from the argument string")?;
 
             cplane.new_node(tenant_id, &node_name, timeline_id, lsn, port, pg_version)?;
         }
         "start" => {
-            let port: Option<u16> = sub_args.get_one::<u16>("port").copied();
+            let port: Option<u16> = match sub_args.value_of("port") {
+                Some(p) => Some(p.parse()?),
+                None => None,
+            };
             let node_name = sub_args
-                .get_one::<String>("node")
+                .value_of("node")
                 .ok_or_else(|| anyhow!("No node name was provided to start"))?;
 
-            let node = cplane.nodes.get(&(tenant_id, node_name.to_string()));
+            let node = cplane.nodes.get(&(tenant_id, node_name.to_owned()));
 
             let auth_token = if matches!(env.pageserver.auth_type, AuthType::NeonJWT) {
                 let claims = Claims::new(Some(tenant_id), Scope::Tenant);
@@ -642,33 +853,36 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> {
             };
 
             if let Some(node) = node {
-                println!("Starting existing postgres {node_name}...");
+                println!("Starting existing postgres {}...", node_name);
                 node.start(&auth_token)?;
             } else {
                 let branch_name = sub_args
-                    .get_one::<String>("branch-name")
-                    .map(|s| s.as_str())
+                    .value_of("branch-name")
                     .unwrap_or(DEFAULT_BRANCH_NAME);
                 let timeline_id = env
                     .get_branch_timeline_id(branch_name, tenant_id)
                     .ok_or_else(|| {
-                        anyhow!("Found no timeline id for branch name '{branch_name}'")
+                        anyhow!("Found no timeline id for branch name '{}'", branch_name)
                     })?;
                 let lsn = sub_args
-                    .get_one::<String>("lsn")
-                    .map(|lsn_str| Lsn::from_str(lsn_str))
+                    .value_of("lsn")
+                    .map(Lsn::from_str)
                     .transpose()
                     .context("Failed to parse Lsn from the request")?;
                 let pg_version = sub_args
-                    .get_one::<u32>("pg-version")
-                    .copied()
-                    .context("Failed to `pg-version` from the argument string")?;
+                    .value_of("pg-version")
+                    .unwrap()
+                    .parse::<u32>()
+                    .context("Failed to parse postgres version from the argument string")?;
                 // when used with custom port this results in non obvious behaviour
                 // port is remembered from first start command, i e
                 // start --port X
                 // stop
                 // start <-- will also use port X even without explicit port argument
-                println!("Starting new postgres (v{pg_version}) {node_name} on timeline {timeline_id} ...");
+                println!(
+                    "Starting new postgres (v{}) {} on timeline {} ...",
+                    pg_version, node_name, timeline_id
+                );
 
                 let node =
                     cplane.new_node(tenant_id, node_name, timeline_id, lsn, port, pg_version)?;
@@ -677,18 +891,18 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> {
         }
         "stop" => {
             let node_name = sub_args
-                .get_one::<String>("node")
+                .value_of("node")
                 .ok_or_else(|| anyhow!("No node name was provided to stop"))?;
-            let destroy = sub_args.get_flag("destroy");
+            let destroy = sub_args.is_present("destroy");
 
             let node = cplane
                 .nodes
-                .get(&(tenant_id, node_name.to_string()))
-                .with_context(|| format!("postgres {node_name} is not found"))?;
+                .get(&(tenant_id, node_name.to_owned()))
+                .with_context(|| format!("postgres {} is not found", node_name))?;
             node.stop(destroy)?;
         }
 
-        _ => bail!("Unexpected pg subcommand '{sub_name}'"),
+        _ => bail!("Unexpected pg subcommand '{}'", sub_name),
     }
 
     Ok(())
@@ -706,10 +920,7 @@ fn handle_pageserver(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul
         }
 
         Some(("stop", stop_match)) => {
-            let immediate = stop_match
-                .get_one::<String>("stop-mode")
-                .map(|s| s.as_str())
-                == Some("immediate");
+            let immediate = stop_match.value_of("stop-mode") == Some("immediate");
 
             if let Err(e) = pageserver.stop(immediate) {
                 eprintln!("pageserver stop failed: {}", e);
@@ -759,7 +970,7 @@ fn handle_safekeeper(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul
     };
 
     // All the commands take an optional safekeeper name argument
-    let sk_id = if let Some(id_str) = sub_args.get_one::<String>("id") {
+    let sk_id = if let Some(id_str) = sub_args.value_of("id") {
         NodeId(id_str.parse().context("while parsing safekeeper id")?)
     } else {
         DEFAULT_SAFEKEEPER_ID
@@ -775,8 +986,7 @@ fn handle_safekeeper(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul
         }
 
         "stop" => {
-            let immediate =
-                sub_args.get_one::<String>("stop-mode").map(|s| s.as_str()) == Some("immediate");
+            let immediate = sub_args.value_of("stop-mode") == Some("immediate");
 
             if let Err(e) = safekeeper.stop(immediate) {
                 eprintln!("safekeeper stop failed: {}", e);
@@ -785,8 +995,7 @@ fn handle_safekeeper(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul
         }
 
         "restart" => {
-            let immediate =
-                sub_args.get_one::<String>("stop-mode").map(|s| s.as_str()) == Some("immediate");
+            let immediate = sub_args.value_of("stop-mode") == Some("immediate");
 
             if let Err(e) = safekeeper.stop(immediate) {
                 eprintln!("safekeeper stop failed: {}", e);
@@ -830,8 +1039,7 @@ fn handle_start_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow
 }
 
 fn handle_stop_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> {
-    let immediate =
-        sub_match.get_one::<String>("stop-mode").map(|s| s.as_str()) == Some("immediate");
+    let immediate = sub_match.value_of("stop-mode") == Some("immediate");
 
     let pageserver = PageServerNode::from_env(env);
 
@@ -864,219 +1072,3 @@ fn try_stop_etcd_process(env: &local_env::LocalEnv) {
         eprintln!("etcd stop failed: {e}");
     }
 }
-
-fn cli() -> Command {
-    let branch_name_arg = Arg::new("branch-name")
-        .long("branch-name")
-        .help("Name of the branch to be created or used as an alias for other services")
-        .required(false);
-
-    let pg_node_arg = Arg::new("node").help("Postgres node name").required(false);
-
-    let safekeeper_id_arg = Arg::new("id").help("safekeeper id").required(false);
-
-    let tenant_id_arg = Arg::new("tenant-id")
-        .long("tenant-id")
-        .help("Tenant id. Represented as a hexadecimal string 32 symbols length")
-        .required(false);
-
-    let timeline_id_arg = Arg::new("timeline-id")
-        .long("timeline-id")
-        .help("Timeline id. Represented as a hexadecimal string 32 symbols length")
-        .required(false);
-
-    let pg_version_arg = Arg::new("pg-version")
-        .long("pg-version")
-        .help("Postgres version to use for the initial tenant")
-        .required(false)
-        .value_parser(value_parser!(u32))
-        .default_value(DEFAULT_PG_VERSION);
-
-    let port_arg = Arg::new("port")
-        .long("port")
-        .required(false)
-        .value_parser(value_parser!(u16))
-        .value_name("port");
-
-    let stop_mode_arg = Arg::new("stop-mode")
-        .short('m')
-        .value_parser(["fast", "immediate"])
-        .help("If 'immediate', don't flush repository data at shutdown")
-        .required(false)
-        .value_name("stop-mode");
-
-    let pageserver_config_args = Arg::new("pageserver-config-override")
-        .long("pageserver-config-override")
-        .num_args(1)
-        .action(ArgAction::Append)
-        .help("Additional pageserver's configuration options or overrides, refer to pageserver's 'config-override' CLI parameter docs for more")
-        .required(false);
-
-    let lsn_arg = Arg::new("lsn")
-        .long("lsn")
-        .help("Specify Lsn on the timeline to start from. By default, end of the timeline would be used.")
-        .required(false);
-
-    Command::new("Neon CLI")
-        .arg_required_else_help(true)
-        .version(GIT_VERSION)
-        .subcommand(
-            Command::new("init")
-                .about("Initialize a new Neon repository")
-                .arg(pageserver_config_args.clone())
-                .arg(timeline_id_arg.clone().help("Use a specific timeline id when creating a tenant and its initial timeline"))
-                .arg(
-                    Arg::new("config")
-                        .long("config")
-                        .required(false)
-                        .value_parser(value_parser!(PathBuf))
-                        .value_name("config"),
-                )
-                .arg(pg_version_arg.clone())
-        )
-        .subcommand(
-            Command::new("timeline")
-            .about("Manage timelines")
-            .subcommand(Command::new("list")
-                .about("List all timelines, available to this pageserver")
-                .arg(tenant_id_arg.clone()))
-            .subcommand(Command::new("branch")
-                .about("Create a new timeline, using another timeline as a base, copying its data")
-                .arg(tenant_id_arg.clone())
-                .arg(branch_name_arg.clone())
-                .arg(Arg::new("ancestor-branch-name").long("ancestor-branch-name")
-                    .help("Use last Lsn of another timeline (and its data) as base when creating the new timeline. The timeline gets resolved by its branch name.").required(false))
-                .arg(Arg::new("ancestor-start-lsn").long("ancestor-start-lsn")
-                    .help("When using another timeline as base, use a specific Lsn in it instead of the latest one").required(false)))
-            .subcommand(Command::new("create")
-                .about("Create a new blank timeline")
-                .arg(tenant_id_arg.clone())
-                .arg(branch_name_arg.clone())
-                .arg(pg_version_arg.clone())
-            )
-            .subcommand(Command::new("import")
-                .about("Import timeline from basebackup directory")
-                .arg(tenant_id_arg.clone())
-                .arg(timeline_id_arg.clone())
-                .arg(Arg::new("node-name").long("node-name")
-                    .help("Name to assign to the imported timeline"))
-                .arg(Arg::new("base-tarfile")
-                    .long("base-tarfile")
-                    .value_parser(value_parser!(PathBuf))
-                    .help("Basebackup tarfile to import")
-                )
-                .arg(Arg::new("base-lsn").long("base-lsn")
-                    .help("Lsn the basebackup starts at"))
-                .arg(Arg::new("wal-tarfile")
-                    .long("wal-tarfile")
-                    .value_parser(value_parser!(PathBuf))
-                    .help("Wal to add after base")
-                )
-                .arg(Arg::new("end-lsn").long("end-lsn")
-                    .help("Lsn the basebackup ends at"))
-                .arg(pg_version_arg.clone())
-            )
-        ).subcommand(
-            Command::new("tenant")
-            .arg_required_else_help(true)
-            .about("Manage tenants")
-            .subcommand(Command::new("list"))
-            .subcommand(Command::new("create")
-                .arg(tenant_id_arg.clone())
-                .arg(timeline_id_arg.clone().help("Use a specific timeline id when creating a tenant and its initial timeline"))
-                .arg(Arg::new("config").short('c').num_args(1).action(ArgAction::Append).required(false))
-                .arg(pg_version_arg.clone())
-                )
-            .subcommand(Command::new("config")
-                .arg(tenant_id_arg.clone())
-                .arg(Arg::new("config").short('c').num_args(1).action(ArgAction::Append).required(false))
-                )
-        )
-        .subcommand(
-            Command::new("pageserver")
-                .arg_required_else_help(true)
-                .about("Manage pageserver")
-                .subcommand(Command::new("status"))
-                .subcommand(Command::new("start").about("Start local pageserver").arg(pageserver_config_args.clone()))
-                .subcommand(Command::new("stop").about("Stop local pageserver")
-                            .arg(stop_mode_arg.clone()))
-                .subcommand(Command::new("restart").about("Restart local pageserver").arg(pageserver_config_args.clone()))
-        )
-        .subcommand(
-            Command::new("safekeeper")
-                .arg_required_else_help(true)
-                .about("Manage safekeepers")
-                .subcommand(Command::new("start")
-                            .about("Start local safekeeper")
-                            .arg(safekeeper_id_arg.clone())
-                )
-                .subcommand(Command::new("stop")
-                            .about("Stop local safekeeper")
-                            .arg(safekeeper_id_arg.clone())
-                            .arg(stop_mode_arg.clone())
-                )
-                .subcommand(Command::new("restart")
-                            .about("Restart local safekeeper")
-                            .arg(safekeeper_id_arg)
-                            .arg(stop_mode_arg.clone())
-                )
-        )
-        .subcommand(
-            Command::new("pg")
-                .arg_required_else_help(true)
-                .about("Manage postgres instances")
-                .subcommand(Command::new("list").arg(tenant_id_arg.clone()))
-                .subcommand(Command::new("create")
-                    .about("Create a postgres compute node")
-                    .arg(pg_node_arg.clone())
-                    .arg(branch_name_arg.clone())
-                    .arg(tenant_id_arg.clone())
-                    .arg(lsn_arg.clone())
-                    .arg(port_arg.clone())
-                    .arg(
-                        Arg::new("config-only")
-                            .help("Don't do basebackup, create compute node with only config files")
-                            .long("config-only")
-                            .required(false))
-                    .arg(pg_version_arg.clone())
-                )
-                .subcommand(Command::new("start")
-                    .about("Start a postgres compute node.\n This command actually creates new node from scratch, but preserves existing config files")
-                    .arg(pg_node_arg.clone())
-                    .arg(tenant_id_arg.clone())
-                    .arg(branch_name_arg)
-                    .arg(timeline_id_arg)
-                    .arg(lsn_arg)
-                    .arg(port_arg)
-                    .arg(pg_version_arg)
-                )
-                .subcommand(
-                    Command::new("stop")
-                    .arg(pg_node_arg)
-                    .arg(tenant_id_arg)
-                    .arg(
-                        Arg::new("destroy")
-                            .help("Also delete data directory (now optional, should be default in future)")
-                            .long("destroy")
-                            .action(ArgAction::SetTrue)
-                            .required(false)
-                        )
-                )
-
-        )
-        .subcommand(
-            Command::new("start")
-                .about("Start page server and safekeepers")
-                .arg(pageserver_config_args)
-        )
-        .subcommand(
-            Command::new("stop")
-                .about("Stop page server and safekeepers")
-                .arg(stop_mode_arg)
-        )
-}
-
-#[test]
-fn verify_cli() {
-    cli().debug_assert();
-}

control_plane/src/compute.rs

@@ -183,18 +183,18 @@ impl PostgresNode {
     }
 
     fn sync_safekeepers(&self, auth_token: &Option<String>, pg_version: u32) -> Result<Lsn> {
-        let pg_path = self.env.pg_bin_dir(pg_version)?.join("postgres");
+        let pg_path = self.env.pg_bin_dir(pg_version).join("postgres");
         let mut cmd = Command::new(&pg_path);
 
         cmd.arg("--sync-safekeepers")
             .env_clear()
             .env(
                 "LD_LIBRARY_PATH",
-                self.env.pg_lib_dir(pg_version)?.to_str().unwrap(),
+                self.env.pg_lib_dir(pg_version).to_str().unwrap(),
             )
             .env(
                 "DYLD_LIBRARY_PATH",
-                self.env.pg_lib_dir(pg_version)?.to_str().unwrap(),
+                self.env.pg_lib_dir(pg_version).to_str().unwrap(),
             )
             .env("PGDATA", self.pgdata().to_str().unwrap())
             .stdout(Stdio::piped())
@@ -422,7 +422,7 @@ impl PostgresNode {
     }
 
     fn pg_ctl(&self, args: &[&str], auth_token: &Option<String>) -> Result<()> {
-        let pg_ctl_path = self.env.pg_bin_dir(self.pg_version)?.join("pg_ctl");
+        let pg_ctl_path = self.env.pg_bin_dir(self.pg_version).join("pg_ctl");
         let mut cmd = Command::new(pg_ctl_path);
         cmd.args(
             [
@@ -440,11 +440,11 @@ impl PostgresNode {
         .env_clear()
         .env(
             "LD_LIBRARY_PATH",
-            self.env.pg_lib_dir(self.pg_version)?.to_str().unwrap(),
+            self.env.pg_lib_dir(self.pg_version).to_str().unwrap(),
         )
         .env(
             "DYLD_LIBRARY_PATH",
-            self.env.pg_lib_dir(self.pg_version)?.to_str().unwrap(),
+            self.env.pg_lib_dir(self.pg_version).to_str().unwrap(),
         );
         if let Some(token) = auth_token {
             cmd.env("ZENITH_AUTH_TOKEN", token);

control_plane/src/local_env.rs

@@ -201,28 +201,28 @@ impl LocalEnv {
         self.pg_distrib_dir.clone()
     }
 
-    pub fn pg_distrib_dir(&self, pg_version: u32) -> anyhow::Result<PathBuf> {
+    pub fn pg_distrib_dir(&self, pg_version: u32) -> PathBuf {
         let path = self.pg_distrib_dir.clone();
 
         match pg_version {
-            14 => Ok(path.join(format!("v{pg_version}"))),
-            15 => Ok(path.join(format!("v{pg_version}"))),
-            _ => bail!("Unsupported postgres version: {}", pg_version),
+            14 => path.join(format!("v{pg_version}")),
+            15 => path.join(format!("v{pg_version}")),
+            _ => panic!("Unsupported postgres version: {}", pg_version),
         }
     }
 
-    pub fn pg_bin_dir(&self, pg_version: u32) -> anyhow::Result<PathBuf> {
+    pub fn pg_bin_dir(&self, pg_version: u32) -> PathBuf {
         match pg_version {
-            14 => Ok(self.pg_distrib_dir(pg_version)?.join("bin")),
-            15 => Ok(self.pg_distrib_dir(pg_version)?.join("bin")),
-            _ => bail!("Unsupported postgres version: {}", pg_version),
+            14 => self.pg_distrib_dir(pg_version).join("bin"),
+            15 => self.pg_distrib_dir(pg_version).join("bin"),
+            _ => panic!("Unsupported postgres version: {}", pg_version),
         }
     }
-    pub fn pg_lib_dir(&self, pg_version: u32) -> anyhow::Result<PathBuf> {
+    pub fn pg_lib_dir(&self, pg_version: u32) -> PathBuf {
         match pg_version {
-            14 => Ok(self.pg_distrib_dir(pg_version)?.join("lib")),
-            15 => Ok(self.pg_distrib_dir(pg_version)?.join("lib")),
-            _ => bail!("Unsupported postgres version: {}", pg_version),
+            14 => self.pg_distrib_dir(pg_version).join("lib"),
+            15 => self.pg_distrib_dir(pg_version).join("lib"),
+            _ => panic!("Unsupported postgres version: {}", pg_version),
         }
     }
 
@@ -422,10 +422,10 @@ impl LocalEnv {
             "directory '{}' already exists. Perhaps already initialized?",
             base_path.display()
         );
-        if !self.pg_bin_dir(pg_version)?.join("postgres").exists() {
+        if !self.pg_bin_dir(pg_version).join("postgres").exists() {
             bail!(
                 "Can't find postgres binary at {}",
-                self.pg_bin_dir(pg_version)?.display()
+                self.pg_bin_dir(pg_version).display()
             );
         }
         for binary in ["pageserver", "safekeeper"] {

docs/SUMMARY.md

@@ -80,6 +80,4 @@
 - [015-storage-messaging](rfcs/015-storage-messaging.md)
 - [016-connection-routing](rfcs/016-connection-routing.md)
 - [017-timeline-data-management](rfcs/017-timeline-data-management.md)
-- [018-storage-messaging-2](rfcs/018-storage-messaging-2.md)
-- [019-tenant-timeline-lifecycles](rfcs/019-tenant-timeline-lifecycles.md)
 - [cluster-size-limits](rfcs/cluster-size-limits.md)

docs/rfcs/019-tenant-timeline-lifecycles.md

@@ -1,91 +0,0 @@
-# Managing Tenant and Timeline lifecycles
-
-## Summary
-
-The pageserver has a Tenant object in memory for each tenant it manages, and a
-Timeline for each timeline. There are a lot of tasks that operate on the tenants
-and timelines with references to those objects. We have some mechanisms to track
-which tasks are operating on each Tenant and Timeline, and to request them to
-shutdown when a tenant or timeline is deleted, but it does not cover all uses,
-and as a result we have many race conditions around tenant/timeline shutdown.
-
-## Motivation
-
-We have a bunch of race conditions that can produce weird errors and can be hard
-to track down.
-
-## Non Goals
-
-This RFC only covers the problem of ensuring that a task/thread isn't operating
-on a Tenant or Timeline. It does not cover what states, aside from Active and
-non-Active, each Tenant and Timeline should have, or when exactly the transitions
-should happen.
-
-## Impacted components (e.g. pageserver, safekeeper, console, etc)
-
-Pageserver. Although I wonder if the safekeeper should have a similar mechanism.
-
-## Current situation
-
-Most pageserver tasks of are managed by task_mgr.rs:
-
-- LibpqEndpointListener
-- HttpEndPointListener
-- WalReceiverManager and -Connection
-- GarbageCollector and Compaction
-- InitialLogicalSizeCalculation
-
-In addition to those tasks, the walreceiver performs some direct tokio::spawn
-calls to spawn tasks that are not registered with 'task_mgr'. And all of these
-tasks can spawn extra operations with tokio spawn_blocking.
-
-Whenever a tenant or timeline is removed from the system, by pageserver
-shutdown, delete_timeline or tenant-detach operation, we rely on the task
-registry in 'task_mgr.rs' to wait until there are no tasks operating on the
-tenant or timeline, before its Tenant/Timeline object is removed. That relies on
-each task to register itself with the tenant/timeline ID in
-'task_mgr.rs'. However, there are many gaps in that. For example,
-GarbageCollection and Compaction tasks are registered with the tenant, but when
-they proceed to operate on a particular timeline of the tenant, they don't
-register with timeline ID. Because of that, the timeline can be deleted while GC
-or compaction is running on it, causing failures in the GC or compaction (see
-https://github.com/neondatabase/neon/issues/2442).
-
-Another problem is that the task registry only works for tokio Tasks. There is
-no way to register a piece of code that runs inside spawn_blocking(), for
-example.
-
-## Proposed implementation
-
-This "voluntary" registration of tasks is fragile. Let's use Rust language features
-to enforce that a tenant/timeline cannot be removed from the system when there is
-still some code operating on it.
-
-Let's introduce new Guard objects for Tenant and Timeline, and do all actions through
-the Guard object. Something like:
-
-TenantActiveGuard: Guard object over Arc<Tenant>. When you acquire the guard,
-the code checks that the tenant is in Active state. If it's not, you get an
-error. You can change the state of the tenant to Stopping while there are
-ActiveTenantGuard objects still on it, to prevent new ActiveTenantGuards from
-being acquired, but the Tenant cannot be removed until all the guards are gone.
-
-TenantMaintenanceGuard: Like ActiveTenantGuard, but can be held even when the
-tenant is not in Active state. Used for operations like attach/detach. Perhaps
-allow only one such guard on a Tenant at a time.
-
-Similarly for Timelines. We don't currentl have a "state" on Timeline, but I think
-we need at least two states: Active and Stopping. The Stopping state is used at
-deletion, to prevent new TimelineActiveGuards from appearing, while you wait for
-existing TimelineActiveGuards to die out.
-
-The shutdown-signaling, using shutdown_watcher() and is_shutdown_requested(),
-probably also needs changes to deal with the new Guards. The rule is that if you
-have a TenantActiveGuard, and the tenant's state changes from Active to
-Stopping, the is_shutdown_requested() function should return true, and
-shutdown_watcher() future should return.
-
-This signaling doesn't neessarily need to cover all cases. For example, if you
-have a block of code in spawn_blocking(), it might be acceptable if
-is_shutdown_requested() doesn't return true even though the tenant is in
-Stopping state, as long as the code finishes reasonably fast.

libs/etcd_broker/src/subscription_value.rs

@@ -29,6 +29,9 @@ pub struct SkTimelineInfo {
     #[serde_as(as = "Option<DisplayFromStr>")]
     #[serde(default)]
     pub peer_horizon_lsn: Option<Lsn>,
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    #[serde(default)]
+    pub local_start_lsn: Option<Lsn>,
     /// A connection string to use for WAL receiving.
     #[serde(default)]
     pub safekeeper_connstr: Option<String>,

libs/postgres_ffi/Cargo.toml

@@ -13,7 +13,7 @@ crc32c = "0.6.0"
 hex = "0.4.3"
 once_cell = "1.13.0"
 log = "0.4.14"
-memoffset = "0.7"
+memoffset = "0.6.2"
 thiserror = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 utils = { path = "../utils" }
@@ -26,4 +26,4 @@ wal_craft = { path = "wal_craft" }
 
 [build-dependencies]
 anyhow = "1.0"
-bindgen = "0.61"
+bindgen = "0.60.1"

libs/postgres_ffi/wal_craft/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 
 [dependencies]
 anyhow = "1.0"
-clap = "4.0"
+clap = "3.0"
 env_logger = "0.9"
 log = "0.4"
 once_cell = "1.13.0"

libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs

@@ -1,19 +1,68 @@
 use anyhow::*;
-use clap::{value_parser, Arg, ArgMatches, Command};
-use std::{path::PathBuf, str::FromStr};
+use clap::{App, Arg, ArgMatches};
+use std::str::FromStr;
 use wal_craft::*;
 
 fn main() -> Result<()> {
     env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("wal_craft=info"))
         .init();
-    let arg_matches = cli().get_matches();
+    let type_arg = &Arg::new("type")
+        .takes_value(true)
+        .help("Type of WAL to craft")
+        .possible_values([
+            Simple::NAME,
+            LastWalRecordXlogSwitch::NAME,
+            LastWalRecordXlogSwitchEndsOnPageBoundary::NAME,
+            WalRecordCrossingSegmentFollowedBySmallOne::NAME,
+            LastWalRecordCrossingSegment::NAME,
+        ])
+        .required(true);
+    let arg_matches = App::new("Postgres WAL crafter")
+        .about("Crafts Postgres databases with specific WAL properties")
+        .subcommand(
+            App::new("print-postgres-config")
+                .about("Print the configuration required for PostgreSQL server before running this script")
+        )
+        .subcommand(
+            App::new("with-initdb")
+                .about("Craft WAL in a new data directory first initialized with initdb")
+                .arg(type_arg)
+                .arg(
+                    Arg::new("datadir")
+                        .takes_value(true)
+                        .help("Data directory for the Postgres server")
+                        .required(true)
+                )
+                .arg(
+                    Arg::new("pg-distrib-dir")
+                        .long("pg-distrib-dir")
+                        .takes_value(true)
+                        .help("Directory with Postgres distributions (bin and lib directories, e.g. pg_install containing subpath `v14/bin/postgresql`)")
+                        .default_value("/usr/local")
+                )
+                .arg(
+                    Arg::new("pg-version")
+                    .long("pg-version")
+                    .help("Postgres version to use for the initial tenant")
+                    .required(true)
+                    .takes_value(true)
+                )
+        )
+        .subcommand(
+            App::new("in-existing")
+                .about("Craft WAL at an existing recently created Postgres database. Note that server may append new WAL entries on shutdown.")
+                .arg(type_arg)
+                .arg(
+                    Arg::new("connection")
+                        .takes_value(true)
+                        .help("Connection string to the Postgres database to populate")
+                        .required(true)
+                )
+        )
+        .get_matches();
 
     let wal_craft = |arg_matches: &ArgMatches, client| {
-        let (intermediate_lsns, end_of_wal_lsn) = match arg_matches
-            .get_one::<String>("type")
-            .map(|s| s.as_str())
-            .context("'type' is required")?
-        {
+        let (intermediate_lsns, end_of_wal_lsn) = match arg_matches.value_of("type").unwrap() {
             Simple::NAME => Simple::craft(client)?,
             LastWalRecordXlogSwitch::NAME => LastWalRecordXlogSwitch::craft(client)?,
             LastWalRecordXlogSwitchEndsOnPageBoundary::NAME => {
@@ -23,12 +72,12 @@ fn main() -> Result<()> {
                 WalRecordCrossingSegmentFollowedBySmallOne::craft(client)?
             }
             LastWalRecordCrossingSegment::NAME => LastWalRecordCrossingSegment::craft(client)?,
-            a => panic!("Unknown --type argument: {a}"),
+            a => panic!("Unknown --type argument: {}", a),
         };
         for lsn in intermediate_lsns {
-            println!("intermediate_lsn = {lsn}");
+            println!("intermediate_lsn = {}", lsn);
         }
-        println!("end_of_wal = {end_of_wal_lsn}");
+        println!("end_of_wal = {}", end_of_wal_lsn);
         Ok(())
     };
 
@@ -36,24 +85,20 @@ fn main() -> Result<()> {
         None => panic!("No subcommand provided"),
         Some(("print-postgres-config", _)) => {
             for cfg in REQUIRED_POSTGRES_CONFIG.iter() {
-                println!("{cfg}");
+                println!("{}", cfg);
             }
             Ok(())
         }
 
         Some(("with-initdb", arg_matches)) => {
             let cfg = Conf {
-                pg_version: *arg_matches
-                    .get_one::<u32>("pg-version")
-                    .context("'pg-version' is required")?,
-                pg_distrib_dir: arg_matches
-                    .get_one::<PathBuf>("pg-distrib-dir")
-                    .context("'pg-distrib-dir' is required")?
-                    .to_owned(),
-                datadir: arg_matches
-                    .get_one::<PathBuf>("datadir")
-                    .context("'datadir' is required")?
-                    .to_owned(),
+                pg_version: arg_matches
+                    .value_of("pg-version")
+                    .unwrap()
+                    .parse::<u32>()
+                    .context("Failed to parse postgres version from the argument string")?,
+                pg_distrib_dir: arg_matches.value_of("pg-distrib-dir").unwrap().into(),
+                datadir: arg_matches.value_of("datadir").unwrap().into(),
             };
             cfg.initdb()?;
             let srv = cfg.start_server()?;
@@ -63,77 +108,9 @@ fn main() -> Result<()> {
         }
         Some(("in-existing", arg_matches)) => wal_craft(
             arg_matches,
-            &mut postgres::Config::from_str(
-                arg_matches
-                    .get_one::<String>("connection")
-                    .context("'connection' is required")?,
-            )
-            .context(
-                "'connection' argument value could not be parsed as a postgres connection string",
-            )?
-            .connect(postgres::NoTls)?,
+            &mut postgres::Config::from_str(arg_matches.value_of("connection").unwrap())?
+                .connect(postgres::NoTls)?,
         ),
         Some(_) => panic!("Unknown subcommand"),
     }
 }
-
-fn cli() -> Command {
-    let type_arg = &Arg::new("type")
-        .help("Type of WAL to craft")
-        .value_parser([
-            Simple::NAME,
-            LastWalRecordXlogSwitch::NAME,
-            LastWalRecordXlogSwitchEndsOnPageBoundary::NAME,
-            WalRecordCrossingSegmentFollowedBySmallOne::NAME,
-            LastWalRecordCrossingSegment::NAME,
-        ])
-        .required(true);
-
-    Command::new("Postgres WAL crafter")
-        .about("Crafts Postgres databases with specific WAL properties")
-        .subcommand(
-            Command::new("print-postgres-config")
-                .about("Print the configuration required for PostgreSQL server before running this script")
-        )
-        .subcommand(
-            Command::new("with-initdb")
-                .about("Craft WAL in a new data directory first initialized with initdb")
-                .arg(type_arg)
-                .arg(
-                    Arg::new("datadir")
-                        .help("Data directory for the Postgres server")
-                        .value_parser(value_parser!(PathBuf))
-                        .required(true)
-                )
-                .arg(
-                    Arg::new("pg-distrib-dir")
-                        .long("pg-distrib-dir")
-                        .value_parser(value_parser!(PathBuf))
-                        .help("Directory with Postgres distributions (bin and lib directories, e.g. pg_install containing subpath `v14/bin/postgresql`)")
-                        .default_value("/usr/local")
-                )
-                .arg(
-                    Arg::new("pg-version")
-                    .long("pg-version")
-                    .help("Postgres version to use for the initial tenant")
-                    .value_parser(value_parser!(u32))
-                    .required(true)
-
-                )
-        )
-        .subcommand(
-            Command::new("in-existing")
-                .about("Craft WAL at an existing recently created Postgres database. Note that server may append new WAL entries on shutdown.")
-                .arg(type_arg)
-                .arg(
-                    Arg::new("connection")
-                        .help("Connection string to the Postgres database to populate")
-                        .required(true)
-                )
-        )
-}
-
-#[test]
-fn verify_cli() {
-    cli().debug_assert();
-}

libs/postgres_ffi/wal_craft/src/lib.rs

@@ -37,22 +37,22 @@ pub static REQUIRED_POSTGRES_CONFIG: Lazy<Vec<&'static str>> = Lazy::new(|| {
 });
 
 impl Conf {
-    pub fn pg_distrib_dir(&self) -> anyhow::Result<PathBuf> {
+    pub fn pg_distrib_dir(&self) -> PathBuf {
         let path = self.pg_distrib_dir.clone();
 
         match self.pg_version {
-            14 => Ok(path.join(format!("v{}", self.pg_version))),
-            15 => Ok(path.join(format!("v{}", self.pg_version))),
-            _ => bail!("Unsupported postgres version: {}", self.pg_version),
+            14 => path.join(format!("v{}", self.pg_version)),
+            15 => path.join(format!("v{}", self.pg_version)),
+            _ => panic!("Unsupported postgres version: {}", self.pg_version),
         }
     }
 
-    fn pg_bin_dir(&self) -> anyhow::Result<PathBuf> {
-        Ok(self.pg_distrib_dir()?.join("bin"))
+    fn pg_bin_dir(&self) -> PathBuf {
+        self.pg_distrib_dir().join("bin")
     }
 
-    fn pg_lib_dir(&self) -> anyhow::Result<PathBuf> {
-        Ok(self.pg_distrib_dir()?.join("lib"))
+    fn pg_lib_dir(&self) -> PathBuf {
+        self.pg_distrib_dir().join("lib")
     }
 
     pub fn wal_dir(&self) -> PathBuf {
@@ -60,12 +60,12 @@ impl Conf {
     }
 
     fn new_pg_command(&self, command: impl AsRef<Path>) -> Result<Command> {
-        let path = self.pg_bin_dir()?.join(command);
+        let path = self.pg_bin_dir().join(command);
         ensure!(path.exists(), "Command {:?} does not exist", path);
         let mut cmd = Command::new(path);
         cmd.env_clear()
-            .env("LD_LIBRARY_PATH", self.pg_lib_dir()?)
-            .env("DYLD_LIBRARY_PATH", self.pg_lib_dir()?);
+            .env("LD_LIBRARY_PATH", self.pg_lib_dir())
+            .env("DYLD_LIBRARY_PATH", self.pg_lib_dir());
         Ok(cmd)
     }
 

libs/remote_storage/src/local_fs.rs

@@ -16,7 +16,7 @@ use tokio::{
     io::{self, AsyncReadExt, AsyncSeekExt, AsyncWriteExt},
 };
 use tracing::*;
-use utils::crashsafe::path_with_suffix_extension;
+use utils::crashsafe_dir::path_with_suffix_extension;
 
 use crate::{Download, DownloadError, RemoteObjectId};
 

libs/utils/src/crashsafe_dir.rs

@@ -12,8 +12,16 @@ pub fn create_dir(path: impl AsRef<Path>) -> io::Result<()> {
     let path = path.as_ref();
 
     fs::create_dir(path)?;
-    fsync_file_and_parent(path)?;
-    Ok(())
+    File::open(path)?.sync_all()?;
+
+    if let Some(parent) = path.parent() {
+        File::open(parent)?.sync_all()
+    } else {
+        Err(io::Error::new(
+            io::ErrorKind::InvalidInput,
+            "can't find parent",
+        ))
+    }
 }
 
 /// Similar to [`std::fs::create_dir_all`], except we fsync all
@@ -57,12 +65,12 @@ pub fn create_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
 
     // Fsync the created directories from child to parent.
     for &path in dirs_to_create.iter() {
-        fsync(path)?;
+        File::open(path)?.sync_all()?;
     }
 
     // If we created any new directories, fsync the parent.
     if !dirs_to_create.is_empty() {
-        fsync(path)?;
+        File::open(path)?.sync_all()?;
     }
 
     Ok(())
@@ -84,33 +92,6 @@ pub fn path_with_suffix_extension(original_path: impl AsRef<Path>, suffix: &str)
         .with_extension(new_extension.as_ref())
 }
 
-pub fn fsync_file_and_parent(file_path: &Path) -> io::Result<()> {
-    let parent = file_path.parent().ok_or_else(|| {
-        io::Error::new(
-            io::ErrorKind::Other,
-            format!("File {file_path:?} has no parent"),
-        )
-    })?;
-
-    fsync(file_path)?;
-    fsync(parent)?;
-    Ok(())
-}
-
-pub fn fsync(path: &Path) -> io::Result<()> {
-    File::open(path)
-        .map_err(|e| io::Error::new(e.kind(), format!("Failed to open the file {path:?}: {e}")))
-        .and_then(|file| {
-            file.sync_all().map_err(|e| {
-                io::Error::new(
-                    e.kind(),
-                    format!("Failed to sync file {path:?} data and metadata: {e}"),
-                )
-            })
-        })
-        .map_err(|e| io::Error::new(e.kind(), format!("Failed to fsync file {path:?}: {e}")))
-}
-
 #[cfg(test)]
 mod tests {
     use tempfile::tempdir;

libs/utils/src/id.rs

@@ -3,6 +3,13 @@ use std::{fmt, str::FromStr};
 use hex::FromHex;
 use rand::Rng;
 use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum IdError {
+    #[error("invalid id length {0}")]
+    VecParseError(usize),
+}
 
 /// Neon ID is a 128-bit random ID.
 /// Used to represent various identifiers. Provides handy utility methods and impls.
@@ -22,6 +29,15 @@ impl Id {
         Id::from(arr)
     }
 
+    pub fn from_vec(src: &Vec<u8>) -> Result<Id, IdError> {
+        if src.len() != 16 {
+            return Err(IdError::VecParseError(src.len()));
+        }
+        let mut zid_slice = [0u8; 16];
+        zid_slice.copy_from_slice(&src);
+        Ok(zid_slice.into())
+    }
+
     pub fn as_arr(&self) -> [u8; 16] {
         self.0
     }
@@ -75,6 +91,12 @@ impl From<[u8; 16]> for Id {
     }
 }
 
+impl From<Id> for u128 {
+    fn from(id: Id) -> Self {
+        u128::from_le_bytes(id.0)
+    }
+}
+
 impl fmt::Display for Id {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_str(&self.hex_encode())
@@ -94,6 +116,10 @@ macro_rules! id_newtype {
                 $t(Id::get_from_buf(buf))
             }
 
+            pub fn from_vec(src: &Vec<u8>) -> Result<$t, IdError> {
+                Ok($t(Id::from_vec(src)?))
+            }
+
             pub fn as_arr(&self) -> [u8; 16] {
                 self.0.as_arr()
             }
@@ -136,6 +162,12 @@ macro_rules! id_newtype {
             }
         }
 
+        impl From<$t> for u128 {
+            fn from(id: $t) -> Self {
+                u128::from(id.0)
+            }
+        }
+
         impl fmt::Display for $t {
             fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
                 self.0.fmt(f)

libs/utils/src/lib.rs

@@ -22,8 +22,8 @@ pub mod pq_proto;
 // dealing with connstring parsing and handy access to it's parts
 pub mod connstring;
 
-// helper functions for creating and fsyncing
-pub mod crashsafe;
+// helper functions for creating and fsyncing directories/trees
+pub mod crashsafe_dir;
 
 // common authentication routines
 pub mod auth;

libs/utils/src/postgres_backend_async.rs

@@ -15,7 +15,7 @@ use std::sync::Arc;
 use std::task::Poll;
 use tracing::{debug, error, trace};
 
-use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, BufReader};
+use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
 use tokio_rustls::TlsAcceptor;
 
 #[async_trait::async_trait]
@@ -66,8 +66,8 @@ pub enum ProcessMsgResult {
 /// Always-writeable sock_split stream.
 /// May not be readable. See [`PostgresBackend::take_stream_in`]
 pub enum Stream {
-    Unencrypted(BufReader<tokio::net::TcpStream>),
-    Tls(Box<tokio_rustls::server::TlsStream<BufReader<tokio::net::TcpStream>>>),
+    Unencrypted(tokio::net::TcpStream),
+    Tls(Box<tokio_rustls::server::TlsStream<tokio::net::TcpStream>>),
     Broken,
 }
 
@@ -157,7 +157,7 @@ impl PostgresBackend {
         let peer_addr = socket.peer_addr()?;
 
         Ok(Self {
-            stream: Stream::Unencrypted(BufReader::new(socket)),
+            stream: Stream::Unencrypted(socket),
             buf_out: BytesMut::with_capacity(10 * 1024),
             state: ProtoState::Initialization,
             md5_salt: [0u8; 4],

pageserver/Cargo.toml

@@ -23,7 +23,7 @@ futures = "0.3.13"
 hex = "0.4.3"
 hyper = "0.14"
 itertools = "0.10.3"
-clap = { version = "4.0", features = ["string"] }
+clap = "3.0"
 daemonize = "0.4.1"
 tokio = { version = "1.17", features = ["process", "sync", "macros", "fs", "rt", "io-util", "time"] }
 tokio-util = { version = "0.7.3", features = ["io", "io-util"] }
@@ -56,7 +56,7 @@ fail = "0.5.0"
 git-version = "0.3.5"
 rstar = "0.9.3"
 num-traits = "0.2.15"
-amplify_num = { git = "https://github.com/hlinnaka/rust-amplify.git", branch = "unsigned-int-perf" }
+amplify_num = "0.4.1"
 
 pageserver_api = { path = "../libs/pageserver_api" }
 postgres_ffi = { path = "../libs/postgres_ffi" }
@@ -69,10 +69,5 @@ close_fds = "0.3.2"
 walkdir = "2.3.2"
 
 [dev-dependencies]
-criterion = "0.4"
 hex-literal = "0.3"
 tempfile = "3.2"
-
-[[bench]]
-name = "bench_layer_map"
-harness = false

pageserver/src/bin/pageserver.rs

@@ -6,7 +6,7 @@ use tracing::*;
 
 use anyhow::{anyhow, bail, Context, Result};
 
-use clap::{Arg, ArgAction, Command};
+use clap::{App, Arg};
 use daemonize::Daemonize;
 
 use fail::FailScenario;
@@ -51,17 +51,57 @@ fn version() -> String {
 }
 
 fn main() -> anyhow::Result<()> {
-    let arg_matches = cli().get_matches();
+    let arg_matches = App::new("Neon page server")
+        .about("Materializes WAL stream to pages and serves them to the postgres")
+        .version(&*version())
+        .arg(
 
-    if arg_matches.get_flag("enabled-features") {
+            Arg::new("daemonize")
+                .short('d')
+                .long("daemonize")
+                .takes_value(false)
+                .help("Run in the background"),
+        )
+        .arg(
+            Arg::new("init")
+                .long("init")
+                .takes_value(false)
+                .help("Initialize pageserver with all given config overrides"),
+        )
+        .arg(
+            Arg::new("workdir")
+                .short('D')
+                .long("workdir")
+                .takes_value(true)
+                .help("Working directory for the pageserver"),
+        )
+        // See `settings.md` for more details on the extra configuration patameters pageserver can process
+        .arg(
+            Arg::new("config-override")
+                .short('c')
+                .takes_value(true)
+                .number_of_values(1)
+                .multiple_occurrences(true)
+                .help("Additional configuration overrides of the ones from the toml config file (or new ones to add there).
+                Any option has to be a valid toml document, example: `-c=\"foo='hey'\"` `-c=\"foo={value=1}\"`"),
+        )
+        .arg(Arg::new("update-config").long("update-config").takes_value(false).help(
+            "Update the config file when started",
+        ))
+        .arg(
+            Arg::new("enabled-features")
+                .long("enabled-features")
+                .takes_value(false)
+                .help("Show enabled compile time features"),
+        )
+        .get_matches();
+
+    if arg_matches.is_present("enabled-features") {
         println!("{{\"features\": {FEATURES:?} }}");
         return Ok(());
     }
 
-    let workdir = arg_matches
-        .get_one::<String>("workdir")
-        .map(Path::new)
-        .unwrap_or_else(|| Path::new(".neon"));
+    let workdir = Path::new(arg_matches.value_of("workdir").unwrap_or(".neon"));
     let workdir = workdir
         .canonicalize()
         .with_context(|| format!("Error opening workdir '{}'", workdir.display()))?;
@@ -75,7 +115,7 @@ fn main() -> anyhow::Result<()> {
         )
     })?;
 
-    let daemonize = arg_matches.get_flag("daemonize");
+    let daemonize = arg_matches.is_present("daemonize");
 
     let conf = match initialize_config(&cfg_file_path, arg_matches, &workdir)? {
         ControlFlow::Continue(conf) => conf,
@@ -87,7 +127,7 @@ fn main() -> anyhow::Result<()> {
 
     let tenants_path = conf.tenants_path();
     if !tenants_path.exists() {
-        utils::crashsafe::create_dir_all(conf.tenants_path()).with_context(|| {
+        utils::crashsafe_dir::create_dir_all(conf.tenants_path()).with_context(|| {
             format!(
                 "Failed to create tenants root dir at '{}'",
                 tenants_path.display()
@@ -113,8 +153,8 @@ fn initialize_config(
     arg_matches: clap::ArgMatches,
     workdir: &Path,
 ) -> anyhow::Result<ControlFlow<(), &'static PageServerConf>> {
-    let init = arg_matches.get_flag("init");
-    let update_config = init || arg_matches.get_flag("update-config");
+    let init = arg_matches.is_present("init");
+    let update_config = init || arg_matches.is_present("update-config");
 
     let (mut toml, config_file_exists) = if cfg_file_path.is_file() {
         if init {
@@ -156,10 +196,13 @@ fn initialize_config(
         )
     };
 
-    if let Some(values) = arg_matches.get_many::<String>("config-override") {
+    if let Some(values) = arg_matches.values_of("config-override") {
         for option_line in values {
             let doc = toml_edit::Document::from_str(option_line).with_context(|| {
-                format!("Option '{option_line}' could not be parsed as a toml document")
+                format!(
+                    "Option '{}' could not be parsed as a toml document",
+                    option_line
+                )
             })?;
 
             for (key, item) in doc.iter() {
@@ -201,7 +244,7 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
     // Initialize logger
     let log_file = logging::init(LOG_FILE_NAME, daemonize)?;
 
-    info!("version: {}", version());
+    info!("version: {GIT_VERSION}");
 
     // TODO: Check that it looks like a valid repository before going further
 
@@ -342,55 +385,3 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
         }
     })
 }
-
-fn cli() -> Command {
-    Command::new("Neon page server")
-        .about("Materializes WAL stream to pages and serves them to the postgres")
-        .version(version())
-        .arg(
-
-            Arg::new("daemonize")
-                .short('d')
-                .long("daemonize")
-                .action(ArgAction::SetTrue)
-                .help("Run in the background"),
-        )
-        .arg(
-            Arg::new("init")
-                .long("init")
-                .action(ArgAction::SetTrue)
-                .help("Initialize pageserver with all given config overrides"),
-        )
-        .arg(
-            Arg::new("workdir")
-                .short('D')
-                .long("workdir")
-                .help("Working directory for the pageserver"),
-        )
-        // See `settings.md` for more details on the extra configuration patameters pageserver can process
-        .arg(
-            Arg::new("config-override")
-                .short('c')
-                .num_args(1)
-                .action(ArgAction::Append)
-                .help("Additional configuration overrides of the ones from the toml config file (or new ones to add there). \
-                Any option has to be a valid toml document, example: `-c=\"foo='hey'\"` `-c=\"foo={value=1}\"`"),
-        )
-        .arg(
-            Arg::new("update-config")
-                .long("update-config")
-                .action(ArgAction::SetTrue)
-                .help("Update the config file when started"),
-        )
-        .arg(
-            Arg::new("enabled-features")
-                .long("enabled-features")
-                .action(ArgAction::SetTrue)
-                .help("Show enabled compile time features"),
-        )
-}
-
-#[test]
-fn verify_cli() {
-    cli().debug_assert();
-}

pageserver/src/bin/pageserver_binutils.rs

@@ -9,7 +9,7 @@ use std::{
 };
 
 use anyhow::Context;
-use clap::{value_parser, Arg, Command};
+use clap::{App, Arg};
 
 use pageserver::{
     page_cache,
@@ -24,14 +24,40 @@ project_git_version!(GIT_VERSION);
 const METADATA_SUBCOMMAND: &str = "metadata";
 
 fn main() -> anyhow::Result<()> {
-    let arg_matches = cli().get_matches();
+    let arg_matches = App::new("Neon Pageserver binutils")
+        .about("Reads pageserver (and related) binary files management utility")
+        .version(GIT_VERSION)
+        .arg(Arg::new("path").help("Input file path").required(false))
+        .subcommand(
+            App::new(METADATA_SUBCOMMAND)
+                .about("Read and update pageserver metadata file")
+                .arg(
+                    Arg::new("metadata_path")
+                        .help("Input metadata file path")
+                        .required(false),
+                )
+                .arg(
+                    Arg::new("disk_consistent_lsn")
+                        .long("disk_consistent_lsn")
+                        .takes_value(true)
+                        .help("Replace disk consistent Lsn"),
+                )
+                .arg(
+                    Arg::new("prev_record_lsn")
+                        .long("prev_record_lsn")
+                        .takes_value(true)
+                        .help("Replace previous record Lsn"),
+                ),
+        )
+        .get_matches();
 
     match arg_matches.subcommand() {
         Some((subcommand_name, subcommand_matches)) => {
-            let path = subcommand_matches
-                .get_one::<PathBuf>("metadata_path")
-                .context("'metadata_path' argument is missing")?
-                .to_path_buf();
+            let path = PathBuf::from(
+                subcommand_matches
+                    .value_of("metadata_path")
+                    .context("'metadata_path' argument is missing")?,
+            );
             anyhow::ensure!(
                 subcommand_name == METADATA_SUBCOMMAND,
                 "Unknown subcommand {subcommand_name}"
@@ -39,10 +65,11 @@ fn main() -> anyhow::Result<()> {
             handle_metadata(&path, subcommand_matches)?;
         }
         None => {
-            let path = arg_matches
-                .get_one::<PathBuf>("path")
-                .context("'path' argument is missing")?
-                .to_path_buf();
+            let path = PathBuf::from(
+                arg_matches
+                    .value_of("path")
+                    .context("'path' argument is missing")?,
+            );
             println!(
                 "No subcommand specified, attempting to guess the format for file {}",
                 path.display()
@@ -83,7 +110,7 @@ fn handle_metadata(path: &Path, arg_matches: &clap::ArgMatches) -> Result<(), an
     let mut meta = TimelineMetadata::from_bytes(&metadata_bytes)?;
     println!("Current metadata:\n{meta:?}");
     let mut update_meta = false;
-    if let Some(disk_consistent_lsn) = arg_matches.get_one::<String>("disk_consistent_lsn") {
+    if let Some(disk_consistent_lsn) = arg_matches.value_of("disk_consistent_lsn") {
         meta = TimelineMetadata::new(
             Lsn::from_str(disk_consistent_lsn)?,
             meta.prev_record_lsn(),
@@ -95,7 +122,7 @@ fn handle_metadata(path: &Path, arg_matches: &clap::ArgMatches) -> Result<(), an
         );
         update_meta = true;
     }
-    if let Some(prev_record_lsn) = arg_matches.get_one::<String>("prev_record_lsn") {
+    if let Some(prev_record_lsn) = arg_matches.value_of("prev_record_lsn") {
         meta = TimelineMetadata::new(
             meta.disk_consistent_lsn(),
             Some(Lsn::from_str(prev_record_lsn)?),
@@ -115,40 +142,3 @@ fn handle_metadata(path: &Path, arg_matches: &clap::ArgMatches) -> Result<(), an
 
     Ok(())
 }
-
-fn cli() -> Command {
-    Command::new("Neon Pageserver binutils")
-        .about("Reads pageserver (and related) binary files management utility")
-        .version(GIT_VERSION)
-        .arg(
-            Arg::new("path")
-                .help("Input file path")
-                .value_parser(value_parser!(PathBuf))
-                .required(false),
-        )
-        .subcommand(
-            Command::new(METADATA_SUBCOMMAND)
-                .about("Read and update pageserver metadata file")
-                .arg(
-                    Arg::new("metadata_path")
-                        .help("Input metadata file path")
-                        .value_parser(value_parser!(PathBuf))
-                        .required(false),
-                )
-                .arg(
-                    Arg::new("disk_consistent_lsn")
-                        .long("disk_consistent_lsn")
-                        .help("Replace disk consistent Lsn"),
-                )
-                .arg(
-                    Arg::new("prev_record_lsn")
-                        .long("prev_record_lsn")
-                        .help("Replace previous record Lsn"),
-                ),
-        )
-}
-
-#[test]
-fn verify_cli() {
-    cli().debug_assert();
-}

pageserver/src/config.rs

@@ -7,7 +7,6 @@
 use anyhow::{anyhow, bail, ensure, Context, Result};
 use remote_storage::RemoteStorageConfig;
 use std::env;
-use utils::crashsafe::path_with_suffix_extension;
 
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
@@ -25,7 +24,6 @@ use crate::tenant_config::{TenantConf, TenantConfOpt};
 
 /// The name of the metadata file pageserver creates per timeline.
 pub const METADATA_FILE_NAME: &str = "metadata";
-pub const TIMELINE_UNINIT_MARK_SUFFIX: &str = "___uninit";
 const TENANT_CONFIG_NAME: &str = "config";
 
 pub mod defaults {
@@ -366,17 +364,6 @@ impl PageServerConf {
         self.timelines_path(tenant_id).join(timeline_id.to_string())
     }
 
-    pub fn timeline_uninit_mark_file_path(
-        &self,
-        tenant_id: TenantId,
-        timeline_id: TimelineId,
-    ) -> PathBuf {
-        path_with_suffix_extension(
-            self.timeline_path(&timeline_id, &tenant_id),
-            TIMELINE_UNINIT_MARK_SUFFIX,
-        )
-    }
-
     /// Points to a place in pageserver's local directory,
     /// where certain timeline's metadata file should be located.
     pub fn metadata_path(&self, timeline_id: TimelineId, tenant_id: TenantId) -> PathBuf {
@@ -387,28 +374,28 @@ impl PageServerConf {
     //
     // Postgres distribution paths
     //
-    pub fn pg_distrib_dir(&self, pg_version: u32) -> anyhow::Result<PathBuf> {
+    pub fn pg_distrib_dir(&self, pg_version: u32) -> PathBuf {
         let path = self.pg_distrib_dir.clone();
 
         match pg_version {
-            14 => Ok(path.join(format!("v{pg_version}"))),
-            15 => Ok(path.join(format!("v{pg_version}"))),
-            _ => bail!("Unsupported postgres version: {}", pg_version),
+            14 => path.join(format!("v{pg_version}")),
+            15 => path.join(format!("v{pg_version}")),
+            _ => panic!("Unsupported postgres version: {}", pg_version),
         }
     }
 
-    pub fn pg_bin_dir(&self, pg_version: u32) -> anyhow::Result<PathBuf> {
+    pub fn pg_bin_dir(&self, pg_version: u32) -> PathBuf {
         match pg_version {
-            14 => Ok(self.pg_distrib_dir(pg_version)?.join("bin")),
-            15 => Ok(self.pg_distrib_dir(pg_version)?.join("bin")),
-            _ => bail!("Unsupported postgres version: {}", pg_version),
+            14 => self.pg_distrib_dir(pg_version).join("bin"),
+            15 => self.pg_distrib_dir(pg_version).join("bin"),
+            _ => panic!("Unsupported postgres version: {}", pg_version),
         }
     }
-    pub fn pg_lib_dir(&self, pg_version: u32) -> anyhow::Result<PathBuf> {
+    pub fn pg_lib_dir(&self, pg_version: u32) -> PathBuf {
         match pg_version {
-            14 => Ok(self.pg_distrib_dir(pg_version)?.join("lib")),
-            15 => Ok(self.pg_distrib_dir(pg_version)?.join("lib")),
-            _ => bail!("Unsupported postgres version: {}", pg_version),
+            14 => self.pg_distrib_dir(pg_version).join("lib"),
+            15 => self.pg_distrib_dir(pg_version).join("lib"),
+            _ => panic!("Unsupported postgres version: {}", pg_version),
         }
     }
 

pageserver/src/http/routes.rs

@@ -386,7 +386,7 @@ async fn tenant_attach_handler(request: Request<Body>) -> Result<Response<Body>,
         }
         return json_response(StatusCode::ACCEPTED, ());
     }
-    // no tenant in the index, release the lock to make the potentially lengthy download operation
+    // no tenant in the index, release the lock to make the potentially lengthy download opetation
     drop(index_accessor);
 
     // download index parts for every tenant timeline
@@ -781,6 +781,11 @@ async fn failpoints_handler(mut request: Request<Body>) -> Result<Response<Body>
 }
 
 // Run GC immediately on given timeline.
+// FIXME: This is just for tests. See test_runner/regress/test_gc.py.
+// This probably should require special authentication or a global flag to
+// enable, I don't think we want to or need to allow regular clients to invoke
+// GC.
+//     @hllinnaka in commits ec44f4b29, 3aca717f3
 #[cfg(feature = "testing")]
 async fn timeline_gc_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
@@ -806,6 +811,9 @@ async fn timeline_gc_handler(mut request: Request<Body>) -> Result<Response<Body
 }
 
 // Run compaction immediately on given timeline.
+// FIXME This is just for tests. Don't expect this to be exposed to
+// the users or the api.
+//     @dhammika in commit a0781f229
 #[cfg(feature = "testing")]
 async fn timeline_compact_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;

pageserver/src/import_datadir.rs

@@ -43,19 +43,19 @@ pub fn get_lsn_from_controlfile(path: &Path) -> Result<Lsn> {
 /// The code that deals with the checkpoint would not work right if the
 /// cluster was not shut down cleanly.
 pub fn import_timeline_from_postgres_datadir(
+    path: &Path,
     tline: &Timeline,
-    pgdata_path: &Path,
-    pgdata_lsn: Lsn,
+    lsn: Lsn,
 ) -> Result<()> {
     let mut pg_control: Option<ControlFileData> = None;
 
     // TODO this shoud be start_lsn, which is not necessarily equal to end_lsn (aka lsn)
     // Then fishing out pg_control would be unnecessary
-    let mut modification = tline.begin_modification(pgdata_lsn);
+    let mut modification = tline.begin_modification(lsn);
     modification.init_empty()?;
 
     // Import all but pg_wal
-    let all_but_wal = WalkDir::new(pgdata_path)
+    let all_but_wal = WalkDir::new(path)
         .into_iter()
         .filter_entry(|entry| !entry.path().ends_with("pg_wal"));
     for entry in all_but_wal {
@@ -63,7 +63,7 @@ pub fn import_timeline_from_postgres_datadir(
         let metadata = entry.metadata().expect("error getting dir entry metadata");
         if metadata.is_file() {
             let absolute_path = entry.path();
-            let relative_path = absolute_path.strip_prefix(pgdata_path)?;
+            let relative_path = absolute_path.strip_prefix(path)?;
 
             let file = File::open(absolute_path)?;
             let len = metadata.len() as usize;
@@ -84,7 +84,7 @@ pub fn import_timeline_from_postgres_datadir(
         "Postgres cluster was not shut down cleanly"
     );
     ensure!(
-        pg_control.checkPointCopy.redo == pgdata_lsn.0,
+        pg_control.checkPointCopy.redo == lsn.0,
         "unexpected checkpoint REDO pointer"
     );
 
@@ -92,10 +92,10 @@ pub fn import_timeline_from_postgres_datadir(
     // this reads the checkpoint record itself, advancing the tip of the timeline to
     // *after* the checkpoint record. And crucially, it initializes the 'prev_lsn'.
     import_wal(
-        &pgdata_path.join("pg_wal"),
+        &path.join("pg_wal"),
         tline,
         Lsn(pg_control.checkPointCopy.redo),
-        pgdata_lsn,
+        lsn,
     )?;
 
     Ok(())

pageserver/src/lib.rs

@@ -46,8 +46,6 @@ pub const DELTA_FILE_MAGIC: u16 = 0x5A61;
 
 pub const LOG_FILE_NAME: &str = "pageserver.log";
 
-static ZERO_PAGE: bytes::Bytes = bytes::Bytes::from_static(&[0u8; 8192]);
-
 /// Config for the Repository checkpointer
 #[derive(Debug, Clone, Copy)]
 pub enum CheckpointConfig {

pageserver/src/metrics.rs

@@ -107,20 +107,18 @@ static CURRENT_LOGICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
 
 // Metrics for cloud upload. These metrics reflect data uploaded to cloud storage,
 // or in testing they estimate how much we would upload if we did.
-static NUM_PERSISTENT_FILES_CREATED: Lazy<IntCounterVec> = Lazy::new(|| {
-    register_int_counter_vec!(
+static NUM_PERSISTENT_FILES_CREATED: Lazy<IntCounter> = Lazy::new(|| {
+    register_int_counter!(
         "pageserver_created_persistent_files_total",
         "Number of files created that are meant to be uploaded to cloud storage",
-        &["tenant_id", "timeline_id"]
     )
     .expect("failed to define a metric")
 });
 
-static PERSISTENT_BYTES_WRITTEN: Lazy<IntCounterVec> = Lazy::new(|| {
-    register_int_counter_vec!(
+static PERSISTENT_BYTES_WRITTEN: Lazy<IntCounter> = Lazy::new(|| {
+    register_int_counter!(
         "pageserver_written_persistent_bytes_total",
         "Total bytes written that are meant to be uploaded to cloud storage",
-        &["tenant_id", "timeline_id"]
     )
     .expect("failed to define a metric")
 });
@@ -277,15 +275,11 @@ pub static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
 /// smallest redo processing times. These buckets allow us to measure down
 /// to 5us, which equates to 200'000 pages/sec, which equates to 1.6GB/sec.
 /// This is much better than the previous 5ms aka 200 pages/sec aka 1.6MB/sec.
-///
-/// Values up to 1s are recorded because metrics show that we have redo
-/// durations and lock times larger than 0.250s.
 macro_rules! redo_histogram_time_buckets {
     () => {
         vec![
             0.000_005, 0.000_010, 0.000_025, 0.000_050, 0.000_100, 0.000_250, 0.000_500, 0.001_000,
-            0.002_500, 0.005_000, 0.010_000, 0.025_000, 0.050_000, 0.100_000, 0.250_000, 0.500_000,
-            1.000_000,
+            0.002_500, 0.005_000, 0.010_000, 0.025_000, 0.050_000, 0.100_000, 0.250_000,
         ]
     };
 }
@@ -300,17 +294,6 @@ macro_rules! redo_histogram_count_buckets {
     };
 }
 
-macro_rules! redo_bytes_histogram_count_buckets {
-    () => {
-        // powers of (2^.5), from 2^4.5 to 2^15 (22 buckets)
-        // rounded up to the next multiple of 8 to capture any MAXALIGNed record of that size, too.
-        vec![
-            24.0, 32.0, 48.0, 64.0, 96.0, 128.0, 184.0, 256.0, 368.0, 512.0, 728.0, 1024.0, 1456.0,
-            2048.0, 2904.0, 4096.0, 5800.0, 8192.0, 11592.0, 16384.0, 23176.0, 32768.0,
-        ]
-    };
-}
-
 pub static WAL_REDO_TIME: Lazy<Histogram> = Lazy::new(|| {
     register_histogram!(
         "pageserver_wal_redo_seconds",
@@ -338,15 +321,6 @@ pub static WAL_REDO_RECORDS_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
     .expect("failed to define a metric")
 });
 
-pub static WAL_REDO_BYTES_HISTOGRAM: Lazy<Histogram> = Lazy::new(|| {
-    register_histogram!(
-        "pageserver_wal_redo_bytes_histogram",
-        "Histogram of number of records replayed per redo",
-        redo_bytes_histogram_count_buckets!(),
-    )
-    .expect("failed to define a metric")
-});
-
 pub static WAL_REDO_RECORD_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
     register_int_counter!(
         "pageserver_replayed_wal_records_total",
@@ -412,12 +386,8 @@ impl TimelineMetrics {
         let current_logical_size_gauge = CURRENT_LOGICAL_SIZE
             .get_metric_with_label_values(&[&tenant_id, &timeline_id])
             .unwrap();
-        let num_persistent_files_created = NUM_PERSISTENT_FILES_CREATED
-            .get_metric_with_label_values(&[&tenant_id, &timeline_id])
-            .unwrap();
-        let persistent_bytes_written = PERSISTENT_BYTES_WRITTEN
-            .get_metric_with_label_values(&[&tenant_id, &timeline_id])
-            .unwrap();
+        let num_persistent_files_created = NUM_PERSISTENT_FILES_CREATED.clone();
+        let persistent_bytes_written = PERSISTENT_BYTES_WRITTEN.clone();
 
         TimelineMetrics {
             tenant_id,
@@ -449,8 +419,6 @@ impl Drop for TimelineMetrics {
         let _ = WAIT_LSN_TIME.remove_label_values(&[tenant_id, timeline_id]);
         let _ = CURRENT_PHYSICAL_SIZE.remove_label_values(&[tenant_id, timeline_id]);
         let _ = CURRENT_LOGICAL_SIZE.remove_label_values(&[tenant_id, timeline_id]);
-        let _ = NUM_PERSISTENT_FILES_CREATED.remove_label_values(&[tenant_id, timeline_id]);
-        let _ = PERSISTENT_BYTES_WRITTEN.remove_label_values(&[tenant_id, timeline_id]);
 
         for op in STORAGE_TIME_OPERATIONS {
             let _ = STORAGE_TIME.remove_label_values(&[op, tenant_id, timeline_id]);

pageserver/src/page_service.rs

@@ -32,7 +32,7 @@ use utils::{
 
 use crate::basebackup;
 use crate::config::{PageServerConf, ProfilingConfig};
-use crate::import_datadir::import_wal_from_tar;
+use crate::import_datadir::{import_basebackup_from_tar, import_wal_from_tar};
 use crate::metrics::{LIVE_CONNECTIONS_COUNT, SMGR_QUERY_TIME};
 use crate::profiling::profpoint_start;
 use crate::reltag::RelTag;
@@ -500,8 +500,11 @@ impl PageServerHandler {
         task_mgr::associate_with(Some(tenant_id), Some(timeline_id));
         // Create empty timeline
         info!("creating new timeline");
-        let tenant = tenant_mgr::get_tenant(tenant_id, true)?;
-        let timeline = tenant.create_empty_timeline(timeline_id, base_lsn, pg_version)?;
+        let timeline = tenant_mgr::get_tenant(tenant_id, true)?.create_empty_timeline(
+            timeline_id,
+            base_lsn,
+            pg_version,
+        )?;
 
         // TODO mark timeline as not ready until it reaches end_lsn.
         // We might have some wal to import as well, and we should prevent compute
@@ -524,8 +527,7 @@ impl PageServerHandler {
         // - use block_in_place()
         let mut copyin_stream = Box::pin(copyin_stream(pgb));
         let reader = SyncIoBridge::new(StreamReader::new(&mut copyin_stream));
-        tokio::task::block_in_place(|| timeline.import_basebackup_from_tar(reader, base_lsn))?;
-        timeline.initialize()?;
+        tokio::task::block_in_place(|| import_basebackup_from_tar(&timeline, reader, base_lsn))?;
 
         // Drain the rest of the Copy data
         let mut bytes_after_tar = 0;
@@ -542,6 +544,12 @@ impl PageServerHandler {
         // It wouldn't work if base came from vanilla postgres though,
         // since we discard some log files.
 
+        // Flush data to disk, then upload to s3
+        info!("flushing layers");
+        timeline.checkpoint(CheckpointConfig::Flush)?;
+
+        timeline.launch_wal_receiver()?;
+
         info!("done");
         Ok(())
     }

pageserver/src/pgdatadir_mapping.rs

@@ -1373,17 +1373,6 @@ fn is_rel_block_key(key: Key) -> bool {
     key.field1 == 0x00 && key.field4 != 0
 }
 
-pub fn is_rel_fsm_block_key(key: Key) -> bool {
-    key.field1 == 0x00 && key.field4 != 0 && key.field5 == FSM_FORKNUM && key.field6 != 0xffffffff
-}
-
-pub fn is_rel_vm_block_key(key: Key) -> bool {
-    key.field1 == 0x00
-        && key.field4 != 0
-        && key.field5 == VISIBILITYMAP_FORKNUM
-        && key.field6 != 0xffffffff
-}
-
 pub fn key_to_slru_block(key: Key) -> Result<(SlruKind, u32, BlockNumber)> {
     Ok(match key.field1 {
         0x01 => {
@@ -1414,9 +1403,7 @@ pub fn create_test_timeline(
     timeline_id: utils::id::TimelineId,
     pg_version: u32,
 ) -> Result<std::sync::Arc<Timeline>> {
-    let tline = tenant
-        .create_empty_timeline(timeline_id, Lsn(8), pg_version)?
-        .initialize()?;
+    let tline = tenant.create_empty_timeline(timeline_id, Lsn(8), pg_version)?;
     let mut m = tline.begin_modification(Lsn(8));
     m.init_empty()?;
     m.commit()?;

pageserver/src/storage_sync.rs

@@ -171,7 +171,7 @@ use self::{
 use crate::{
     config::PageServerConf,
     exponential_backoff,
-    storage_sync::index::{LayerFileMetadata, RemoteIndex},
+    storage_sync::index::RemoteIndex,
     task_mgr,
     task_mgr::TaskKind,
     task_mgr::BACKGROUND_RUNTIME,
@@ -193,7 +193,7 @@ static SYNC_QUEUE: OnceCell<SyncQueue> = OnceCell::new();
 
 /// A timeline status to share with pageserver's sync counterpart,
 /// after comparing local and remote timeline state.
-#[derive(Clone, PartialEq, Eq)]
+#[derive(Clone)]
 pub enum LocalTimelineInitStatus {
     /// The timeline has every remote layer present locally.
     /// There could be some layers requiring uploading,
@@ -316,7 +316,7 @@ impl SyncQueue {
 
 /// A task to run in the async download/upload loop.
 /// Limited by the number of retries, after certain threshold the failing task gets evicted and the timeline disabled.
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone)]
 enum SyncTask {
     /// A checkpoint outcome with possible local file updates that need actualization in the remote storage.
     /// Not necessary more fresh than the one already uploaded.
@@ -427,7 +427,7 @@ impl SyncTaskBatch {
                             .extend(new_delete.data.deleted_layers.iter().cloned());
                     }
                     if let Some(batch_upload) = &mut self.upload {
-                        let not_deleted = |layer: &PathBuf, _: &mut LayerFileMetadata| {
+                        let not_deleted = |layer: &PathBuf| {
                             !new_delete.data.layers_to_delete.contains(layer)
                                 && !new_delete.data.deleted_layers.contains(layer)
                         };
@@ -455,35 +455,21 @@ impl SyncTaskBatch {
 #[derive(Debug, Clone, PartialEq, Eq)]
 struct LayersUpload {
     /// Layer file path in the pageserver workdir, that were added for the corresponding checkpoint.
-    layers_to_upload: HashMap<PathBuf, LayerFileMetadata>,
+    layers_to_upload: HashSet<PathBuf>,
     /// Already uploaded layers. Used to store the data about the uploads between task retries
     /// and to record the data into the remote index after the task got completed or evicted.
-    uploaded_layers: HashMap<PathBuf, LayerFileMetadata>,
+    uploaded_layers: HashSet<PathBuf>,
     metadata: Option<TimelineMetadata>,
 }
 
 /// A timeline download task.
 /// Does not contain the file list to download, to allow other
 /// parts of the pageserer code to schedule the task
-/// without using the remote index or any other ways to list the remote timeline files.
+/// without using the remote index or any other ways to list the remote timleine files.
 /// Skips the files that are already downloaded.
 #[derive(Debug, Clone, PartialEq, Eq)]
 struct LayersDownload {
     layers_to_skip: HashSet<PathBuf>,
-
-    /// Paths which have been downloaded, and had their metadata verified or generated.
-    ///
-    /// Metadata generation happens when upgrading from past version of `IndexPart`.
-    gathered_metadata: HashMap<PathBuf, LayerFileMetadata>,
-}
-
-impl LayersDownload {
-    fn from_skipped_layers(layers_to_skip: HashSet<PathBuf>) -> Self {
-        LayersDownload {
-            layers_to_skip,
-            gathered_metadata: HashMap::default(),
-        }
-    }
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -505,7 +491,7 @@ struct LayersDeletion {
 pub fn schedule_layer_upload(
     tenant_id: TenantId,
     timeline_id: TimelineId,
-    layers_to_upload: HashMap<PathBuf, LayerFileMetadata>,
+    layers_to_upload: HashSet<PathBuf>,
     metadata: Option<TimelineMetadata>,
 ) {
     let sync_queue = match SYNC_QUEUE.get() {
@@ -522,7 +508,7 @@ pub fn schedule_layer_upload(
         },
         SyncTask::upload(LayersUpload {
             layers_to_upload,
-            uploaded_layers: HashMap::new(),
+            uploaded_layers: HashSet::new(),
             metadata,
         }),
     );
@@ -580,44 +566,21 @@ pub fn schedule_layer_download(tenant_id: TenantId, timeline_id: TimelineId) {
             tenant_id,
             timeline_id,
         },
-        SyncTask::download(LayersDownload::from_skipped_layers(HashSet::new())),
+        SyncTask::download(LayersDownload {
+            layers_to_skip: HashSet::new(),
+        }),
     );
     debug!("Download task for tenant {tenant_id}, timeline {timeline_id} sent")
 }
 
-/// Local existing timeline files
-///
-/// Values of this type serve different meanings in different contexts. On startup, collected
-/// timelines come with the full collected information and when signalling readyness to attach
-/// after completed download. After the download the file information is no longer carried, because
-/// it is already merged into [`RemoteTimeline`].
-#[derive(Debug)]
-pub struct TimelineLocalFiles(TimelineMetadata, HashMap<PathBuf, LayerFileMetadata>);
-
-impl TimelineLocalFiles {
-    pub fn metadata(&self) -> &TimelineMetadata {
-        &self.0
-    }
-
-    /// Called during startup, for all of the local files with full metadata.
-    pub(crate) fn collected(
-        metadata: TimelineMetadata,
-        timeline_files: HashMap<PathBuf, LayerFileMetadata>,
-    ) -> TimelineLocalFiles {
-        TimelineLocalFiles(metadata, timeline_files)
-    }
-
-    /// Called near the end of tenant initialization, to signal readyness to attach tenants.
-    pub(crate) fn ready(metadata: TimelineMetadata) -> Self {
-        TimelineLocalFiles(metadata, HashMap::new())
-    }
-}
-
 /// Launch a thread to perform remote storage sync tasks.
 /// See module docs for loop step description.
 pub fn spawn_storage_sync_task(
     conf: &'static PageServerConf,
-    local_timeline_files: HashMap<TenantId, HashMap<TimelineId, TimelineLocalFiles>>,
+    local_timeline_files: HashMap<
+        TenantId,
+        HashMap<TimelineId, (TimelineMetadata, HashSet<PathBuf>)>,
+    >,
     storage: GenericRemoteStorage,
     max_concurrent_timelines_sync: NonZeroUsize,
     max_sync_errors: NonZeroU32,
@@ -775,7 +738,7 @@ async fn storage_sync_loop(
                                     tenant_entry
                                         .iter()
                                         .map(|(&id, entry)| {
-                                            (id, TimelineLocalFiles::ready(entry.metadata.clone()))
+                                            (id, (entry.metadata.clone(), HashSet::new()))
                                         })
                                         .collect(),
                                 ),
@@ -1020,27 +983,15 @@ async fn download_timeline_data(
         }
         DownloadedTimeline::Successful(mut download_data) => {
             match update_local_metadata(conf, sync_id, current_remote_timeline).await {
-                Ok(()) => {
-                    let mut g = index.write().await;
-
-                    match g.set_awaits_download(&sync_id, false) {
-                        Ok(()) => {
-                            let timeline = g
-                                .timeline_entry_mut(&sync_id)
-                                .expect("set_awaits_download verified existence");
-
-                            timeline.merge_metadata_from_downloaded(
-                                &download_data.data.gathered_metadata,
-                            );
-
-                            register_sync_status(sync_id, sync_start, TASK_NAME, Some(true));
-                            return DownloadStatus::Downloaded;
-                        }
-                        Err(e) => {
-                            error!("Timeline {sync_id} was expected to be in the remote index after a successful download, but it's absent: {e:?}");
-                        }
-                    };
-                }
+                Ok(()) => match index.write().await.set_awaits_download(&sync_id, false) {
+                    Ok(()) => {
+                        register_sync_status(sync_id, sync_start, TASK_NAME, Some(true));
+                        return DownloadStatus::Downloaded;
+                    }
+                    Err(e) => {
+                        error!("Timeline {sync_id} was expected to be in the remote index after a successful download, but it's absent: {e:?}");
+                    }
+                },
                 Err(e) => {
                     error!("Failed to update local timeline metadata: {e:?}");
                     download_data.retries += 1;
@@ -1243,18 +1194,11 @@ async fn update_remote_data(
                         }
                         if upload_failed {
                             existing_entry.add_upload_failures(
-                                uploaded_data
-                                    .layers_to_upload
-                                    .iter()
-                                    .map(|(k, v)| (k.to_owned(), v.to_owned())),
+                                uploaded_data.layers_to_upload.iter().cloned(),
                             );
                         } else {
-                            existing_entry.add_timeline_layers(
-                                uploaded_data
-                                    .uploaded_layers
-                                    .iter()
-                                    .map(|(k, v)| (k.to_owned(), v.to_owned())),
-                            );
+                            existing_entry
+                                .add_timeline_layers(uploaded_data.uploaded_layers.iter().cloned());
                         }
                     }
                     RemoteDataUpdate::Delete(layers_to_remove) => {
@@ -1274,19 +1218,11 @@ async fn update_remote_data(
                     };
                     let mut new_remote_timeline = RemoteTimeline::new(new_metadata.clone());
                     if upload_failed {
-                        new_remote_timeline.add_upload_failures(
-                            uploaded_data
-                                .layers_to_upload
-                                .iter()
-                                .map(|(k, v)| (k.to_owned(), v.to_owned())),
-                        );
+                        new_remote_timeline
+                            .add_upload_failures(uploaded_data.layers_to_upload.iter().cloned());
                     } else {
-                        new_remote_timeline.add_timeline_layers(
-                            uploaded_data
-                                .uploaded_layers
-                                .iter()
-                                .map(|(k, v)| (k.to_owned(), v.to_owned())),
-                        );
+                        new_remote_timeline
+                            .add_timeline_layers(uploaded_data.uploaded_layers.iter().cloned());
                     }
 
                     index_accessor.add_timeline_entry(sync_id, new_remote_timeline.clone());
@@ -1334,14 +1270,13 @@ async fn validate_task_retries(
 fn schedule_first_sync_tasks(
     index: &mut RemoteTimelineIndex,
     sync_queue: &SyncQueue,
-    local_timeline_files: HashMap<TenantTimelineId, TimelineLocalFiles>,
+    local_timeline_files: HashMap<TenantTimelineId, (TimelineMetadata, HashSet<PathBuf>)>,
 ) -> TenantTimelineValues<LocalTimelineInitStatus> {
     let mut local_timeline_init_statuses = TenantTimelineValues::new();
 
     let mut new_sync_tasks = VecDeque::with_capacity(local_timeline_files.len());
 
-    for (sync_id, local_timeline) in local_timeline_files {
-        let TimelineLocalFiles(local_metadata, local_files) = local_timeline;
+    for (sync_id, (local_metadata, local_files)) in local_timeline_files {
         match index.timeline_entry_mut(&sync_id) {
             Some(remote_timeline) => {
                 let (timeline_status, awaits_download) = compare_local_and_remote_timeline(
@@ -1385,7 +1320,7 @@ fn schedule_first_sync_tasks(
                     sync_id,
                     SyncTask::upload(LayersUpload {
                         layers_to_upload: local_files,
-                        uploaded_layers: HashMap::new(),
+                        uploaded_layers: HashSet::new(),
                         metadata: Some(local_metadata.clone()),
                     }),
                 ));
@@ -1412,46 +1347,20 @@ fn compare_local_and_remote_timeline(
     new_sync_tasks: &mut VecDeque<(TenantTimelineId, SyncTask)>,
     sync_id: TenantTimelineId,
     local_metadata: TimelineMetadata,
-    local_files: HashMap<PathBuf, LayerFileMetadata>,
+    local_files: HashSet<PathBuf>,
     remote_entry: &RemoteTimeline,
 ) -> (LocalTimelineInitStatus, bool) {
     let _entered = info_span!("compare_local_and_remote_timeline", sync_id = %sync_id).entered();
 
-    let needed_to_download_files = remote_entry
-        .stored_files()
-        .iter()
-        .filter_map(|(layer_file, remote_metadata)| {
-            if let Some(local_metadata) = local_files.get(layer_file) {
-                match (remote_metadata.file_size(), local_metadata.file_size()) {
-                    (Some(x), Some(y)) if x == y => { None },
-                    (None, Some(_)) => {
-                        // upgrading from an earlier IndexPart without metadata
-                        None
-                    },
-                    _ => {
-                        // having to deal with other than (Some(x), Some(y)) where x != y here is a
-                        // bummer, but see #2582 and #2610 for attempts and discussion.
-                        warn!("Redownloading locally existing {layer_file:?} due to size mismatch, size on index: {:?}, on disk: {:?}", remote_metadata.file_size(), local_metadata.file_size());
-                        Some(layer_file)
-                    },
-                }
-            } else {
-                // doesn't exist locally
-                Some(layer_file)
-            }
-        })
-        .collect::<HashSet<_>>();
+    let remote_files = remote_entry.stored_files();
 
-    let (initial_timeline_status, awaits_download) = if !needed_to_download_files.is_empty() {
+    let number_of_layers_to_download = remote_files.difference(&local_files).count();
+    let (initial_timeline_status, awaits_download) = if number_of_layers_to_download > 0 {
         new_sync_tasks.push_back((
             sync_id,
-            SyncTask::download(LayersDownload::from_skipped_layers(
-                local_files
-                    .keys()
-                    .filter(|path| !needed_to_download_files.contains(path))
-                    .cloned()
-                    .collect(),
-            )),
+            SyncTask::download(LayersDownload {
+                layers_to_skip: local_files.clone(),
+            }),
         ));
         info!("NeedsSync");
         (LocalTimelineInitStatus::NeedsSync, true)
@@ -1466,22 +1375,15 @@ fn compare_local_and_remote_timeline(
     };
 
     let layers_to_upload = local_files
-        .iter()
-        .filter_map(|(local_file, metadata)| {
-            if !remote_entry.stored_files().contains_key(local_file) {
-                Some((local_file.to_owned(), metadata.to_owned()))
-            } else {
-                None
-            }
-        })
-        .collect::<HashMap<_, _>>();
-
+        .difference(remote_files)
+        .cloned()
+        .collect::<HashSet<_>>();
     if !layers_to_upload.is_empty() {
         new_sync_tasks.push_back((
             sync_id,
             SyncTask::upload(LayersUpload {
                 layers_to_upload,
-                uploaded_layers: HashMap::new(),
+                uploaded_layers: HashSet::new(),
                 metadata: Some(local_metadata),
             }),
         ));
@@ -1537,12 +1439,11 @@ mod test_utils {
         let timeline_path = harness.timeline_path(&timeline_id);
         fs::create_dir_all(&timeline_path).await?;
 
-        let mut layers_to_upload = HashMap::with_capacity(filenames.len());
+        let mut layers_to_upload = HashSet::with_capacity(filenames.len());
         for &file in filenames {
             let file_path = timeline_path.join(file);
             fs::write(&file_path, dummy_contents(file).into_bytes()).await?;
-            let metadata = LayerFileMetadata::new(file_path.metadata()?.len());
-            layers_to_upload.insert(file_path, metadata);
+            layers_to_upload.insert(file_path);
         }
 
         fs::write(
@@ -1553,7 +1454,7 @@ mod test_utils {
 
         Ok(LayersUpload {
             layers_to_upload,
-            uploaded_layers: HashMap::new(),
+            uploaded_layers: HashSet::new(),
             metadata: Some(metadata),
         })
     }
@@ -1608,13 +1509,12 @@ mod tests {
         assert!(sync_id_2 != sync_id_3);
         assert!(sync_id_3 != TEST_SYNC_ID);
 
-        let download_task =
-            SyncTask::download(LayersDownload::from_skipped_layers(HashSet::from([
-                PathBuf::from("sk"),
-            ])));
+        let download_task = SyncTask::download(LayersDownload {
+            layers_to_skip: HashSet::from([PathBuf::from("sk")]),
+        });
         let upload_task = SyncTask::upload(LayersUpload {
-            layers_to_upload: HashMap::from([(PathBuf::from("up"), LayerFileMetadata::new(123))]),
-            uploaded_layers: HashMap::from([(PathBuf::from("upl"), LayerFileMetadata::new(123))]),
+            layers_to_upload: HashSet::from([PathBuf::from("up")]),
+            uploaded_layers: HashSet::from([PathBuf::from("upl")]),
             metadata: Some(dummy_metadata(Lsn(2))),
         });
         let delete_task = SyncTask::delete(LayersDeletion {
@@ -1658,10 +1558,12 @@ mod tests {
         let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
         assert_eq!(sync_queue.len(), 0);
 
-        let download = LayersDownload::from_skipped_layers(HashSet::from([PathBuf::from("sk")]));
+        let download = LayersDownload {
+            layers_to_skip: HashSet::from([PathBuf::from("sk")]),
+        };
         let upload = LayersUpload {
-            layers_to_upload: HashMap::from([(PathBuf::from("up"), LayerFileMetadata::new(123))]),
-            uploaded_layers: HashMap::from([(PathBuf::from("upl"), LayerFileMetadata::new(123))]),
+            layers_to_upload: HashSet::from([PathBuf::from("up")]),
+            uploaded_layers: HashSet::from([PathBuf::from("upl")]),
             metadata: Some(dummy_metadata(Lsn(2))),
         };
         let delete = LayersDeletion {
@@ -1709,10 +1611,18 @@ mod tests {
     #[tokio::test]
     async fn same_task_id_same_tasks_batch() {
         let sync_queue = SyncQueue::new(NonZeroUsize::new(1).unwrap());
-        let download_1 = LayersDownload::from_skipped_layers(HashSet::from([PathBuf::from("sk1")]));
-        let download_2 = LayersDownload::from_skipped_layers(HashSet::from([PathBuf::from("sk2")]));
-        let download_3 = LayersDownload::from_skipped_layers(HashSet::from([PathBuf::from("sk3")]));
-        let download_4 = LayersDownload::from_skipped_layers(HashSet::from([PathBuf::from("sk4")]));
+        let download_1 = LayersDownload {
+            layers_to_skip: HashSet::from([PathBuf::from("sk1")]),
+        };
+        let download_2 = LayersDownload {
+            layers_to_skip: HashSet::from([PathBuf::from("sk2")]),
+        };
+        let download_3 = LayersDownload {
+            layers_to_skip: HashSet::from([PathBuf::from("sk3")]),
+        };
+        let download_4 = LayersDownload {
+            layers_to_skip: HashSet::from([PathBuf::from("sk4")]),
+        };
 
         let sync_id_2 = TenantTimelineId {
             tenant_id: TenantId::from_array(hex!("22223344556677881122334455667788")),
@@ -1736,15 +1646,15 @@ mod tests {
             Some(SyncTaskBatch {
                 download: Some(SyncData {
                     retries: 0,
-                    data: LayersDownload::from_skipped_layers(
-                        {
+                    data: LayersDownload {
+                        layers_to_skip: {
                             let mut set = HashSet::new();
                             set.extend(download_1.layers_to_skip.into_iter());
                             set.extend(download_2.layers_to_skip.into_iter());
                             set.extend(download_4.layers_to_skip.into_iter());
                             set
                         },
-                    )
+                    }
                 }),
                 upload: None,
                 delete: None,
@@ -1760,148 +1670,4 @@ mod tests {
             "Should have one task left out of the batch"
         );
     }
-
-    mod local_and_remote_comparisons {
-        use super::*;
-
-        #[test]
-        fn ready() {
-            let mut new_sync_tasks = VecDeque::default();
-            let sync_id = TenantTimelineId::generate();
-            let local_metadata = dummy_metadata(0x02.into());
-            let local_files =
-                HashMap::from([(PathBuf::from("first_file"), LayerFileMetadata::new(123))]);
-            let mut remote_entry = RemoteTimeline::new(local_metadata.clone());
-            remote_entry
-                .add_timeline_layers([(PathBuf::from("first_file"), LayerFileMetadata::new(123))]);
-
-            let (status, sync_needed) = compare_local_and_remote_timeline(
-                &mut new_sync_tasks,
-                sync_id,
-                local_metadata.clone(),
-                local_files,
-                &remote_entry,
-            );
-
-            assert_eq!(
-                status,
-                LocalTimelineInitStatus::LocallyComplete(local_metadata)
-            );
-            assert!(!sync_needed);
-
-            assert!(new_sync_tasks.is_empty(), "{:?}", new_sync_tasks);
-        }
-
-        #[test]
-        fn needs_download() {
-            let mut new_sync_tasks = VecDeque::default();
-            let sync_id = TenantTimelineId::generate();
-            let local_metadata = dummy_metadata(0x02.into());
-            let local_files = HashMap::default();
-            let mut remote_entry = RemoteTimeline::new(local_metadata.clone());
-            remote_entry
-                .add_timeline_layers([(PathBuf::from("first_file"), LayerFileMetadata::new(123))]);
-
-            let (status, sync_needed) = compare_local_and_remote_timeline(
-                &mut new_sync_tasks,
-                sync_id,
-                local_metadata,
-                local_files.clone(),
-                &remote_entry,
-            );
-
-            assert_eq!(status, LocalTimelineInitStatus::NeedsSync);
-            assert!(sync_needed);
-
-            let new_sync_tasks = new_sync_tasks.into_iter().collect::<Vec<_>>();
-
-            assert_eq!(
-                &new_sync_tasks,
-                &[(
-                    sync_id,
-                    SyncTask::download(LayersDownload::from_skipped_layers(
-                        local_files.keys().cloned().collect()
-                    ))
-                )]
-            );
-        }
-
-        #[test]
-        fn redownload_is_not_needed_on_upgrade() {
-            // originally the implementation missed the `(None, Some(_))` case in the match, and
-            // proceeded to always redownload if the remote metadata was not available.
-
-            let mut new_sync_tasks = VecDeque::default();
-            let sync_id = TenantTimelineId::generate();
-
-            let local_metadata = dummy_metadata(0x02.into());
-
-            // type system would in general allow that LayerFileMetadata would be created with
-            // file_size: None, however `LayerFileMetadata::default` is only allowed from tests,
-            // and so everywhere within the system valid LayerFileMetadata is being created, it is
-            // created through `::new`.
-            let local_files =
-                HashMap::from([(PathBuf::from("first_file"), LayerFileMetadata::new(123))]);
-
-            let mut remote_entry = RemoteTimeline::new(local_metadata.clone());
-
-            // RemoteTimeline is constructed out of an older version IndexPart, which didn't carry
-            // any metadata.
-            remote_entry
-                .add_timeline_layers([(PathBuf::from("first_file"), LayerFileMetadata::default())]);
-
-            let (status, sync_needed) = compare_local_and_remote_timeline(
-                &mut new_sync_tasks,
-                sync_id,
-                local_metadata.clone(),
-                local_files,
-                &remote_entry,
-            );
-
-            assert_eq!(
-                status,
-                LocalTimelineInitStatus::LocallyComplete(local_metadata)
-            );
-            assert!(!sync_needed);
-        }
-
-        #[test]
-        fn needs_upload() {
-            let mut new_sync_tasks = VecDeque::default();
-            let sync_id = TenantTimelineId::generate();
-            let local_metadata = dummy_metadata(0x02.into());
-            let local_files =
-                HashMap::from([(PathBuf::from("first_file"), LayerFileMetadata::new(123))]);
-            let mut remote_entry = RemoteTimeline::new(local_metadata.clone());
-            remote_entry.add_timeline_layers([]);
-
-            let (status, sync_needed) = compare_local_and_remote_timeline(
-                &mut new_sync_tasks,
-                sync_id,
-                local_metadata.clone(),
-                local_files.clone(),
-                &remote_entry,
-            );
-
-            assert_eq!(
-                status,
-                LocalTimelineInitStatus::LocallyComplete(local_metadata.clone())
-            );
-            assert!(!sync_needed);
-
-            let new_sync_tasks = new_sync_tasks.into_iter().collect::<Vec<_>>();
-
-            assert_eq!(
-                &new_sync_tasks,
-                &[(
-                    sync_id,
-                    SyncTask::upload(LayersUpload {
-                        layers_to_upload: local_files,
-                        uploaded_layers: HashMap::default(),
-                        metadata: Some(local_metadata),
-                    })
-                )]
-            );
-        }
-    }
 }

pageserver/src/storage_sync/delete.rs

@@ -171,7 +171,7 @@ mod tests {
         let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
         let timeline_upload =
             create_local_timeline(&harness, TIMELINE_ID, &layer_files, metadata.clone()).await?;
-        for (local_path, _metadata) in timeline_upload.layers_to_upload {
+        for local_path in timeline_upload.layers_to_upload {
             let remote_path =
                 local_storage.resolve_in_storage(&local_storage.remote_object_id(&local_path)?)?;
             let remote_parent_dir = remote_path.parent().unwrap();

pageserver/src/storage_sync/download.rs

@@ -16,13 +16,9 @@ use tokio::{
 };
 use tracing::{debug, error, info, warn};
 
-use crate::{
-    config::PageServerConf,
-    storage_sync::{index::LayerFileMetadata, SyncTask},
-    TEMP_FILE_SUFFIX,
-};
+use crate::{config::PageServerConf, storage_sync::SyncTask, TEMP_FILE_SUFFIX};
 use utils::{
-    crashsafe::path_with_suffix_extension,
+    crashsafe_dir::path_with_suffix_extension,
     id::{TenantId, TenantTimelineId, TimelineId},
 };
 
@@ -223,14 +219,8 @@ pub(super) async fn download_timeline_layers<'a>(
 
     let layers_to_download = remote_timeline
         .stored_files()
-        .iter()
-        .filter_map(|(layer_path, metadata)| {
-            if !download.layers_to_skip.contains(layer_path) {
-                Some((layer_path.to_owned(), metadata.to_owned()))
-            } else {
-                None
-            }
-        })
+        .difference(&download.layers_to_skip)
+        .cloned()
         .collect::<Vec<_>>();
 
     debug!("Layers to download: {layers_to_download:?}");
@@ -243,129 +233,89 @@ pub(super) async fn download_timeline_layers<'a>(
 
     let mut download_tasks = layers_to_download
         .into_iter()
-        .map(|(layer_destination_path, metadata)| async move {
+        .map(|layer_destination_path| async move {
+            if layer_destination_path.exists() {
+                debug!(
+                    "Layer already exists locally, skipping download: {}",
+                    layer_destination_path.display()
+                );
+            } else {
+                // Perform a rename inspired by durable_rename from file_utils.c.
+                // The sequence:
+                //     write(tmp)
+                //     fsync(tmp)
+                //     rename(tmp, new)
+                //     fsync(new)
+                //     fsync(parent)
+                // For more context about durable_rename check this email from postgres mailing list:
+                // https://www.postgresql.org/message-id/56583BDD.9060302@2ndquadrant.com
+                // If pageserver crashes the temp file will be deleted on startup and re-downloaded.
+                let temp_file_path =
+                    path_with_suffix_extension(&layer_destination_path, TEMP_FILE_SUFFIX);
 
-            match layer_destination_path.metadata() {
-                Ok(m) if m.is_file() => {
-                    // the file exists from earlier round when we failed after renaming it as
-                    // layer_destination_path
-                    let verified = if let Some(expected) = metadata.file_size() {
-                        m.len() == expected
-                    } else {
-                        // behaviour before recording metadata was to accept any existing
-                        true
-                    };
+                let mut destination_file =
+                    fs::File::create(&temp_file_path).await.with_context(|| {
+                        format!(
+                            "Failed to create a destination file for layer '{}'",
+                            temp_file_path.display()
+                        )
+                    })?;
 
-                    if verified {
-                        debug!(
-                            "Layer already exists locally, skipping download: {}",
-                            layer_destination_path.display()
-                        );
-                        return Ok((layer_destination_path, LayerFileMetadata::new(m.len())))
-                    } else {
-                        // no need to remove it, it will be overwritten by fs::rename
-                        // after successful download
-                        warn!("Downloaded layer exists already but layer file metadata mismatches: {}, metadata {:?}", layer_destination_path.display(), metadata);
-                    }
-                }
-                Ok(m) => {
-                    return Err(anyhow::anyhow!("Downloaded layer destination exists but is not a file: {m:?}, target needs to be removed/archived manually: {layer_destination_path:?}"));
-                }
-                Err(_) => {
-                    // behave as the file didn't exist
-                }
+                let mut layer_download = storage.download_storage_object(None, &layer_destination_path)
+                    .await
+                    .with_context(|| {
+                        format!(
+                            "Failed to initiate the download the layer for {sync_id} into file '{}'",
+                            temp_file_path.display()
+                        )
+                    })?;
+                io::copy(&mut layer_download.download_stream, &mut destination_file)
+                    .await
+                    .with_context(|| {
+                        format!(
+                            "Failed to download the layer for {sync_id} into file '{}'",
+                            temp_file_path.display()
+                        )
+                    })?;
+
+                // Tokio doc here: https://docs.rs/tokio/1.17.0/tokio/fs/struct.File.html states that:
+                // A file will not be closed immediately when it goes out of scope if there are any IO operations
+                // that have not yet completed. To ensure that a file is closed immediately when it is dropped,
+                // you should call flush before dropping it.
+                //
+                // From the tokio code I see that it waits for pending operations to complete. There shouldn't be any because
+                // we assume that `destination_file` file is fully written. I.e there is no pending .write(...).await operations.
+                // But for additional safety let's check/wait for any pending operations.
+                destination_file.flush().await.with_context(|| {
+                    format!(
+                        "failed to flush source file at {}",
+                        temp_file_path.display()
+                    )
+                })?;
+
+                // not using sync_data because it can lose file size update
+                destination_file.sync_all().await.with_context(|| {
+                    format!(
+                        "failed to fsync source file at {}",
+                        temp_file_path.display()
+                    )
+                })?;
+                drop(destination_file);
+
+                fail::fail_point!("remote-storage-download-pre-rename", |_| {
+                    anyhow::bail!("remote-storage-download-pre-rename failpoint triggered")
+                });
+
+                fs::rename(&temp_file_path, &layer_destination_path).await?;
+
+                fsync_path(&layer_destination_path).await.with_context(|| {
+                    format!(
+                        "Cannot fsync layer destination path {}",
+                        layer_destination_path.display(),
+                    )
+                })?;
             }
-
-            // Perform a rename inspired by durable_rename from file_utils.c.
-            // The sequence:
-            //     write(tmp)
-            //     fsync(tmp)
-            //     rename(tmp, new)
-            //     fsync(new)
-            //     fsync(parent)
-            // For more context about durable_rename check this email from postgres mailing list:
-            // https://www.postgresql.org/message-id/56583BDD.9060302@2ndquadrant.com
-            // If pageserver crashes the temp file will be deleted on startup and re-downloaded.
-            let temp_file_path =
-                path_with_suffix_extension(&layer_destination_path, TEMP_FILE_SUFFIX);
-
-            // TODO: this doesn't use the cached fd for some reason?
-            let mut destination_file =
-                fs::File::create(&temp_file_path).await.with_context(|| {
-                    format!(
-                        "Failed to create a destination file for layer '{}'",
-                        temp_file_path.display()
-                    )
-                })?;
-
-            let mut layer_download = storage.download_storage_object(None, &layer_destination_path)
-                .await
-                .with_context(|| {
-                    format!(
-                        "Failed to initiate the download the layer for {sync_id} into file '{}'",
-                        temp_file_path.display()
-                    )
-                })?;
-
-            let bytes_amount = io::copy(&mut layer_download.download_stream, &mut destination_file)
-                .await
-                .with_context(|| {
-                    format!(
-                        "Failed to download the layer for {sync_id} into file '{}'",
-                        temp_file_path.display()
-                    )
-                })?;
-
-            // Tokio doc here: https://docs.rs/tokio/1.17.0/tokio/fs/struct.File.html states that:
-            // A file will not be closed immediately when it goes out of scope if there are any IO operations
-            // that have not yet completed. To ensure that a file is closed immediately when it is dropped,
-            // you should call flush before dropping it.
-            //
-            // From the tokio code I see that it waits for pending operations to complete. There shouldn't be any because
-            // we assume that `destination_file` file is fully written. I.e there is no pending .write(...).await operations.
-            // But for additional safety let's check/wait for any pending operations.
-            destination_file.flush().await.with_context(|| {
-                format!(
-                    "failed to flush source file at {}",
-                    temp_file_path.display()
-                )
-            })?;
-
-            match metadata.file_size() {
-                Some(expected) if expected != bytes_amount => {
-                    anyhow::bail!(
-                        "According to layer file metadata should had downloaded {expected} bytes but downloaded {bytes_amount} bytes into file '{}'",
-                        temp_file_path.display()
-                    );
-                },
-                Some(_) | None => {
-                    // matches, or upgrading from an earlier IndexPart version
-                }
-            }
-
-            // not using sync_data because it can lose file size update
-            destination_file.sync_all().await.with_context(|| {
-                format!(
-                    "failed to fsync source file at {}",
-                    temp_file_path.display()
-                )
-            })?;
-            drop(destination_file);
-
-            fail::fail_point!("remote-storage-download-pre-rename", |_| {
-                anyhow::bail!("remote-storage-download-pre-rename failpoint triggered")
-            });
-
-            fs::rename(&temp_file_path, &layer_destination_path).await?;
-
-            fsync_path(&layer_destination_path).await.with_context(|| {
-                format!(
-                    "Cannot fsync layer destination path {}",
-                    layer_destination_path.display(),
-                )
-            })?;
-
-            Ok::<_, anyhow::Error>((layer_destination_path, LayerFileMetadata::new(bytes_amount)))
+            Ok::<_, anyhow::Error>(layer_destination_path)
         })
         .collect::<FuturesUnordered<_>>();
 
@@ -374,12 +324,9 @@ pub(super) async fn download_timeline_layers<'a>(
     let mut undo = HashSet::new();
     while let Some(download_result) = download_tasks.next().await {
         match download_result {
-            Ok((downloaded_path, metadata)) => {
+            Ok(downloaded_path) => {
                 undo.insert(downloaded_path.clone());
-                download.layers_to_skip.insert(downloaded_path.clone());
-                // what if the key existed already? ignore, because then we would had
-                // downloaded a partial file, and had to retry
-                download.gathered_metadata.insert(downloaded_path, metadata);
+                download.layers_to_skip.insert(downloaded_path);
             }
             Err(e) => {
                 errors_happened = true;
@@ -402,8 +349,6 @@ pub(super) async fn download_timeline_layers<'a>(
         );
         for item in undo {
             download.layers_to_skip.remove(&item);
-            // intentionally don't clear the gathered_metadata because it exists for fsync_path
-            // failure on parent directory
         }
         errors_happened = true;
     }
@@ -508,9 +453,9 @@ mod tests {
         let timeline_upload =
             create_local_timeline(&harness, TIMELINE_ID, &layer_files, metadata.clone()).await?;
 
-        for local_path in timeline_upload.layers_to_upload.keys() {
+        for local_path in timeline_upload.layers_to_upload {
             let remote_path =
-                local_storage.resolve_in_storage(&storage.remote_object_id(local_path)?)?;
+                local_storage.resolve_in_storage(&storage.remote_object_id(&local_path)?)?;
             let remote_parent_dir = remote_path.parent().unwrap();
             if !remote_parent_dir.exists() {
                 fs::create_dir_all(&remote_parent_dir).await?;
@@ -528,19 +473,11 @@ mod tests {
 
         let mut remote_timeline = RemoteTimeline::new(metadata.clone());
         remote_timeline.awaits_download = true;
-        remote_timeline.add_timeline_layers(layer_files.iter().map(|layer| {
-            let layer_path = local_timeline_path.join(layer);
-
-            // this could had also been LayerFileMetadata::default(), but since in this test we
-            // don't do the merge operation done by storage_sync::download_timeline_data, it would
-            // not be merged back to timeline.
-            let metadata_from_upload = timeline_upload
-                .layers_to_upload
-                .get(&layer_path)
-                .expect("layer must exist in previously uploaded paths")
-                .to_owned();
-            (layer_path, metadata_from_upload)
-        }));
+        remote_timeline.add_timeline_layers(
+            layer_files
+                .iter()
+                .map(|layer| local_timeline_path.join(layer)),
+        );
 
         let download_data = match download_timeline_layers(
             harness.conf,
@@ -550,9 +487,9 @@ mod tests {
             sync_id,
             SyncData::new(
                 current_retries,
-                LayersDownload::from_skipped_layers(HashSet::from([
-                    local_timeline_path.join("layer_to_skip")
-                ])),
+                LayersDownload {
+                    layers_to_skip: HashSet::from([local_timeline_path.join("layer_to_skip")]),
+                },
             ),
         )
         .await
@@ -615,7 +552,12 @@ mod tests {
             &sync_queue,
             None,
             sync_id,
-            SyncData::new(0, LayersDownload::from_skipped_layers(HashSet::new())),
+            SyncData::new(
+                0,
+                LayersDownload {
+                    layers_to_skip: HashSet::new(),
+                },
+            ),
         )
         .await;
         assert!(
@@ -634,7 +576,12 @@ mod tests {
             &sync_queue,
             Some(&not_expecting_download_remote_timeline),
             sync_id,
-            SyncData::new(0, LayersDownload::from_skipped_layers(HashSet::new())),
+            SyncData::new(
+                0,
+                LayersDownload {
+                    layers_to_skip: HashSet::new(),
+                },
+            ),
         )
         .await;
         assert!(

pageserver/src/storage_sync/index.rs

@@ -212,8 +212,8 @@ impl RemoteTimelineIndex {
 /// Restored index part data about the timeline, stored in the remote index.
 #[derive(Debug, Clone)]
 pub struct RemoteTimeline {
-    timeline_layers: HashMap<PathBuf, LayerFileMetadata>,
-    missing_layers: HashMap<PathBuf, LayerFileMetadata>,
+    timeline_layers: HashSet<PathBuf>,
+    missing_layers: HashSet<PathBuf>,
 
     pub metadata: TimelineMetadata,
     pub awaits_download: bool,
@@ -222,161 +222,62 @@ pub struct RemoteTimeline {
 impl RemoteTimeline {
     pub fn new(metadata: TimelineMetadata) -> Self {
         Self {
-            timeline_layers: HashMap::default(),
-            missing_layers: HashMap::default(),
+            timeline_layers: HashSet::new(),
+            missing_layers: HashSet::new(),
             metadata,
             awaits_download: false,
         }
     }
 
-    pub fn add_timeline_layers(
-        &mut self,
-        new_layers: impl IntoIterator<Item = (PathBuf, LayerFileMetadata)>,
-    ) {
-        self.timeline_layers.extend(new_layers);
+    pub fn add_timeline_layers(&mut self, new_layers: impl IntoIterator<Item = PathBuf>) {
+        self.timeline_layers.extend(new_layers.into_iter());
     }
 
-    pub fn add_upload_failures(
-        &mut self,
-        upload_failures: impl IntoIterator<Item = (PathBuf, LayerFileMetadata)>,
-    ) {
-        self.missing_layers.extend(upload_failures);
+    pub fn add_upload_failures(&mut self, upload_failures: impl IntoIterator<Item = PathBuf>) {
+        self.missing_layers.extend(upload_failures.into_iter());
     }
 
     pub fn remove_layers(&mut self, layers_to_remove: &HashSet<PathBuf>) {
         self.timeline_layers
-            .retain(|layer, _| !layers_to_remove.contains(layer));
+            .retain(|layer| !layers_to_remove.contains(layer));
         self.missing_layers
-            .retain(|layer, _| !layers_to_remove.contains(layer));
+            .retain(|layer| !layers_to_remove.contains(layer));
     }
 
     /// Lists all layer files in the given remote timeline. Omits the metadata file.
-    pub fn stored_files(&self) -> &HashMap<PathBuf, LayerFileMetadata> {
+    pub fn stored_files(&self) -> &HashSet<PathBuf> {
         &self.timeline_layers
     }
 
-    /// Combines metadata gathered or verified during downloading needed layer files to metadata on
-    /// the [`RemoteIndex`], so it can be uploaded later.
-    pub fn merge_metadata_from_downloaded(
-        &mut self,
-        downloaded: &HashMap<PathBuf, LayerFileMetadata>,
-    ) {
-        downloaded.iter().for_each(|(path, metadata)| {
-            if let Some(upgraded) = self.timeline_layers.get_mut(path) {
-                upgraded.merge(metadata);
-            }
-        });
-    }
-
     pub fn from_index_part(timeline_path: &Path, index_part: IndexPart) -> anyhow::Result<Self> {
         let metadata = TimelineMetadata::from_bytes(&index_part.metadata_bytes)?;
-        let default_metadata = &IndexLayerMetadata::default();
-
-        let find_metadata = |key: &RelativePath| -> LayerFileMetadata {
-            index_part
-                .layer_metadata
-                .get(key)
-                .unwrap_or(default_metadata)
-                .into()
-        };
-
         Ok(Self {
-            timeline_layers: index_part
-                .timeline_layers
-                .iter()
-                .map(|layer_path| (layer_path.as_path(timeline_path), find_metadata(layer_path)))
-                .collect(),
-            missing_layers: index_part
-                .missing_layers
-                .iter()
-                .map(|layer_path| (layer_path.as_path(timeline_path), find_metadata(layer_path)))
-                .collect(),
+            timeline_layers: to_local_paths(timeline_path, index_part.timeline_layers),
+            missing_layers: to_local_paths(timeline_path, index_part.missing_layers),
             metadata,
             awaits_download: false,
         })
     }
 }
 
-/// Metadata gathered for each of the layer files.
-///
-/// Fields have to be `Option`s because remote [`IndexPart`]'s can be from different version, which
-/// might have less or more metadata depending if upgrading or rolling back an upgrade.
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
-#[cfg_attr(test, derive(Default))]
-pub struct LayerFileMetadata {
-    file_size: Option<u64>,
-}
-
-impl From<&'_ IndexLayerMetadata> for LayerFileMetadata {
-    fn from(other: &IndexLayerMetadata) -> Self {
-        LayerFileMetadata {
-            file_size: other.file_size,
-        }
-    }
-}
-
-impl LayerFileMetadata {
-    pub fn new(file_size: u64) -> Self {
-        LayerFileMetadata {
-            file_size: Some(file_size),
-        }
-    }
-
-    pub fn file_size(&self) -> Option<u64> {
-        self.file_size
-    }
-
-    /// Metadata has holes due to version upgrades. This method is called to upgrade self with the
-    /// other value.
-    ///
-    /// This is called on the possibly outdated version.
-    pub fn merge(&mut self, other: &Self) {
-        self.file_size = other.file_size.or(self.file_size);
-    }
-}
-
 /// Part of the remote index, corresponding to a certain timeline.
 /// Contains the data about all files in the timeline, present remotely and its metadata.
-///
-/// This type needs to be backwards and forwards compatible. When changing the fields,
-/// remember to add a test case for the changed version.
 #[serde_as]
 #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
 pub struct IndexPart {
-    /// Debugging aid describing the version of this type.
-    #[serde(default)]
-    version: usize,
-
-    /// Each of the layers present on remote storage.
-    ///
-    /// Additional metadata can might exist in `layer_metadata`.
     timeline_layers: HashSet<RelativePath>,
-
     /// Currently is not really used in pageserver,
     /// present to manually keep track of the layer files that pageserver might never retrieve.
     ///
     /// Such "holes" might appear if any upload task was evicted on an error threshold:
     /// the this layer will only be rescheduled for upload on pageserver restart.
     missing_layers: HashSet<RelativePath>,
-
-    /// Per layer file metadata, which can be present for a present or missing layer file.
-    ///
-    /// Older versions of `IndexPart` will not have this property or have only a part of metadata
-    /// that latest version stores.
-    #[serde(default)]
-    layer_metadata: HashMap<RelativePath, IndexLayerMetadata>,
-
     #[serde_as(as = "DisplayFromStr")]
     disk_consistent_lsn: Lsn,
     metadata_bytes: Vec<u8>,
 }
 
 impl IndexPart {
-    /// When adding or modifying any parts of `IndexPart`, increment the version so that it can be
-    /// used to understand later versions.
-    ///
-    /// Version is currently informative only.
-    const LATEST_VERSION: usize = 1;
     pub const FILE_NAME: &'static str = "index_part.json";
 
     #[cfg(test)]
@@ -387,10 +288,8 @@ impl IndexPart {
         metadata_bytes: Vec<u8>,
     ) -> Self {
         Self {
-            version: Self::LATEST_VERSION,
             timeline_layers,
             missing_layers,
-            layer_metadata: HashMap::default(),
             disk_consistent_lsn,
             metadata_bytes,
         }
@@ -405,68 +304,35 @@ impl IndexPart {
         remote_timeline: RemoteTimeline,
     ) -> anyhow::Result<Self> {
         let metadata_bytes = remote_timeline.metadata.to_bytes()?;
-
-        let mut layer_metadata = HashMap::new();
-
-        let mut missing_layers = HashSet::new();
-
-        separate_paths_and_metadata(
-            timeline_path,
-            &remote_timeline.missing_layers,
-            &mut missing_layers,
-            &mut layer_metadata,
-        )
-        .context("Failed to convert missing layers' paths to relative ones")?;
-
-        let mut timeline_layers = HashSet::new();
-
-        separate_paths_and_metadata(
-            timeline_path,
-            &remote_timeline.timeline_layers,
-            &mut timeline_layers,
-            &mut layer_metadata,
-        )
-        .context("Failed to convert timeline layers' paths to relative ones")?;
-
         Ok(Self {
-            version: Self::LATEST_VERSION,
-            timeline_layers,
-            missing_layers,
-            layer_metadata,
+            timeline_layers: to_relative_paths(timeline_path, remote_timeline.timeline_layers)
+                .context("Failed to convert timeline layers' paths to relative ones")?,
+            missing_layers: to_relative_paths(timeline_path, remote_timeline.missing_layers)
+                .context("Failed to convert missing layers' paths to relative ones")?,
             disk_consistent_lsn: remote_timeline.metadata.disk_consistent_lsn(),
             metadata_bytes,
         })
     }
 }
 
-/// Serialized form of [`LayerFileMetadata`].
-#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Default)]
-pub struct IndexLayerMetadata {
-    file_size: Option<u64>,
-}
-
-impl From<&'_ LayerFileMetadata> for IndexLayerMetadata {
-    fn from(other: &'_ LayerFileMetadata) -> Self {
-        IndexLayerMetadata {
-            file_size: other.file_size,
-        }
-    }
-}
-
-fn separate_paths_and_metadata(
+fn to_local_paths(
     timeline_path: &Path,
-    input: &HashMap<PathBuf, LayerFileMetadata>,
-    output: &mut HashSet<RelativePath>,
-    layer_metadata: &mut HashMap<RelativePath, IndexLayerMetadata>,
-) -> anyhow::Result<()> {
-    for (path, metadata) in input {
-        let rel_path = RelativePath::new(timeline_path, path)?;
-        let metadata = IndexLayerMetadata::from(metadata);
+    paths: impl IntoIterator<Item = RelativePath>,
+) -> HashSet<PathBuf> {
+    paths
+        .into_iter()
+        .map(|path| path.as_path(timeline_path))
+        .collect()
+}
 
-        layer_metadata.insert(rel_path.clone(), metadata);
-        output.insert(rel_path);
-    }
-    Ok(())
+fn to_relative_paths(
+    timeline_path: &Path,
+    paths: impl IntoIterator<Item = PathBuf>,
+) -> anyhow::Result<HashSet<RelativePath>> {
+    paths
+        .into_iter()
+        .map(|path| RelativePath::new(timeline_path, path))
+        .collect()
 }
 
 #[cfg(test)]
@@ -491,13 +357,13 @@ mod tests {
             DEFAULT_PG_VERSION,
         );
         let remote_timeline = RemoteTimeline {
-            timeline_layers: HashMap::from([
-                (timeline_path.join("layer_1"), LayerFileMetadata::new(1)),
-                (timeline_path.join("layer_2"), LayerFileMetadata::new(2)),
+            timeline_layers: HashSet::from([
+                timeline_path.join("layer_1"),
+                timeline_path.join("layer_2"),
             ]),
-            missing_layers: HashMap::from([
-                (timeline_path.join("missing_1"), LayerFileMetadata::new(3)),
-                (timeline_path.join("missing_2"), LayerFileMetadata::new(4)),
+            missing_layers: HashSet::from([
+                timeline_path.join("missing_1"),
+                timeline_path.join("missing_2"),
             ]),
             metadata: metadata.clone(),
             awaits_download: false,
@@ -619,13 +485,13 @@ mod tests {
         let conversion_result = IndexPart::from_remote_timeline(
             &timeline_path,
             RemoteTimeline {
-                timeline_layers: HashMap::from([
-                    (PathBuf::from("bad_path"), LayerFileMetadata::new(1)),
-                    (timeline_path.join("layer_2"), LayerFileMetadata::new(2)),
+                timeline_layers: HashSet::from([
+                    PathBuf::from("bad_path"),
+                    timeline_path.join("layer_2"),
                 ]),
-                missing_layers: HashMap::from([
-                    (timeline_path.join("missing_1"), LayerFileMetadata::new(3)),
-                    (timeline_path.join("missing_2"), LayerFileMetadata::new(4)),
+                missing_layers: HashSet::from([
+                    timeline_path.join("missing_1"),
+                    timeline_path.join("missing_2"),
                 ]),
                 metadata: metadata.clone(),
                 awaits_download: false,
@@ -636,13 +502,13 @@ mod tests {
         let conversion_result = IndexPart::from_remote_timeline(
             &timeline_path,
             RemoteTimeline {
-                timeline_layers: HashMap::from([
-                    (timeline_path.join("layer_1"), LayerFileMetadata::new(1)),
-                    (timeline_path.join("layer_2"), LayerFileMetadata::new(2)),
+                timeline_layers: HashSet::from([
+                    timeline_path.join("layer_1"),
+                    timeline_path.join("layer_2"),
                 ]),
-                missing_layers: HashMap::from([
-                    (PathBuf::from("bad_path"), LayerFileMetadata::new(3)),
-                    (timeline_path.join("missing_2"), LayerFileMetadata::new(4)),
+                missing_layers: HashSet::from([
+                    PathBuf::from("bad_path"),
+                    timeline_path.join("missing_2"),
                 ]),
                 metadata,
                 awaits_download: false,
@@ -650,63 +516,4 @@ mod tests {
         );
         assert!(conversion_result.is_err(), "Should not be able to convert metadata with missing layer paths that are not in the timeline directory");
     }
-
-    #[test]
-    fn v0_indexpart_is_parsed() {
-        let example = r#"{
-            "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
-            "missing_layers":["not_a_real_layer_but_adding_coverage"],
-            "disk_consistent_lsn":"0/16960E8",
-            "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-        }"#;
-
-        let expected = IndexPart {
-            version: 0,
-            timeline_layers: [RelativePath("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".to_owned())].into_iter().collect(),
-            missing_layers: [RelativePath("not_a_real_layer_but_adding_coverage".to_owned())].into_iter().collect(),
-            layer_metadata: HashMap::default(),
-            disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
-            metadata_bytes: [113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0].to_vec(),
-        };
-
-        let part = serde_json::from_str::<IndexPart>(example).unwrap();
-        assert_eq!(part, expected);
-    }
-
-    #[test]
-    fn v1_indexpart_is_parsed() {
-        let example = r#"{
-            "version":1,
-            "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
-            "missing_layers":["not_a_real_layer_but_adding_coverage"],
-            "layer_metadata":{
-                "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
-                "not_a_real_layer_but_adding_coverage": { "file_size": 9007199254741001 }
-            },
-            "disk_consistent_lsn":"0/16960E8",
-            "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-        }"#;
-
-        let expected = IndexPart {
-            // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
-            version: 1,
-            timeline_layers: [RelativePath("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".to_owned())].into_iter().collect(),
-            missing_layers: [RelativePath("not_a_real_layer_but_adding_coverage".to_owned())].into_iter().collect(),
-            layer_metadata: HashMap::from([
-                (RelativePath("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".to_owned()), IndexLayerMetadata {
-                    file_size: Some(25600000),
-                }),
-                (RelativePath("not_a_real_layer_but_adding_coverage".to_owned()), IndexLayerMetadata {
-                    // serde_json should always parse this but this might be a double with jq for
-                    // example.
-                    file_size: Some(9007199254741001),
-                })
-            ]),
-            disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
-            metadata_bytes: [113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0].to_vec(),
-        };
-
-        let part = serde_json::from_str::<IndexPart>(example).unwrap();
-        assert_eq!(part, expected);
-    }
 }

pageserver/src/storage_sync/upload.rs

@@ -69,25 +69,14 @@ pub(super) async fn upload_timeline_layers<'a>(
         .map(|meta| meta.disk_consistent_lsn());
 
     let already_uploaded_layers = remote_timeline
-        .map(|timeline| {
-            timeline
-                .stored_files()
-                .keys()
-                .cloned()
-                .collect::<std::collections::HashSet<_>>()
-        })
+        .map(|timeline| timeline.stored_files())
+        .cloned()
         .unwrap_or_default();
 
     let layers_to_upload = upload
         .layers_to_upload
-        .iter()
-        .filter_map(|(k, v)| {
-            if !already_uploaded_layers.contains(k) {
-                Some((k.to_owned(), v.to_owned()))
-            } else {
-                None
-            }
-        })
+        .difference(&already_uploaded_layers)
+        .cloned()
         .collect::<Vec<_>>();
 
     if layers_to_upload.is_empty() {
@@ -109,7 +98,7 @@ pub(super) async fn upload_timeline_layers<'a>(
 
     let mut upload_tasks = layers_to_upload
         .into_iter()
-        .map(|(source_path, known_metadata)| async move {
+        .map(|source_path| async move {
             let source_file = match fs::File::open(&source_path).await.with_context(|| {
                 format!(
                     "Failed to upen a source file for layer '{}'",
@@ -120,7 +109,7 @@ pub(super) async fn upload_timeline_layers<'a>(
                 Err(e) => return Err(UploadError::MissingLocalFile(source_path, e)),
             };
 
-            let fs_size = source_file
+            let source_size = source_file
                 .metadata()
                 .await
                 .with_context(|| {
@@ -130,24 +119,10 @@ pub(super) async fn upload_timeline_layers<'a>(
                     )
                 })
                 .map_err(UploadError::Other)?
-                .len();
-
-            // FIXME: this looks bad
-            if let Some(metadata_size) = known_metadata.file_size() {
-                if metadata_size != fs_size {
-                    return Err(UploadError::Other(anyhow::anyhow!(
-                        "File {source_path:?} has its current FS size {fs_size} diferent from initially determined {metadata_size}"
-                    )));
-                }
-            } else {
-                // this is a silly state we would like to avoid
-            }
-
-            let fs_size = usize::try_from(fs_size).with_context(|| format!("File {source_path:?} size {fs_size} could not be converted to usize"))
-                .map_err(UploadError::Other)?;
+                .len() as usize;
 
             match storage
-                .upload_storage_object(Box::new(source_file), fs_size, &source_path)
+                .upload_storage_object(Box::new(source_file), source_size, &source_path)
                 .await
                 .with_context(|| format!("Failed to upload layer file for {sync_id}"))
             {
@@ -161,11 +136,8 @@ pub(super) async fn upload_timeline_layers<'a>(
     while let Some(upload_result) = upload_tasks.next().await {
         match upload_result {
             Ok(uploaded_path) => {
-                let metadata = upload
-                    .layers_to_upload
-                    .remove(&uploaded_path)
-                    .expect("metadata should always exist, assuming no double uploads");
-                upload.uploaded_layers.insert(uploaded_path, metadata);
+                upload.layers_to_upload.remove(&uploaded_path);
+                upload.uploaded_layers.insert(uploaded_path);
             }
             Err(e) => match e {
                 UploadError::Other(e) => {
@@ -290,7 +262,7 @@ mod tests {
         assert_eq!(
             upload
                 .uploaded_layers
-                .keys()
+                .iter()
                 .cloned()
                 .collect::<BTreeSet<_>>(),
             layer_files
@@ -385,7 +357,7 @@ mod tests {
         assert_eq!(
             upload
                 .uploaded_layers
-                .keys()
+                .iter()
                 .cloned()
                 .collect::<BTreeSet<_>>(),
             layer_files

pageserver/src/tenant/layer_map.rs

@@ -62,8 +62,6 @@ pub struct LayerMap {
 
 struct LayerRTreeObject {
     layer: Arc<dyn Layer>,
-
-    envelope: AABB<[IntKey; 2]>,
 }
 
 // Representation of Key as numeric type.
@@ -199,16 +197,9 @@ impl PartialEq for LayerRTreeObject {
 impl RTreeObject for LayerRTreeObject {
     type Envelope = AABB<[IntKey; 2]>;
     fn envelope(&self) -> Self::Envelope {
-        self.envelope
-    }
-}
-
-impl LayerRTreeObject {
-    fn new(layer: Arc<dyn Layer>) -> Self {
-        let key_range = layer.get_key_range();
-        let lsn_range = layer.get_lsn_range();
-
-        let envelope = AABB::from_corners(
+        let key_range = self.layer.get_key_range();
+        let lsn_range = self.layer.get_lsn_range();
+        AABB::from_corners(
             [
                 IntKey::from(key_range.start.to_i128()),
                 IntKey::from(lsn_range.start.0 as i128),
@@ -217,8 +208,7 @@ impl LayerRTreeObject {
                 IntKey::from(key_range.end.to_i128() - 1),
                 IntKey::from(lsn_range.end.0 as i128 - 1),
             ], // AABB::upper is inclusive, while `key_range.end` and `lsn_range.end` are exclusive
-        );
-        LayerRTreeObject { layer, envelope }
+        )
     }
 }
 
@@ -348,7 +338,7 @@ impl LayerMap {
         if layer.get_key_range() == (Key::MIN..Key::MAX) {
             self.l0_delta_layers.push(layer.clone());
         }
-        self.historic_layers.insert(LayerRTreeObject::new(layer));
+        self.historic_layers.insert(LayerRTreeObject { layer });
         NUM_ONDISK_LAYERS.inc();
     }
 
@@ -372,7 +362,7 @@ impl LayerMap {
         }
         assert!(self
             .historic_layers
-            .remove(&LayerRTreeObject::new(layer))
+            .remove(&LayerRTreeObject { layer })
             .is_some());
         NUM_ONDISK_LAYERS.dec();
     }

pageserver/src/tenant/timeline.rs

@@ -1,6 +1,6 @@
 //!
 
-use anyhow::{anyhow, bail, ensure, Context};
+use anyhow::{anyhow, bail, ensure, Context, Result};
 use bytes::Bytes;
 use fail::fail_point;
 use itertools::Itertools;
@@ -34,7 +34,6 @@ use crate::keyspace::{KeyPartitioning, KeySpace};
 use crate::metrics::TimelineMetrics;
 use crate::pgdatadir_mapping::BlockNumber;
 use crate::pgdatadir_mapping::LsnForTimestamp;
-use crate::pgdatadir_mapping::{is_rel_fsm_block_key, is_rel_vm_block_key};
 use crate::reltag::RelTag;
 use crate::tenant_config::TenantConfOpt;
 
@@ -53,11 +52,7 @@ use crate::task_mgr::TaskKind;
 use crate::walreceiver::{is_etcd_client_initialized, spawn_connection_manager_task};
 use crate::walredo::WalRedoManager;
 use crate::CheckpointConfig;
-use crate::ZERO_PAGE;
-use crate::{
-    page_cache,
-    storage_sync::{self, index::LayerFileMetadata},
-};
+use crate::{page_cache, storage_sync};
 
 pub struct Timeline {
     conf: &'static PageServerConf,
@@ -307,6 +302,10 @@ pub struct GcInfo {
 
 /// Public interface functions
 impl Timeline {
+    //------------------------------------------------------------------------------
+    // Public GET functions
+    //------------------------------------------------------------------------------
+
     /// Get the LSN where this branch was created
     pub fn get_ancestor_lsn(&self) -> Lsn {
         self.ancestor_lsn
@@ -441,7 +440,7 @@ impl Timeline {
         &self,
         lsn: Lsn,
         latest_gc_cutoff_lsn: &RcuReadGuard<Lsn>,
-    ) -> anyhow::Result<()> {
+    ) -> Result<()> {
         ensure!(
             lsn >= **latest_gc_cutoff_lsn,
             "LSN {} is earlier than latest GC horizon {} (we might've already garbage collected needed data)",
@@ -451,6 +450,12 @@ impl Timeline {
         Ok(())
     }
 
+    //------------------------------------------------------------------------------
+    // Public PUT functions, to update the repository with new page versions.
+    //
+    // These are called by the WAL receiver to digest WAL records.
+    //------------------------------------------------------------------------------
+
     /// Flush to disk all data that was written with the put_* functions
     ///
     /// NOTE: This has nothing to do with checkpoint in PostgreSQL. We don't
@@ -469,91 +474,6 @@ impl Timeline {
         }
     }
 
-    pub fn compact(&self) -> anyhow::Result<()> {
-        let last_record_lsn = self.get_last_record_lsn();
-
-        // Last record Lsn could be zero in case the timelie was just created
-        if !last_record_lsn.is_valid() {
-            warn!("Skipping compaction for potentially just initialized timeline, it has invalid last record lsn: {last_record_lsn}");
-            return Ok(());
-        }
-
-        //
-        // High level strategy for compaction / image creation:
-        //
-        // 1. First, calculate the desired "partitioning" of the
-        // currently in-use key space. The goal is to partition the
-        // key space into roughly fixed-size chunks, but also take into
-        // account any existing image layers, and try to align the
-        // chunk boundaries with the existing image layers to avoid
-        // too much churn. Also try to align chunk boundaries with
-        // relation boundaries.  In principle, we don't know about
-        // relation boundaries here, we just deal with key-value
-        // pairs, and the code in pgdatadir_mapping.rs knows how to
-        // map relations into key-value pairs. But in practice we know
-        // that 'field6' is the block number, and the fields 1-5
-        // identify a relation. This is just an optimization,
-        // though.
-        //
-        // 2. Once we know the partitioning, for each partition,
-        // decide if it's time to create a new image layer. The
-        // criteria is: there has been too much "churn" since the last
-        // image layer? The "churn" is fuzzy concept, it's a
-        // combination of too many delta files, or too much WAL in
-        // total in the delta file. Or perhaps: if creating an image
-        // file would allow to delete some older files.
-        //
-        // 3. After that, we compact all level0 delta files if there
-        // are too many of them.  While compacting, we also garbage
-        // collect any page versions that are no longer needed because
-        // of the new image layers we created in step 2.
-        //
-        // TODO: This high level strategy hasn't been implemented yet.
-        // Below are functions compact_level0() and create_image_layers()
-        // but they are a bit ad hoc and don't quite work like it's explained
-        // above. Rewrite it.
-        let _layer_removal_cs = self.layer_removal_cs.lock().unwrap();
-
-        let target_file_size = self.get_checkpoint_distance();
-
-        // Define partitioning schema if needed
-
-        match self.repartition(
-            self.get_last_record_lsn(),
-            self.get_compaction_target_size(),
-        ) {
-            Ok((partitioning, lsn)) => {
-                // 2. Create new image layers for partitions that have been modified
-                // "enough".
-                let layer_paths_to_upload = self.create_image_layers(&partitioning, lsn, false)?;
-                if !layer_paths_to_upload.is_empty()
-                    && self.upload_layers.load(atomic::Ordering::Relaxed)
-                {
-                    storage_sync::schedule_layer_upload(
-                        self.tenant_id,
-                        self.timeline_id,
-                        layer_paths_to_upload,
-                        None,
-                    );
-                }
-
-                // 3. Compact
-                let timer = self.metrics.compact_time_histo.start_timer();
-                self.compact_level0(target_file_size)?;
-                timer.stop_and_record();
-            }
-            Err(err) => {
-                // no partitioning? This is normal, if the timeline was just created
-                // as an empty timeline. Also in unit tests, when we use the timeline
-                // as a simple key-value store, ignoring the datadir layout. Log the
-                // error but continue.
-                error!("could not compact, repartitioning keyspace failed: {err:?}");
-            }
-        };
-
-        Ok(())
-    }
-
     /// Mutate the timeline with a [`TimelineWriter`].
     pub fn writer(&self) -> TimelineWriter<'_> {
         TimelineWriter {
@@ -561,80 +481,6 @@ impl Timeline {
             _write_guard: self.write_lock.lock().unwrap(),
         }
     }
-
-    /// Retrieve current logical size of the timeline.
-    ///
-    /// The size could be lagging behind the actual number, in case
-    /// the initial size calculation has not been run (gets triggered on the first size access).
-    pub fn get_current_logical_size(self: &Arc<Self>) -> anyhow::Result<u64> {
-        let current_size = self.current_logical_size.current_size()?;
-        debug!("Current size: {current_size:?}");
-
-        let size = current_size.size();
-        if let (CurrentLogicalSize::Approximate(_), Some(init_lsn)) =
-            (current_size, self.current_logical_size.initial_part_end)
-        {
-            self.try_spawn_size_init_task(init_lsn);
-        }
-
-        Ok(size)
-    }
-
-    /// Check if more than 'checkpoint_distance' of WAL has been accumulated in
-    /// the in-memory layer, and initiate flushing it if so.
-    ///
-    /// Also flush after a period of time without new data -- it helps
-    /// safekeepers to regard pageserver as caught up and suspend activity.
-    pub fn check_checkpoint_distance(self: &Arc<Timeline>) -> anyhow::Result<()> {
-        let last_lsn = self.get_last_record_lsn();
-        let layers = self.layers.read().unwrap();
-        if let Some(open_layer) = &layers.open_layer {
-            let open_layer_size = open_layer.size()?;
-            drop(layers);
-            let last_freeze_at = self.last_freeze_at.load();
-            let last_freeze_ts = *(self.last_freeze_ts.read().unwrap());
-            let distance = last_lsn.widening_sub(last_freeze_at);
-            // Checkpointing the open layer can be triggered by layer size or LSN range.
-            // S3 has a 5 GB limit on the size of one upload (without multi-part upload), and
-            // we want to stay below that with a big margin.  The LSN distance determines how
-            // much WAL the safekeepers need to store.
-            if distance >= self.get_checkpoint_distance().into()
-                || open_layer_size > self.get_checkpoint_distance()
-                || (distance > 0 && last_freeze_ts.elapsed() >= self.get_checkpoint_timeout())
-            {
-                info!(
-                    "check_checkpoint_distance {}, layer size {}, elapsed since last flush {:?}",
-                    distance,
-                    open_layer_size,
-                    last_freeze_ts.elapsed()
-                );
-
-                self.freeze_inmem_layer(true);
-                self.last_freeze_at.store(last_lsn);
-                *(self.last_freeze_ts.write().unwrap()) = Instant::now();
-
-                // Launch a task to flush the frozen layer to disk, unless
-                // a task was already running. (If the task was running
-                // at the time that we froze the layer, it must've seen the
-                // the layer we just froze before it exited; see comments
-                // in flush_frozen_layers())
-                if let Ok(guard) = self.layer_flush_lock.try_lock() {
-                    drop(guard);
-                    let self_clone = Arc::clone(self);
-                    task_mgr::spawn(
-                        task_mgr::BACKGROUND_RUNTIME.handle(),
-                        task_mgr::TaskKind::LayerFlushTask,
-                        Some(self.tenant_id),
-                        Some(self.timeline_id),
-                        "layer flush task",
-                        false,
-                        async move { self_clone.flush_frozen_layers(false) },
-                    );
-                }
-            }
-        }
-        Ok(())
-    }
 }
 
 // Private functions
@@ -678,7 +524,7 @@ impl Timeline {
     ///
     /// Loads the metadata for the timeline into memory, but not the layer map.
     #[allow(clippy::too_many_arguments)]
-    pub(super) fn new(
+    pub fn new(
         conf: &'static PageServerConf,
         tenant_conf: Arc<RwLock<TenantConfOpt>>,
         metadata: TimelineMetadata,
@@ -751,11 +597,11 @@ impl Timeline {
         result
     }
 
-    pub(super) fn launch_wal_receiver(self: &Arc<Self>) {
+    pub fn launch_wal_receiver(self: &Arc<Self>) -> anyhow::Result<()> {
         if !is_etcd_client_initialized() {
             if cfg!(test) {
                 info!("not launching WAL receiver because etcd client hasn't been initialized");
-                return;
+                return Ok(());
             } else {
                 panic!("etcd client not initialized");
             }
@@ -783,14 +629,16 @@ impl Timeline {
             walreceiver_connect_timeout,
             lagging_wal_timeout,
             max_lsn_wal_lag,
-        );
+        )?;
+
+        Ok(())
     }
 
     ///
     /// Scan the timeline directory to populate the layer map.
     /// Returns all timeline-related files that were found and loaded.
     ///
-    pub(super) fn load_layer_map(&self, disk_consistent_lsn: Lsn) -> anyhow::Result<()> {
+    pub fn load_layer_map(&self, disk_consistent_lsn: Lsn) -> anyhow::Result<()> {
         let mut layers = self.layers.write().unwrap();
         let mut num_layers = 0;
 
@@ -876,12 +724,30 @@ impl Timeline {
         Ok(())
     }
 
-    pub(super) fn layer_removal_guard(&self) -> anyhow::Result<MutexGuard<()>> {
+    pub fn layer_removal_guard(&self) -> anyhow::Result<MutexGuard<()>> {
         self.layer_removal_cs
             .try_lock()
             .map_err(|e| anyhow!("cannot lock compaction critical section {e}"))
     }
 
+    /// Retrieve current logical size of the timeline.
+    ///
+    /// The size could be lagging behind the actual number, in case
+    /// the initial size calculation has not been run (gets triggered on the first size access).
+    pub fn get_current_logical_size(self: &Arc<Self>) -> anyhow::Result<u64> {
+        let current_size = self.current_logical_size.current_size()?;
+        debug!("Current size: {current_size:?}");
+
+        let size = current_size.size();
+        if let (CurrentLogicalSize::Approximate(_), Some(init_lsn)) =
+            (current_size, self.current_logical_size.initial_part_end)
+        {
+            self.try_spawn_size_init_task(init_lsn);
+        }
+
+        Ok(size)
+    }
+
     fn try_spawn_size_init_task(self: &Arc<Self>, init_lsn: Lsn) {
         let timeline_id = self.timeline_id;
 
@@ -1102,7 +968,7 @@ impl Timeline {
         Some((lsn, img))
     }
 
-    fn get_ancestor_timeline(&self) -> anyhow::Result<Arc<Timeline>> {
+    fn get_ancestor_timeline(&self) -> Result<Arc<Timeline>> {
         let ancestor = self.ancestor_timeline.as_ref().with_context(|| {
             format!(
                 "Ancestor is missing. Timeline id: {} Ancestor id {:?}",
@@ -1161,14 +1027,14 @@ impl Timeline {
         Ok(layer)
     }
 
-    fn put_value(&self, key: Key, lsn: Lsn, val: &Value) -> anyhow::Result<()> {
+    fn put_value(&self, key: Key, lsn: Lsn, val: &Value) -> Result<()> {
         //info!("PUT: key {} at {}", key, lsn);
         let layer = self.get_layer_for_write(lsn)?;
         layer.put_value(key, lsn, val)?;
         Ok(())
     }
 
-    fn put_tombstone(&self, key_range: Range<Key>, lsn: Lsn) -> anyhow::Result<()> {
+    fn put_tombstone(&self, key_range: Range<Key>, lsn: Lsn) -> Result<()> {
         let layer = self.get_layer_for_write(lsn)?;
         layer.put_tombstone(key_range, lsn)?;
 
@@ -1207,6 +1073,64 @@ impl Timeline {
         drop(layers);
     }
 
+    ///
+    /// Check if more than 'checkpoint_distance' of WAL has been accumulated in
+    /// the in-memory layer, and initiate flushing it if so.
+    ///
+    /// Also flush after a period of time without new data -- it helps
+    /// safekeepers to regard pageserver as caught up and suspend activity.
+    ///
+    pub fn check_checkpoint_distance(self: &Arc<Timeline>) -> Result<()> {
+        let last_lsn = self.get_last_record_lsn();
+        let layers = self.layers.read().unwrap();
+        if let Some(open_layer) = &layers.open_layer {
+            let open_layer_size = open_layer.size()?;
+            drop(layers);
+            let last_freeze_at = self.last_freeze_at.load();
+            let last_freeze_ts = *(self.last_freeze_ts.read().unwrap());
+            let distance = last_lsn.widening_sub(last_freeze_at);
+            // Checkpointing the open layer can be triggered by layer size or LSN range.
+            // S3 has a 5 GB limit on the size of one upload (without multi-part upload), and
+            // we want to stay below that with a big margin.  The LSN distance determines how
+            // much WAL the safekeepers need to store.
+            if distance >= self.get_checkpoint_distance().into()
+                || open_layer_size > self.get_checkpoint_distance()
+                || (distance > 0 && last_freeze_ts.elapsed() >= self.get_checkpoint_timeout())
+            {
+                info!(
+                    "check_checkpoint_distance {}, layer size {}, elapsed since last flush {:?}",
+                    distance,
+                    open_layer_size,
+                    last_freeze_ts.elapsed()
+                );
+
+                self.freeze_inmem_layer(true);
+                self.last_freeze_at.store(last_lsn);
+                *(self.last_freeze_ts.write().unwrap()) = Instant::now();
+
+                // Launch a task to flush the frozen layer to disk, unless
+                // a task was already running. (If the task was running
+                // at the time that we froze the layer, it must've seen the
+                // the layer we just froze before it exited; see comments
+                // in flush_frozen_layers())
+                if let Ok(guard) = self.layer_flush_lock.try_lock() {
+                    drop(guard);
+                    let self_clone = Arc::clone(self);
+                    task_mgr::spawn(
+                        task_mgr::BACKGROUND_RUNTIME.handle(),
+                        task_mgr::TaskKind::LayerFlushTask,
+                        Some(self.tenant_id),
+                        Some(self.timeline_id),
+                        "layer flush task",
+                        false,
+                        async move { self_clone.flush_frozen_layers(false) },
+                    );
+                }
+            }
+        }
+        Ok(())
+    }
+
     /// Flush all frozen layers to disk.
     ///
     /// Only one task at a time can be doing layer-flushing for a
@@ -1214,7 +1138,7 @@ impl Timeline {
     /// currently doing the flushing, this function will wait for it
     /// to finish. If 'wait' is false, this function will return
     /// immediately instead.
-    fn flush_frozen_layers(&self, wait: bool) -> anyhow::Result<()> {
+    fn flush_frozen_layers(&self, wait: bool) -> Result<()> {
         let flush_lock_guard = if wait {
             self.layer_flush_lock.lock().unwrap()
         } else {
@@ -1253,7 +1177,7 @@ impl Timeline {
     }
 
     /// Flush one frozen in-memory layer to disk, as a new delta layer.
-    fn flush_frozen_layer(&self, frozen_layer: Arc<InMemoryLayer>) -> anyhow::Result<()> {
+    fn flush_frozen_layer(&self, frozen_layer: Arc<InMemoryLayer>) -> Result<()> {
         // As a special case, when we have just imported an image into the repository,
         // instead of writing out a L0 delta layer, we directly write out image layer
         // files instead. This is possible as long as *all* the data imported into the
@@ -1266,8 +1190,8 @@ impl Timeline {
                 self.create_image_layers(&partitioning, self.initdb_lsn, true)?
             } else {
                 // normal case, write out a L0 delta layer file.
-                let (delta_path, metadata) = self.create_delta_layer(&frozen_layer)?;
-                HashMap::from([(delta_path, metadata)])
+                let delta_path = self.create_delta_layer(&frozen_layer)?;
+                HashSet::from([delta_path])
             };
 
         fail_point!("flush-frozen-before-sync");
@@ -1293,86 +1217,85 @@ impl Timeline {
         // TODO: This perhaps should be done in 'flush_frozen_layers', after flushing
         // *all* the layers, to avoid fsyncing the file multiple times.
         let disk_consistent_lsn = Lsn(lsn_range.end.0 - 1);
-        let old_disk_consistent_lsn = self.disk_consistent_lsn.load();
+        self.update_disk_consistent_lsn(disk_consistent_lsn, layer_paths_to_upload)?;
 
-        // If we were able to advance 'disk_consistent_lsn', save it the metadata file.
-        // After crash, we will restart WAL streaming and processing from that point.
-        if disk_consistent_lsn != old_disk_consistent_lsn {
-            assert!(disk_consistent_lsn > old_disk_consistent_lsn);
-            self.update_metadata_file(disk_consistent_lsn, layer_paths_to_upload)?;
-            // Also update the in-memory copy
-            self.disk_consistent_lsn.store(disk_consistent_lsn);
-        }
         Ok(())
     }
 
     /// Update metadata file
-    fn update_metadata_file(
+    fn update_disk_consistent_lsn(
         &self,
         disk_consistent_lsn: Lsn,
-        layer_paths_to_upload: HashMap<PathBuf, LayerFileMetadata>,
-    ) -> anyhow::Result<()> {
-        // We can only save a valid 'prev_record_lsn' value on disk if we
-        // flushed *all* in-memory changes to disk. We only track
-        // 'prev_record_lsn' in memory for the latest processed record, so we
-        // don't remember what the correct value that corresponds to some old
-        // LSN is. But if we flush everything, then the value corresponding
-        // current 'last_record_lsn' is correct and we can store it on disk.
-        let RecordLsn {
-            last: last_record_lsn,
-            prev: prev_record_lsn,
-        } = self.last_record_lsn.load();
-        let ondisk_prev_record_lsn = if disk_consistent_lsn == last_record_lsn {
-            Some(prev_record_lsn)
-        } else {
-            None
-        };
+        layer_paths_to_upload: HashSet<PathBuf>,
+    ) -> Result<()> {
+        // If we were able to advance 'disk_consistent_lsn', save it the metadata file.
+        // After crash, we will restart WAL streaming and processing from that point.
+        let old_disk_consistent_lsn = self.disk_consistent_lsn.load();
+        if disk_consistent_lsn != old_disk_consistent_lsn {
+            assert!(disk_consistent_lsn > old_disk_consistent_lsn);
 
-        let ancestor_timeline_id = self
-            .ancestor_timeline
-            .as_ref()
-            .map(|ancestor| ancestor.timeline_id);
+            // We can only save a valid 'prev_record_lsn' value on disk if we
+            // flushed *all* in-memory changes to disk. We only track
+            // 'prev_record_lsn' in memory for the latest processed record, so we
+            // don't remember what the correct value that corresponds to some old
+            // LSN is. But if we flush everything, then the value corresponding
+            // current 'last_record_lsn' is correct and we can store it on disk.
+            let RecordLsn {
+                last: last_record_lsn,
+                prev: prev_record_lsn,
+            } = self.last_record_lsn.load();
+            let ondisk_prev_record_lsn = if disk_consistent_lsn == last_record_lsn {
+                Some(prev_record_lsn)
+            } else {
+                None
+            };
 
-        let metadata = TimelineMetadata::new(
-            disk_consistent_lsn,
-            ondisk_prev_record_lsn,
-            ancestor_timeline_id,
-            self.ancestor_lsn,
-            *self.latest_gc_cutoff_lsn.read(),
-            self.initdb_lsn,
-            self.pg_version,
-        );
+            let ancestor_timeline_id = self
+                .ancestor_timeline
+                .as_ref()
+                .map(|ancestor| ancestor.timeline_id);
 
-        fail_point!("checkpoint-before-saving-metadata", |x| bail!(
-            "{}",
-            x.unwrap()
-        ));
-
-        save_metadata(
-            self.conf,
-            self.timeline_id,
-            self.tenant_id,
-            &metadata,
-            false,
-        )?;
-
-        if self.upload_layers.load(atomic::Ordering::Relaxed) {
-            storage_sync::schedule_layer_upload(
-                self.tenant_id,
-                self.timeline_id,
-                layer_paths_to_upload,
-                Some(metadata),
+            let metadata = TimelineMetadata::new(
+                disk_consistent_lsn,
+                ondisk_prev_record_lsn,
+                ancestor_timeline_id,
+                self.ancestor_lsn,
+                *self.latest_gc_cutoff_lsn.read(),
+                self.initdb_lsn,
+                self.pg_version,
             );
+
+            fail_point!("checkpoint-before-saving-metadata", |x| bail!(
+                "{}",
+                x.unwrap()
+            ));
+
+            save_metadata(
+                self.conf,
+                self.timeline_id,
+                self.tenant_id,
+                &metadata,
+                false,
+            )?;
+
+            if self.upload_layers.load(atomic::Ordering::Relaxed) {
+                storage_sync::schedule_layer_upload(
+                    self.tenant_id,
+                    self.timeline_id,
+                    layer_paths_to_upload,
+                    Some(metadata),
+                );
+            }
+
+            // Also update the in-memory copy
+            self.disk_consistent_lsn.store(disk_consistent_lsn);
         }
 
         Ok(())
     }
 
     // Write out the given frozen in-memory layer as a new L0 delta file
-    fn create_delta_layer(
-        &self,
-        frozen_layer: &InMemoryLayer,
-    ) -> anyhow::Result<(PathBuf, LayerFileMetadata)> {
+    fn create_delta_layer(&self, frozen_layer: &InMemoryLayer) -> Result<PathBuf> {
         // Write it out
         let new_delta = frozen_layer.write_to_disk()?;
         let new_delta_path = new_delta.path();
@@ -1398,16 +1321,100 @@ impl Timeline {
 
         // update the timeline's physical size
         let sz = new_delta_path.metadata()?.len();
-
         self.metrics.current_physical_size_gauge.add(sz);
         // update metrics
         self.metrics.num_persistent_files_created.inc_by(1);
         self.metrics.persistent_bytes_written.inc_by(sz);
 
-        Ok((new_delta_path, LayerFileMetadata::new(sz)))
+        Ok(new_delta_path)
     }
 
-    fn repartition(&self, lsn: Lsn, partition_size: u64) -> anyhow::Result<(KeyPartitioning, Lsn)> {
+    pub fn compact(&self) -> anyhow::Result<()> {
+        let last_record_lsn = self.get_last_record_lsn();
+
+        // Last record Lsn could be zero in case the timelie was just created
+        if !last_record_lsn.is_valid() {
+            warn!("Skipping compaction for potentially just initialized timeline, it has invalid last record lsn: {last_record_lsn}");
+            return Ok(());
+        }
+
+        //
+        // High level strategy for compaction / image creation:
+        //
+        // 1. First, calculate the desired "partitioning" of the
+        // currently in-use key space. The goal is to partition the
+        // key space into roughly fixed-size chunks, but also take into
+        // account any existing image layers, and try to align the
+        // chunk boundaries with the existing image layers to avoid
+        // too much churn. Also try to align chunk boundaries with
+        // relation boundaries.  In principle, we don't know about
+        // relation boundaries here, we just deal with key-value
+        // pairs, and the code in pgdatadir_mapping.rs knows how to
+        // map relations into key-value pairs. But in practice we know
+        // that 'field6' is the block number, and the fields 1-5
+        // identify a relation. This is just an optimization,
+        // though.
+        //
+        // 2. Once we know the partitioning, for each partition,
+        // decide if it's time to create a new image layer. The
+        // criteria is: there has been too much "churn" since the last
+        // image layer? The "churn" is fuzzy concept, it's a
+        // combination of too many delta files, or too much WAL in
+        // total in the delta file. Or perhaps: if creating an image
+        // file would allow to delete some older files.
+        //
+        // 3. After that, we compact all level0 delta files if there
+        // are too many of them.  While compacting, we also garbage
+        // collect any page versions that are no longer needed because
+        // of the new image layers we created in step 2.
+        //
+        // TODO: This high level strategy hasn't been implemented yet.
+        // Below are functions compact_level0() and create_image_layers()
+        // but they are a bit ad hoc and don't quite work like it's explained
+        // above. Rewrite it.
+        let _layer_removal_cs = self.layer_removal_cs.lock().unwrap();
+
+        let target_file_size = self.get_checkpoint_distance();
+
+        // Define partitioning schema if needed
+
+        match self.repartition(
+            self.get_last_record_lsn(),
+            self.get_compaction_target_size(),
+        ) {
+            Ok((partitioning, lsn)) => {
+                // 2. Create new image layers for partitions that have been modified
+                // "enough".
+                let layer_paths_to_upload = self.create_image_layers(&partitioning, lsn, false)?;
+                if !layer_paths_to_upload.is_empty()
+                    && self.upload_layers.load(atomic::Ordering::Relaxed)
+                {
+                    storage_sync::schedule_layer_upload(
+                        self.tenant_id,
+                        self.timeline_id,
+                        HashSet::from_iter(layer_paths_to_upload),
+                        None,
+                    );
+                }
+
+                // 3. Compact
+                let timer = self.metrics.compact_time_histo.start_timer();
+                self.compact_level0(target_file_size)?;
+                timer.stop_and_record();
+            }
+            Err(err) => {
+                // no partitioning? This is normal, if the timeline was just created
+                // as an empty timeline. Also in unit tests, when we use the timeline
+                // as a simple key-value store, ignoring the datadir layout. Log the
+                // error but continue.
+                error!("could not compact, repartitioning keyspace failed: {err:?}");
+            }
+        };
+
+        Ok(())
+    }
+
+    fn repartition(&self, lsn: Lsn, partition_size: u64) -> Result<(KeyPartitioning, Lsn)> {
         let mut partitioning_guard = self.partitioning.lock().unwrap();
         if partitioning_guard.1 == Lsn(0)
             || lsn.0 - partitioning_guard.1 .0 > self.repartition_threshold
@@ -1421,7 +1428,7 @@ impl Timeline {
     }
 
     // Is it time to create a new image layer for the given partition?
-    fn time_for_new_image_layer(&self, partition: &KeySpace, lsn: Lsn) -> anyhow::Result<bool> {
+    fn time_for_new_image_layer(&self, partition: &KeySpace, lsn: Lsn) -> Result<bool> {
         let layers = self.layers.read().unwrap();
 
         for part_range in &partition.ranges {
@@ -1466,9 +1473,10 @@ impl Timeline {
         partitioning: &KeyPartitioning,
         lsn: Lsn,
         force: bool,
-    ) -> anyhow::Result<HashMap<PathBuf, LayerFileMetadata>> {
+    ) -> Result<HashSet<PathBuf>> {
         let timer = self.metrics.create_images_time_histo.start_timer();
         let mut image_layers: Vec<ImageLayer> = Vec::new();
+        let mut layer_paths_to_upload = HashSet::new();
         for partition in partitioning.parts.iter() {
             if force || self.time_for_new_image_layer(partition, lsn)? {
                 let img_range =
@@ -1484,37 +1492,13 @@ impl Timeline {
                 for range in &partition.ranges {
                     let mut key = range.start;
                     while key < range.end {
-                        let img = match self.get(key, lsn) {
-                            Ok(img) => img,
-                            Err(err) => {
-                                // If we fail to reconstruct a VM or FSM page, we can zero the
-                                // page without losing any actual user data. That seems better
-                                // than failing repeatedly and getting stuck.
-                                //
-                                // We had a bug at one point, where we truncated the FSM and VM
-                                // in the pageserver, but the Postgres didn't know about that
-                                // and continued to generate incremental WAL records for pages
-                                // that didn't exist in the pageserver. Trying to replay those
-                                // WAL records failed to find the previous image of the page.
-                                // This special case allows us to recover from that situation.
-                                // See https://github.com/neondatabase/neon/issues/2601.
-                                //
-                                // Unfortunately we cannot do this for the main fork, or for
-                                // any metadata keys, keys, as that would lead to actual data
-                                // loss.
-                                if is_rel_fsm_block_key(key) || is_rel_vm_block_key(key) {
-                                    warn!("could not reconstruct FSM or VM key {key}, filling with zeros: {err:?}");
-                                    ZERO_PAGE.clone()
-                                } else {
-                                    return Err(err);
-                                }
-                            }
-                        };
+                        let img = self.get(key, lsn)?;
                         image_layer_writer.put_image(key, &img)?;
                         key = key.next();
                     }
                 }
                 let image_layer = image_layer_writer.finish()?;
+                layer_paths_to_upload.insert(image_layer.path());
                 image_layers.push(image_layer);
             }
         }
@@ -1528,25 +1512,15 @@ impl Timeline {
         //
         // Compaction creates multiple image layers. It would be better to create them all
         // and fsync them all in parallel.
-        let all_paths = image_layers
-            .iter()
-            .map(|layer| layer.path())
-            .chain(std::iter::once(
-                self.conf.timeline_path(&self.timeline_id, &self.tenant_id),
-            ))
-            .collect::<Vec<_>>();
+        let mut all_paths = Vec::from_iter(layer_paths_to_upload.clone());
+        all_paths.push(self.conf.timeline_path(&self.timeline_id, &self.tenant_id));
         par_fsync::par_fsync(&all_paths)?;
 
-        let mut layer_paths_to_upload = HashMap::with_capacity(image_layers.len());
-
         let mut layers = self.layers.write().unwrap();
         for l in image_layers {
-            let path = l.path();
-            let metadata = path.metadata()?;
-
-            layer_paths_to_upload.insert(path, LayerFileMetadata::new(metadata.len()));
-
-            self.metrics.current_physical_size_gauge.add(metadata.len());
+            self.metrics
+                .current_physical_size_gauge
+                .add(l.path().metadata()?.len());
             layers.insert_historic(Arc::new(l));
         }
         drop(layers);
@@ -1559,7 +1533,7 @@ impl Timeline {
     /// Collect a bunch of Level 0 layer files, and compact and reshuffle them as
     /// as Level 1 files.
     ///
-    fn compact_level0(&self, target_file_size: u64) -> anyhow::Result<()> {
+    fn compact_level0(&self, target_file_size: u64) -> Result<()> {
         let layers = self.layers.read().unwrap();
         let mut level0_deltas = layers.get_level0_deltas()?;
         drop(layers);
@@ -1797,16 +1771,16 @@ impl Timeline {
         }
 
         let mut layers = self.layers.write().unwrap();
-        let mut new_layer_paths = HashMap::with_capacity(new_layers.len());
+        let mut new_layer_paths = HashSet::with_capacity(new_layers.len());
         for l in new_layers {
             let new_delta_path = l.path();
 
-            let metadata = new_delta_path.metadata()?;
-
             // update the timeline's physical size
-            self.metrics.current_physical_size_gauge.add(metadata.len());
+            self.metrics
+                .current_physical_size_gauge
+                .add(new_delta_path.metadata()?.len());
 
-            new_layer_paths.insert(new_delta_path, LayerFileMetadata::new(metadata.len()));
+            new_layer_paths.insert(new_delta_path);
             layers.insert_historic(Arc::new(l));
         }
 
@@ -1869,12 +1843,12 @@ impl Timeline {
     ///
     /// The 'pitr' duration is used to calculate a 'pitr_cutoff', which can be used to determine
     /// whether a record is needed for PITR.
-    pub(super) fn update_gc_info(
+    pub fn update_gc_info(
         &self,
         retain_lsns: Vec<Lsn>,
         cutoff_horizon: Lsn,
         pitr: Duration,
-    ) -> anyhow::Result<()> {
+    ) -> Result<()> {
         let mut gc_info = self.gc_info.write().unwrap();
 
         gc_info.horizon_cutoff = cutoff_horizon;
@@ -1929,7 +1903,7 @@ impl Timeline {
     /// within a layer file. We can only remove the whole file if it's fully
     /// obsolete.
     ///
-    pub(super) fn gc(&self) -> anyhow::Result<GcResult> {
+    pub fn gc(&self) -> Result<GcResult> {
         let mut result: GcResult = Default::default();
         let now = SystemTime::now();
 
@@ -1973,9 +1947,6 @@ impl Timeline {
             );
             write_guard.store_and_unlock(new_gc_cutoff).wait();
         }
-        // Persist the new GC cutoff value in the metadata file, before
-        // we actually remove anything.
-        self.update_metadata_file(self.disk_consistent_lsn.load(), HashMap::new())?;
 
         info!("GC starting");
 
@@ -2101,15 +2072,6 @@ impl Timeline {
             result.layers_removed += 1;
         }
 
-        info!(
-            "GC completed removing {} layers, cutoff {}",
-            result.layers_removed, new_gc_cutoff
-        );
-
-        if result.layers_removed != 0 {
-            fail_point!("after-timeline-gc-removed-layers");
-        }
-
         if self.upload_layers.load(atomic::Ordering::Relaxed) {
             storage_sync::schedule_layer_delete(
                 self.tenant_id,
@@ -2249,11 +2211,11 @@ impl<'a> TimelineWriter<'a> {
     ///
     /// This will implicitly extend the relation, if the page is beyond the
     /// current end-of-file.
-    pub fn put(&self, key: Key, lsn: Lsn, value: &Value) -> anyhow::Result<()> {
+    pub fn put(&self, key: Key, lsn: Lsn, value: &Value) -> Result<()> {
         self.tl.put_value(key, lsn, value)
     }
 
-    pub fn delete(&self, key_range: Range<Key>, lsn: Lsn) -> anyhow::Result<()> {
+    pub fn delete(&self, key_range: Range<Key>, lsn: Lsn) -> Result<()> {
         self.tl.put_tombstone(key_range, lsn)
     }
 

pageserver/src/tenant_mgr.rs

@@ -1,7 +1,7 @@
 //! This module acts as a switchboard to access different repositories managed by this
 //! page server.
 
-use std::collections::{hash_map, HashMap};
+use std::collections::{hash_map, HashMap, HashSet};
 use std::ffi::OsStr;
 use std::fs;
 use std::path::{Path, PathBuf};
@@ -12,10 +12,10 @@ use tracing::*;
 
 use remote_storage::GenericRemoteStorage;
 
-use crate::config::{PageServerConf, METADATA_FILE_NAME, TIMELINE_UNINIT_MARK_SUFFIX};
+use crate::config::{PageServerConf, METADATA_FILE_NAME};
 use crate::http::models::TenantInfo;
-use crate::storage_sync::index::{LayerFileMetadata, RemoteIndex, RemoteTimelineIndex};
-use crate::storage_sync::{self, LocalTimelineInitStatus, SyncStartupData, TimelineLocalFiles};
+use crate::storage_sync::index::{RemoteIndex, RemoteTimelineIndex};
+use crate::storage_sync::{self, LocalTimelineInitStatus, SyncStartupData};
 use crate::task_mgr::{self, TaskKind};
 use crate::tenant::{
     ephemeral_file::is_ephemeral_file, metadata::TimelineMetadata, Tenant, TenantState,
@@ -24,7 +24,7 @@ use crate::tenant_config::TenantConfOpt;
 use crate::walredo::PostgresRedoManager;
 use crate::TEMP_FILE_SUFFIX;
 
-use utils::crashsafe::{self, path_with_suffix_extension};
+use utils::crashsafe_dir::{self, path_with_suffix_extension};
 use utils::id::{TenantId, TimelineId};
 
 mod tenants_state {
@@ -104,7 +104,7 @@ pub fn init_tenant_mgr(
                 if let TenantAttachData::Ready(t) = new_timeline_values {
                     for (timeline_id, old_value) in old_values {
                         if let LocalTimelineInitStatus::LocallyComplete(metadata) = old_value {
-                            t.insert(timeline_id, TimelineLocalFiles::ready(metadata));
+                            t.insert(timeline_id, (metadata, HashSet::new()));
                         }
                     }
                 }
@@ -189,7 +189,7 @@ pub fn attach_local_tenants(
                 let has_timelines = !timelines.is_empty();
                 let timelines_to_attach = timelines
                     .iter()
-                    .map(|(&k, v)| (k, v.metadata().to_owned()))
+                    .map(|(&k, (v, _))| (k, v.clone()))
                     .collect();
                 match tenant.init_attach_timelines(timelines_to_attach) {
                     Ok(()) => {
@@ -265,98 +265,58 @@ fn create_tenant_files(
         temporary_tenant_dir.display()
     );
 
+    let temporary_tenant_timelines_dir = rebase_directory(
+        &conf.timelines_path(&tenant_id),
+        &target_tenant_directory,
+        &temporary_tenant_dir,
+    )?;
+    let temporary_tenant_config_path = rebase_directory(
+        &conf.tenant_config_path(tenant_id),
+        &target_tenant_directory,
+        &temporary_tenant_dir,
+    )?;
+
     // top-level dir may exist if we are creating it through CLI
-    crashsafe::create_dir_all(&temporary_tenant_dir).with_context(|| {
+    crashsafe_dir::create_dir_all(&temporary_tenant_dir).with_context(|| {
         format!(
             "could not create temporary tenant directory {}",
             temporary_tenant_dir.display()
         )
     })?;
-
-    let creation_result = try_create_target_tenant_dir(
-        conf,
-        tenant_conf,
-        tenant_id,
-        &temporary_tenant_dir,
-        &target_tenant_directory,
-    );
-
-    if creation_result.is_err() {
-        error!("Failed to create directory structure for tenant {tenant_id}, cleaning tmp data");
-        if let Err(e) = fs::remove_dir_all(&temporary_tenant_dir) {
-            error!("Failed to remove temporary tenant directory {temporary_tenant_dir:?}: {e}")
-        } else if let Err(e) = crashsafe::fsync(&temporary_tenant_dir) {
-            error!(
-                "Failed to fsync removed temporary tenant directory {temporary_tenant_dir:?}: {e}"
-            )
-        }
-    }
-
-    creation_result
-}
-
-fn try_create_target_tenant_dir(
-    conf: &'static PageServerConf,
-    tenant_conf: TenantConfOpt,
-    tenant_id: TenantId,
-    temporary_tenant_dir: &Path,
-    target_tenant_directory: &Path,
-) -> Result<(), anyhow::Error> {
-    let temporary_tenant_timelines_dir = rebase_directory(
-        &conf.timelines_path(&tenant_id),
-        target_tenant_directory,
-        temporary_tenant_dir,
-    )
-    .with_context(|| format!("Failed to resolve tenant {tenant_id} temporary timelines dir"))?;
-    let temporary_tenant_config_path = rebase_directory(
-        &conf.tenant_config_path(tenant_id),
-        target_tenant_directory,
-        temporary_tenant_dir,
-    )
-    .with_context(|| format!("Failed to resolve tenant {tenant_id} temporary config path"))?;
-
-    Tenant::persist_tenant_config(&temporary_tenant_config_path, tenant_conf, true).with_context(
-        || {
-            format!(
-                "Failed to write tenant {} config to {}",
-                tenant_id,
-                temporary_tenant_config_path.display()
-            )
-        },
-    )?;
-    crashsafe::create_dir(&temporary_tenant_timelines_dir).with_context(|| {
+    // first, create a config in the top-level temp directory, fsync the file
+    Tenant::persist_tenant_config(&temporary_tenant_config_path, tenant_conf, true)?;
+    // then, create a subdirectory in the top-level temp directory, fsynced
+    crashsafe_dir::create_dir(&temporary_tenant_timelines_dir).with_context(|| {
         format!(
-            "could not create tenant {} temporary timelines directory {}",
-            tenant_id,
+            "could not create temporary tenant timelines directory {}",
             temporary_tenant_timelines_dir.display()
         )
     })?;
+
     fail::fail_point!("tenant-creation-before-tmp-rename", |_| {
         anyhow::bail!("failpoint tenant-creation-before-tmp-rename");
     });
 
-    fs::rename(&temporary_tenant_dir, target_tenant_directory).with_context(|| {
+    // move-rename tmp directory with all files synced into a permanent directory, fsync its parent
+    fs::rename(&temporary_tenant_dir, &target_tenant_directory).with_context(|| {
         format!(
-            "failed to move tenant {} temporary directory {} into the permanent one {}",
-            tenant_id,
+            "failed to move temporary tenant directory {} into the permanent one {}",
             temporary_tenant_dir.display(),
             target_tenant_directory.display()
         )
     })?;
     let target_dir_parent = target_tenant_directory.parent().with_context(|| {
         format!(
-            "Failed to get tenant {} dir parent for {}",
-            tenant_id,
+            "Failed to get tenant dir parent for {}",
             target_tenant_directory.display()
         )
     })?;
-    crashsafe::fsync(target_dir_parent).with_context(|| {
-        format!(
-            "Failed to fsync renamed directory's parent {} for tenant {}",
-            target_dir_parent.display(),
-            tenant_id,
-        )
-    })?;
+    fs::File::open(target_dir_parent)?.sync_all()?;
+
+    info!(
+        "created tenant directory structure in {}",
+        target_tenant_directory.display()
+    );
 
     Ok(())
 }
@@ -523,7 +483,7 @@ pub fn list_tenant_info(remote_index: &RemoteTimelineIndex) -> Vec<TenantInfo> {
 
 #[derive(Debug)]
 pub enum TenantAttachData {
-    Ready(HashMap<TimelineId, TimelineLocalFiles>),
+    Ready(HashMap<TimelineId, (TimelineMetadata, HashSet<PathBuf>)>),
     Broken(anyhow::Error),
 }
 /// Attempts to collect information about all tenant and timelines, existing on the local FS.
@@ -642,15 +602,7 @@ fn is_temporary(path: &Path) -> bool {
     }
 }
 
-fn is_uninit_mark(path: &Path) -> bool {
-    match path.file_name() {
-        Some(name) => name
-            .to_string_lossy()
-            .ends_with(TIMELINE_UNINIT_MARK_SUFFIX),
-        None => false,
-    }
-}
-
+#[allow(clippy::type_complexity)]
 fn collect_timelines_for_tenant(
     config: &'static PageServerConf,
     tenant_path: &Path,
@@ -693,74 +645,25 @@ fn collect_timelines_for_tenant(
                             e
                         );
                     }
-                } else if is_uninit_mark(&timeline_dir) {
-                    let timeline_uninit_mark_file = &timeline_dir;
-                    info!(
-                        "Found an uninit mark file {}, removing the timeline and its uninit mark",
-                        timeline_uninit_mark_file.display()
-                    );
-                    let timeline_id = timeline_uninit_mark_file
-                        .file_stem()
-                        .and_then(OsStr::to_str)
-                        .unwrap_or_default()
-                        .parse::<TimelineId>()
-                        .with_context(|| {
-                            format!(
-                                "Could not parse timeline id out of the timeline uninit mark name {}",
-                                timeline_uninit_mark_file.display()
-                            )
-                        })?;
-                    let timeline_dir = config.timeline_path(&timeline_id, &tenant_id);
-                    if let Err(e) =
-                        remove_timeline_and_uninit_mark(&timeline_dir, timeline_uninit_mark_file)
-                    {
-                        error!("Failed to clean up uninit marked timeline: {e:?}");
-                    }
                 } else {
-                    let timeline_id = timeline_dir
-                        .file_name()
-                        .and_then(OsStr::to_str)
-                        .unwrap_or_default()
-                        .parse::<TimelineId>()
-                        .with_context(|| {
-                            format!(
-                                "Could not parse timeline id out of the timeline dir name {}",
-                                timeline_dir.display()
-                            )
-                        })?;
-                    let timeline_uninit_mark_file =
-                        config.timeline_uninit_mark_file_path(tenant_id, timeline_id);
-                    if timeline_uninit_mark_file.exists() {
-                        info!("Found an uninit mark file for timeline {tenant_id}/{timeline_id}, removing the timeline and its uninit mark");
-                        if let Err(e) = remove_timeline_and_uninit_mark(
-                            &timeline_dir,
-                            &timeline_uninit_mark_file,
-                        ) {
-                            error!("Failed to clean up uninit marked timeline: {e:?}");
+                    match collect_timeline_files(&timeline_dir) {
+                        Ok((timeline_id, metadata, timeline_files)) => {
+                            tenant_timelines.insert(timeline_id, (metadata, timeline_files));
                         }
-                    } else {
-                        match collect_timeline_files(&timeline_dir) {
-                            Ok((metadata, timeline_files)) => {
-                                tenant_timelines.insert(
-                                    timeline_id,
-                                    TimelineLocalFiles::collected(metadata, timeline_files),
-                                );
-                            }
-                            Err(e) => {
-                                error!(
-                                    "Failed to process timeline dir contents at '{}', reason: {:?}",
-                                    timeline_dir.display(),
-                                    e
-                                );
-                                match remove_if_empty(&timeline_dir) {
-                                    Ok(true) => info!(
-                                        "Removed empty timeline directory {}",
-                                        timeline_dir.display()
-                                    ),
-                                    Ok(false) => (),
-                                    Err(e) => {
-                                        error!("Failed to remove empty timeline directory: {e:?}")
-                                    }
+                        Err(e) => {
+                            error!(
+                                "Failed to process timeline dir contents at '{}', reason: {:?}",
+                                timeline_dir.display(),
+                                e
+                            );
+                            match remove_if_empty(&timeline_dir) {
+                                Ok(true) => info!(
+                                    "Removed empty timeline directory {}",
+                                    timeline_dir.display()
+                                ),
+                                Ok(false) => (),
+                                Err(e) => {
+                                    error!("Failed to remove empty timeline directory: {e:?}")
                                 }
                             }
                         }
@@ -783,48 +686,25 @@ fn collect_timelines_for_tenant(
     Ok((tenant_id, TenantAttachData::Ready(tenant_timelines)))
 }
 
-fn remove_timeline_and_uninit_mark(timeline_dir: &Path, uninit_mark: &Path) -> anyhow::Result<()> {
-    fs::remove_dir_all(&timeline_dir)
-        .or_else(|e| {
-            if e.kind() == std::io::ErrorKind::NotFound {
-                // we can leave the uninit mark without a timeline dir,
-                // just remove the mark then
-                Ok(())
-            } else {
-                Err(e)
-            }
-        })
-        .with_context(|| {
-            format!(
-                "Failed to remove unit marked timeline directory {}",
-                timeline_dir.display()
-            )
-        })?;
-    fs::remove_file(&uninit_mark).with_context(|| {
-        format!(
-            "Failed to remove timeline uninit mark file {}",
-            uninit_mark.display()
-        )
-    })?;
-
-    Ok(())
-}
-
 // discover timeline files and extract timeline metadata
 //  NOTE: ephemeral files are excluded from the list
 fn collect_timeline_files(
     timeline_dir: &Path,
-) -> anyhow::Result<(TimelineMetadata, HashMap<PathBuf, LayerFileMetadata>)> {
-    let mut timeline_files = HashMap::new();
+) -> anyhow::Result<(TimelineId, TimelineMetadata, HashSet<PathBuf>)> {
+    let mut timeline_files = HashSet::new();
     let mut timeline_metadata_path = None;
 
+    let timeline_id = timeline_dir
+        .file_name()
+        .and_then(OsStr::to_str)
+        .unwrap_or_default()
+        .parse::<TimelineId>()
+        .context("Could not parse timeline id out of the timeline dir name")?;
     let timeline_dir_entries =
         fs::read_dir(&timeline_dir).context("Failed to list timeline dir contents")?;
     for entry in timeline_dir_entries {
         let entry_path = entry.context("Failed to list timeline dir entry")?.path();
-        let metadata = entry_path.metadata()?;
-
-        if metadata.is_file() {
+        if entry_path.is_file() {
             if entry_path.file_name().and_then(OsStr::to_str) == Some(METADATA_FILE_NAME) {
                 timeline_metadata_path = Some(entry_path);
             } else if is_ephemeral_file(&entry_path.file_name().unwrap().to_string_lossy()) {
@@ -839,8 +719,7 @@ fn collect_timeline_files(
                     )
                 })?;
             } else {
-                let layer_metadata = LayerFileMetadata::new(metadata.len());
-                timeline_files.insert(entry_path, layer_metadata);
+                timeline_files.insert(entry_path);
             }
         }
     }
@@ -866,5 +745,5 @@ fn collect_timeline_files(
         "Timeline has no ancestor and no layer files"
     );
 
-    Ok((metadata, timeline_files))
+    Ok((timeline_id, metadata, timeline_files))
 }

pageserver/src/tenant_tasks.rs

@@ -70,10 +70,8 @@ async fn compaction_loop(tenant_id: TenantId) {
             // Run compaction
             let mut sleep_duration = tenant.get_compaction_period();
             if let Err(e) = tenant.compaction_iteration() {
+                error!("Compaction failed, retrying: {e:#}");
                 sleep_duration = wait_duration;
-                error!("Compaction failed, retrying in {:?}: {e:#}", sleep_duration);
-                #[cfg(feature = "testing")]
-                std::process::abort();
             }
 
             // Sleep
@@ -121,10 +119,8 @@ async fn gc_loop(tenant_id: TenantId) {
             if gc_horizon > 0 {
                 if let Err(e) = tenant.gc_iteration(None, gc_horizon, tenant.get_pitr_interval(), false)
                 {
+                    error!("Gc failed, retrying: {e:#}");
                     sleep_duration = wait_duration;
-                    error!("Gc failed, retrying in {:?}: {e:#}", sleep_duration);
-                    #[cfg(feature = "testing")]
-                    std::process::abort();
                 }
             }
 

pageserver/src/walingest.rs

@@ -34,7 +34,6 @@ use crate::pgdatadir_mapping::*;
 use crate::reltag::{RelTag, SlruKind};
 use crate::tenant::Timeline;
 use crate::walrecord::*;
-use crate::ZERO_PAGE;
 use postgres_ffi::pg_constants;
 use postgres_ffi::relfile_utils::{FSM_FORKNUM, MAIN_FORKNUM, VISIBILITYMAP_FORKNUM};
 use postgres_ffi::v14::nonrelfile_utils::mx_offset_to_member_segment;
@@ -44,6 +43,8 @@ use postgres_ffi::TransactionId;
 use postgres_ffi::BLCKSZ;
 use utils::lsn::Lsn;
 
+static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]);
+
 pub struct WalIngest<'a> {
     timeline: &'a Timeline,
 

pageserver/src/walreceiver/connection_manager.rs

@@ -47,7 +47,7 @@ pub fn spawn_connection_manager_task(
     wal_connect_timeout: Duration,
     lagging_wal_timeout: Duration,
     max_lsn_wal_lag: NonZeroU64,
-) {
+) -> anyhow::Result<()> {
     let mut etcd_client = get_etcd_client().clone();
 
     let tenant_id = timeline.tenant_id;
@@ -95,6 +95,7 @@ pub fn spawn_connection_manager_task(
             info_span!("wal_connection_manager", tenant = %tenant_id, timeline = %timeline_id),
         ),
     );
+    Ok(())
 }
 
 /// Attempts to subscribe for timeline updates, pushed by safekeepers into the broker.
@@ -801,6 +802,7 @@ mod tests {
                         backup_lsn: None,
                         remote_consistent_lsn: None,
                         peer_horizon_lsn: None,
+                        local_start_lsn: None,
                         safekeeper_connstr: None,
                     },
                     etcd_version: 0,
@@ -817,6 +819,8 @@ mod tests {
                         backup_lsn: None,
                         remote_consistent_lsn: None,
                         peer_horizon_lsn: None,
+                        local_start_lsn: None,
+
                         safekeeper_connstr: Some("no commit_lsn".to_string()),
                     },
                     etcd_version: 0,
@@ -833,6 +837,7 @@ mod tests {
                         backup_lsn: None,
                         remote_consistent_lsn: None,
                         peer_horizon_lsn: None,
+                        local_start_lsn: None,
                         safekeeper_connstr: Some("no commit_lsn".to_string()),
                     },
                     etcd_version: 0,
@@ -849,6 +854,7 @@ mod tests {
                         backup_lsn: None,
                         remote_consistent_lsn: None,
                         peer_horizon_lsn: None,
+                        local_start_lsn: None,
                         safekeeper_connstr: None,
                     },
                     etcd_version: 0,
@@ -908,6 +914,8 @@ mod tests {
                         backup_lsn: None,
                         remote_consistent_lsn: None,
                         peer_horizon_lsn: None,
+                        local_start_lsn: None,
+
                         safekeeper_connstr: Some(DUMMY_SAFEKEEPER_CONNSTR.to_string()),
                     },
                     etcd_version: 0,
@@ -924,6 +932,8 @@ mod tests {
                         backup_lsn: None,
                         remote_consistent_lsn: None,
                         peer_horizon_lsn: None,
+                        local_start_lsn: None,
+
                         safekeeper_connstr: Some("not advanced Lsn".to_string()),
                     },
                     etcd_version: 0,
@@ -940,6 +950,8 @@ mod tests {
                         backup_lsn: None,
                         remote_consistent_lsn: None,
                         peer_horizon_lsn: None,
+                        local_start_lsn: None,
+
                         safekeeper_connstr: Some("not enough advanced Lsn".to_string()),
                     },
                     etcd_version: 0,
@@ -974,6 +986,8 @@ mod tests {
                     backup_lsn: None,
                     remote_consistent_lsn: None,
                     peer_horizon_lsn: None,
+                    local_start_lsn: None,
+
                     safekeeper_connstr: Some(DUMMY_SAFEKEEPER_CONNSTR.to_string()),
                 },
                 etcd_version: 0,
@@ -1006,6 +1020,8 @@ mod tests {
                         backup_lsn: None,
                         remote_consistent_lsn: None,
                         peer_horizon_lsn: None,
+                        local_start_lsn: None,
+
                         safekeeper_connstr: Some("smaller commit_lsn".to_string()),
                     },
                     etcd_version: 0,
@@ -1022,6 +1038,8 @@ mod tests {
                         backup_lsn: None,
                         remote_consistent_lsn: None,
                         peer_horizon_lsn: None,
+                        local_start_lsn: None,
+
                         safekeeper_connstr: Some(DUMMY_SAFEKEEPER_CONNSTR.to_string()),
                     },
                     etcd_version: 0,
@@ -1038,6 +1056,8 @@ mod tests {
                         backup_lsn: None,
                         remote_consistent_lsn: None,
                         peer_horizon_lsn: None,
+                        local_start_lsn: None,
+
                         safekeeper_connstr: None,
                     },
                     etcd_version: 0,
@@ -1083,6 +1103,8 @@ mod tests {
                         backup_lsn: None,
                         remote_consistent_lsn: None,
                         peer_horizon_lsn: None,
+                        local_start_lsn: None,
+
                         safekeeper_connstr: Some(DUMMY_SAFEKEEPER_CONNSTR.to_string()),
                     },
                     etcd_version: 0,
@@ -1099,6 +1121,8 @@ mod tests {
                         backup_lsn: None,
                         remote_consistent_lsn: None,
                         peer_horizon_lsn: None,
+                        local_start_lsn: None,
+
                         safekeeper_connstr: Some(DUMMY_SAFEKEEPER_CONNSTR.to_string()),
                     },
                     etcd_version: 0,
@@ -1168,6 +1192,8 @@ mod tests {
                         backup_lsn: None,
                         remote_consistent_lsn: None,
                         peer_horizon_lsn: None,
+                        local_start_lsn: None,
+
                         safekeeper_connstr: Some(DUMMY_SAFEKEEPER_CONNSTR.to_string()),
                     },
                     etcd_version: 0,
@@ -1184,6 +1210,8 @@ mod tests {
                         backup_lsn: None,
                         remote_consistent_lsn: None,
                         peer_horizon_lsn: None,
+                        local_start_lsn: None,
+
                         safekeeper_connstr: Some("advanced by Lsn safekeeper".to_string()),
                     },
                     etcd_version: 0,
@@ -1255,6 +1283,8 @@ mod tests {
                     backup_lsn: None,
                     remote_consistent_lsn: None,
                     peer_horizon_lsn: None,
+                    local_start_lsn: None,
+
                     safekeeper_connstr: Some(DUMMY_SAFEKEEPER_CONNSTR.to_string()),
                 },
                 etcd_version: 0,
@@ -1326,6 +1356,8 @@ mod tests {
                     backup_lsn: None,
                     remote_consistent_lsn: None,
                     peer_horizon_lsn: None,
+                    local_start_lsn: None,
+
                     safekeeper_connstr: Some(DUMMY_SAFEKEEPER_CONNSTR.to_string()),
                 },
                 etcd_version: 0,
@@ -1373,9 +1405,7 @@ mod tests {
             timeline: harness
                 .load()
                 .create_empty_timeline(TIMELINE_ID, Lsn(0), crate::DEFAULT_PG_VERSION)
-                .expect("Failed to create an empty timeline for dummy wal connection manager")
-                .initialize()
-                .unwrap(),
+                .expect("Failed to create an empty timeline for dummy wal connection manager"),
             wal_connect_timeout: Duration::from_secs(1),
             lagging_wal_timeout: Duration::from_secs(1),
             max_lsn_wal_lag: NonZeroU64::new(1024 * 1024).unwrap(),

pageserver/src/walredo.rs

@@ -35,12 +35,11 @@ use std::sync::Mutex;
 use std::time::Duration;
 use std::time::Instant;
 use tracing::*;
-use utils::crashsafe::path_with_suffix_extension;
+use utils::crashsafe_dir::path_with_suffix_extension;
 use utils::{bin_ser::BeSer, id::TenantId, lsn::Lsn, nonblock::set_nonblock};
 
 use crate::metrics::{
-    WAL_REDO_BYTES_HISTOGRAM, WAL_REDO_RECORDS_HISTOGRAM, WAL_REDO_RECORD_COUNTER, WAL_REDO_TIME,
-    WAL_REDO_WAIT_TIME,
+    WAL_REDO_RECORDS_HISTOGRAM, WAL_REDO_RECORD_COUNTER, WAL_REDO_TIME, WAL_REDO_WAIT_TIME,
 };
 use crate::pgdatadir_mapping::{key_to_rel_block, key_to_slru_block};
 use crate::reltag::{RelTag, SlruKind};
@@ -245,23 +244,12 @@ impl PostgresRedoManager {
         let end_time = Instant::now();
         let duration = end_time.duration_since(lock_time);
 
-        let len = records.len();
-        let nbytes = records.iter().fold(0, |acumulator, record| {
-            acumulator
-                + match &record.1 {
-                    NeonWalRecord::Postgres { rec, .. } => rec.len(),
-                    _ => unreachable!("Only PostgreSQL records are accepted in this batch"),
-                }
-        });
-
         WAL_REDO_TIME.observe(duration.as_secs_f64());
-        WAL_REDO_RECORDS_HISTOGRAM.observe(len as f64);
-        WAL_REDO_BYTES_HISTOGRAM.observe(nbytes as f64);
+        WAL_REDO_RECORDS_HISTOGRAM.observe(records.len() as f64);
 
         debug!(
-            "postgres applied {} WAL records ({} bytes) in {} us to reconstruct page image at LSN {}",
-            len,
-            nbytes,
+            "postgres applied {} WAL records in {} us to reconstruct page image at LSN {}",
+            records.len(),
             duration.as_micros(),
             lsn
         );
@@ -270,9 +258,8 @@ impl PostgresRedoManager {
         // next request will launch a new one.
         if result.is_err() {
             error!(
-                "error applying {} WAL records ({} bytes) to reconstruct page image at LSN {}",
+                "error applying {} WAL records to reconstruct page image at LSN {}",
                 records.len(),
-                nbytes,
                 lsn
             );
             let process = process_guard.take().unwrap();
@@ -610,26 +597,13 @@ impl PostgresRedoProcess {
             );
             fs::remove_dir_all(&datadir)?;
         }
-        let pg_bin_dir_path = conf.pg_bin_dir(pg_version).map_err(|e| {
-            Error::new(
-                ErrorKind::Other,
-                format!("incorrect pg_bin_dir path: {}", e),
-            )
-        })?;
-        let pg_lib_dir_path = conf.pg_lib_dir(pg_version).map_err(|e| {
-            Error::new(
-                ErrorKind::Other,
-                format!("incorrect pg_lib_dir path: {}", e),
-            )
-        })?;
-
         info!("running initdb in {}", datadir.display());
-        let initdb = Command::new(pg_bin_dir_path.join("initdb"))
+        let initdb = Command::new(conf.pg_bin_dir(pg_version).join("initdb"))
             .args(&["-D", &datadir.to_string_lossy()])
             .arg("-N")
             .env_clear()
-            .env("LD_LIBRARY_PATH", &pg_lib_dir_path)
-            .env("DYLD_LIBRARY_PATH", &pg_lib_dir_path) // macOS
+            .env("LD_LIBRARY_PATH", conf.pg_lib_dir(pg_version))
+            .env("DYLD_LIBRARY_PATH", conf.pg_lib_dir(pg_version))
             .close_fds()
             .output()
             .map_err(|e| Error::new(e.kind(), format!("failed to execute initdb: {e}")))?;
@@ -655,14 +629,14 @@ impl PostgresRedoProcess {
         }
 
         // Start postgres itself
-        let mut child = Command::new(pg_bin_dir_path.join("postgres"))
+        let mut child = Command::new(conf.pg_bin_dir(pg_version).join("postgres"))
             .arg("--wal-redo")
             .stdin(Stdio::piped())
             .stderr(Stdio::piped())
             .stdout(Stdio::piped())
             .env_clear()
-            .env("LD_LIBRARY_PATH", &pg_lib_dir_path)
-            .env("DYLD_LIBRARY_PATH", &pg_lib_dir_path)
+            .env("LD_LIBRARY_PATH", conf.pg_lib_dir(pg_version))
+            .env("DYLD_LIBRARY_PATH", conf.pg_lib_dir(pg_version))
             .env("PGDATA", &datadir)
             // The redo process is not trusted, so it runs in seccomp mode
             // (see seccomp in zenith_wal_redo.c). We have to make sure it doesn't

pgxn/neon/libpqwalproposer.c

@@ -10,12 +10,51 @@ struct WalProposerConn
 	PGconn	   *pg_conn;
 	bool		is_nonblocking; /* whether the connection is non-blocking */
 	char	   *recvbuf;		/* last received data from
-								 * walprop_async_read */
+								 * libpqprop_async_read */
 };
 
+/* Prototypes for exported functions */
+static char *libpqprop_error_message(WalProposerConn * conn);
+static WalProposerConnStatusType libpqprop_status(WalProposerConn * conn);
+static WalProposerConn * libpqprop_connect_start(char *conninfo);
+static WalProposerConnectPollStatusType libpqprop_connect_poll(WalProposerConn * conn);
+static bool libpqprop_send_query(WalProposerConn * conn, char *query);
+static WalProposerExecStatusType libpqprop_get_query_result(WalProposerConn * conn);
+static pgsocket libpqprop_socket(WalProposerConn * conn);
+static int	libpqprop_flush(WalProposerConn * conn);
+static void libpqprop_finish(WalProposerConn * conn);
+static PGAsyncReadResult libpqprop_async_read(WalProposerConn * conn, char **buf, int *amount);
+static PGAsyncWriteResult libpqprop_async_write(WalProposerConn * conn, void const *buf, size_t size);
+static bool libpqprop_blocking_write(WalProposerConn * conn, void const *buf, size_t size);
+
+static WalProposerFunctionsType PQWalProposerFunctions =
+{
+	libpqprop_error_message,
+		libpqprop_status,
+		libpqprop_connect_start,
+		libpqprop_connect_poll,
+		libpqprop_send_query,
+		libpqprop_get_query_result,
+		libpqprop_socket,
+		libpqprop_flush,
+		libpqprop_finish,
+		libpqprop_async_read,
+		libpqprop_async_write,
+		libpqprop_blocking_write,
+};
+
+/* Module initialization */
+void
+pg_init_libpqwalproposer(void)
+{
+	if (WalProposerFunctions != NULL)
+		elog(ERROR, "libpqwalproposer already loaded");
+	WalProposerFunctions = &PQWalProposerFunctions;
+}
+
 /* Helper function */
 static bool
-ensure_nonblocking_status(WalProposerConn *conn, bool is_nonblocking)
+ensure_nonblocking_status(WalProposerConn * conn, bool is_nonblocking)
 {
 	/* If we're already correctly blocking or nonblocking, all good */
 	if (is_nonblocking == conn->is_nonblocking)
@@ -30,14 +69,14 @@ ensure_nonblocking_status(WalProposerConn *conn, bool is_nonblocking)
 }
 
 /* Exported function definitions */
-char *
-walprop_error_message(WalProposerConn *conn)
+static char *
+libpqprop_error_message(WalProposerConn * conn)
 {
 	return PQerrorMessage(conn->pg_conn);
 }
 
-WalProposerConnStatusType
-walprop_status(WalProposerConn *conn)
+static WalProposerConnStatusType
+libpqprop_status(WalProposerConn * conn)
 {
 	switch (PQstatus(conn->pg_conn))
 	{
@@ -50,8 +89,8 @@ walprop_status(WalProposerConn *conn)
 	}
 }
 
-WalProposerConn *
-walprop_connect_start(char *conninfo)
+static WalProposerConn *
+libpqprop_connect_start(char *conninfo)
 {
 	WalProposerConn *conn;
 	PGconn	   *pg_conn;
@@ -80,8 +119,8 @@ walprop_connect_start(char *conninfo)
 	return conn;
 }
 
-WalProposerConnectPollStatusType
-walprop_connect_poll(WalProposerConn *conn)
+static WalProposerConnectPollStatusType
+libpqprop_connect_poll(WalProposerConn * conn)
 {
 	WalProposerConnectPollStatusType return_val;
 
@@ -121,8 +160,8 @@ walprop_connect_poll(WalProposerConn *conn)
 	return return_val;
 }
 
-bool
-walprop_send_query(WalProposerConn *conn, char *query)
+static bool
+libpqprop_send_query(WalProposerConn * conn, char *query)
 {
 	/*
 	 * We need to be in blocking mode for sending the query to run without
@@ -138,8 +177,8 @@ walprop_send_query(WalProposerConn *conn, char *query)
 	return true;
 }
 
-WalProposerExecStatusType
-walprop_get_query_result(WalProposerConn *conn)
+static WalProposerExecStatusType
+libpqprop_get_query_result(WalProposerConn * conn)
 {
 	PGresult   *result;
 	WalProposerExecStatusType return_val;
@@ -216,20 +255,20 @@ walprop_get_query_result(WalProposerConn *conn)
 	return return_val;
 }
 
-pgsocket
-walprop_socket(WalProposerConn *conn)
+static pgsocket
+libpqprop_socket(WalProposerConn * conn)
 {
 	return PQsocket(conn->pg_conn);
 }
 
-int
-walprop_flush(WalProposerConn *conn)
+static int
+libpqprop_flush(WalProposerConn * conn)
 {
 	return (PQflush(conn->pg_conn));
 }
 
-void
-walprop_finish(WalProposerConn *conn)
+static void
+libpqprop_finish(WalProposerConn * conn)
 {
 	if (conn->recvbuf != NULL)
 		PQfreemem(conn->recvbuf);
@@ -243,8 +282,8 @@ walprop_finish(WalProposerConn *conn)
  * On success, the data is placed in *buf. It is valid until the next call
  * to this function.
  */
-PGAsyncReadResult
-walprop_async_read(WalProposerConn *conn, char **buf, int *amount)
+static PGAsyncReadResult
+libpqprop_async_read(WalProposerConn * conn, char **buf, int *amount)
 {
 	int			result;
 
@@ -314,8 +353,8 @@ walprop_async_read(WalProposerConn *conn, char **buf, int *amount)
 	}
 }
 
-PGAsyncWriteResult
-walprop_async_write(WalProposerConn *conn, void const *buf, size_t size)
+static PGAsyncWriteResult
+libpqprop_async_write(WalProposerConn * conn, void const *buf, size_t size)
 {
 	int			result;
 
@@ -369,12 +408,8 @@ walprop_async_write(WalProposerConn *conn, void const *buf, size_t size)
 	}
 }
 
-/*
- * This function is very similar to walprop_async_write. For more
- * information, refer to the comments there.
- */
-bool
-walprop_blocking_write(WalProposerConn *conn, void const *buf, size_t size)
+static bool
+libpqprop_blocking_write(WalProposerConn * conn, void const *buf, size_t size)
 {
 	int			result;
 
@@ -382,6 +417,10 @@ walprop_blocking_write(WalProposerConn *conn, void const *buf, size_t size)
 	if (!ensure_nonblocking_status(conn, false))
 		return false;
 
+	/*
+	 * Ths function is very similar to libpqprop_async_write. For more
+	 * information, refer to the comments there
+	 */
 	if ((result = PQputCopyData(conn->pg_conn, buf, size)) == -1)
 		return false;
 

pgxn/neon/neon.c

@@ -32,6 +32,7 @@ void
 _PG_init(void)
 {
 	pg_init_libpagestore();
+	pg_init_libpqwalproposer();
 	pg_init_walproposer();
 
 	EmitWarningsOnPlaceholders("neon");

pgxn/neon/neon.h

@@ -13,6 +13,7 @@
 #define NEON_H
 
 extern void pg_init_libpagestore(void);
+extern void pg_init_libpqwalproposer(void);
 extern void pg_init_walproposer(void);
 
 #endif							/* NEON_H */

pgxn/neon/walproposer.c

@@ -79,6 +79,9 @@ bool		am_wal_proposer;
 char	   *neon_timeline_walproposer = NULL;
 char	   *neon_tenant_walproposer = NULL;
 
+/* Declared in walproposer.h, defined here, initialized in libpqwalproposer.c */
+WalProposerFunctionsType *WalProposerFunctions = NULL;
+
 #define WAL_PROPOSER_SLOT_NAME "wal_proposer_slot"
 
 static int	n_safekeepers = 0;
@@ -435,6 +438,10 @@ WalProposerInitImpl(XLogRecPtr flushRecPtr, uint64 systemId)
 	char	   *sep;
 	char	   *port;
 
+	/* Load the libpq-specific functions */
+	if (WalProposerFunctions == NULL)
+		elog(ERROR, "libpqwalproposer didn't initialize correctly");
+
 	load_file("libpqwalreceiver", false);
 	if (WalReceiverFunctions == NULL)
 		elog(ERROR, "libpqwalreceiver didn't initialize correctly");

pgxn/neon/walproposer.h

@@ -446,31 +446,31 @@ typedef enum
 }			WalProposerConnStatusType;
 
 /* Re-exported PQerrorMessage */
-extern char *walprop_error_message(WalProposerConn *conn);
+typedef char *(*walprop_error_message_fn) (WalProposerConn * conn);
 
 /* Re-exported PQstatus */
-extern WalProposerConnStatusType walprop_status(WalProposerConn *conn);
+typedef WalProposerConnStatusType(*walprop_status_fn) (WalProposerConn * conn);
 
 /* Re-exported PQconnectStart */
-extern WalProposerConn * walprop_connect_start(char *conninfo);
+typedef WalProposerConn * (*walprop_connect_start_fn) (char *conninfo);
 
 /* Re-exported PQconectPoll */
-extern WalProposerConnectPollStatusType walprop_connect_poll(WalProposerConn *conn);
+typedef WalProposerConnectPollStatusType(*walprop_connect_poll_fn) (WalProposerConn * conn);
 
 /* Blocking wrapper around PQsendQuery */
-extern bool walprop_send_query(WalProposerConn *conn, char *query);
+typedef bool (*walprop_send_query_fn) (WalProposerConn * conn, char *query);
 
 /* Wrapper around PQconsumeInput + PQisBusy + PQgetResult */
-extern WalProposerExecStatusType walprop_get_query_result(WalProposerConn *conn);
+typedef WalProposerExecStatusType(*walprop_get_query_result_fn) (WalProposerConn * conn);
 
 /* Re-exported PQsocket */
-extern pgsocket walprop_socket(WalProposerConn *conn);
+typedef pgsocket (*walprop_socket_fn) (WalProposerConn * conn);
 
 /* Wrapper around PQconsumeInput (if socket's read-ready) + PQflush */
-extern int	walprop_flush(WalProposerConn *conn);
+typedef int (*walprop_flush_fn) (WalProposerConn * conn);
 
 /* Re-exported PQfinish */
-extern void walprop_finish(WalProposerConn *conn);
+typedef void (*walprop_finish_fn) (WalProposerConn * conn);
 
 /*
  * Ergonomic wrapper around PGgetCopyData
@@ -486,7 +486,9 @@ extern void walprop_finish(WalProposerConn *conn);
  * performs a bit of extra checking work that's always required and is normally
  * somewhat verbose.
  */
-extern PGAsyncReadResult walprop_async_read(WalProposerConn *conn, char **buf, int *amount);
+typedef PGAsyncReadResult(*walprop_async_read_fn) (WalProposerConn * conn,
+												   char **buf,
+												   int *amount);
 
 /*
  * Ergonomic wrapper around PQputCopyData + PQflush
@@ -495,14 +497,69 @@ extern PGAsyncReadResult walprop_async_read(WalProposerConn *conn, char **buf, i
  *
  * For information on the meaning of return codes, refer to PGAsyncWriteResult.
  */
-extern PGAsyncWriteResult walprop_async_write(WalProposerConn *conn, void const *buf, size_t size);
+typedef PGAsyncWriteResult(*walprop_async_write_fn) (WalProposerConn * conn,
+													 void const *buf,
+													 size_t size);
 
 /*
  * Blocking equivalent to walprop_async_write_fn
  *
  * Returns 'true' if successful, 'false' on failure.
  */
-extern bool walprop_blocking_write(WalProposerConn *conn, void const *buf, size_t size);
+typedef bool (*walprop_blocking_write_fn) (WalProposerConn * conn, void const *buf, size_t size);
+
+/* All libpqwalproposer exported functions collected together. */
+typedef struct WalProposerFunctionsType
+{
+	walprop_error_message_fn walprop_error_message;
+	walprop_status_fn walprop_status;
+	walprop_connect_start_fn walprop_connect_start;
+	walprop_connect_poll_fn walprop_connect_poll;
+	walprop_send_query_fn walprop_send_query;
+	walprop_get_query_result_fn walprop_get_query_result;
+	walprop_socket_fn walprop_socket;
+	walprop_flush_fn walprop_flush;
+	walprop_finish_fn walprop_finish;
+	walprop_async_read_fn walprop_async_read;
+	walprop_async_write_fn walprop_async_write;
+	walprop_blocking_write_fn walprop_blocking_write;
+}			WalProposerFunctionsType;
+
+/* Allow the above functions to be "called" with normal syntax */
+#define walprop_error_message(conn) \
+	WalProposerFunctions->walprop_error_message(conn)
+#define walprop_status(conn) \
+	WalProposerFunctions->walprop_status(conn)
+#define walprop_connect_start(conninfo) \
+	WalProposerFunctions->walprop_connect_start(conninfo)
+#define walprop_connect_poll(conn) \
+	WalProposerFunctions->walprop_connect_poll(conn)
+#define walprop_send_query(conn, query) \
+	WalProposerFunctions->walprop_send_query(conn, query)
+#define walprop_get_query_result(conn) \
+	WalProposerFunctions->walprop_get_query_result(conn)
+#define walprop_set_nonblocking(conn, arg) \
+	WalProposerFunctions->walprop_set_nonblocking(conn, arg)
+#define walprop_socket(conn) \
+	WalProposerFunctions->walprop_socket(conn)
+#define walprop_flush(conn) \
+	WalProposerFunctions->walprop_flush(conn)
+#define walprop_finish(conn) \
+	WalProposerFunctions->walprop_finish(conn)
+#define walprop_async_read(conn, buf, amount) \
+	WalProposerFunctions->walprop_async_read(conn, buf, amount)
+#define walprop_async_write(conn, buf, size) \
+	WalProposerFunctions->walprop_async_write(conn, buf, size)
+#define walprop_blocking_write(conn, buf, size) \
+	WalProposerFunctions->walprop_blocking_write(conn, buf, size)
+
+/*
+ * The runtime location of the libpqwalproposer functions.
+ *
+ * This pointer is set by the initializer in libpqwalproposer, so that we
+ * can use it later.
+ */
+extern PGDLLIMPORT WalProposerFunctionsType * WalProposerFunctions;
 
 extern uint64 BackpressureThrottlingTime(void);
 

proxy/Cargo.toml

@@ -7,9 +7,9 @@ edition = "2021"
 anyhow = "1.0"
 atty = "0.2.14"
 base64 = "0.13.0"
-bstr = "1.0"
+bstr = "0.2.17"
 bytes = { version = "1.0.1", features = ['serde'] }
-clap = "4.0"
+clap = "3.0"
 futures = "0.3.13"
 git-version = "0.3.5"
 hashbrown = "0.12"
@@ -22,11 +22,7 @@ once_cell = "1.13.0"
 parking_lot = "0.12"
 pin-project-lite = "0.2.7"
 rand = "0.8.3"
-reqwest = { version = "0.11", default-features = false, features = [
-    "blocking",
-    "json",
-    "rustls-tls",
-] }
+reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "rustls-tls"] }
 routerify = "3"
 rustls = "0.20.0"
 rustls-pemfile = "1"
@@ -37,13 +33,13 @@ sha2 = "0.10.2"
 socket2 = "0.4.4"
 thiserror = "1.0.30"
 tokio = { version = "1.17", features = ["macros"] }
-tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev = "d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
 tokio-rustls = "0.23.0"
 tracing = "0.1.36"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 url = "2.2.2"
-uuid = { version = "1.2", features = ["v4", "serde"] }
-x509-parser = "0.14"
+uuid = { version = "0.8.2", features = ["v4", "serde"]}
+x509-parser = "0.13.2"
 
 utils = { path = "../libs/utils" }
 metrics = { path = "../libs/metrics" }
@@ -51,6 +47,6 @@ workspace_hack = { version = "0.1", path = "../workspace_hack" }
 
 [dev-dependencies]
 async-trait = "0.1"
-rcgen = "0.10"
-rstest = "0.15"
+rcgen = "0.8.14"
+rstest = "0.12"
 tokio-postgres-rustls = "0.9.0"

proxy/src/main.rs

@@ -45,43 +45,98 @@ async fn main() -> anyhow::Result<()> {
         .with_target(false)
         .init();
 
-    let arg_matches = cli().get_matches();
+    let arg_matches = clap::App::new("Neon proxy/router")
+        .version(GIT_VERSION)
+        .arg(
+            Arg::new("proxy")
+                .short('p')
+                .long("proxy")
+                .takes_value(true)
+                .help("listen for incoming client connections on ip:port")
+                .default_value("127.0.0.1:4432"),
+        )
+        .arg(
+            Arg::new("auth-backend")
+                .long("auth-backend")
+                .takes_value(true)
+                .possible_values(["console", "postgres", "link"])
+                .default_value("link"),
+        )
+        .arg(
+            Arg::new("mgmt")
+                .short('m')
+                .long("mgmt")
+                .takes_value(true)
+                .help("listen for management callback connection on ip:port")
+                .default_value("127.0.0.1:7000"),
+        )
+        .arg(
+            Arg::new("http")
+                .short('h')
+                .long("http")
+                .takes_value(true)
+                .help("listen for incoming http connections (metrics, etc) on ip:port")
+                .default_value("127.0.0.1:7001"),
+        )
+        .arg(
+            Arg::new("uri")
+                .short('u')
+                .long("uri")
+                .takes_value(true)
+                .help("redirect unauthenticated users to the given uri in case of link auth")
+                .default_value("http://localhost:3000/psql_session/"),
+        )
+        .arg(
+            Arg::new("auth-endpoint")
+                .short('a')
+                .long("auth-endpoint")
+                .takes_value(true)
+                .help("cloud API endpoint for authenticating users")
+                .default_value("http://localhost:3000/authenticate_proxy_request/"),
+        )
+        .arg(
+            Arg::new("tls-key")
+                .short('k')
+                .long("tls-key")
+                .alias("ssl-key") // backwards compatibility
+                .takes_value(true)
+                .help("path to TLS key for client postgres connections"),
+        )
+        .arg(
+            Arg::new("tls-cert")
+                .short('c')
+                .long("tls-cert")
+                .alias("ssl-cert") // backwards compatibility
+                .takes_value(true)
+                .help("path to TLS cert for client postgres connections"),
+        )
+        .get_matches();
 
     let tls_config = match (
-        arg_matches.get_one::<String>("tls-key"),
-        arg_matches.get_one::<String>("tls-cert"),
+        arg_matches.value_of("tls-key"),
+        arg_matches.value_of("tls-cert"),
     ) {
         (Some(key_path), Some(cert_path)) => Some(config::configure_tls(key_path, cert_path)?),
         (None, None) => None,
         _ => bail!("either both or neither tls-key and tls-cert must be specified"),
     };
 
-    let proxy_address: SocketAddr = arg_matches.get_one::<String>("proxy").unwrap().parse()?;
-    let mgmt_address: SocketAddr = arg_matches.get_one::<String>("mgmt").unwrap().parse()?;
-    let http_address: SocketAddr = arg_matches.get_one::<String>("http").unwrap().parse()?;
+    let proxy_address: SocketAddr = arg_matches.value_of("proxy").unwrap().parse()?;
+    let mgmt_address: SocketAddr = arg_matches.value_of("mgmt").unwrap().parse()?;
+    let http_address: SocketAddr = arg_matches.value_of("http").unwrap().parse()?;
 
-    let auth_backend = match arg_matches
-        .get_one::<String>("auth-backend")
-        .unwrap()
-        .as_str()
-    {
+    let auth_backend = match arg_matches.value_of("auth-backend").unwrap() {
         "console" => {
-            let url = arg_matches
-                .get_one::<String>("auth-endpoint")
-                .unwrap()
-                .parse()?;
+            let url = arg_matches.value_of("auth-endpoint").unwrap().parse()?;
             let endpoint = http::Endpoint::new(url, reqwest::Client::new());
             auth::BackendType::Console(Cow::Owned(endpoint), ())
         }
         "postgres" => {
-            let url = arg_matches
-                .get_one::<String>("auth-endpoint")
-                .unwrap()
-                .parse()?;
+            let url = arg_matches.value_of("auth-endpoint").unwrap().parse()?;
             auth::BackendType::Postgres(Cow::Owned(url), ())
         }
         "link" => {
-            let url = arg_matches.get_one::<String>("uri").unwrap().parse()?;
+            let url = arg_matches.value_of("uri").unwrap().parse()?;
             auth::BackendType::Link(Cow::Owned(url))
         }
         other => bail!("unsupported auth backend: {other}"),
@@ -119,68 +174,3 @@ async fn main() -> anyhow::Result<()> {
 
     Ok(())
 }
-
-fn cli() -> clap::Command {
-    clap::Command::new("Neon proxy/router")
-        .disable_help_flag(true)
-        .version(GIT_VERSION)
-        .arg(
-            Arg::new("proxy")
-                .short('p')
-                .long("proxy")
-                .help("listen for incoming client connections on ip:port")
-                .default_value("127.0.0.1:4432"),
-        )
-        .arg(
-            Arg::new("auth-backend")
-                .long("auth-backend")
-                .value_parser(["console", "postgres", "link"])
-                .default_value("link"),
-        )
-        .arg(
-            Arg::new("mgmt")
-                .short('m')
-                .long("mgmt")
-                .help("listen for management callback connection on ip:port")
-                .default_value("127.0.0.1:7000"),
-        )
-        .arg(
-            Arg::new("http")
-                .long("http")
-                .help("listen for incoming http connections (metrics, etc) on ip:port")
-                .default_value("127.0.0.1:7001"),
-        )
-        .arg(
-            Arg::new("uri")
-                .short('u')
-                .long("uri")
-                .help("redirect unauthenticated users to the given uri in case of link auth")
-                .default_value("http://localhost:3000/psql_session/"),
-        )
-        .arg(
-            Arg::new("auth-endpoint")
-                .short('a')
-                .long("auth-endpoint")
-                .help("cloud API endpoint for authenticating users")
-                .default_value("http://localhost:3000/authenticate_proxy_request/"),
-        )
-        .arg(
-            Arg::new("tls-key")
-                .short('k')
-                .long("tls-key")
-                .alias("ssl-key") // backwards compatibility
-                .help("path to TLS key for client postgres connections"),
-        )
-        .arg(
-            Arg::new("tls-cert")
-                .short('c')
-                .long("tls-cert")
-                .alias("ssl-cert") // backwards compatibility
-                .help("path to TLS cert for client postgres connections"),
-        )
-}
-
-#[test]
-fn verify_cli() {
-    cli().debug_assert();
-}

safekeeper/Cargo.toml

@@ -11,7 +11,7 @@ hyper = "0.14"
 fs2 = "0.4.3"
 serde_json = "1"
 tracing = "0.1.27"
-clap = "4.0"
+clap = "3.0"
 daemonize = "0.4.1"
 tokio = { version = "1.17", features = ["macros", "fs"] }
 postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }

safekeeper/src/bin/safekeeper.rs

@@ -2,7 +2,7 @@
 // Main entry point for the safekeeper executable
 //
 use anyhow::{bail, Context, Result};
-use clap::{value_parser, Arg, ArgAction, Command};
+use clap::{App, Arg};
 use const_format::formatcp;
 use daemonize::Daemonize;
 use fs2::FileExt;
@@ -21,7 +21,8 @@ use metrics::set_build_info_metric;
 use safekeeper::broker;
 use safekeeper::control_file;
 use safekeeper::defaults::{
-    DEFAULT_HTTP_LISTEN_ADDR, DEFAULT_PG_LISTEN_ADDR, DEFAULT_WAL_BACKUP_RUNTIME_THREADS,
+    DEFAULT_HEARTBEAT_TIMEOUT, DEFAULT_HTTP_LISTEN_ADDR, DEFAULT_MAX_OFFLOADER_LAG,
+    DEFAULT_PG_LISTEN_ADDR, DEFAULT_WAL_BACKUP_RUNTIME_THREADS,
 };
 use safekeeper::http;
 use safekeeper::remove_wal;
@@ -40,44 +41,147 @@ const ID_FILE_NAME: &str = "safekeeper.id";
 project_git_version!(GIT_VERSION);
 
 fn main() -> anyhow::Result<()> {
-    let arg_matches = cli().get_matches();
+    let arg_matches = App::new("Neon safekeeper")
+        .about("Store WAL stream to local file system and push it to WAL receivers")
+        .version(GIT_VERSION)
+        .arg(
+            Arg::new("datadir")
+                .short('D')
+                .long("dir")
+                .takes_value(true)
+                .help("Path to the safekeeper data directory"),
+        )
+        .arg(
+            Arg::new("init")
+                .long("init")
+                .takes_value(false)
+                .help("Initialize safekeeper with ID"),
+        )
+        .arg(
+            Arg::new("listen-pg")
+                .short('l')
+                .long("listen-pg")
+                .alias("listen") // for compatibility
+                .takes_value(true)
+                .help(formatcp!("listen for incoming WAL data connections on ip:port (default: {DEFAULT_PG_LISTEN_ADDR})")),
+        )
+        .arg(
+            Arg::new("listen-http")
+                .long("listen-http")
+                .takes_value(true)
+                .help(formatcp!("http endpoint address for metrics on ip:port (default: {DEFAULT_HTTP_LISTEN_ADDR})")),
+        )
+        // FIXME this argument is no longer needed since pageserver address is forwarded from compute.
+        // However because this argument is in use by console's e2e tests let's keep it for now and remove separately.
+        // So currently it is a noop.
+        .arg(
+            Arg::new("pageserver")
+                .short('p')
+                .long("pageserver")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::new("daemonize")
+                .short('d')
+                .long("daemonize")
+                .takes_value(false)
+                .help("Run in the background"),
+        )
+        .arg(
+            Arg::new("no-sync")
+                .short('n')
+                .long("no-sync")
+                .takes_value(false)
+                .help("Do not wait for changes to be written safely to disk"),
+        )
+        .arg(
+            Arg::new("dump-control-file")
+                .long("dump-control-file")
+                .takes_value(true)
+                .help("Dump control file at path specified by this argument and exit"),
+        )
+        .arg(
+            Arg::new("id").long("id").takes_value(true).help("safekeeper node id: integer")
+        ).arg(
+            Arg::new("broker-endpoints")
+            .long("broker-endpoints")
+            .takes_value(true)
+            .help("a comma separated broker (etcd) endpoints for storage nodes coordination, e.g. 'http://127.0.0.1:2379'"),
+        )
+        .arg(
+            Arg::new("broker-etcd-prefix")
+            .long("broker-etcd-prefix")
+            .takes_value(true)
+            .help("a prefix to always use when polling/pusing data in etcd from this safekeeper"),
+        )
+        .arg(
+            Arg::new("heartbeat-timeout")
+                .long("heartbeat-timeout")
+                .takes_value(true)
+                .help(formatcp!("Peer is considered dead after not receiving heartbeats from it during this period (default {}s), passed as a human readable duration.", DEFAULT_HEARTBEAT_TIMEOUT.as_secs()))
+        )
+        .arg(
+            Arg::new("wal-backup-threads").long("backup-threads").takes_value(true).help(formatcp!("number of threads for wal backup (default {DEFAULT_WAL_BACKUP_RUNTIME_THREADS}")),
+        ).arg(
+            Arg::new("remote-storage")
+                .long("remote-storage")
+                .takes_value(true)
+                .help("Remote storage configuration for WAL backup (offloading to s3) as TOML inline table, e.g. {\"max_concurrent_syncs\" = 17, \"max_sync_errors\": 13, \"bucket_name\": \"<BUCKETNAME>\", \"bucket_region\":\"<REGION>\", \"concurrency_limit\": 119}.\nSafekeeper offloads WAL to [prefix_in_bucket/]<tenant_id>/<timeline_id>/<segment_file>, mirroring structure on the file system.")
+        )
+        .arg(
+            Arg::new("max-offloader-lag")
+                .long("max-offloader-lag")
+                .takes_value(true)
+                .help(formatcp!("Safekeeper won't be elected for WAL offloading if it is lagging for more than this value (default {}MB) in bytes", DEFAULT_MAX_OFFLOADER_LAG / (1 << 20)))
+        )
+        .arg(
+            Arg::new("enable-wal-backup")
+                .long("enable-wal-backup")
+                .takes_value(true)
+                .default_value("true")
+                .default_missing_value("true")
+                .help("Enable/disable WAL backup to s3. When disabled, safekeeper removes WAL ignoring WAL backup horizon."),
+        )
+        .arg(
+            Arg::new("auth-validation-public-key-path")
+                .long("auth-validation-public-key-path")
+                .takes_value(true)
+                .help("Path to an RSA .pem public key which is used to check JWT tokens")
+        )
+        .get_matches();
 
-    if let Some(addr) = arg_matches.get_one::<String>("dump-control-file") {
+    if let Some(addr) = arg_matches.value_of("dump-control-file") {
         let state = control_file::FileStorage::load_control_file(Path::new(addr))?;
         let json = serde_json::to_string(&state)?;
-        print!("{json}");
+        print!("{}", json);
         return Ok(());
     }
 
     let mut conf = SafeKeeperConf::default();
 
-    if let Some(dir) = arg_matches.get_one::<PathBuf>("datadir") {
+    if let Some(dir) = arg_matches.value_of("datadir") {
         // change into the data directory.
-        std::env::set_current_dir(dir)?;
+        std::env::set_current_dir(PathBuf::from(dir))?;
     }
 
-    if arg_matches.get_flag("no-sync") {
+    if arg_matches.is_present("no-sync") {
         conf.no_sync = true;
     }
 
-    if arg_matches.get_flag("daemonize") {
+    if arg_matches.is_present("daemonize") {
         conf.daemonize = true;
     }
 
-    if let Some(addr) = arg_matches.get_one::<String>("listen-pg") {
-        conf.listen_pg_addr = addr.to_string();
+    if let Some(addr) = arg_matches.value_of("listen-pg") {
+        conf.listen_pg_addr = addr.to_owned();
     }
 
-    if let Some(addr) = arg_matches.get_one::<String>("listen-http") {
-        conf.listen_http_addr = addr.to_string();
-    }
-
-    if let Some(recall) = arg_matches.get_one::<String>("recall") {
-        conf.recall_period = humantime::parse_duration(recall)?;
+    if let Some(addr) = arg_matches.value_of("listen-http") {
+        conf.listen_http_addr = addr.to_owned();
     }
 
     let mut given_id = None;
-    if let Some(given_id_str) = arg_matches.get_one::<String>("id") {
+    if let Some(given_id_str) = arg_matches.value_of("id") {
         given_id = Some(NodeId(
             given_id_str
                 .parse()
@@ -85,38 +189,56 @@ fn main() -> anyhow::Result<()> {
         ));
     }
 
-    if let Some(addr) = arg_matches.get_one::<String>("broker-endpoints") {
+    if let Some(addr) = arg_matches.value_of("broker-endpoints") {
         let collected_ep: Result<Vec<Url>, ParseError> = addr.split(',').map(Url::parse).collect();
         conf.broker_endpoints = collected_ep.context("Failed to parse broker endpoint urls")?;
     }
-    if let Some(prefix) = arg_matches.get_one::<String>("broker-etcd-prefix") {
+    if let Some(prefix) = arg_matches.value_of("broker-etcd-prefix") {
         conf.broker_etcd_prefix = prefix.to_string();
     }
 
-    if let Some(backup_threads) = arg_matches.get_one::<String>("wal-backup-threads") {
+    if let Some(heartbeat_timeout_str) = arg_matches.value_of("heartbeat-timeout") {
+        conf.heartbeat_timeout =
+            humantime::parse_duration(heartbeat_timeout_str).with_context(|| {
+                format!(
+                    "failed to parse heartbeat-timeout {}",
+                    heartbeat_timeout_str
+                )
+            })?;
+    }
+
+    if let Some(backup_threads) = arg_matches.value_of("wal-backup-threads") {
         conf.backup_runtime_threads = backup_threads
             .parse()
             .with_context(|| format!("Failed to parse backup threads {}", backup_threads))?;
     }
-    if let Some(storage_conf) = arg_matches.get_one::<String>("remote-storage") {
+    if let Some(storage_conf) = arg_matches.value_of("remote-storage") {
         // funny toml doesn't consider plain inline table as valid document, so wrap in a key to parse
         let storage_conf_toml = format!("remote_storage = {}", storage_conf);
         let parsed_toml = storage_conf_toml.parse::<Document>()?; // parse
         let (_, storage_conf_parsed_toml) = parsed_toml.iter().next().unwrap(); // and strip key off again
         conf.remote_storage = Some(RemoteStorageConfig::from_toml(storage_conf_parsed_toml)?);
     }
+    if let Some(max_offloader_lag_str) = arg_matches.value_of("max-offloader-lag") {
+        conf.max_offloader_lag = max_offloader_lag_str.parse().with_context(|| {
+            format!(
+                "failed to parse max offloader lag {}",
+                max_offloader_lag_str
+            )
+        })?;
+    }
     // Seems like there is no better way to accept bool values explicitly in clap.
     conf.wal_backup_enabled = arg_matches
-        .get_one::<String>("enable-wal-backup")
+        .value_of("enable-wal-backup")
         .unwrap()
         .parse()
         .context("failed to parse bool enable-s3-offload bool")?;
 
     conf.auth_validation_public_key_path = arg_matches
-        .get_one::<String>("auth-validation-public-key-path")
+        .value_of("auth-validation-public-key-path")
         .map(PathBuf::from);
 
-    start_safekeeper(conf, given_id, arg_matches.get_flag("init"))
+    start_safekeeper(conf, given_id, arg_matches.is_present("init"))
 }
 
 fn start_safekeeper(mut conf: SafeKeeperConf, given_id: Option<NodeId>, init: bool) -> Result<()> {
@@ -323,102 +445,3 @@ fn set_id(conf: &mut SafeKeeperConf, given_id: Option<NodeId>) -> Result<()> {
     conf.my_id = my_id;
     Ok(())
 }
-
-fn cli() -> Command {
-    Command::new("Neon safekeeper")
-        .about("Store WAL stream to local file system and push it to WAL receivers")
-        .version(GIT_VERSION)
-        .arg(
-            Arg::new("datadir")
-                .short('D')
-                .long("dir")
-                .value_parser(value_parser!(PathBuf))
-                .help("Path to the safekeeper data directory"),
-        )
-        .arg(
-            Arg::new("init")
-                .long("init")
-                .action(ArgAction::SetTrue)
-                .help("Initialize safekeeper with ID"),
-        )
-        .arg(
-            Arg::new("listen-pg")
-                .short('l')
-                .long("listen-pg")
-                .alias("listen") // for compatibility
-                .help(formatcp!("listen for incoming WAL data connections on ip:port (default: {DEFAULT_PG_LISTEN_ADDR})")),
-        )
-        .arg(
-            Arg::new("listen-http")
-                .long("listen-http")
-                .help(formatcp!("http endpoint address for metrics on ip:port (default: {DEFAULT_HTTP_LISTEN_ADDR})")),
-        )
-        // FIXME this argument is no longer needed since pageserver address is forwarded from compute.
-        // However because this argument is in use by console's e2e tests let's keep it for now and remove separately.
-        // So currently it is a noop.
-        .arg(
-            Arg::new("pageserver")
-                .short('p')
-                .long("pageserver"),
-        )
-        .arg(
-            Arg::new("recall")
-                .long("recall")
-                .help("Period for requestion pageserver to call for replication"),
-        )
-        .arg(
-            Arg::new("daemonize")
-                .short('d')
-                .long("daemonize")
-                .action(ArgAction::SetTrue)
-                .help("Run in the background"),
-        )
-        .arg(
-            Arg::new("no-sync")
-                .short('n')
-                .long("no-sync")
-                .action(ArgAction::SetTrue)
-                .help("Do not wait for changes to be written safely to disk"),
-        )
-        .arg(
-            Arg::new("dump-control-file")
-                .long("dump-control-file")
-                .help("Dump control file at path specified by this argument and exit"),
-        )
-        .arg(
-            Arg::new("id").long("id").help("safekeeper node id: integer")
-        ).arg(
-            Arg::new("broker-endpoints")
-            .long("broker-endpoints")
-            .help("a comma separated broker (etcd) endpoints for storage nodes coordination, e.g. 'http://127.0.0.1:2379'"),
-        )
-        .arg(
-            Arg::new("broker-etcd-prefix")
-            .long("broker-etcd-prefix")
-            .help("a prefix to always use when polling/pusing data in etcd from this safekeeper"),
-        )
-        .arg(
-            Arg::new("wal-backup-threads").long("backup-threads").help(formatcp!("number of threads for wal backup (default {DEFAULT_WAL_BACKUP_RUNTIME_THREADS}")),
-        ).arg(
-            Arg::new("remote-storage")
-                .long("remote-storage")
-                .help("Remote storage configuration for WAL backup (offloading to s3) as TOML inline table, e.g. {\"max_concurrent_syncs\" = 17, \"max_sync_errors\": 13, \"bucket_name\": \"<BUCKETNAME>\", \"bucket_region\":\"<REGION>\", \"concurrency_limit\": 119}.\nSafekeeper offloads WAL to [prefix_in_bucket/]<tenant_id>/<timeline_id>/<segment_file>, mirroring structure on the file system.")
-        )
-        .arg(
-            Arg::new("enable-wal-backup")
-                .long("enable-wal-backup")
-                .default_value("true")
-                .default_missing_value("true")
-                .help("Enable/disable WAL backup to s3. When disabled, safekeeper removes WAL ignoring WAL backup horizon."),
-        )
-        .arg(
-            Arg::new("auth-validation-public-key-path")
-                .long("auth-validation-public-key-path")
-                .help("Path to an RSA .pem public key which is used to check JWT tokens")
-        )
-}
-
-#[test]
-fn verify_cli() {
-    cli().debug_assert();
-}

safekeeper/src/broker.rs

@@ -1,6 +1,5 @@
 //! Communication with etcd, providing safekeeper peers and pageserver coordination.
 
-use anyhow::anyhow;
 use anyhow::Context;
 use anyhow::Error;
 use anyhow::Result;
@@ -12,11 +11,9 @@ use std::collections::hash_map::Entry;
 use std::collections::HashMap;
 use std::collections::HashSet;
 use std::time::Duration;
-use tokio::spawn;
 use tokio::task::JoinHandle;
 use tokio::{runtime, time::sleep};
 use tracing::*;
-use url::Url;
 
 use crate::GlobalTimelines;
 use crate::SafeKeeperConf;
@@ -56,113 +53,6 @@ fn timeline_safekeeper_path(
     )
 }
 
-pub struct Election {
-    pub election_name: String,
-    pub candidate_name: String,
-    pub broker_endpoints: Vec<Url>,
-}
-
-impl Election {
-    pub fn new(election_name: String, candidate_name: String, broker_endpoints: Vec<Url>) -> Self {
-        Self {
-            election_name,
-            candidate_name,
-            broker_endpoints,
-        }
-    }
-}
-
-pub struct ElectionLeader {
-    client: Client,
-    keep_alive: JoinHandle<Result<()>>,
-}
-
-impl ElectionLeader {
-    pub async fn check_am_i(
-        &mut self,
-        election_name: String,
-        candidate_name: String,
-    ) -> Result<bool> {
-        let resp = self.client.leader(election_name).await?;
-
-        let kv = resp
-            .kv()
-            .ok_or_else(|| anyhow!("failed to get leader response"))?;
-        let leader = kv.value_str()?;
-
-        Ok(leader == candidate_name)
-    }
-
-    pub async fn give_up(self) {
-        self.keep_alive.abort();
-        // TODO: it'll be wise to resign here but it'll happen after lease expiration anyway
-        // should we await for keep alive termination?
-        let _ = self.keep_alive.await;
-    }
-}
-
-pub async fn get_leader(req: &Election, leader: &mut Option<ElectionLeader>) -> Result<()> {
-    let mut client = Client::connect(req.broker_endpoints.clone(), None)
-        .await
-        .context("Could not connect to etcd")?;
-
-    let lease = client
-        .lease_grant(LEASE_TTL_SEC, None)
-        .await
-        .context("Could not acquire a lease");
-
-    let lease_id = lease.map(|l| l.id()).unwrap();
-
-    // kill previous keepalive, if any
-    if let Some(l) = leader.take() {
-        l.give_up().await;
-    }
-
-    let keep_alive = spawn::<_>(lease_keep_alive(client.clone(), lease_id));
-    // immediately save handle to kill task if we get canceled below
-    *leader = Some(ElectionLeader {
-        client: client.clone(),
-        keep_alive,
-    });
-
-    client
-        .campaign(
-            req.election_name.clone(),
-            req.candidate_name.clone(),
-            lease_id,
-        )
-        .await?;
-
-    Ok(())
-}
-
-async fn lease_keep_alive(mut client: Client, lease_id: i64) -> Result<()> {
-    let (mut keeper, mut ka_stream) = client
-        .lease_keep_alive(lease_id)
-        .await
-        .context("failed to create keepalive stream")?;
-
-    loop {
-        let push_interval = Duration::from_millis(PUSH_INTERVAL_MSEC);
-
-        keeper
-            .keep_alive()
-            .await
-            .context("failed to send LeaseKeepAliveRequest")?;
-
-        ka_stream
-            .message()
-            .await
-            .context("failed to receive LeaseKeepAliveResponse")?;
-
-        sleep(push_interval).await;
-    }
-}
-
-pub fn get_candiate_name(system_id: NodeId) -> String {
-    format!("id_{system_id}")
-}
-
 async fn push_sk_info(
     ttid: TenantTimelineId,
     mut client: Client,
@@ -236,7 +126,7 @@ async fn push_loop(conf: SafeKeeperConf) -> anyhow::Result<()> {
         let handles = active_tlis
             .iter()
             .map(|tli| {
-                let sk_info = tli.get_public_info(&conf);
+                let sk_info = tli.get_safekeer_info(&conf);
                 let key =
                     timeline_safekeeper_path(conf.broker_etcd_prefix.clone(), tli.ttid, conf.my_id);
                 let lease = leases.remove(&tli.ttid).unwrap();
@@ -282,6 +172,9 @@ async fn pull_loop(conf: SafeKeeperConf) -> Result<()> {
             Some(new_info) => {
                 // note: there are blocking operations below, but it's considered fine for now
                 if let Ok(tli) = GlobalTimelines::get(new_info.key.id) {
+                    // Note that we also receive *our own* info. That's
+                    // important, as it is used as an indication of live
+                    // connection to the broker.
                     tli.record_safekeeper_info(&new_info.value, new_info.key.node_id)
                         .await?
                 }

safekeeper/src/control_file_upgrade.rs

@@ -1,6 +1,7 @@
 //! Code to deal with safekeeper control file upgrades
 use crate::safekeeper::{
-    AcceptorState, Peers, PgUuid, SafeKeeperState, ServerInfo, Term, TermHistory, TermSwitchEntry,
+    AcceptorState, PersistedPeers, PgUuid, SafeKeeperState, ServerInfo, Term, TermHistory,
+    TermSwitchEntry,
 };
 use anyhow::{bail, Result};
 use serde::{Deserialize, Serialize};
@@ -134,7 +135,7 @@ pub struct SafeKeeperStateV4 {
     // fundamental; but state is saved here only for informational purposes and
     // obviously can be stale. (Currently not saved at all, but let's provision
     // place to have less file version upgrades).
-    pub peers: Peers,
+    pub peers: PersistedPeers,
 }
 
 pub fn upgrade_control_file(buf: &[u8], version: u32) -> Result<SafeKeeperState> {
@@ -165,7 +166,7 @@ pub fn upgrade_control_file(buf: &[u8], version: u32) -> Result<SafeKeeperState>
             backup_lsn: Lsn(0),
             peer_horizon_lsn: oldstate.truncate_lsn,
             remote_consistent_lsn: Lsn(0),
-            peers: Peers(vec![]),
+            peers: PersistedPeers(vec![]),
         });
     // migrate to hexing some ids
     } else if version == 2 {
@@ -188,7 +189,7 @@ pub fn upgrade_control_file(buf: &[u8], version: u32) -> Result<SafeKeeperState>
             backup_lsn: Lsn(0),
             peer_horizon_lsn: oldstate.truncate_lsn,
             remote_consistent_lsn: Lsn(0),
-            peers: Peers(vec![]),
+            peers: PersistedPeers(vec![]),
         });
     // migrate to moving tenant_id/timeline_id to the top and adding some lsns
     } else if version == 3 {
@@ -211,7 +212,7 @@ pub fn upgrade_control_file(buf: &[u8], version: u32) -> Result<SafeKeeperState>
             backup_lsn: Lsn(0),
             peer_horizon_lsn: oldstate.truncate_lsn,
             remote_consistent_lsn: Lsn(0),
-            peers: Peers(vec![]),
+            peers: PersistedPeers(vec![]),
         });
     // migrate to having timeline_start_lsn
     } else if version == 4 {
@@ -234,7 +235,7 @@ pub fn upgrade_control_file(buf: &[u8], version: u32) -> Result<SafeKeeperState>
             backup_lsn: Lsn::INVALID,
             peer_horizon_lsn: oldstate.peer_horizon_lsn,
             remote_consistent_lsn: Lsn(0),
-            peers: Peers(vec![]),
+            peers: PersistedPeers(vec![]),
         });
     } else if version == 5 {
         info!("reading safekeeper control file version {}", version);

safekeeper/src/lib.rs

@@ -1,4 +1,6 @@
-use defaults::DEFAULT_WAL_BACKUP_RUNTIME_THREADS;
+use defaults::{
+    DEFAULT_HEARTBEAT_TIMEOUT, DEFAULT_MAX_OFFLOADER_LAG, DEFAULT_WAL_BACKUP_RUNTIME_THREADS,
+};
 //
 use remote_storage::RemoteStorageConfig;
 use std::path::PathBuf;
@@ -36,6 +38,8 @@ pub mod defaults {
 
     pub const DEFAULT_RECALL_PERIOD: Duration = Duration::from_secs(10);
     pub const DEFAULT_WAL_BACKUP_RUNTIME_THREADS: usize = 8;
+    pub const DEFAULT_HEARTBEAT_TIMEOUT: Duration = Duration::from_secs(5);
+    pub const DEFAULT_MAX_OFFLOADER_LAG: u64 = 128 * (1 << 20);
 }
 
 #[derive(Debug, Clone)]
@@ -60,6 +64,8 @@ pub struct SafeKeeperConf {
     pub broker_endpoints: Vec<Url>,
     pub broker_etcd_prefix: String,
     pub auth_validation_public_key_path: Option<PathBuf>,
+    pub heartbeat_timeout: Duration,
+    pub max_offloader_lag: u64,
 }
 
 impl SafeKeeperConf {
@@ -92,6 +98,8 @@ impl Default for SafeKeeperConf {
             backup_runtime_threads: DEFAULT_WAL_BACKUP_RUNTIME_THREADS,
             wal_backup_enabled: true,
             auth_validation_public_key_path: None,
+            heartbeat_timeout: DEFAULT_HEARTBEAT_TIMEOUT,
+            max_offloader_lag: DEFAULT_MAX_OFFLOADER_LAG,
         }
     }
 }

safekeeper/src/safekeeper.rs

@@ -11,6 +11,7 @@ use std::cmp::max;
 use std::cmp::min;
 use std::fmt;
 use std::io::Read;
+
 use tracing::*;
 
 use crate::control_file;
@@ -132,9 +133,8 @@ pub struct ServerInfo {
     pub wal_seg_size: u32,
 }
 
-/// Data published by safekeeper to the peers
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct PeerInfo {
+pub struct PersistedPeerInfo {
     /// LSN up to which safekeeper offloaded WAL to s3.
     backup_lsn: Lsn,
     /// Term of the last entry.
@@ -145,7 +145,7 @@ pub struct PeerInfo {
     commit_lsn: Lsn,
 }
 
-impl PeerInfo {
+impl PersistedPeerInfo {
     fn new() -> Self {
         Self {
             backup_lsn: Lsn::INVALID,
@@ -156,10 +156,8 @@ impl PeerInfo {
     }
 }
 
-// vector-based node id -> peer state map with very limited functionality we
-// need/
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct Peers(pub Vec<(NodeId, PeerInfo)>);
+pub struct PersistedPeers(pub Vec<(NodeId, PersistedPeerInfo)>);
 
 /// Persistent information stored on safekeeper node
 /// On disk data is prefixed by magic and format version and followed by checksum.
@@ -203,7 +201,7 @@ pub struct SafeKeeperState {
     // fundamental; but state is saved here only for informational purposes and
     // obviously can be stale. (Currently not saved at all, but let's provision
     // place to have less file version upgrades).
-    pub peers: Peers,
+    pub peers: PersistedPeers,
 }
 
 #[derive(Debug, Clone)]
@@ -240,7 +238,12 @@ impl SafeKeeperState {
             backup_lsn: local_start_lsn,
             peer_horizon_lsn: local_start_lsn,
             remote_consistent_lsn: Lsn(0),
-            peers: Peers(peers.iter().map(|p| (*p, PeerInfo::new())).collect()),
+            peers: PersistedPeers(
+                peers
+                    .iter()
+                    .map(|p| (*p, PersistedPeerInfo::new()))
+                    .collect(),
+            ),
         }
     }
 

safekeeper/src/timeline.rs

@@ -7,7 +7,7 @@ use etcd_broker::subscription_value::SkTimelineInfo;
 
 use postgres_ffi::XLogSegNo;
 
-use tokio::sync::watch;
+use tokio::{sync::watch, time::Instant};
 
 use std::cmp::{max, min};
 
@@ -26,7 +26,7 @@ use utils::{
 
 use crate::safekeeper::{
     AcceptorProposerMessage, ProposerAcceptorMessage, SafeKeeper, SafeKeeperState,
-    SafekeeperMemState, ServerInfo,
+    SafekeeperMemState, ServerInfo, Term,
 };
 use crate::send_wal::HotStandbyFeedback;
 use crate::{control_file, safekeeper::UNKNOWN_SERVER_VERSION};
@@ -36,6 +36,53 @@ use crate::wal_storage;
 use crate::wal_storage::Storage as wal_storage_iface;
 use crate::SafeKeeperConf;
 
+/// Things safekeeper should know about timeline state on peers.
+#[derive(Debug, Clone)]
+pub struct PeerInfo {
+    pub sk_id: NodeId,
+    /// Term of the last entry.
+    _last_log_term: Term,
+    /// LSN of the last record.
+    _flush_lsn: Lsn,
+    pub commit_lsn: Lsn,
+    /// Since which LSN safekeeper has WAL. TODO: remove this once we fill new
+    /// sk since backup_lsn.
+    pub local_start_lsn: Lsn,
+    /// When info was received.
+    ts: Instant,
+}
+
+impl PeerInfo {
+    fn from_sk_info(sk_id: NodeId, sk_info: &SkTimelineInfo, ts: Instant) -> PeerInfo {
+        PeerInfo {
+            sk_id,
+            _last_log_term: sk_info.last_log_term.unwrap_or(0),
+            _flush_lsn: sk_info.flush_lsn.unwrap_or(Lsn::INVALID),
+            commit_lsn: sk_info.commit_lsn.unwrap_or(Lsn::INVALID),
+            local_start_lsn: sk_info.local_start_lsn.unwrap_or(Lsn::INVALID),
+            ts,
+        }
+    }
+}
+
+// vector-based node id -> peer state map with very limited functionality we
+// need.
+#[derive(Debug, Clone, Default)]
+pub struct PeersInfo(pub Vec<PeerInfo>);
+
+impl PeersInfo {
+    fn get(&mut self, id: NodeId) -> Option<&mut PeerInfo> {
+        self.0.iter_mut().find(|p| p.sk_id == id)
+    }
+
+    fn upsert(&mut self, p: &PeerInfo) {
+        match self.get(p.sk_id) {
+            Some(rp) => *rp = p.clone(),
+            None => self.0.push(p.clone()),
+        }
+    }
+}
+
 /// Replica status update + hot standby feedback
 #[derive(Debug, Clone, Copy)]
 pub struct ReplicaState {
@@ -74,6 +121,8 @@ impl ReplicaState {
 pub struct SharedState {
     /// Safekeeper object
     sk: SafeKeeper<control_file::FileStorage, wal_storage::PhysicalStorage>,
+    /// State of peers as we know it.
+    peers_info: PeersInfo,
     /// State of replicas
     replicas: Vec<Option<ReplicaState>>,
     /// True when WAL backup launcher oversees the timeline, making sure WAL is
@@ -123,7 +172,8 @@ impl SharedState {
 
         Ok(Self {
             sk,
-            replicas: Vec::new(),
+            peers_info: PeersInfo(vec![]),
+            replicas: vec![],
             wal_backup_active: false,
             active: false,
             num_computes: 0,
@@ -142,6 +192,7 @@ impl SharedState {
 
         Ok(Self {
             sk: SafeKeeper::new(control_store, wal_store, conf.my_id)?,
+            peers_info: PeersInfo(vec![]),
             replicas: Vec::new(),
             wal_backup_active: false,
             active: false,
@@ -268,6 +319,24 @@ impl SharedState {
         self.replicas.push(Some(state));
         pos
     }
+
+    fn get_safekeeper_info(&self, conf: &SafeKeeperConf) -> SkTimelineInfo {
+        SkTimelineInfo {
+            last_log_term: Some(self.sk.get_epoch()),
+            flush_lsn: Some(self.sk.wal_store.flush_lsn()),
+            // note: this value is not flushed to control file yet and can be lost
+            commit_lsn: Some(self.sk.inmem.commit_lsn),
+            // TODO: rework feedbacks to avoid max here
+            remote_consistent_lsn: Some(max(
+                self.get_replicas_state().remote_consistent_lsn,
+                self.sk.inmem.remote_consistent_lsn,
+            )),
+            peer_horizon_lsn: Some(self.sk.inmem.peer_horizon_lsn),
+            safekeeper_connstr: Some(conf.listen_pg_addr.clone()),
+            backup_lsn: Some(self.sk.inmem.backup_lsn),
+            local_start_lsn: Some(self.sk.state.local_start_lsn),
+        }
+    }
 }
 
 #[derive(Debug, thiserror::Error)]
@@ -632,36 +701,25 @@ impl Timeline {
         Ok(())
     }
 
-    /// Return public safekeeper info for broadcasting to broker and other peers.
-    pub fn get_public_info(&self, conf: &SafeKeeperConf) -> SkTimelineInfo {
+    /// Get safekeeper info for broadcasting to broker and other peers.
+    pub fn get_safekeer_info(&self, conf: &SafeKeeperConf) -> SkTimelineInfo {
         let shared_state = self.write_shared_state();
-        SkTimelineInfo {
-            last_log_term: Some(shared_state.sk.get_epoch()),
-            flush_lsn: Some(shared_state.sk.wal_store.flush_lsn()),
-            // note: this value is not flushed to control file yet and can be lost
-            commit_lsn: Some(shared_state.sk.inmem.commit_lsn),
-            // TODO: rework feedbacks to avoid max here
-            remote_consistent_lsn: Some(max(
-                shared_state.get_replicas_state().remote_consistent_lsn,
-                shared_state.sk.inmem.remote_consistent_lsn,
-            )),
-            peer_horizon_lsn: Some(shared_state.sk.inmem.peer_horizon_lsn),
-            safekeeper_connstr: Some(conf.listen_pg_addr.clone()),
-            backup_lsn: Some(shared_state.sk.inmem.backup_lsn),
-        }
+        shared_state.get_safekeeper_info(conf)
     }
 
     /// Update timeline state with peer safekeeper data.
     pub async fn record_safekeeper_info(
         &self,
         sk_info: &SkTimelineInfo,
-        _sk_id: NodeId,
+        sk_id: NodeId,
     ) -> Result<()> {
         let is_wal_backup_action_pending: bool;
         let commit_lsn: Lsn;
         {
             let mut shared_state = self.write_shared_state();
             shared_state.sk.record_safekeeper_info(sk_info)?;
+            let peer_info = PeerInfo::from_sk_info(sk_id, sk_info, Instant::now());
+            shared_state.peers_info.upsert(&peer_info);
             is_wal_backup_action_pending = shared_state.update_status(self.ttid);
             commit_lsn = shared_state.sk.inmem.commit_lsn;
         }
@@ -673,6 +731,22 @@ impl Timeline {
         Ok(())
     }
 
+    /// Get our latest view of alive peers status on the timeline.
+    /// We pass our own info through the broker as well, so when we don't have connection
+    /// to the broker returned vec is empty.
+    pub fn get_peers(&self, conf: &SafeKeeperConf) -> Vec<PeerInfo> {
+        let shared_state = self.write_shared_state();
+        let now = Instant::now();
+        shared_state
+            .peers_info
+            .0
+            .iter()
+            // Regard peer as absent if we haven't heard from it within heartbeat_timeout.
+            .filter(|p| now.duration_since(p.ts) <= conf.heartbeat_timeout)
+            .cloned()
+            .collect()
+    }
+
     /// Add send_wal replica to the in-memory vector of replicas.
     pub fn add_replica(&self, state: ReplicaState) -> usize {
         self.write_shared_state().add_replica(state)

safekeeper/src/wal_backup.rs

@@ -1,8 +1,7 @@
 use anyhow::{Context, Result};
-use etcd_broker::subscription_key::{
-    NodeKind, OperationKind, SkOperationKind, SubscriptionKey, SubscriptionKind,
-};
+
 use tokio::task::JoinHandle;
+use utils::id::NodeId;
 
 use std::cmp::min;
 use std::collections::HashMap;
@@ -26,14 +25,11 @@ use tracing::*;
 
 use utils::{id::TenantTimelineId, lsn::Lsn};
 
-use crate::broker::{Election, ElectionLeader};
-use crate::timeline::Timeline;
-use crate::{broker, GlobalTimelines, SafeKeeperConf};
+use crate::timeline::{PeerInfo, Timeline};
+use crate::{GlobalTimelines, SafeKeeperConf};
 
 use once_cell::sync::OnceCell;
 
-const BROKER_CONNECTION_RETRY_DELAY_MS: u64 = 1000;
-
 const UPLOAD_FAILURE_RETRY_MIN_MS: u64 = 10;
 const UPLOAD_FAILURE_RETRY_MAX_MS: u64 = 5000;
 
@@ -70,47 +66,100 @@ struct WalBackupTimelineEntry {
     handle: Option<WalBackupTaskHandle>,
 }
 
-/// Start per timeline task, if it makes sense for this safekeeper to offload.
-fn consider_start_task(
+async fn shut_down_task(ttid: TenantTimelineId, entry: &mut WalBackupTimelineEntry) {
+    if let Some(wb_handle) = entry.handle.take() {
+        // Tell the task to shutdown. Error means task exited earlier, that's ok.
+        let _ = wb_handle.shutdown_tx.send(()).await;
+        // Await the task itself. TODO: restart panicked tasks earlier.
+        if let Err(e) = wb_handle.handle.await {
+            warn!("WAL backup task for {} panicked: {}", ttid, e);
+        }
+    }
+}
+
+/// The goal is to ensure that normally only one safekeepers offloads. However,
+/// it is fine (and inevitable, as s3 doesn't provide CAS) that for some short
+/// time we have several ones as they PUT the same files. Also,
+/// - frequently changing the offloader would be bad;
+/// - electing seriously lagging safekeeper is undesirable;
+/// So we deterministically choose among the reasonably caught up candidates.
+fn determine_offloader(
+    alive_peers: &[PeerInfo],
+    wal_backup_lsn: Lsn,
+    ttid: TenantTimelineId,
+    conf: &SafeKeeperConf,
+) -> (Option<NodeId>, String) {
+    // TODO: remove this once we fill newly joined safekeepers since backup_lsn.
+    let capable_peers = alive_peers
+        .iter()
+        .filter(|p| p.local_start_lsn <= wal_backup_lsn);
+    match alive_peers.iter().map(|p| p.commit_lsn).max() {
+        None => (None, "no connected peers to elect from".to_string()),
+        Some(max_commit_lsn) => {
+            let threshold = max_commit_lsn
+                .checked_sub(conf.max_offloader_lag)
+                .unwrap_or(Lsn(0));
+            let mut caughtup_peers = capable_peers
+                .clone()
+                .filter(|p| p.commit_lsn >= threshold)
+                .collect::<Vec<_>>();
+            caughtup_peers.sort_by(|p1, p2| p1.sk_id.cmp(&p2.sk_id));
+
+            // To distribute the load, shift by timeline_id.
+            let offloader = caughtup_peers
+                [(u128::from(ttid.timeline_id) % caughtup_peers.len() as u128) as usize]
+                .sk_id;
+
+            (
+                Some(offloader),
+                format!(
+                    "elected {} among {:?} peers, with {} of them being caughtup",
+                    offloader,
+                    capable_peers
+                        .map(|p| (p.sk_id, p.commit_lsn))
+                        .collect::<Vec<_>>(),
+                    caughtup_peers.len()
+                ),
+            )
+        }
+    }
+}
+
+/// Based on peer information determine which safekeeper should offload; if it
+/// is me, run (per timeline) task, if not yet. OTOH, if it is not me and task
+/// is running, kill it.
+async fn update_task(
     conf: &SafeKeeperConf,
     ttid: TenantTimelineId,
-    task: &mut WalBackupTimelineEntry,
+    entry: &mut WalBackupTimelineEntry,
 ) {
-    if !task.timeline.can_wal_backup() {
-        return;
+    let alive_peers = entry.timeline.get_peers(conf);
+    let wal_backup_lsn = entry.timeline.get_wal_backup_lsn();
+    let (offloader, election_dbg_str) =
+        determine_offloader(&alive_peers, wal_backup_lsn, ttid, conf);
+    let elected_me = Some(conf.my_id) == offloader;
+
+    if elected_me != (entry.handle.is_some()) {
+        if elected_me {
+            info!("elected for backup {}: {}", ttid, election_dbg_str);
+
+            let (shutdown_tx, shutdown_rx) = mpsc::channel(1);
+            let timeline_dir = conf.timeline_dir(&ttid);
+
+            let handle = tokio::spawn(
+                backup_task_main(ttid, timeline_dir, shutdown_rx)
+                    .instrument(info_span!("WAL backup task", ttid = %ttid)),
+            );
+
+            entry.handle = Some(WalBackupTaskHandle {
+                shutdown_tx,
+                handle,
+            });
+        } else {
+            info!("stepping down from backup {}: {}", ttid, election_dbg_str);
+            shut_down_task(ttid, entry).await;
+        }
     }
-    info!("starting WAL backup task for {}", ttid);
-
-    // TODO: decide who should offload right here by simply checking current
-    // state instead of running elections in offloading task.
-    let election_name = SubscriptionKey {
-        cluster_prefix: conf.broker_etcd_prefix.clone(),
-        kind: SubscriptionKind::Operation(
-            ttid,
-            NodeKind::Safekeeper,
-            OperationKind::Safekeeper(SkOperationKind::WalBackup),
-        ),
-    }
-    .watch_key();
-    let my_candidate_name = broker::get_candiate_name(conf.my_id);
-    let election = broker::Election::new(
-        election_name,
-        my_candidate_name,
-        conf.broker_endpoints.clone(),
-    );
-
-    let (shutdown_tx, shutdown_rx) = mpsc::channel(1);
-    let timeline_dir = conf.timeline_dir(&ttid);
-
-    let handle = tokio::spawn(
-        backup_task_main(ttid, timeline_dir, shutdown_rx, election)
-            .instrument(info_span!("WAL backup task", ttid = %ttid)),
-    );
-
-    task.handle = Some(WalBackupTaskHandle {
-        shutdown_tx,
-        handle,
-    });
 }
 
 const CHECK_TASKS_INTERVAL_MSEC: u64 = 1000;
@@ -158,27 +207,20 @@ async fn wal_backup_launcher_main_loop(
                             timeline,
                             handle: None,
                         });
-                        consider_start_task(&conf, ttid, entry);
+                        update_task(&conf, ttid, entry).await;
                     } else {
                         // need to stop the task
                         info!("stopping WAL backup task for {}", ttid);
-
-                        let entry = tasks.remove(&ttid).unwrap();
-                        if let Some(wb_handle) = entry.handle {
-                            // Tell the task to shutdown. Error means task exited earlier, that's ok.
-                            let _ = wb_handle.shutdown_tx.send(()).await;
-                            // Await the task itself. TODO: restart panicked tasks earlier.
-                            if let Err(e) = wb_handle.handle.await {
-                                warn!("WAL backup task for {} panicked: {}", ttid, e);
-                            }
-                        }
+                        let mut entry = tasks.remove(&ttid).unwrap();
+                        shut_down_task(ttid, &mut entry).await;
                     }
                 }
             }
-            // Start known tasks, if needed and possible.
+            // For each timeline needing offloading, check if this safekeeper
+            // should do the job and start/stop the task accordingly.
             _ = ticker.tick() => {
-                for (ttid, entry) in tasks.iter_mut().filter(|(_, entry)| entry.handle.is_none()) {
-                    consider_start_task(&conf, *ttid, entry);
+                for (ttid, entry) in tasks.iter_mut() {
+                    update_task(&conf, *ttid, entry).await;
                 }
             }
         }
@@ -190,17 +232,13 @@ struct WalBackupTask {
     timeline_dir: PathBuf,
     wal_seg_size: usize,
     commit_lsn_watch_rx: watch::Receiver<Lsn>,
-    leader: Option<ElectionLeader>,
-    election: Election,
 }
 
-/// Offload single timeline. Called only after we checked that backup
-/// is required (wal_backup_attend) and possible (can_wal_backup).
+/// Offload single timeline.
 async fn backup_task_main(
     ttid: TenantTimelineId,
     timeline_dir: PathBuf,
     mut shutdown_rx: Receiver<()>,
-    election: Election,
 ) {
     info!("started");
     let res = GlobalTimelines::get(ttid);
@@ -215,8 +253,6 @@ async fn backup_task_main(
         commit_lsn_watch_rx: tli.get_commit_lsn_watch_rx(),
         timeline: tli,
         timeline_dir,
-        leader: None,
-        election,
     };
 
     // task is spinned up only when wal_seg_size already initialized
@@ -229,9 +265,6 @@ async fn backup_task_main(
             canceled = true;
         }
     }
-    if let Some(l) = wb.leader {
-        l.give_up().await;
-    }
     info!("task {}", if canceled { "canceled" } else { "terminated" });
 }
 
@@ -239,107 +272,68 @@ impl WalBackupTask {
     async fn run(&mut self) {
         let mut backup_lsn = Lsn(0);
 
-        // election loop
+        let mut retry_attempt = 0u32;
+        // offload loop
         loop {
-            let mut retry_attempt = 0u32;
+            if retry_attempt == 0 {
+                // wait for new WAL to arrive
+                if let Err(e) = self.commit_lsn_watch_rx.changed().await {
+                    // should never happen, as we hold Arc to timeline.
+                    error!("commit_lsn watch shut down: {:?}", e);
+                    return;
+                }
+            } else {
+                // or just sleep if we errored previously
+                let mut retry_delay = UPLOAD_FAILURE_RETRY_MAX_MS;
+                if let Some(backoff_delay) = UPLOAD_FAILURE_RETRY_MIN_MS.checked_shl(retry_attempt)
+                {
+                    retry_delay = min(retry_delay, backoff_delay);
+                }
+                sleep(Duration::from_millis(retry_delay)).await;
+            }
 
-            info!("acquiring leadership");
-            if let Err(e) = broker::get_leader(&self.election, &mut self.leader).await {
-                error!("error during leader election {:?}", e);
-                sleep(Duration::from_millis(BROKER_CONNECTION_RETRY_DELAY_MS)).await;
+            let commit_lsn = *self.commit_lsn_watch_rx.borrow();
+
+            // Note that backup_lsn can be higher than commit_lsn if we
+            // don't have much local WAL and others already uploaded
+            // segments we don't even have.
+            if backup_lsn.segment_number(self.wal_seg_size)
+                >= commit_lsn.segment_number(self.wal_seg_size)
+            {
+                continue; /* nothing to do, common case as we wake up on every commit_lsn bump */
+            }
+            // Perhaps peers advanced the position, check shmem value.
+            backup_lsn = self.timeline.get_wal_backup_lsn();
+            if backup_lsn.segment_number(self.wal_seg_size)
+                >= commit_lsn.segment_number(self.wal_seg_size)
+            {
                 continue;
             }
-            info!("acquired leadership");
 
-            // offload loop
-            loop {
-                if retry_attempt == 0 {
-                    // wait for new WAL to arrive
-                    if let Err(e) = self.commit_lsn_watch_rx.changed().await {
-                        // should never happen, as we hold Arc to timeline.
-                        error!("commit_lsn watch shut down: {:?}", e);
+            match backup_lsn_range(
+                backup_lsn,
+                commit_lsn,
+                self.wal_seg_size,
+                &self.timeline_dir,
+            )
+            .await
+            {
+                Ok(backup_lsn_result) => {
+                    backup_lsn = backup_lsn_result;
+                    let res = self.timeline.set_wal_backup_lsn(backup_lsn_result);
+                    if let Err(e) = res {
+                        error!("backup error: {}", e);
                         return;
                     }
-                } else {
-                    // or just sleep if we errored previously
-                    let mut retry_delay = UPLOAD_FAILURE_RETRY_MAX_MS;
-                    if let Some(backoff_delay) =
-                        UPLOAD_FAILURE_RETRY_MIN_MS.checked_shl(retry_attempt)
-                    {
-                        retry_delay = min(retry_delay, backoff_delay);
-                    }
-                    sleep(Duration::from_millis(retry_delay)).await;
+                    retry_attempt = 0;
                 }
+                Err(e) => {
+                    error!(
+                        "failed while offloading range {}-{}: {:?}",
+                        backup_lsn, commit_lsn, e
+                    );
 
-                let commit_lsn = *self.commit_lsn_watch_rx.borrow();
-
-                // Note that backup_lsn can be higher than commit_lsn if we
-                // don't have much local WAL and others already uploaded
-                // segments we don't even have.
-                if backup_lsn.segment_number(self.wal_seg_size)
-                    >= commit_lsn.segment_number(self.wal_seg_size)
-                {
-                    continue; /* nothing to do, common case as we wake up on every commit_lsn bump */
-                }
-                // Perhaps peers advanced the position, check shmem value.
-                backup_lsn = self.timeline.get_wal_backup_lsn();
-                if backup_lsn.segment_number(self.wal_seg_size)
-                    >= commit_lsn.segment_number(self.wal_seg_size)
-                {
-                    continue;
-                }
-
-                if let Some(l) = self.leader.as_mut() {
-                    // Optimization idea for later:
-                    //  Avoid checking election leader every time by returning current lease grant expiration time
-                    //  Re-check leadership only after expiration time,
-                    //  such approach would reduce overhead on write-intensive workloads
-
-                    match l
-                        .check_am_i(
-                            self.election.election_name.clone(),
-                            self.election.candidate_name.clone(),
-                        )
-                        .await
-                    {
-                        Ok(leader) => {
-                            if !leader {
-                                info!("lost leadership");
-                                break;
-                            }
-                        }
-                        Err(e) => {
-                            warn!("error validating leader, {:?}", e);
-                            break;
-                        }
-                    }
-                }
-
-                match backup_lsn_range(
-                    backup_lsn,
-                    commit_lsn,
-                    self.wal_seg_size,
-                    &self.timeline_dir,
-                )
-                .await
-                {
-                    Ok(backup_lsn_result) => {
-                        backup_lsn = backup_lsn_result;
-                        let res = self.timeline.set_wal_backup_lsn(backup_lsn_result);
-                        if let Err(e) = res {
-                            error!("backup error: {}", e);
-                            return;
-                        }
-                        retry_attempt = 0;
-                    }
-                    Err(e) => {
-                        error!(
-                            "failed while offloading range {}-{}: {:?}",
-                            backup_lsn, commit_lsn, e
-                        );
-
-                        retry_attempt = min(retry_attempt + 1, u32::MAX);
-                    }
+                    retry_attempt = min(retry_attempt + 1, u32::MAX);
                 }
             }
         }

test_runner/fixtures/compare_fixtures.py

@@ -130,12 +130,11 @@ class NeonCompare(PgCompare):
             "size", timeline_size / (1024 * 1024), "MB", report=MetricReport.LOWER_IS_BETTER
         )
 
-        params = f'{{tenant_id="{self.env.initial_tenant}",timeline_id="{self.timeline}"}}'
         total_files = self.zenbenchmark.get_int_counter_value(
-            self.env.pageserver, "pageserver_created_persistent_files_total" + params
+            self.env.pageserver, "pageserver_created_persistent_files_total"
         )
         total_bytes = self.zenbenchmark.get_int_counter_value(
-            self.env.pageserver, "pageserver_written_persistent_bytes_total" + params
+            self.env.pageserver, "pageserver_written_persistent_bytes_total"
         )
         self.zenbenchmark.record(
             "data_uploaded", total_bytes / (1024 * 1024), "MB", report=MetricReport.LOWER_IS_BETTER

test_runner/fixtures/metrics.py

@@ -60,6 +60,4 @@ PAGESERVER_PER_TENANT_METRICS = [
     "pageserver_wait_lsn_seconds_bucket",
     "pageserver_wait_lsn_seconds_count",
     "pageserver_wait_lsn_seconds_sum",
-    "pageserver_created_persistent_files_total",
-    "pageserver_written_persistent_bytes_total",
 ]

test_runner/fixtures/neon_fixtures.py

@@ -149,6 +149,19 @@ def pytest_configure(config):
         raise Exception('neon binaries not found at "{}"'.format(neon_binpath))
 
 
+def profiling_supported():
+    """Return True if the pageserver was compiled with the 'profiling' feature"""
+    bin_pageserver = os.path.join(str(neon_binpath), "pageserver")
+    res = subprocess.run(
+        [bin_pageserver, "--version"],
+        check=True,
+        universal_newlines=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    return "profiling:true" in res.stdout
+
+
 def shareable_scope(fixture_name, config) -> Literal["session", "function"]:
     """Return either session of function scope, depending on TEST_SHARED_FIXTURES envvar.
 
@@ -861,17 +874,6 @@ class NeonEnv:
         """Get a timeline directory's path based on the repo directory of the test environment"""
         return self.repo_dir / "tenants" / str(tenant_id) / "timelines" / str(timeline_id)
 
-    def get_pageserver_version(self) -> str:
-        bin_pageserver = os.path.join(str(neon_binpath), "pageserver")
-        res = subprocess.run(
-            [bin_pageserver, "--version"],
-            check=True,
-            universal_newlines=True,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-        )
-        return res.stdout
-
     @cached_property
     def auth_keys(self) -> AuthKeys:
         pub = (Path(self.repo_dir) / "auth_public_key.pem").read_bytes()
@@ -970,11 +972,10 @@ class NeonPageserverApiException(Exception):
 
 
 class NeonPageserverHttpClient(requests.Session):
-    def __init__(self, port: int, is_testing_enabled_or_skip, auth_token: Optional[str] = None):
+    def __init__(self, port: int, auth_token: Optional[str] = None):
         super().__init__()
         self.port = port
         self.auth_token = auth_token
-        self.is_testing_enabled_or_skip = is_testing_enabled_or_skip
 
         if auth_token is not None:
             self.headers["Authorization"] = f"Bearer {auth_token}"
@@ -993,8 +994,6 @@ class NeonPageserverHttpClient(requests.Session):
         self.get(f"http://localhost:{self.port}/v1/status").raise_for_status()
 
     def configure_failpoints(self, config_strings: tuple[str, str] | list[tuple[str, str]]) -> None:
-        self.is_testing_enabled_or_skip()
-
         if isinstance(config_strings, tuple):
             pairs = [config_strings]
         else:
@@ -1112,8 +1111,6 @@ class NeonPageserverHttpClient(requests.Session):
     def timeline_gc(
         self, tenant_id: TenantId, timeline_id: TimelineId, gc_horizon: Optional[int]
     ) -> dict[str, Any]:
-        self.is_testing_enabled_or_skip()
-
         log.info(
             f"Requesting GC: tenant {tenant_id}, timeline {timeline_id}, gc_horizon {repr(gc_horizon)}"
         )
@@ -1129,8 +1126,6 @@ class NeonPageserverHttpClient(requests.Session):
         return res_json
 
     def timeline_compact(self, tenant_id: TenantId, timeline_id: TimelineId):
-        self.is_testing_enabled_or_skip()
-
         log.info(f"Requesting compact: tenant {tenant_id}, timeline {timeline_id}")
         res = self.put(
             f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/compact"
@@ -1155,8 +1150,6 @@ class NeonPageserverHttpClient(requests.Session):
         return res_json
 
     def timeline_checkpoint(self, tenant_id: TenantId, timeline_id: TimelineId):
-        self.is_testing_enabled_or_skip()
-
         log.info(f"Requesting checkpoint: tenant {tenant_id}, timeline {timeline_id}")
         res = self.put(
             f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/checkpoint"
@@ -1476,6 +1469,21 @@ class NeonCli(AbstractNeonCli):
             res.check_returncode()
             return res
 
+    def pageserver_enabled_features(self) -> Any:
+        bin_pageserver = os.path.join(str(neon_binpath), "pageserver")
+        args = [bin_pageserver, "--enabled-features"]
+        log.info('Running command "{}"'.format(" ".join(args)))
+
+        res = subprocess.run(
+            args,
+            check=True,
+            universal_newlines=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        log.info(f"pageserver_enabled_features success: {res.stdout}")
+        return json.loads(res.stdout)
+
     def pageserver_start(
         self,
         overrides=(),
@@ -1634,7 +1642,6 @@ class NeonPageserver(PgProtocol):
         self.running = False
         self.service_port = port
         self.config_override = config_override
-        self.version = env.get_pageserver_version()
 
     def start(self, overrides=()) -> "NeonPageserver":
         """
@@ -1664,19 +1671,10 @@ class NeonPageserver(PgProtocol):
     def __exit__(self, exc_type, exc, tb):
         self.stop(immediate=True)
 
-    def is_testing_enabled_or_skip(self):
-        if '"testing"' not in self.version:
-            pytest.skip("pageserver was built without 'testing' feature")
-
-    def is_profiling_enabled_or_skip(self):
-        if '"profiling"' not in self.version:
-            pytest.skip("pageserver was built without 'profiling' feature")
-
     def http_client(self, auth_token: Optional[str] = None) -> NeonPageserverHttpClient:
         return NeonPageserverHttpClient(
             port=self.service_port.http,
             auth_token=auth_token,
-            is_testing_enabled_or_skip=self.is_testing_enabled_or_skip,
         )
 
 

test_runner/performance/test_perf_pgbench.py

@@ -9,6 +9,7 @@ from typing import Dict, List
 import pytest
 from fixtures.benchmark_fixture import MetricReport, PgBenchInitResult, PgBenchRunResult
 from fixtures.compare_fixtures import NeonCompare, PgCompare
+from fixtures.neon_fixtures import profiling_supported
 from fixtures.utils import get_scale_for_db
 
 
@@ -186,8 +187,10 @@ def test_pgbench_flamegraph(zenbenchmark, pg_bin, neon_env_builder, scale: int,
     neon_env_builder.pageserver_config_override = """
 profiling="page_requests"
 """
+    if not profiling_supported():
+        pytest.skip("pageserver was built without 'profiling' feature")
+
     env = neon_env_builder.init_start()
-    env.pageserver.is_profiling_enabled_or_skip()
     env.neon_cli.create_branch("empty", "main")
 
     neon_compare = NeonCompare(zenbenchmark, env, pg_bin, "pgbench")

test_runner/regress/test_broken_timeline.py

@@ -111,20 +111,18 @@ def test_create_multiple_timelines_parallel(neon_simple_env: NeonEnv):
             future.result()
 
 
-def test_timeline_init_break_before_checkpoint(neon_simple_env: NeonEnv):
+def test_fix_broken_timelines_on_startup(neon_simple_env: NeonEnv):
     env = neon_simple_env
     pageserver_http = env.pageserver.http_client()
 
     tenant_id, _ = env.neon_cli.create_tenant()
 
-    timelines_dir = env.repo_dir / "tenants" / str(tenant_id) / "timelines"
     old_tenant_timelines = env.neon_cli.list_timelines(tenant_id)
-    initial_timeline_dirs = [d for d in timelines_dir.iterdir()]
 
-    # Introduce failpoint during timeline init (some intermediate files are on disk), before it's checkpointed.
+    # Introduce failpoint when creating a new timeline
     pageserver_http.configure_failpoints(("before-checkpoint-new-timeline", "return"))
     with pytest.raises(Exception, match="before-checkpoint-new-timeline"):
-        _ = env.neon_cli.create_timeline("test_timeline_init_break_before_checkpoint", tenant_id)
+        _ = env.neon_cli.create_timeline("test_fix_broken_timelines", tenant_id)
 
     # Restart the page server
     env.neon_cli.pageserver_stop(immediate=True)
@@ -135,36 +133,3 @@ def test_timeline_init_break_before_checkpoint(neon_simple_env: NeonEnv):
     assert (
         new_tenant_timelines == old_tenant_timelines
     ), f"Pageserver after restart should ignore non-initialized timelines for tenant {tenant_id}"
-
-    timeline_dirs = [d for d in timelines_dir.iterdir()]
-    assert (
-        timeline_dirs == initial_timeline_dirs
-    ), "pageserver should clean its temp timeline files on timeline creation failure"
-
-
-def test_timeline_create_break_after_uninit_mark(neon_simple_env: NeonEnv):
-    env = neon_simple_env
-    pageserver_http = env.pageserver.http_client()
-
-    tenant_id, _ = env.neon_cli.create_tenant()
-
-    timelines_dir = env.repo_dir / "tenants" / str(tenant_id) / "timelines"
-    old_tenant_timelines = env.neon_cli.list_timelines(tenant_id)
-    initial_timeline_dirs = [d for d in timelines_dir.iterdir()]
-
-    # Introduce failpoint when creating a new timeline uninit mark, before any other files were created
-    pageserver_http.configure_failpoints(("after-timeline-uninit-mark-creation", "return"))
-    with pytest.raises(Exception, match="after-timeline-uninit-mark-creation"):
-        _ = env.neon_cli.create_timeline("test_timeline_create_break_after_uninit_mark", tenant_id)
-
-    # Creating the timeline didn't finish. The other timelines on tenant should still be present and work normally.
-    # "New" timeline is not present in the list, allowing pageserver to retry the same request
-    new_tenant_timelines = env.neon_cli.list_timelines(tenant_id)
-    assert (
-        new_tenant_timelines == old_tenant_timelines
-    ), f"Pageserver after restart should ignore non-initialized timelines for tenant {tenant_id}"
-
-    timeline_dirs = [d for d in timelines_dir.iterdir()]
-    assert (
-        timeline_dirs == initial_timeline_dirs
-    ), "pageserver should clean its temp timeline files on timeline creation failure"

test_runner/regress/test_gc_cutoff.py

@@ -1,39 +0,0 @@
-from fixtures.neon_fixtures import NeonEnvBuilder, PgBin
-
-
-# Test gc_cutoff
-#
-# This test sets fail point at the end of GC, and checks that pageserver
-# normally restarts after it. Also, there should be GC ERRORs in the log,
-# but the fixture checks the log for any unexpected ERRORs after every
-# test anyway, so it doesn't need any special attention here.
-def test_gc_cutoff(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin):
-    env = neon_env_builder.init_start()
-    pageserver_http = env.pageserver.http_client()
-
-    # Use aggressive GC and checkpoint settings, so that we also exercise GC during the test
-    tenant_id, _ = env.neon_cli.create_tenant(
-        conf={
-            "gc_period": "10 s",
-            "gc_horizon": f"{1024 ** 2}",
-            "checkpoint_distance": f"{1024 ** 2}",
-            "compaction_period": "5 s",
-            # set PITR interval to be small, so we can do GC
-            "pitr_interval": "1 s",
-            "compaction_threshold": "3",
-            "image_creation_threshold": "2",
-        }
-    )
-    pg = env.postgres.create_start("main", tenant_id=tenant_id)
-    connstr = pg.connstr(options="-csynchronous_commit=off")
-    pg_bin.run_capture(["pgbench", "-i", "-s10", connstr])
-
-    pageserver_http.configure_failpoints(("after-timeline-gc-removed-layers", "exit"))
-
-    for i in range(5):
-        try:
-            pg_bin.run_capture(["pgbench", "-N", "-c5", "-T100", "-Mprepared", connstr])
-        except Exception:
-            env.pageserver.stop()
-            env.pageserver.start()
-            pageserver_http.configure_failpoints(("after-timeline-gc-removed-layers", "exit"))

test_runner/regress/test_import.py

@@ -105,11 +105,15 @@ def test_import_from_vanilla(test_output_dir, pg_bin, vanilla_pg, neon_env_build
     with pytest.raises(Exception):
         import_tar(corrupt_base_tar, wal_tar)
 
+    # Clean up
+    # TODO it should clean itself
+    client = env.pageserver.http_client()
+    client.timeline_delete(tenant, timeline)
+
     # Importing correct backup works
     import_tar(base_tar, wal_tar)
 
     # Wait for data to land in s3
-    client = env.pageserver.http_client()
     wait_for_last_record_lsn(client, tenant, timeline, Lsn(end_lsn))
     wait_for_upload(client, tenant, timeline, Lsn(end_lsn))
 

test_runner/regress/test_recovery.py

@@ -13,8 +13,13 @@ def test_pageserver_recovery(neon_env_builder: NeonEnvBuilder):
     neon_env_builder.pageserver_config_override = "tenant_config={checkpoint_distance = 1048576}"
 
     env = neon_env_builder.init()
-    env.pageserver.is_testing_enabled_or_skip()
 
+    # Check if failpoints enables. Otherwise the test doesn't make sense
+    f = env.neon_cli.pageserver_enabled_features()
+
+    assert (
+        "testing" in f["features"]
+    ), "Build pageserver with --features=testing option to run this test"
     neon_env_builder.start()
 
     # Create a branch for us

test_runner/regress/test_remote_storage.py

@@ -10,8 +10,8 @@ import pytest
 from fixtures.log_helper import log
 from fixtures.neon_fixtures import (
     NeonEnvBuilder,
+    NeonPageserverHttpClient,
     RemoteStorageKind,
-    assert_no_in_progress_downloads_for_tenant,
     available_remote_storages,
     wait_for_last_record_lsn,
     wait_for_upload,
@@ -125,7 +125,7 @@ def test_remote_storage_backup_and_restore(
     wait_until(
         number_of_iterations=20,
         interval=1,
-        func=lambda: assert_no_in_progress_downloads_for_tenant(client, tenant_id),
+        func=lambda: expect_tenant_to_download_timeline(client, tenant_id),
     )
 
     detail = client.timeline_detail(tenant_id, timeline_id)
@@ -142,3 +142,16 @@ def test_remote_storage_backup_and_restore(
                 query_scalar(cur, f"SELECT secret FROM t{checkpoint_number} WHERE id = {data_id};")
                 == f"{data_secret}|{checkpoint_number}"
             )
+
+
+def expect_tenant_to_download_timeline(
+    client: NeonPageserverHttpClient,
+    tenant_id: TenantId,
+):
+    for tenant in client.tenant_list():
+        if tenant["id"] == str(tenant_id):
+            assert not tenant.get(
+                "has_in_progress_downloads", True
+            ), f"Tenant {tenant_id} should have no downloads in progress"
+            return
+    assert False, f"Tenant {tenant_id} is missing on pageserver"

test_runner/regress/test_tenant_relocation.py

@@ -346,11 +346,7 @@ def test_tenant_relocation(
     log.info("new pageserver ports pg %s http %s", new_pageserver_pg_port, new_pageserver_http_port)
     pageserver_bin = pathlib.Path(neon_binpath) / "pageserver"
 
-    new_pageserver_http = NeonPageserverHttpClient(
-        port=new_pageserver_http_port,
-        auth_token=None,
-        is_testing_enabled_or_skip=env.pageserver.is_testing_enabled_or_skip,
-    )
+    new_pageserver_http = NeonPageserverHttpClient(port=new_pageserver_http_port, auth_token=None)
 
     with new_pageserver_helper(
         new_pageserver_dir,

test_runner/regress/test_tenants.py

@@ -23,7 +23,7 @@ def test_tenant_creation_fails(neon_simple_env: NeonEnv):
     initial_tenants = sorted(
         map(lambda t: t.split()[0], neon_simple_env.neon_cli.list_tenants().stdout.splitlines())
     )
-    initial_tenant_dirs = [d for d in tenants_dir.iterdir()]
+    initial_tenant_dirs = set([d for d in tenants_dir.iterdir()])
 
     pageserver_http = neon_simple_env.pageserver.http_client()
     pageserver_http.configure_failpoints(("tenant-creation-before-tmp-rename", "return"))
@@ -35,10 +35,26 @@ def test_tenant_creation_fails(neon_simple_env: NeonEnv):
     )
     assert initial_tenants == new_tenants, "should not create new tenants"
 
-    new_tenant_dirs = [d for d in tenants_dir.iterdir()]
+    new_tenant_dirs = list(set([d for d in tenants_dir.iterdir()]) - initial_tenant_dirs)
+    assert len(new_tenant_dirs) == 1, "should have new tenant directory created"
+    tmp_tenant_dir = new_tenant_dirs[0]
+    assert str(tmp_tenant_dir).endswith(
+        ".___temp"
+    ), "new tenant directory created should be a temporary one"
+
+    neon_simple_env.pageserver.stop()
+    neon_simple_env.pageserver.start()
+
+    tenants_after_restart = sorted(
+        map(lambda t: t.split()[0], neon_simple_env.neon_cli.list_tenants().stdout.splitlines())
+    )
+    dirs_after_restart = set([d for d in tenants_dir.iterdir()])
     assert (
-        new_tenant_dirs == initial_tenant_dirs
-    ), "pageserver should clean its temp tenant dirs on tenant creation failure"
+        tenants_after_restart == initial_tenants
+    ), "should load all non-corrupt tenants after restart"
+    assert (
+        dirs_after_restart == initial_tenant_dirs
+    ), "pageserver should clean its temp tenant dirs on restart"
 
 
 def test_tenants_normal_work(neon_env_builder: NeonEnvBuilder):

test_runner/regress/test_tenants_with_remote_storage.py

@@ -7,21 +7,18 @@
 #
 
 import asyncio
-import json
 import os
-import shutil
 from pathlib import Path
 from typing import List, Tuple
 
 import pytest
 from fixtures.log_helper import log
 from fixtures.neon_fixtures import (
-    LocalFsStorage,
     NeonEnv,
     NeonEnvBuilder,
+    NeonPageserverHttpClient,
     Postgres,
     RemoteStorageKind,
-    assert_no_in_progress_downloads_for_tenant,
     available_remote_storages,
     wait_for_last_record_lsn,
     wait_for_upload,
@@ -168,7 +165,7 @@ def test_tenants_attached_after_download(
     wait_until(
         number_of_iterations=5,
         interval=1,
-        func=lambda: assert_no_in_progress_downloads_for_tenant(client, tenant_id),
+        func=lambda: expect_tenant_to_download_timeline(client, tenant_id),
     )
 
     restored_timelines = client.timeline_list(tenant_id)
@@ -181,244 +178,14 @@ def test_tenants_attached_after_download(
     ), f"Tenant {tenant_id} should have its old timeline {timeline_id} restored from the remote storage"
 
 
-@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS])
-def test_tenant_upgrades_index_json_from_v0(
-    neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind
+def expect_tenant_to_download_timeline(
+    client: NeonPageserverHttpClient,
+    tenant_id: TenantId,
 ):
-    # the "image" for the v0 index_part.json. the fields themselves are
-    # replaced with values read from the later version because of #2592 (initdb
-    # lsn not reproducible).
-    v0_skeleton = json.loads(
-        """{
-        "timeline_layers":[
-            "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"
-        ],
-        "missing_layers":[],
-        "disk_consistent_lsn":"0/16960E8",
-        "metadata_bytes":[]
-    }"""
-    )
-
-    # getting a too eager compaction happening for this test would not play
-    # well with the strict assertions.
-    neon_env_builder.pageserver_config_override = "tenant_config.compaction_period='1h'"
-
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind, "test_tenant_upgrades_index_json_from_v0"
-    )
-
-    # launch pageserver, populate the default tenants timeline, wait for it to be uploaded,
-    # then go ahead and modify the "remote" version as if it was downgraded, needing upgrade
-    env = neon_env_builder.init_start()
-    pageserver_http = env.pageserver.http_client()
-    pg = env.postgres.create_start("main")
-
-    tenant_id = TenantId(pg.safe_psql("show neon.tenant_id")[0][0])
-    timeline_id = TimelineId(pg.safe_psql("show neon.timeline_id")[0][0])
-
-    with pg.cursor() as cur:
-        cur.execute("CREATE TABLE t0 AS VALUES (123, 'second column as text');")
-        current_lsn = Lsn(query_scalar(cur, "SELECT pg_current_wal_flush_lsn()"))
-
-    # flush, wait until in remote storage
-    wait_for_last_record_lsn(pageserver_http, tenant_id, timeline_id, current_lsn)
-    pageserver_http.timeline_checkpoint(tenant_id, timeline_id)
-    wait_for_upload(pageserver_http, tenant_id, timeline_id, current_lsn)
-
-    env.postgres.stop_all()
-    env.pageserver.stop()
-
-    # remove all local data for the tenant to force redownloading and subsequent upgrade
-    shutil.rmtree(Path(env.repo_dir) / "tenants" / str(tenant_id))
-
-    # downgrade the remote file
-    timeline_path = local_fs_index_part_path(env, tenant_id, timeline_id)
-    with open(timeline_path, "r+") as timeline_file:
-        # keep the deserialized for later inspection
-        orig_index_part = json.load(timeline_file)
-
-        v0_index_part = {key: orig_index_part[key] for key in v0_skeleton}
-
-        timeline_file.seek(0)
-        json.dump(v0_index_part, timeline_file)
-
-    env.pageserver.start()
-    pageserver_http = env.pageserver.http_client()
-    pageserver_http.tenant_attach(tenant_id)
-
-    wait_until(
-        number_of_iterations=5,
-        interval=1,
-        func=lambda: assert_no_in_progress_downloads_for_tenant(pageserver_http, tenant_id),
-    )
-
-    pg = env.postgres.create_start("main")
-
-    with pg.cursor() as cur:
-        cur.execute("INSERT INTO t0 VALUES (234, 'test data');")
-        current_lsn = Lsn(query_scalar(cur, "SELECT pg_current_wal_flush_lsn()"))
-
-    wait_for_last_record_lsn(pageserver_http, tenant_id, timeline_id, current_lsn)
-    pageserver_http.timeline_checkpoint(tenant_id, timeline_id)
-    wait_for_upload(pageserver_http, tenant_id, timeline_id, current_lsn)
-
-    # not needed anymore
-    env.postgres.stop_all()
-    env.pageserver.stop()
-
-    # make sure the file has been upgraded back to how it started
-    index_part = local_fs_index_part(env, tenant_id, timeline_id)
-    assert index_part["version"] == orig_index_part["version"]
-    assert index_part["missing_layers"] == orig_index_part["missing_layers"]
-
-    # expect one more layer because of the forced checkpoint
-    assert len(index_part["timeline_layers"]) == len(orig_index_part["timeline_layers"]) + 1
-
-    # all of the same layer files are there, but they might be shuffled around
-    orig_layers = set(orig_index_part["timeline_layers"])
-    later_layers = set(index_part["timeline_layers"])
-    assert later_layers.issuperset(orig_layers)
-
-    added_layers = later_layers - orig_layers
-    assert len(added_layers) == 1
-
-    # all of metadata has been regenerated (currently just layer file size)
-    all_metadata_keys = set()
-    for layer in orig_layers:
-        orig_metadata = orig_index_part["layer_metadata"][layer]
-        new_metadata = index_part["layer_metadata"][layer]
-        assert (
-            orig_metadata == new_metadata
-        ), f"metadata for layer {layer} should not have changed {orig_metadata} vs. {new_metadata}"
-        all_metadata_keys |= set(orig_metadata.keys())
-
-    one_new_layer = next(iter(added_layers))
-    assert one_new_layer in index_part["layer_metadata"], "new layer should have metadata"
-
-    only_new_metadata = index_part["layer_metadata"][one_new_layer]
-
-    assert (
-        set(only_new_metadata.keys()).symmetric_difference(all_metadata_keys) == set()
-    ), "new layer metadata has same metadata as others"
-
-
-# FIXME: test index_part.json getting downgraded from imaginary new version
-
-
-@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS])
-def test_tenant_redownloads_truncated_file_on_startup(
-    neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind
-):
-    # since we now store the layer file length metadata, we notice on startup that a layer file is of wrong size, and proceed to redownload it.
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-        test_name="test_tenant_redownloads_truncated_file_on_startup",
-    )
-
-    env = neon_env_builder.init_start()
-    pageserver_http = env.pageserver.http_client()
-    pg = env.postgres.create_start("main")
-
-    tenant_id = TenantId(pg.safe_psql("show neon.tenant_id")[0][0])
-    timeline_id = TimelineId(pg.safe_psql("show neon.timeline_id")[0][0])
-
-    with pg.cursor() as cur:
-        cur.execute("CREATE TABLE t1 AS VALUES (123, 'foobar');")
-        current_lsn = Lsn(query_scalar(cur, "SELECT pg_current_wal_flush_lsn()"))
-
-    wait_for_last_record_lsn(pageserver_http, tenant_id, timeline_id, current_lsn)
-    pageserver_http.timeline_checkpoint(tenant_id, timeline_id)
-    wait_for_upload(pageserver_http, tenant_id, timeline_id, current_lsn)
-
-    env.postgres.stop_all()
-    env.pageserver.stop()
-
-    timeline_dir = Path(env.repo_dir) / "tenants" / str(tenant_id) / "timelines" / str(timeline_id)
-    local_layer_truncated = None
-    for path in Path.iterdir(timeline_dir):
-        if path.name.startswith("00000"):
-            correct_size = os.stat(path).st_size
-            os.truncate(path, 0)
-            local_layer_truncated = (path, correct_size)
-            break
-    assert (
-        local_layer_truncated is not None
-    ), f"Found no local layer files to delete in directory {timeline_dir}"
-
-    (path, expected_size) = local_layer_truncated
-
-    # ensure the same size is found from the index_part.json
-    index_part = local_fs_index_part(env, tenant_id, timeline_id)
-    assert index_part["layer_metadata"][path.name]["file_size"] == expected_size
-
-    ##### Start the pageserver, forcing it to download the layer file and load the timeline into memory
-    env.pageserver.start()
-    client = env.pageserver.http_client()
-
-    wait_until(
-        number_of_iterations=5,
-        interval=1,
-        func=lambda: assert_no_in_progress_downloads_for_tenant(client, tenant_id),
-    )
-
-    restored_timelines = client.timeline_list(tenant_id)
-    assert (
-        len(restored_timelines) == 1
-    ), f"Tenant {tenant_id} should have its timeline reattached after its layer is downloaded from the remote storage"
-    retored_timeline = restored_timelines[0]
-    assert retored_timeline["timeline_id"] == str(
-        timeline_id
-    ), f"Tenant {tenant_id} should have its old timeline {timeline_id} restored from the remote storage"
-
-    assert os.stat(path).st_size == expected_size, "truncated layer should had been re-downloaded"
-
-    # the remote side of local_layer_truncated
-    remote_layer_path = local_fs_index_part_path(env, tenant_id, timeline_id).parent / path.name
-
-    # if the upload ever was ongoing, this check would be racy, but at least one
-    # extra http request has been made in between so assume it's enough delay
-    assert (
-        os.stat(remote_layer_path).st_size == expected_size
-    ), "truncated file should not had been uploaded around re-download"
-
-    pg = env.postgres.create_start("main")
-
-    with pg.cursor() as cur:
-        cur.execute("INSERT INTO t1 VALUES (234, 'test data');")
-        current_lsn = Lsn(query_scalar(cur, "SELECT pg_current_wal_flush_lsn()"))
-
-    wait_for_last_record_lsn(pageserver_http, tenant_id, timeline_id, current_lsn)
-    pageserver_http.timeline_checkpoint(tenant_id, timeline_id)
-    wait_for_upload(pageserver_http, tenant_id, timeline_id, current_lsn)
-
-    # now that the upload is complete, make sure the file hasn't been
-    # re-uploaded truncated. this is a rather bogus check given the current
-    # implementation, but it's critical it doesn't happen so wasting a few
-    # lines of python to do this.
-    assert (
-        os.stat(remote_layer_path).st_size == expected_size
-    ), "truncated file should not had been uploaded after next checkpoint"
-
-
-def local_fs_index_part(env, tenant_id, timeline_id):
-    """
-    Return json.load parsed index_part.json of tenant and timeline from LOCAL_FS
-    """
-    timeline_path = local_fs_index_part_path(env, tenant_id, timeline_id)
-    with open(timeline_path, "r") as timeline_file:
-        return json.load(timeline_file)
-
-
-def local_fs_index_part_path(env, tenant_id, timeline_id):
-    """
-    Return path to the LOCAL_FS index_part.json of the tenant and timeline.
-    """
-    assert isinstance(env.remote_storage, LocalFsStorage)
-    return (
-        env.remote_storage.root
-        / "tenants"
-        / str(tenant_id)
-        / "timelines"
-        / str(timeline_id)
-        / "index_part.json"
-    )
+    for tenant in client.tenant_list():
+        if tenant["id"] == str(tenant_id):
+            assert not tenant.get(
+                "has_in_progress_downloads", True
+            ), f"Tenant {tenant_id} should have no downloads in progress"
+            return
+    assert False, f"Tenant {tenant_id} is missing on pageserver"

test_runner/regress/test_timeline_size.py

@@ -270,15 +270,9 @@ def test_timeline_physical_size_post_compaction(neon_env_builder: NeonEnvBuilder
     new_timeline_id = env.neon_cli.create_branch("test_timeline_physical_size_post_compaction")
     pg = env.postgres.create_start("test_timeline_physical_size_post_compaction")
 
-    # We don't want autovacuum to run on the table, while we are calculating the
-    # physical size, because that could cause a new layer to be created and a
-    # mismatch between the incremental and non-incremental size. (If that still
-    # happens, because of some other background activity or autovacuum on other
-    # tables, we could simply retry the size calculations. It's unlikely that
-    # that would happen more than once.)
     pg.safe_psql_many(
         [
-            "CREATE TABLE foo (t text) WITH (autovacuum_enabled = off)",
+            "CREATE TABLE foo (t text)",
             """INSERT INTO foo
            SELECT 'long string to consume some space' || g
            FROM generate_series(1, 100000) g""",
@@ -303,10 +297,9 @@ def test_timeline_physical_size_post_gc(neon_env_builder: NeonEnvBuilder):
     new_timeline_id = env.neon_cli.create_branch("test_timeline_physical_size_post_gc")
     pg = env.postgres.create_start("test_timeline_physical_size_post_gc")
 
-    # Like in test_timeline_physical_size_post_compaction, disable autovacuum
     pg.safe_psql_many(
         [
-            "CREATE TABLE foo (t text) WITH (autovacuum_enabled = off)",
+            "CREATE TABLE foo (t text)",
             """INSERT INTO foo
            SELECT 'long string to consume some space' || g
            FROM generate_series(1, 100000) g""",

test_runner/regress/test_wal_acceptor.py

@@ -1114,7 +1114,10 @@ def test_delete_force(neon_env_builder: NeonEnvBuilder, auth_enabled: bool):
                 cur.execute("INSERT INTO t (key) VALUES (1)")
 
     # Remove initial tenant's br1 (active)
-    assert sk_http.timeline_delete_force(tenant_id, timeline_id_1)["dir_existed"]
+    assert sk_http.timeline_delete_force(tenant_id, timeline_id_1) == {
+        "dir_existed": True,
+        "was_active": True,
+    }
     assert not (sk_data_dir / str(tenant_id) / str(timeline_id_1)).exists()
     assert (sk_data_dir / str(tenant_id) / str(timeline_id_2)).is_dir()
     assert (sk_data_dir / str(tenant_id) / str(timeline_id_3)).is_dir()
@@ -1122,7 +1125,10 @@ def test_delete_force(neon_env_builder: NeonEnvBuilder, auth_enabled: bool):
     assert (sk_data_dir / str(tenant_id_other) / str(timeline_id_other)).is_dir()
 
     # Ensure repeated deletion succeeds
-    assert not sk_http.timeline_delete_force(tenant_id, timeline_id_1)["dir_existed"]
+    assert sk_http.timeline_delete_force(tenant_id, timeline_id_1) == {
+        "dir_existed": False,
+        "was_active": False,
+    }
     assert not (sk_data_dir / str(tenant_id) / str(timeline_id_1)).exists()
     assert (sk_data_dir / str(tenant_id) / str(timeline_id_2)).is_dir()
     assert (sk_data_dir / str(tenant_id) / str(timeline_id_3)).is_dir()
@@ -1139,7 +1145,10 @@ def test_delete_force(neon_env_builder: NeonEnvBuilder, auth_enabled: bool):
         assert (sk_data_dir / str(tenant_id_other) / str(timeline_id_other)).is_dir()
 
     # Remove initial tenant's br2 (inactive)
-    assert sk_http.timeline_delete_force(tenant_id, timeline_id_2)["dir_existed"]
+    assert sk_http.timeline_delete_force(tenant_id, timeline_id_2) == {
+        "dir_existed": True,
+        "was_active": False,
+    }
     assert not (sk_data_dir / str(tenant_id) / str(timeline_id_1)).exists()
     assert not (sk_data_dir / str(tenant_id) / str(timeline_id_2)).exists()
     assert (sk_data_dir / str(tenant_id) / str(timeline_id_3)).is_dir()
@@ -1147,7 +1156,10 @@ def test_delete_force(neon_env_builder: NeonEnvBuilder, auth_enabled: bool):
     assert (sk_data_dir / str(tenant_id_other) / str(timeline_id_other)).is_dir()
 
     # Remove non-existing branch, should succeed
-    assert not sk_http.timeline_delete_force(tenant_id, TimelineId("00" * 16))["dir_existed"]
+    assert sk_http.timeline_delete_force(tenant_id, TimelineId("00" * 16)) == {
+        "dir_existed": False,
+        "was_active": False,
+    }
     assert not (sk_data_dir / str(tenant_id) / str(timeline_id_1)).exists()
     assert not (sk_data_dir / str(tenant_id) / str(timeline_id_2)).exists()
     assert (sk_data_dir / str(tenant_id) / str(timeline_id_3)).exists()
@@ -1156,7 +1168,10 @@ def test_delete_force(neon_env_builder: NeonEnvBuilder, auth_enabled: bool):
 
     # Remove initial tenant fully (two branches are active)
     response = sk_http.tenant_delete_force(tenant_id)
-    assert response[str(timeline_id_3)]["dir_existed"]
+    assert response[str(timeline_id_3)] == {
+        "dir_existed": True,
+        "was_active": True,
+    }
     assert not (sk_data_dir / str(tenant_id)).exists()
     assert (sk_data_dir / str(tenant_id_other) / str(timeline_id_other)).is_dir()