Check postgres version and ensure that public schema exists

before running GRANT query on it
Merge pull request #2668 from neondatabase/main
2026-07-03 04:00:37 +00:00 · 2022-10-22 02:14:26 +03:00 · 2022-10-21 15:21:42 +03:00 · 2022-10-07 16:50:55 +03:00 · 2022-10-07 11:07:06 +03:00 · 2022-09-01 20:20:44 +03:00
235 changed files with 9524 additions and 18439 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -14,7 +14,6 @@
 !pgxn/
 !proxy/
 !safekeeper/
-!storage_broker/
 !vendor/postgres-v14/
 !vendor/postgres-v15/
 !workspace_hack/
--- a/.github/actions/run-python-test-set/action.yml
+++ b/.github/actions/run-python-test-set/action.yml
@@ -55,22 +55,6 @@ runs:
        name: neon-${{ runner.os }}-${{ inputs.build_type }}-artifact
        path: /tmp/neon

-    - name: Download Neon binaries for the previous release
-      if: inputs.build_type != 'remote'
-      uses: ./.github/actions/download
-      with:
-        name: neon-${{ runner.os }}-${{ inputs.build_type }}-artifact
-        path: /tmp/neon-previous
-        prefix: latest
-
-    - name: Download compatibility snapshot for Postgres 14
-      if: inputs.build_type != 'remote'
-      uses: ./.github/actions/download
-      with:
-        name: compatibility-snapshot-${{ inputs.build_type }}-pg14
-        path: /tmp/compatibility_snapshot_pg14
-        prefix: latest
-
    - name: Checkout
      if: inputs.needs_postgres_source == 'true'
      uses: actions/checkout@v3
@@ -92,15 +76,10 @@ runs:
    - name: Run pytest
      env:
        NEON_BIN: /tmp/neon/bin
-        COMPATIBILITY_NEON_BIN: /tmp/neon-previous/bin
-        COMPATIBILITY_POSTGRES_DISTRIB_DIR: /tmp/neon-previous/pg_install
        TEST_OUTPUT: /tmp/test_output
        BUILD_TYPE: ${{ inputs.build_type }}
        AWS_ACCESS_KEY_ID: ${{ inputs.real_s3_access_key_id }}
        AWS_SECRET_ACCESS_KEY: ${{ inputs.real_s3_secret_access_key }}
-        COMPATIBILITY_SNAPSHOT_DIR: /tmp/compatibility_snapshot_pg14
-        ALLOW_BACKWARD_COMPATIBILITY_BREAKAGE: contains(github.event.pull_request.labels.*.name, 'backward compatibility breakage')
-        ALLOW_FORWARD_COMPATIBILITY_BREAKAGE: contains(github.event.pull_request.labels.*.name, 'forward compatibility breakage')
      shell: bash -euxo pipefail {0}
      run: |
        # PLATFORM will be embedded in the perf test report
@@ -123,12 +102,7 @@ runs:
          exit 1
        fi
        if [[ "${{ inputs.run_in_parallel }}" == "true" ]]; then
-          # -n4 uses four processes to run tests via pytest-xdist
          EXTRA_PARAMS="-n4 $EXTRA_PARAMS"
-
-          # --dist=loadgroup points tests marked with @pytest.mark.xdist_group
-          # to the same worker to make @pytest.mark.order work with xdist
-          EXTRA_PARAMS="--dist=loadgroup $EXTRA_PARAMS"
        fi

        if [[ "${{ inputs.run_with_real_s3 }}" == "true" ]]; then
@@ -163,9 +137,9 @@ runs:
        # --verbose prints name of each test (helpful when there are
        # multiple tests in one file)
        # -rA prints summary in the end
+        # -n4 uses four processes to run tests via pytest-xdist
        # -s is not used to prevent pytest from capturing output, because tests are running
        # in parallel and logs are mixed between different tests
-        #
        mkdir -p $TEST_OUTPUT/allure/results
        "${cov_prefix[@]}" ./scripts/pytest \
          --junitxml=$TEST_OUTPUT/junit.xml \
@@ -180,17 +154,8 @@ runs:
          scripts/generate_and_push_perf_report.sh
        fi

-    - name: Upload compatibility snapshot for Postgres 14
-      if: github.ref_name == 'release'
-      uses: ./.github/actions/upload
-      with:
-        name: compatibility-snapshot-${{ inputs.build_type }}-pg14-${{ github.run_id }}
-        # The path includes a test name (test_create_snapshot) and directory that the test creates (compatibility_snapshot_pg14), keep the path in sync with the test
-        path: /tmp/test_output/test_create_snapshot/compatibility_snapshot_pg14/
-        prefix: latest
-
    - name: Create Allure report
-      if: success() || failure()
+      if: always()
      uses: ./.github/actions/allure-report
      with:
        action: store
--- a/.github/ansible/.gitignore
+++ b/.github/ansible/.gitignore
@@ -1,3 +1,5 @@
+zenith_install.tar.gz
+.zenith_current_version
 neon_install.tar.gz
 .neon_current_version

--- a/.github/ansible/get_binaries.sh
+++ b/.github/ansible/get_binaries.sh
@@ -25,7 +25,6 @@ mkdir neon_install/bin/
 docker cp ${ID}:/usr/local/bin/pageserver neon_install/bin/
 docker cp ${ID}:/usr/local/bin/pageserver_binutils neon_install/bin/
 docker cp ${ID}:/usr/local/bin/safekeeper neon_install/bin/
-docker cp ${ID}:/usr/local/bin/storage_broker neon_install/bin/
 docker cp ${ID}:/usr/local/bin/proxy neon_install/bin/
 docker cp ${ID}:/usr/local/v14/bin/ neon_install/v14/bin/
 docker cp ${ID}:/usr/local/v15/bin/ neon_install/v15/bin/
--- a/.github/ansible/neon-stress.hosts.yaml
+++ b/.github/ansible/neon-stress.hosts.yaml
@@ -3,6 +3,7 @@ storage:
    bucket_name: neon-storage-ireland
    bucket_region: eu-west-1
    console_mgmt_base_url: http://neon-stress-console.local
+    env_name: neon-stress
    etcd_endpoints: neon-stress-etcd.local:2379
    safekeeper_enable_s3_offload: 'false'
    pageserver_config_stub:
@@ -11,7 +12,6 @@ storage:
        bucket_name: "{{ bucket_name }}"
        bucket_region: "{{ bucket_region }}"
        prefix_in_bucket: "{{ inventory_hostname }}"
-    safekeeper_s3_prefix: neon-stress/wal
    hostname_suffix: ".local"
    remote_user: admin
  children:
--- a/.github/ansible/prod.ap-southeast-1.hosts.yaml
+++ b/.github/ansible/prod.ap-southeast-1.hosts.yaml
@@ -1,35 +0,0 @@
-storage:
-  vars:
-    bucket_name: neon-prod-storage-ap-southeast-1
-    bucket_region: ap-southeast-1
-    console_mgmt_base_url: http://console-release.local
-    etcd_endpoints: etcd-0.ap-southeast-1.aws.neon.tech:2379
-    pageserver_config_stub:
-      pg_distrib_dir: /usr/local
-      remote_storage:
-        bucket_name: "{{ bucket_name }}"
-        bucket_region: "{{ bucket_region }}"
-        prefix_in_bucket: "pageserver/v1"
-    safekeeper_s3_prefix: safekeeper/v1/wal
-    hostname_suffix: ""
-    remote_user: ssm-user
-    ansible_aws_ssm_region: ap-southeast-1
-    ansible_aws_ssm_bucket_name: neon-prod-storage-ap-southeast-1
-    console_region_id: aws-ap-southeast-1
-
-  children:
-    pageservers:
-      hosts:
-        pageserver-0.ap-southeast-1.aws.neon.tech:
-          ansible_host:  i-064de8ea28bdb495b
-        pageserver-1.ap-southeast-1.aws.neon.tech:
-          ansible_host:  i-0b180defcaeeb6b93
-
-    safekeepers:
-      hosts:
-        safekeeper-0.ap-southeast-1.aws.neon.tech:
-          ansible_host:  i-0d6f1dc5161eef894
-        safekeeper-1.ap-southeast-1.aws.neon.tech:
-          ansible_host:  i-0e338adda8eb2d19f
-        safekeeper-2.ap-southeast-1.aws.neon.tech:
-          ansible_host:  i-04fb63634e4679eb9
--- a/.github/ansible/prod.eu-central-1.hosts.yaml
+++ b/.github/ansible/prod.eu-central-1.hosts.yaml
@@ -1,35 +0,0 @@
-storage:
-  vars:
-    bucket_name: neon-prod-storage-eu-central-1
-    bucket_region: eu-central-1
-    console_mgmt_base_url: http://console-release.local
-    etcd_endpoints: etcd-0.eu-central-1.aws.neon.tech:2379
-    pageserver_config_stub:
-      pg_distrib_dir: /usr/local
-      remote_storage:
-        bucket_name: "{{ bucket_name }}"
-        bucket_region: "{{ bucket_region }}"
-        prefix_in_bucket: "pageserver/v1"
-    safekeeper_s3_prefix: safekeeper/v1/wal
-    hostname_suffix: ""
-    remote_user: ssm-user
-    ansible_aws_ssm_region: eu-central-1
-    ansible_aws_ssm_bucket_name: neon-prod-storage-eu-central-1
-    console_region_id: aws-eu-central-1
-
-  children:
-    pageservers:
-      hosts:
-        pageserver-0.eu-central-1.aws.neon.tech:
-          ansible_host:  i-0cd8d316ecbb715be
-        pageserver-1.eu-central-1.aws.neon.tech:
-          ansible_host:  i-090044ed3d383fef0
-
-    safekeepers:
-      hosts:
-        safekeeper-0.eu-central-1.aws.neon.tech:
-          ansible_host:  i-0b238612d2318a050
-        safekeeper-1.eu-central-1.aws.neon.tech:
-          ansible_host:  i-07b9c45e5c2637cd4
-        safekeeper-2.eu-central-1.aws.neon.tech:
-          ansible_host:  i-020257302c3c93d88
--- a/.github/ansible/prod.us-east-2.hosts.yaml
+++ b/.github/ansible/prod.us-east-2.hosts.yaml
@@ -1,36 +0,0 @@
-storage:
-  vars:
-    bucket_name: neon-prod-storage-us-east-2
-    bucket_region: us-east-2
-    console_mgmt_base_url: http://console-release.local
-    etcd_endpoints: etcd-0.us-east-2.aws.neon.tech:2379
-    pageserver_config_stub:
-      pg_distrib_dir: /usr/local
-      remote_storage:
-        bucket_name: "{{ bucket_name }}"
-        bucket_region: "{{ bucket_region }}"
-        prefix_in_bucket: "pageserver/v1"
-    safekeeper_s3_prefix: safekeeper/v1/wal
-    hostname_suffix: ""
-    remote_user: ssm-user
-    ansible_aws_ssm_region: us-east-2
-    ansible_aws_ssm_bucket_name: neon-prod-storage-us-east-2
-    console_region_id: aws-us-east-2
-
-  children:
-    pageservers:
-      hosts:
-        pageserver-0.us-east-2.aws.neon.tech:
-          ansible_host:  i-062227ba7f119eb8c
-        pageserver-1.us-east-2.aws.neon.tech:
-          ansible_host:  i-0b3ec0afab5968938
-
-    safekeepers:
-      hosts:
-        safekeeper-0.us-east-2.aws.neon.tech:
-          ansible_host:  i-0e94224750c57d346
-        safekeeper-1.us-east-2.aws.neon.tech:
-          ansible_host:  i-06d113fb73bfddeb0
-        safekeeper-2.us-east-2.aws.neon.tech:
-          ansible_host:  i-09f66c8e04afff2e8
-          
--- a/.github/ansible/production.hosts.yaml
+++ b/.github/ansible/production.hosts.yaml
@@ -1,6 +1,7 @@
 ---
 storage:
  vars:
+    env_name: prod-1
    console_mgmt_base_url: http://console-release.local
    bucket_name: zenith-storage-oregon
    bucket_region: us-west-2
@@ -11,7 +12,6 @@ storage:
        bucket_name: "{{ bucket_name }}"
        bucket_region: "{{ bucket_region }}"
        prefix_in_bucket: "{{ inventory_hostname }}"
-    safekeeper_s3_prefix: prod-1/wal
    hostname_suffix: ".local"
    remote_user: admin

@@ -22,10 +22,6 @@ storage:
          console_region_id: aws-us-west-2
        zenith-1-ps-3:
          console_region_id: aws-us-west-2
-        zenith-1-ps-4:
-          console_region_id: aws-us-west-2
-        zenith-1-ps-5:
-          console_region_id: aws-us-west-2

    safekeepers:
      hosts:
--- a/.github/ansible/ssm_config
+++ b/.github/ansible/ssm_config
@@ -1,2 +1,3 @@
 ansible_connection: aws_ssm
+ansible_aws_ssm_bucket_name: neon-dev-bucket
 ansible_python_interpreter: /usr/bin/python3
--- a/.github/ansible/staging.eu-west-1.hosts.yaml
+++ b/.github/ansible/staging.eu-west-1.hosts.yaml
@@ -1,33 +0,0 @@
-storage:
-  vars:
-    bucket_name: neon-dev-storage-eu-west-1
-    bucket_region: eu-west-1
-    console_mgmt_base_url: http://console-staging.local
-    etcd_endpoints: etcd-0.eu-west-1.aws.neon.build:2379
-    pageserver_config_stub:
-      pg_distrib_dir: /usr/local
-      remote_storage:
-        bucket_name: "{{ bucket_name }}"
-        bucket_region: "{{ bucket_region }}"
-        prefix_in_bucket: "pageserver/v1"
-    safekeeper_s3_prefix: safekeeper/v1/wal
-    hostname_suffix: ""
-    remote_user: ssm-user
-    ansible_aws_ssm_region: eu-west-1
-    ansible_aws_ssm_bucket_name: neon-dev-storage-eu-west-1
-    console_region_id: aws-eu-west-1
-
-  children:
-    pageservers:
-      hosts:
-        pageserver-0.eu-west-1.aws.neon.build:
-          ansible_host: i-01d496c5041c7f34c
-
-    safekeepers:
-      hosts:
-        safekeeper-0.eu-west-1.aws.neon.build:
-          ansible_host: i-05226ef85722831bf
-        safekeeper-1.eu-west-1.aws.neon.build:
-          ansible_host: i-06969ee1bf2958bfc
-        safekeeper-2.eu-west-1.aws.neon.build:
-          ansible_host: i-087892e9625984a0b
--- a/.github/ansible/staging.hosts.yaml
+++ b/.github/ansible/staging.hosts.yaml
@@ -3,14 +3,14 @@ storage:
    bucket_name: zenith-staging-storage-us-east-1
    bucket_region: us-east-1
    console_mgmt_base_url: http://console-staging.local
-    etcd_endpoints: etcd-0.us-east-2.aws.neon.build:2379
+    env_name: us-stage
+    etcd_endpoints: zenith-us-stage-etcd.local:2379
    pageserver_config_stub:
      pg_distrib_dir: /usr/local
      remote_storage:
        bucket_name: "{{ bucket_name }}"
        bucket_region: "{{ bucket_region }}"
        prefix_in_bucket: "{{ inventory_hostname }}"
-    safekeeper_s3_prefix: us-stage/wal
    hostname_suffix: ".local"
    remote_user: admin

--- a/.github/ansible/staging.us-east-2.hosts.yaml
+++ b/.github/ansible/staging.us-east-2.hosts.yaml
@@ -3,6 +3,7 @@ storage:
    bucket_name: neon-staging-storage-us-east-2
    bucket_region: us-east-2
    console_mgmt_base_url: http://console-staging.local
+    env_name: us-stage
    etcd_endpoints: etcd-0.us-east-2.aws.neon.build:2379
    pageserver_config_stub:
      pg_distrib_dir: /usr/local
@@ -10,11 +11,9 @@ storage:
        bucket_name: "{{ bucket_name }}"
        bucket_region: "{{ bucket_region }}"
        prefix_in_bucket: "pageserver/v1"
-    safekeeper_s3_prefix: safekeeper/v1/wal
    hostname_suffix: ""
    remote_user: ssm-user
    ansible_aws_ssm_region: us-east-2
-    ansible_aws_ssm_bucket_name: neon-staging-storage-us-east-2
    console_region_id: aws-us-east-2

  children:
@@ -22,8 +21,6 @@ storage:
      hosts:
        pageserver-0.us-east-2.aws.neon.build:
          ansible_host: i-0c3e70929edb5d691
-        pageserver-1.us-east-2.aws.neon.build:
-          ansible_host: i-0565a8b4008aa3f40

    safekeepers:
      hosts:
--- a/.github/ansible/systemd/safekeeper.service
+++ b/.github/ansible/systemd/safekeeper.service
@@ -6,7 +6,7 @@ After=network.target auditd.service
 Type=simple
 User=safekeeper
 Environment=RUST_BACKTRACE=1 NEON_REPO_DIR=/storage/safekeeper/data LD_LIBRARY_PATH=/usr/local/v14/lib
-ExecStart=/usr/local/bin/safekeeper -l {{ inventory_hostname }}{{ hostname_suffix }}:6500 --listen-http {{ inventory_hostname }}{{ hostname_suffix }}:7676 -D /storage/safekeeper/data --broker-endpoints={{ etcd_endpoints }} --remote-storage='{bucket_name="{{bucket_name}}", bucket_region="{{bucket_region}}", prefix_in_bucket="{{ safekeeper_s3_prefix }}"}'
+ExecStart=/usr/local/bin/safekeeper -l {{ inventory_hostname }}{{ hostname_suffix }}:6500 --listen-http {{ inventory_hostname }}{{ hostname_suffix }}:7676 -D /storage/safekeeper/data --broker-endpoints={{ etcd_endpoints }} --remote-storage='{bucket_name="{{bucket_name}}", bucket_region="{{bucket_region}}", prefix_in_bucket="{{ env_name }}/wal"}'
 ExecReload=/bin/kill -HUP $MAINPID
 KillMode=mixed
 KillSignal=SIGINT
--- a/.github/helm-values/dev-eu-west-1-zeta.neon-proxy-scram.yaml
+++ b/.github/helm-values/dev-eu-west-1-zeta.neon-proxy-scram.yaml
@@ -1,31 +0,0 @@
-# Helm chart values for neon-proxy-scram.
-# This is a YAML-formatted file.
-
-image:
-  repository: neondatabase/neon
-
-settings:
-  authBackend: "console"
-  authEndpoint: "http://console-staging.local/management/api/v2"
-  domain: "*.eu-west-1.aws.neon.build"
-
-# -- Additional labels for neon-proxy pods
-podLabels:
-  zenith_service: proxy-scram
-  zenith_env: dev
-  zenith_region: eu-west-1
-  zenith_region_slug: eu-west-1
-
-exposedService:
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: eu-west-1.aws.neon.build
-
-#metrics:
-#  enabled: true
-#  serviceMonitor:
-#    enabled: true
-#    selector:
-#      release: kube-prometheus-stack
--- a/.github/helm-values/dev-us-east-2-beta.neon-proxy-link.yaml
+++ b/.github/helm-values/dev-us-east-2-beta.neon-proxy-link.yaml
@@ -1,39 +0,0 @@
-# Helm chart values for neon-proxy-link.
-# This is a YAML-formatted file.
-
-image:
-  repository: neondatabase/neon
-
-settings:
-  authBackend: "link"
-  authEndpoint: "https://console.stage.neon.tech/authenticate_proxy_request/"
-  uri: "https://console.stage.neon.tech/psql_session/"
-
-# -- Additional labels for neon-proxy-link pods
-podLabels:
-  zenith_service: proxy
-  zenith_env: dev
-  zenith_region: us-east-2
-  zenith_region_slug: us-east-2
-
-service:
-  type: LoadBalancer
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internal
-    external-dns.alpha.kubernetes.io/hostname: neon-proxy-link-mgmt.beta.us-east-2.aws.neon.build
-
-exposedService:
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: neon-proxy-link.beta.us-east-2.aws.neon.build
-
-#metrics:
-#  enabled: true
-#  serviceMonitor:
-#    enabled: true
-#    selector:
-#      release: kube-prometheus-stack
--- a/.github/helm-values/prod-ap-southeast-1-epsilon.neon-proxy-scram.yaml
+++ b/.github/helm-values/prod-ap-southeast-1-epsilon.neon-proxy-scram.yaml
@@ -1,31 +0,0 @@
-# Helm chart values for neon-proxy-scram.
-# This is a YAML-formatted file.
-
-image:
-  repository: neondatabase/neon
-
-settings:
-  authBackend: "console"
-  authEndpoint: "http://console-release.local/management/api/v2"
-  domain: "*.ap-southeast-1.aws.neon.tech"
-
-# -- Additional labels for neon-proxy pods
-podLabels:
-  zenith_service: proxy-scram
-  zenith_env: prod
-  zenith_region: ap-southeast-1
-  zenith_region_slug: ap-southeast-1
-
-exposedService:
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: ap-southeast-1.aws.neon.tech
-
-#metrics:
-#  enabled: true
-#  serviceMonitor:
-#    enabled: true
-#    selector:
-#      release: kube-prometheus-stack
--- a/.github/helm-values/prod-eu-central-1-gamma.neon-proxy-scram.yaml
+++ b/.github/helm-values/prod-eu-central-1-gamma.neon-proxy-scram.yaml
@@ -1,31 +0,0 @@
-# Helm chart values for neon-proxy-scram.
-# This is a YAML-formatted file.
-
-image:
-  repository: neondatabase/neon
-
-settings:
-  authBackend: "console"
-  authEndpoint: "http://console-release.local/management/api/v2"
-  domain: "*.eu-central-1.aws.neon.tech"
-
-# -- Additional labels for neon-proxy pods
-podLabels:
-  zenith_service: proxy-scram
-  zenith_env: prod
-  zenith_region: eu-central-1
-  zenith_region_slug: eu-central-1
-
-exposedService:
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: eu-central-1.aws.neon.tech
-
-#metrics:
-#  enabled: true
-#  serviceMonitor:
-#    enabled: true
-#    selector:
-#      release: kube-prometheus-stack
--- a/.github/helm-values/prod-us-east-2-delta.neon-proxy-scram.yaml
+++ b/.github/helm-values/prod-us-east-2-delta.neon-proxy-scram.yaml
@@ -1,31 +0,0 @@
-# Helm chart values for neon-proxy-scram.
-# This is a YAML-formatted file.
-
-image:
-  repository: neondatabase/neon
-
-settings:
-  authBackend: "console"
-  authEndpoint: "http://console-release.local/management/api/v2"
-  domain: "*.us-east-2.aws.neon.tech"
-
-# -- Additional labels for neon-proxy pods
-podLabels:
-  zenith_service: proxy-scram
-  zenith_env: prod
-  zenith_region: us-east-2
-  zenith_region_slug: us-east-2
-
-exposedService:
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: us-east-2.aws.neon.tech
-
-#metrics:
-#  enabled: true
-#  serviceMonitor:
-#    enabled: true
-#    selector:
-#      release: kube-prometheus-stack
--- a/.github/workflows/benchmarking.yml
+++ b/.github/workflows/benchmarking.yml
@@ -144,9 +144,7 @@ jobs:
        # neon-captest-new: Run pgbench in a freshly created project
        # neon-captest-reuse: Same, but reusing existing project
        # neon-captest-prefetch: Same, with prefetching enabled (new project)
-        # rds-aurora: Aurora Postgres Serverless v2 with autoscaling from 0.5 to 2 ACUs
-        # rds-postgres: RDS Postgres db.m5.large instance (2 vCPU, 8 GiB) with gp3 EBS storage
-        platform: [ neon-captest-new, neon-captest-reuse, neon-captest-prefetch, rds-postgres ]
+        platform: [ neon-captest-new, neon-captest-reuse, neon-captest-prefetch ]
        db_size: [ 10gb ]
        include:
          - platform: neon-captest-new
@@ -166,7 +164,7 @@ jobs:
      SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref == 'refs/heads/main' ) }}
      PLATFORM: ${{ matrix.platform }}

-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
    container:
      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rustlegacy:pinned
      options: --init
@@ -209,11 +207,8 @@ jobs:
          rds-aurora)
            CONNSTR=${{ secrets.BENCHMARK_RDS_CONNSTR }}
            ;;
-          rds-postgres)
-            CONNSTR=${{ secrets.BENCHMARK_RDS_POSTGRES_CONNSTR }}
-            ;;
          *)
-            echo 2>&1 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neon-captest-reuse', 'neon-captest-new', 'neon-captest-prefetch', 'rds-aurora', or 'rds-postgres'"
+            echo 2>&1 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neon-captest-reuse', 'neon-captest-new', 'neon-captest-prefetch' or 'rds-aurora'"
            exit 1
            ;;
        esac
@@ -270,7 +265,7 @@ jobs:
        PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"

    - name: Create Allure report
-      if: success() || failure()
+      if: always()
      uses: ./.github/actions/allure-report
      with:
        action: generate
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -18,8 +18,8 @@ env:

 jobs:
  tag:
-    runs-on: [ self-hosted, dev, x64 ]
-    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned
+    runs-on: dev
+    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:latest
    outputs:
      build-tag: ${{steps.build-tag.outputs.tag}}

@@ -46,7 +46,7 @@ jobs:
        id: build-tag

  build-neon:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
    container:
      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
      options: --init
@@ -127,8 +127,8 @@ jobs:
            target/
          # Fall back to older versions of the key, if no cache for current Cargo.lock was found
          key: |
-            v10-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('Cargo.lock') }}
-            v10-${{ runner.os }}-${{ matrix.build_type }}-cargo-
+            v9-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('Cargo.lock') }}
+            v9-${{ runner.os }}-${{ matrix.build_type }}-cargo-

      - name: Cache postgres v14 build
        id: cache_pg_14
@@ -236,7 +236,7 @@ jobs:
        uses: ./.github/actions/save-coverage-data

  regress-tests:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
    container:
      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
      options: --init
@@ -268,8 +268,34 @@ jobs:
        if: matrix.build_type == 'debug'
        uses: ./.github/actions/save-coverage-data

+  upload-latest-artifacts:
+    runs-on: dev
+    container:
+      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
+      options: --init
+    needs: [ regress-tests ]
+    if: github.ref_name == 'main'
+    steps:
+      - name: Copy Neon artifact to the latest directory
+        shell: bash -euxo pipefail {0}
+        env:
+          BUCKET: neon-github-public-dev
+          PREFIX: artifacts/${{ github.run_id }}
+        run: |
+          for build_type in debug release; do
+            FILENAME=neon-${{ runner.os }}-${build_type}-artifact.tar.zst
+
+            S3_KEY=$(aws s3api list-objects-v2 --bucket ${BUCKET} --prefix ${PREFIX} | jq -r '.Contents[].Key' | grep ${FILENAME} | sort --version-sort | tail -1 || true)
+            if [ -z "${S3_KEY}" ]; then
+              echo 2>&1 "Neither s3://${BUCKET}/${PREFIX}/${FILENAME} nor its version from previous attempts exist"
+              exit 1
+            fi
+
+            time aws s3 cp --only-show-errors s3://${BUCKET}/${S3_KEY} s3://${BUCKET}/artifacts/latest/${FILENAME}
+          done
+
  benchmarks:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
    container:
      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
      options: --init
@@ -300,12 +326,12 @@ jobs:
      # while coverage is currently collected for the debug ones

  merge-allure-report:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
    container:
      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
      options: --init
    needs: [ regress-tests, benchmarks ]
-    if: success() || failure()
+    if: always()
    strategy:
      fail-fast: false
      matrix:
@@ -338,7 +364,7 @@ jobs:
          DATABASE_URL="$TEST_RESULT_CONNSTR" poetry run python3 scripts/ingest_regress_test_result.py --revision ${SHA} --reference ${GITHUB_REF} --build-type ${BUILD_TYPE} --ingest suites.json

  coverage-report:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
    container:
      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
      options: --init
@@ -363,7 +389,7 @@ jobs:
            !~/.cargo/registry/src
            ~/.cargo/git/
            target/
-          key: v10-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('Cargo.lock') }}
+          key: v9-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('Cargo.lock') }}

      - name: Get Neon artifact
        uses: ./.github/actions/download
@@ -415,19 +441,15 @@ jobs:
        shell: bash -euxo pipefail {0}

  trigger-e2e-tests:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
    container:
      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned
      options: --init
-    needs: [ push-docker-hub, tag ]
+    needs: [ build-neon ]
    steps:
      - name: Set PR's status to pending and request a remote CI test
        run: |
-          # For pull requests, GH Actions set "github.sha" variable to point at a fake merge commit
-          # but we need to use a real sha of a latest commit in the PR's branch for the e2e job,
-          # to place a job run status update later.
          COMMIT_SHA=${{ github.event.pull_request.head.sha }}
-          # For non-PR kinds of runs, the above will produce an empty variable, pick the original sha value for those
          COMMIT_SHA=${COMMIT_SHA:-${{ github.sha }}}

          REMOTE_REPO="${{ github.repository_owner }}/cloud"
@@ -453,14 +475,12 @@ jobs:
              \"inputs\": {
                \"ci_job_name\": \"neon-cloud-e2e\",
                \"commit_hash\": \"$COMMIT_SHA\",
-                \"remote_repo\": \"${{ github.repository }}\",
-                \"storage_image_tag\": \"${{ needs.tag.outputs.build-tag }}\",
-                \"compute_image_tag\": \"${{ needs.tag.outputs.build-tag }}\"
+                \"remote_repo\": \"${{ github.repository }}\"
              }
            }"

  neon-image:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
    needs: [ tag ]
    container: gcr.io/kaniko-project/executor:v1.9.0-debug

@@ -478,7 +498,7 @@ jobs:
        run: /kaniko/executor --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --build-arg GIT_VERSION=${{ github.sha }} --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}}

  compute-tools-image:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
    needs: [ tag ]
    container: gcr.io/kaniko-project/executor:v1.9.0-debug

@@ -492,8 +512,28 @@ jobs:
      - name: Kaniko build compute tools
        run: /kaniko/executor --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-tools --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}}

+  compute-node-image:
+    runs-on: dev
+    container: gcr.io/kaniko-project/executor:v1.9.0-debug
+    needs: [ tag ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1 # v3 won't work with kaniko
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Configure ECR login
+        run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json
+
+        # compute-node uses postgres 14, which is default now
+        # cloud repo depends on this image name, thus duplicating it
+        # remove compute-node when cloud repo is updated
+      - name: Kaniko build compute node with extensions v14 (compatibility)
+        run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-node-v14 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:${{needs.tag.outputs.build-tag}}
+
  compute-node-image-v14:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
    container: gcr.io/kaniko-project/executor:v1.9.0-debug
    needs: [ tag ]
    steps:
@@ -509,8 +549,9 @@ jobs:
      - name: Kaniko build compute node with extensions v14
        run: /kaniko/executor --skip-unused-stages  --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache  --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-node-v14 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}}

+
  compute-node-image-v15:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
    container: gcr.io/kaniko-project/executor:v1.9.0-debug
    needs: [ tag ]
    steps:
@@ -526,58 +567,18 @@ jobs:
      - name: Kaniko build compute node with extensions v15
        run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-node-v15 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:${{needs.tag.outputs.build-tag}}

-  test-images:
-    needs: [ tag, neon-image, compute-node-image-v14, compute-node-image-v15, compute-tools-image ]
-    runs-on: [ self-hosted, dev, x64 ]
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      # `neondatabase/neon` contains multiple binaries, all of them use the same input for the version into the same version formatting library.
-      # Pick pageserver as currently the only binary with extra "version" features printed in the string to verify.
-      # Regular pageserver version string looks like
-      #   Neon page server git-env:32d14403bd6ab4f4520a94cbfd81a6acef7a526c failpoints: true, features: []
-      # Bad versions might loop like:
-      #   Neon page server git-env:local failpoints: true, features: ["testing"]
-      # Ensure that we don't have bad versions.
-      - name: Verify image versions
-        shell: bash # ensure no set -e for better error messages
-        run: |
-          pageserver_version=$(docker run --rm 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}} "/bin/sh" "-c" "/usr/local/bin/pageserver --version")
-
-          echo "Pageserver version string: $pageserver_version"
-
-          if ! echo "$pageserver_version" | grep -qv 'git-env:local' ; then
-            echo "Pageserver version should not be the default Dockerfile one"
-            exit 1
-          fi
-
-          if ! echo "$pageserver_version" | grep -qv '"testing"' ; then
-            echo "Pageserver version should have no testing feature enabled"
-            exit 1
-          fi
-
-      - name: Verify docker-compose example
-        run: env REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com TAG=${{needs.tag.outputs.build-tag}} ./docker-compose/docker_compose_test.sh
-
-      - name: Print logs and clean up
-        if: always()
-        run: |
-          docker compose -f ./docker-compose/docker-compose.yml logs || 0
-          docker compose -f ./docker-compose/docker-compose.yml down
-
  promote-images:
-    runs-on: [ self-hosted, dev, x64 ]
-    needs: [ tag, test-images ]
+    runs-on: dev
+    needs: [ tag, neon-image, compute-node-image, compute-node-image-v14, compute-node-image-v15, compute-tools-image ]
    if: github.event_name != 'workflow_dispatch'
    container: amazon/aws-cli
    strategy:
      fail-fast: false
      matrix:
-        name: [ neon, compute-node-v14, compute-node-v15, compute-tools ]
+        # compute-node uses postgres 14, which is default now
+        # cloud repo depends on this image name, thus duplicating it
+        # remove compute-node when cloud repo is updated
+        name: [ neon, compute-node, compute-node-v14, compute-node-v15, compute-tools ]

    steps:
      - name: Promote image to latest
@@ -586,7 +587,7 @@ jobs:
          aws ecr put-image --repository-name ${{ matrix.name }} --image-tag latest --image-manifest "$MANIFEST"

  push-docker-hub:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
    needs: [ promote-images, tag ]
    container: golang:1.19-bullseye

@@ -607,6 +608,9 @@ jobs:
      - name: Pull compute tools image from ECR
        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} compute-tools

+      - name: Pull compute node image from ECR
+        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:${{needs.tag.outputs.build-tag}} compute-node
+
      - name: Pull compute node v14 image from ECR
        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}} compute-node-v14

@@ -621,10 +625,11 @@ jobs:
          (github.ref_name == 'main' || github.ref_name == 'release') &&
          github.event_name != 'workflow_dispatch'
        run: |
-          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/neon:latest
-          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:latest
-          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:latest
-          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:latest
+          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.us-east-2.amazonaws.com/neon:latest
+          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.us-east-2.amazonaws.com/compute-tools:latest
+          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.us-east-2.amazonaws.com/compute-node:latest
+          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.us-east-2.amazonaws.com/compute-node-v14:latest
+          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.us-east-2.amazonaws.com/compute-node-v15:latest

      - name: Configure Docker Hub login
        run: |
@@ -638,6 +643,9 @@ jobs:
      - name: Push compute tools image to Docker Hub
        run: crane push compute-tools neondatabase/compute-tools:${{needs.tag.outputs.build-tag}}

+      - name: Push compute node image to Docker Hub
+        run: crane push compute-node neondatabase/compute-node:${{needs.tag.outputs.build-tag}}
+
      - name: Push compute node v14 image to Docker Hub
        run: crane push compute-node-v14 neondatabase/compute-node-v14:${{needs.tag.outputs.build-tag}}

@@ -654,6 +662,7 @@ jobs:
        run: |
          crane tag neondatabase/neon:${{needs.tag.outputs.build-tag}} latest
          crane tag neondatabase/compute-tools:${{needs.tag.outputs.build-tag}} latest
+          crane tag neondatabase/compute-node:${{needs.tag.outputs.build-tag}} latest
          crane tag neondatabase/compute-node-v14:${{needs.tag.outputs.build-tag}} latest
          crane tag neondatabase/compute-node-v15:${{needs.tag.outputs.build-tag}} latest

@@ -736,7 +745,7 @@ jobs:
          rm -f neon_install.tar.gz .neon_current_version

  deploy-new:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
    # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version.
    # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly
@@ -747,80 +756,9 @@ jobs:
    defaults:
      run:
        shell: bash
-    strategy:
-      matrix:
-        target_region: [ us-east-2 ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          fetch-depth: 0
-
-      - name: Redeploy
-        run: |
-          export DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          cd "$(pwd)/.github/ansible"
-          if [[ "$GITHUB_REF_NAME" == "main" ]]; then
-            ./get_binaries.sh
-          elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
-            RELEASE=true ./get_binaries.sh
-          else
-            echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
-            exit 1
-          fi
-          ansible-galaxy collection install sivel.toiletwater
-          ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_STAGING_API_KEY}}
-          rm -f neon_install.tar.gz .neon_current_version
-
-  deploy-pr-test-new:
-    runs-on: [ self-hosted, dev, x64 ]
-    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
-    # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version.
-    # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly
-    needs: [ push-docker-hub, tag, regress-tests ]
-    if: |
-      contains(github.event.pull_request.labels.*.name, 'deploy-test-storage') && 
-      github.event_name != 'workflow_dispatch'
-    defaults:
-      run:
-        shell: bash
-    strategy:
-      matrix:
-        target_region: [ eu-west-1 ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          fetch-depth: 0
-
-      - name: Redeploy
-        run: |
-          export DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          cd "$(pwd)/.github/ansible"
-
-          ./get_binaries.sh
-
-          ansible-galaxy collection install sivel.toiletwater
-          ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_STAGING_API_KEY}}
-          rm -f neon_install.tar.gz .neon_current_version
-
-  deploy-prod-new:
-    runs-on: prod
-    container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
-    # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version.
-    # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly
-    needs: [ push-docker-hub, tag, regress-tests ]
-    if: |
-      (github.ref_name == 'release') &&
-      github.event_name != 'workflow_dispatch'
-    defaults:
-      run:
-        shell: bash
-    strategy:
-      matrix:
-        target_region: [ us-east-2, eu-central-1, ap-southeast-1 ]
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
@@ -843,11 +781,11 @@ jobs:
          fi

          ansible-galaxy collection install sivel.toiletwater
-          ansible-playbook deploy.yaml -i prod.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_PRODUCTION_API_KEY}}
+          ansible-playbook deploy.yaml -i staging.us-east-2.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_STAGING_API_KEY}}
          rm -f neon_install.tar.gz .neon_current_version

  deploy-proxy:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:latest
    # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
    needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
@@ -889,25 +827,16 @@ jobs:
          helm upgrade ${{ matrix.proxy_job }}-scram neondatabase/neon-proxy --namespace neon-proxy --install -f .github/helm-values/${{ matrix.proxy_config }}-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s

  deploy-proxy-new:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
    # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
-    needs: [ push-docker-hub, tag, regress-tests ]
+    needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
    if: |
-      (github.ref_name == 'main') &&
+      (github.ref_name == 'main' || github.ref_name == 'release') &&
      github.event_name != 'workflow_dispatch'
    defaults:
      run:
        shell: bash
-    strategy:
-      matrix:
-        include:
-          - target_region:  us-east-2
-            target_cluster: dev-us-east-2-beta
-            deploy_link_proxy: true
-          - target_region:  eu-west-1
-            target_cluster: dev-eu-west-1-zeta
-            deploy_link_proxy: false
    steps:
      - name: Checkout
        uses: actions/checkout@v3
@@ -915,91 +844,12 @@ jobs:
          submodules: true
          fetch-depth: 0

-      - name: Configure environment
+      - name: Configure environment 
        run: |
          helm repo add neondatabase https://neondatabase.github.io/helm-charts
-          aws --region ${{ matrix.target_region }} eks update-kubeconfig --name  ${{ matrix.target_cluster }}
-
-      - name: Re-deploy scram proxy
-        run: |
-          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
-
-      - name: Re-deploy link proxy
-        if: matrix.deploy_link_proxy
-        run: |
-          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
-
-  deploy-proxy-prod-new:
-    runs-on: prod
-    container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
-    # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
-    needs: [ push-docker-hub, tag, regress-tests ]
-    if: |
-      (github.ref_name == 'release') &&
-      github.event_name != 'workflow_dispatch'
-    defaults:
-      run:
-        shell: bash
-    strategy:
-      matrix:
-        include:
-          - target_region:  us-east-2
-            target_cluster: prod-us-east-2-delta
-          - target_region: eu-central-1
-            target_cluster: prod-eu-central-1-gamma
-          - target_region: ap-southeast-1
-            target_cluster: prod-ap-southeast-1-epsilon
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          fetch-depth: 0
-
-      - name: Configure environment
-        run: |
-          helm repo add neondatabase https://neondatabase.github.io/helm-charts
-          aws --region ${{ matrix.target_region }} eks update-kubeconfig --name  ${{ matrix.target_cluster }}
+          aws --region us-east-2 eks update-kubeconfig --name dev-us-east-2-beta --role-arn arn:aws:iam::369495373322:role/github-runner

      - name: Re-deploy proxy
        run: |
          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
-
-  promote-compatibility-data:
-    runs-on: [ self-hosted, dev, x64 ]
-    container:
-      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
-      options: --init
-    needs: [ deploy, deploy-proxy ]
-    if: github.ref_name == 'release' && github.event_name != 'workflow_dispatch'
-    steps:
-      - name: Promote compatibility snapshot for the release
-        shell: bash -euxo pipefail {0}
-        env:
-          BUCKET: neon-github-public-dev
-          PREFIX: artifacts/latest
-        run: |
-          # Update compatibility snapshot for the release
-          for build_type in debug release; do
-            OLD_FILENAME=compatibility-snapshot-${build_type}-pg14-${GITHUB_RUN_ID}.tar.zst
-            NEW_FILENAME=compatibility-snapshot-${build_type}-pg14.tar.zst
-
-            time aws s3 mv --only-show-errors s3://${BUCKET}/${PREFIX}/${OLD_FILENAME} s3://${BUCKET}/${PREFIX}/${NEW_FILENAME}
-          done
-
-          # Update Neon artifact for the release (reuse already uploaded artifact)
-          for build_type in debug release; do
-            OLD_PREFIX=artifacts/${GITHUB_RUN_ID}
-            FILENAME=neon-${{ runner.os }}-${build_type}-artifact.tar.zst
-
-            S3_KEY=$(aws s3api list-objects-v2 --bucket ${BUCKET} --prefix ${OLD_PREFIX} | jq -r '.Contents[].Key' | grep ${FILENAME} | sort --version-sort | tail -1 || true)
-            if [ -z "${S3_KEY}" ]; then
-              echo 2>&1 "Neither s3://${BUCKET}/${OLD_PREFIX}/${FILENAME} nor its version from previous attempts exist"
-              exit 1
-            fi
-
-            time aws s3 cp --only-show-errors s3://${BUCKET}/${S3_KEY} s3://${BUCKET}/${PREFIX}/${FILENAME}
-          done
+          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/dev-us-east-2-beta.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
--- a/.github/workflows/codestyle.yml
+++ b/.github/workflows/codestyle.yml
@@ -48,11 +48,11 @@ jobs:
        if: matrix.os == 'ubuntu-latest'
        run: |
          sudo apt update
-          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev libssl-dev protobuf-compiler
+          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev libssl-dev

      - name: Install macOS postgres dependencies
        if: matrix.os == 'macos-latest'
-        run: brew install flex bison openssl protobuf
+        run: brew install flex bison openssl

      - name: Set pg 14 revision for caching
        id: pg_v14_rev
@@ -106,7 +106,7 @@ jobs:
            !~/.cargo/registry/src
            ~/.cargo/git
            target
-          key: v6-${{ runner.os }}-cargo-${{ hashFiles('./Cargo.lock') }}-rust
+          key: v5-${{ runner.os }}-cargo-${{ hashFiles('./Cargo.lock') }}-rust

      - name: Run cargo clippy
        run: ./run_clippy.sh
@@ -115,7 +115,7 @@ jobs:
        run: cargo build --locked --all --all-targets

  check-rust-dependencies:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
    container:
      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
      options: --init
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,7 +1,7 @@
 [submodule "vendor/postgres-v14"]
 	path = vendor/postgres-v14
 	url = https://github.com/neondatabase/postgres.git
-	branch = REL_14_STABLE_neon
+	branch = main
 [submodule "vendor/postgres-v15"]
 	path = vendor/postgres-v15
 	url = https://github.com/neondatabase/postgres.git
--- a/11
+++ b/11
@@ -1,11 +0,0 @@
-/compute_tools/ @neondatabase/control-plane
-/control_plane/ @neondatabase/compute @neondatabase/storage
-/libs/pageserver_api/ @neondatabase/compute @neondatabase/storage
-/libs/postgres_ffi/ @neondatabase/compute 
-/libs/remote_storage/ @neondatabase/storage 
-/libs/safekeeper_api/ @neondatabase/safekeepers  
-/pageserver/ @neondatabase/compute @neondatabase/storage 
-/pgxn/ @neondatabase/compute
-/proxy/ @neondatabase/control-plane 
-/safekeeper/ @neondatabase/safekeepers
-/vendor/ @neondatabase/compute
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -317,6 +317,12 @@ dependencies = [
 "generic-array",
 ]

+[[package]]
+name = "boxfnonce"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5988cb1d626264ac94100be357308f29ff7cbdd3b36bda27f450a4ee3f713426"
+
 [[package]]
 name = "bstr"
 version = "1.0.1"
@@ -457,26 +463,11 @@ checksum = "6bf8832993da70a4c6d13c581f4463c2bdda27b9bf1c5498dc4365543abe6d6f"
 dependencies = [
 "atty",
 "bitflags",
- "clap_derive",
 "clap_lex 0.3.0",
- "once_cell",
 "strsim",
 "termcolor",
 ]

-[[package]]
-name = "clap_derive"
-version = "4.0.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c42f169caba89a7d512b5418b09864543eeb4d497416c917d7137863bd2076ad"
-dependencies = [
- "heck",
- "proc-macro-error",
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "clap_lex"
 version = "0.2.4"
@@ -601,7 +592,6 @@ dependencies = [
 "once_cell",
 "pageserver_api",
 "postgres",
- "postgres_connection",
 "regex",
 "reqwest",
 "safekeeper_api",
@@ -610,7 +600,6 @@ dependencies = [
 "tar",
 "thiserror",
 "toml",
- "url",
 "utils",
 "workspace_hack",
 ]
@@ -860,6 +849,16 @@ dependencies = [
 "syn",
 ]

+[[package]]
+name = "daemonize"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70c24513e34f53b640819f0ac9f705b673fcf4006d7aab8778bee72ebfc89815"
+dependencies = [
+ "boxfnonce",
+ "libc",
+]
+
 [[package]]
 name = "darling"
 version = "0.14.1"
@@ -1021,11 +1020,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9fb8664f6ea68aba5503d42dd1be786b0f1bd9b7972e7f40208c83ef74db91bf"
 dependencies = [
 "http",
- "prost 0.10.4",
+ "prost",
 "tokio",
 "tokio-stream",
- "tonic 0.7.2",
- "tonic-build 0.7.2",
+ "tonic",
+ "tonic-build",
 "tower",
 "tower-service",
 ]
@@ -2141,6 +2140,7 @@ dependencies = [
 "crc32c",
 "criterion",
 "crossbeam-utils",
+ "daemonize",
 "etcd_broker",
 "fail",
 "futures",
@@ -2156,14 +2156,11 @@ dependencies = [
 "num-traits",
 "once_cell",
 "pageserver_api",
- "pin-project-lite",
 "postgres",
 "postgres-protocol",
 "postgres-types",
- "postgres_connection",
 "postgres_ffi",
 "pprof",
- "pq_proto",
 "rand",
 "regex",
 "remote_storage",
@@ -2173,10 +2170,8 @@ dependencies = [
 "serde_json",
 "serde_with",
 "signal-hook",
- "svg_fmt",
 "tar",
 "tempfile",
- "tenant_size_model",
 "thiserror",
 "tokio",
 "tokio-postgres",
@@ -2193,11 +2188,7 @@ dependencies = [
 name = "pageserver_api"
 version = "0.1.0"
 dependencies = [
- "anyhow",
- "byteorder",
- "bytes",
 "const_format",
- "postgres_ffi",
 "serde",
 "serde_with",
 "utils",
@@ -2408,19 +2399,6 @@ dependencies = [
 "postgres-protocol",
 ]

-[[package]]
-name = "postgres_connection"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "itertools",
- "once_cell",
- "postgres",
- "tokio-postgres",
- "url",
- "workspace_hack",
-]
-
 [[package]]
 name = "postgres_ffi"
 version = "0.1.0"
@@ -2470,21 +2448,6 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"

-[[package]]
-name = "pq_proto"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "bytes",
- "pin-project-lite",
- "postgres-protocol",
- "rand",
- "serde",
- "tokio",
- "tracing",
- "workspace_hack",
-]
-
 [[package]]
 name = "prettyplease"
 version = "0.1.21"
@@ -2495,30 +2458,6 @@ dependencies = [
 "syn",
 ]

-[[package]]
-name = "proc-macro-error"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
-dependencies = [
- "proc-macro-error-attr",
- "proc-macro2",
- "quote",
- "syn",
- "version_check",
-]
-
-[[package]]
-name = "proc-macro-error-attr"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
-dependencies = [
- "proc-macro2",
- "quote",
- "version_check",
-]
-
 [[package]]
 name = "proc-macro-hack"
 version = "0.5.19"
@@ -2570,17 +2509,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71adf41db68aa0daaefc69bb30bcd68ded9b9abaad5d1fbb6304c4fb390e083e"
 dependencies = [
 "bytes",
- "prost-derive 0.10.1",
-]
-
-[[package]]
-name = "prost"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0841812012b2d4a6145fae9a6af1534873c32aa67fff26bd09f8fa42c83f95a"
-dependencies = [
- "bytes",
- "prost-derive 0.11.2",
+ "prost-derive",
 ]

 [[package]]
@@ -2598,35 +2527,13 @@ dependencies = [
 "log",
 "multimap",
 "petgraph",
- "prost 0.10.4",
- "prost-types 0.10.1",
+ "prost",
+ "prost-types",
 "regex",
 "tempfile",
 "which",
 ]

-[[package]]
-name = "prost-build"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d8b442418ea0822409d9e7d047cbf1e7e9e1760b172bf9982cf29d517c93511"
-dependencies = [
- "bytes",
- "heck",
- "itertools",
- "lazy_static",
- "log",
- "multimap",
- "petgraph",
- "prettyplease",
- "prost 0.11.2",
- "prost-types 0.11.2",
- "regex",
- "syn",
- "tempfile",
- "which",
-]
-
 [[package]]
 name = "prost-derive"
 version = "0.10.1"
@@ -2640,19 +2547,6 @@ dependencies = [
 "syn",
 ]

-[[package]]
-name = "prost-derive"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "164ae68b6587001ca506d3bf7f1000bfa248d0e1217b618108fba4ec1d0cc306"
-dependencies = [
- "anyhow",
- "itertools",
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "prost-types"
 version = "0.10.1"
@@ -2660,17 +2554,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68"
 dependencies = [
 "bytes",
- "prost 0.10.4",
-]
-
-[[package]]
-name = "prost-types"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "747761bc3dc48f9a34553bf65605cf6cb6288ba219f3450b4275dbd81539551a"
-dependencies = [
- "bytes",
- "prost 0.11.2",
+ "prost",
 ]

 [[package]]
@@ -2696,7 +2580,6 @@ dependencies = [
 "once_cell",
 "parking_lot 0.12.1",
 "pin-project-lite",
- "pq_proto",
 "rand",
 "rcgen",
 "reqwest",
@@ -3200,6 +3083,7 @@ dependencies = [
 "clap 4.0.15",
 "const_format",
 "crc32c",
+ "daemonize",
 "etcd_broker",
 "fs2",
 "git-version",
@@ -3207,13 +3091,11 @@ dependencies = [
 "humantime",
 "hyper",
 "metrics",
- "nix 0.25.0",
 "once_cell",
 "parking_lot 0.12.1",
 "postgres",
 "postgres-protocol",
 "postgres_ffi",
- "pq_proto",
 "regex",
 "remote_storage",
 "safekeeper_api",
@@ -3532,32 +3414,6 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"

-[[package]]
-name = "storage_broker"
-version = "0.1.0"
-dependencies = [
- "async-stream",
- "bytes",
- "clap 4.0.15",
- "futures",
- "futures-core",
- "futures-util",
- "git-version",
- "humantime",
- "hyper",
- "metrics",
- "once_cell",
- "parking_lot 0.12.1",
- "prost 0.11.2",
- "tokio",
- "tokio-stream",
- "tonic 0.8.2",
- "tonic-build 0.8.2",
- "tracing",
- "utils",
- "workspace_hack",
-]
-
 [[package]]
 name = "str_stack"
 version = "0.1.0"
@@ -3605,12 +3461,6 @@ version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"

-[[package]]
-name = "svg_fmt"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fb1df15f412ee2e9dfc1c504260fa695c1c3f10fe9f4a6ee2d2184d7d6450e2"
-
 [[package]]
 name = "symbolic-common"
 version = "8.8.0"
@@ -3688,13 +3538,6 @@ dependencies = [
 "winapi",
 ]

-[[package]]
-name = "tenant_size_model"
-version = "0.1.0"
-dependencies = [
- "workspace_hack",
-]
-
 [[package]]
 name = "termcolor"
 version = "1.1.3"
@@ -3958,40 +3801,8 @@ dependencies = [
 "hyper-timeout",
 "percent-encoding",
 "pin-project",
- "prost 0.10.4",
- "prost-derive 0.10.1",
- "tokio",
- "tokio-stream",
- "tokio-util",
- "tower",
- "tower-layer",
- "tower-service",
- "tracing",
- "tracing-futures",
-]
-
-[[package]]
-name = "tonic"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55b9af819e54b8f33d453655bef9b9acc171568fb49523078d0cc4e7484200ec"
-dependencies = [
- "async-stream",
- "async-trait",
- "axum",
- "base64",
- "bytes",
- "futures-core",
- "futures-util",
- "h2",
- "http",
- "http-body",
- "hyper",
- "hyper-timeout",
- "percent-encoding",
- "pin-project",
- "prost 0.11.2",
- "prost-derive 0.11.2",
+ "prost",
+ "prost-derive",
 "tokio",
 "tokio-stream",
 "tokio-util",
@@ -4010,20 +3821,7 @@ checksum = "d9263bf4c9bfaae7317c1c2faf7f18491d2fe476f70c414b73bf5d445b00ffa1"
 dependencies = [
 "prettyplease",
 "proc-macro2",
- "prost-build 0.10.4",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "tonic-build"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48c6fd7c2581e36d63388a9e04c350c21beb7a8b059580b2e93993c526899ddc"
-dependencies = [
- "prettyplease",
- "proc-macro2",
- "prost-build 0.11.2",
+ "prost-build",
 "quote",
 "syn",
 ]
@@ -4134,16 +3932,6 @@ dependencies = [
 "tracing-core",
 ]

-[[package]]
-name = "tracing-serde"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1"
-dependencies = [
- "serde",
- "tracing-core",
-]
-
 [[package]]
 name = "tracing-subscriber"
 version = "0.3.16"
@@ -4154,15 +3942,12 @@ dependencies = [
 "nu-ansi-term",
 "once_cell",
 "regex",
- "serde",
- "serde_json",
 "sharded-slab",
 "smallvec",
 "thread_local",
 "tracing",
 "tracing-core",
 "tracing-log",
- "tracing-serde",
 ]

 [[package]]
@@ -4245,7 +4030,9 @@ dependencies = [
 "metrics",
 "nix 0.25.0",
 "once_cell",
- "pq_proto",
+ "pin-project-lite",
+ "postgres",
+ "postgres-protocol",
 "rand",
 "routerify",
 "rustls",
@@ -4255,8 +4042,6 @@ dependencies = [
 "serde_json",
 "serde_with",
 "signal-hook",
- "strum",
- "strum_macros",
 "tempfile",
 "thiserror",
 "tokio",
@@ -4570,9 +4355,6 @@ dependencies = [
 "crossbeam-utils",
 "either",
 "fail",
- "futures-channel",
- "futures-task",
- "futures-util",
 "hashbrown",
 "indexmap",
 "libc",
@@ -4582,12 +4364,10 @@ dependencies = [
 "num-bigint",
 "num-integer",
 "num-traits",
- "prost 0.10.4",
- "prost 0.11.2",
+ "prost",
 "rand",
 "regex",
 "regex-syntax",
- "reqwest",
 "scopeguard",
 "serde",
 "stable_deref_trait",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -16,7 +16,6 @@ members = [
    "pageserver",
    "proxy",
    "safekeeper",
-    "storage_broker",
    "workspace_hack",
    "libs/*",
 ]
@@ -26,10 +25,6 @@ members = [
 # Besides, debug info should not affect the performance.
 debug = true

-# disable debug symbols for all packages except this one to decrease binaries size
-[profile.release.package."*"]
-debug = false
-
 [profile.release-line-debug]
 inherits = "release"
 debug = 1 # true = 2 = all symbols, 1 = line only
--- a/4
+++ b/4
@@ -44,7 +44,7 @@ COPY . .
 # Show build caching stats to check if it was used in the end.
 # Has to be the part of the same RUN since cachepot daemon is killed in the end of this RUN, losing the compilation stats.
 RUN set -e \
-&& mold -run cargo build --bin pageserver --bin pageserver_binutils --bin draw_timeline_dir --bin safekeeper --bin storage_broker --bin proxy --locked --release \
+&& mold -run cargo build --bin pageserver --bin pageserver_binutils --bin safekeeper --bin proxy --locked --release \
    && cachepot -s

 # Build final image
@@ -65,9 +65,7 @@ RUN set -e \

 COPY --from=build --chown=neon:neon /home/nonroot/target/release/pageserver          /usr/local/bin
 COPY --from=build --chown=neon:neon /home/nonroot/target/release/pageserver_binutils /usr/local/bin
-COPY --from=build --chown=neon:neon /home/nonroot/target/release/draw_timeline_dir   /usr/local/bin
 COPY --from=build --chown=neon:neon /home/nonroot/target/release/safekeeper          /usr/local/bin
-COPY --from=build --chown=neon:neon /home/nonroot/target/release/storage_broker         /usr/local/bin
 COPY --from=build --chown=neon:neon /home/nonroot/target/release/proxy               /usr/local/bin

 COPY --from=pg-build /home/nonroot/pg_install/v14 /usr/local/v14/
--- a/Dockerfile.compute-node-v14
+++ b/Dockerfile.compute-node-v14
@@ -1,50 +1,50 @@
-#
-# This file is identical to the Dockerfile.compute-node-v15 file
-# except for the version of Postgres that is built.
-#
-
 ARG TAG=pinned
+# apparently, ARGs don't get replaced in RUN commands in kaniko
+# ARG POSTGIS_VERSION=3.3.0
+# ARG PLV8_VERSION=3.1.4
+# ARG PG_VERSION=v14

-#########################################################################################
 #
 # Layer "build-deps"
 #
-#########################################################################################
 FROM debian:bullseye-slim AS build-deps
+RUN echo "deb http://ftp.debian.org/debian testing main" >> /etc/apt/sources.list && \
+    echo "APT::Default-Release \"stable\";" > /etc/apt/apt.conf.d/default-release && \
+    apt update
 RUN apt update &&  \
-    apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev \
-    zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libssl-dev
+    apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \
+    libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libglib2.0-dev

-#########################################################################################
 #
 # Layer "pg-build"
 # Build Postgres from the neon postgres repository.
 #
-#########################################################################################
 FROM build-deps AS pg-build
 COPY vendor/postgres-v14 postgres
 RUN cd postgres && \
-    ./configure CFLAGS='-O2 -g3' --enable-debug --with-openssl --with-uuid=ossp && \
+    ./configure CFLAGS='-O2 -g3' --enable-debug --with-uuid=ossp && \
    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s install && \
    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C contrib/ install && \
    # Install headers
    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/include install && \
    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/interfaces/libpq install

-#########################################################################################
 #
 # Layer "postgis-build"
 # Build PostGIS from the upstream PostGIS mirror.
 #
-#########################################################################################
+# PostGIS compiles against neon postgres sources without changes. Perhaps we
+# could even use the upstream binaries, compiled against vanilla Postgres, but
+# it would require some investigation to check that it works, and also keeps
+# working in the future. So for now, we compile our own binaries.
 FROM build-deps AS postgis-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
 RUN apt update && \
    apt install -y gdal-bin libgdal-dev libprotobuf-c-dev protobuf-c-compiler xsltproc

-RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.1.tar.gz && \
-    tar xvzf postgis-3.3.1.tar.gz && \
-    cd postgis-3.3.1 && \
+RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.0.tar.gz && \
+    tar xvzf postgis-3.3.0.tar.gz && \
+    cd postgis-3.3.0 && \
    ./autogen.sh && \
    export PATH="/usr/local/pgsql/bin:$PATH" && \
    ./configure && \
@@ -57,29 +57,19 @@ RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.1.tar.gz && \
    echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_tiger_geocoder.control && \
    echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_topology.control

-#########################################################################################
 #
 # Layer "plv8-build"
 # Build plv8
 #
-#########################################################################################
 FROM build-deps AS plv8-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
 RUN apt update && \
-    apt install -y ninja-build python3-dev libc++-dev libc++abi-dev libncurses5 binutils
+    apt install -y ninja-build python3-dev libc++-dev libc++abi-dev libncurses5

-# https://github.com/plv8/plv8/issues/475:
-#   v8 uses gold for linking and sets `--thread-count=4` which breaks
-#   gold version <= 1.35 (https://sourceware.org/bugzilla/show_bug.cgi?id=23607)
-# Install newer gold version manually as debian-testing binutils version updates
-# libc version, which in turn breaks other extension built against non-testing libc.
-RUN wget https://ftp.gnu.org/gnu/binutils/binutils-2.38.tar.gz && \
-    tar xvzf binutils-2.38.tar.gz && \
-    cd binutils-2.38 && \
-    cd libiberty && ./configure && make -j $(getconf _NPROCESSORS_ONLN) && \
-    cd ../bfd && ./configure && make bfdver.h && \
-    cd ../gold && ./configure && make -j $(getconf _NPROCESSORS_ONLN) && make install && \
-    cp /usr/local/bin/ld.gold /usr/bin/gold
+# https://github.com/plv8/plv8/issues/475
+# Debian bullseye provides binutils 2.35 when >= 2.38 is necessary
+RUN apt update && \
+    apt install -y --no-install-recommends -t testing binutils

 # Sed is used to patch for https://github.com/plv8/plv8/issues/503
 RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.4.tar.gz && \
@@ -87,25 +77,21 @@ RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.4.tar.gz && \
    cd plv8-3.1.4 && \
    export PATH="/usr/local/pgsql/bin:$PATH" && \
    sed -i 's/MemoryContextAlloc(/MemoryContextAllocZero(/' plv8.cc && \
-    make DOCKER=1 -j $(getconf _NPROCESSORS_ONLN) install && \
+    make -j $(getconf _NPROCESSORS_ONLN) && \
+    make -j $(getconf _NPROCESSORS_ONLN) install && \
    rm -rf /plv8-* && \
    echo 'trusted = true' >> /usr/local/pgsql/share/extension/plv8.control

-#########################################################################################
 #
 # Layer "h3-pg-build"
 # Build h3_pg
 #
-#########################################################################################
 FROM build-deps AS h3-pg-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/

 # packaged cmake is too old
-RUN wget https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2-linux-x86_64.sh \
-      -q -O /tmp/cmake-install.sh \
-      && chmod u+x /tmp/cmake-install.sh \
-      && /tmp/cmake-install.sh --skip-license --prefix=/usr/local/ \
-      && rm /tmp/cmake-install.sh
+RUN apt update && \
+    apt install -y --no-install-recommends -t testing cmake

 RUN wget https://github.com/uber/h3/archive/refs/tags/v4.0.1.tar.gz -O h3.tgz && \
    tar xvzf h3.tgz  && \
@@ -124,15 +110,12 @@ RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.0.1.tar.gz -O h3
    export PATH="/usr/local/pgsql/bin:$PATH" && \
    make -j $(getconf _NPROCESSORS_ONLN) && \
    make -j $(getconf _NPROCESSORS_ONLN) install && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3_postgis.control
+    echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3.control

-#########################################################################################
 #
 # Layer "neon-pg-ext-build"
 # compile neon extensions
 #
-#########################################################################################
 FROM build-deps AS neon-pg-ext-build
 COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/
@@ -145,22 +128,16 @@ RUN make -j $(getconf _NPROCESSORS_ONLN) \
        -C pgxn/neon \
        -s install

-#########################################################################################
-#
 # Compile and run the Neon-specific `compute_ctl` binary
-#
-#########################################################################################
 FROM 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:$TAG AS compute-tools
 USER nonroot
 # Copy entire project to get Cargo.* files with proper dependencies for the whole project
 COPY --chown=nonroot . .
 RUN cd compute_tools && cargo build --locked --profile release-line-debug-size-lto

-#########################################################################################
 #
 # Clean up postgres folder before inclusion
 #
-#########################################################################################
 FROM neon-pg-ext-build AS postgres-cleanup-layer
 COPY --from=neon-pg-ext-build /usr/local/pgsql /usr/local/pgsql

@@ -178,12 +155,10 @@ RUN rm -r /usr/local/pgsql/lib/pgxs/src
 # if they were to be used by other libraries.
 RUN rm /usr/local/pgsql/lib/lib*.a

-#########################################################################################
 #
 # Final layer
 # Put it all together into the final image
 #
-#########################################################################################
 FROM debian:bullseye-slim
 # Add user postgres
 RUN mkdir /var/db && useradd -m -d /var/db/postgres postgres && \
@@ -200,6 +175,11 @@ COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-deb
 # libreadline8 for psql
 # libossp-uuid16 for extension ossp-uuid
 # libgeos, libgdal, libproj and libprotobuf-c1 for PostGIS
+# GLIBC 2.34 for plv8.
+#     Debian bullseye provides GLIBC 2.31, so we install the library from testing
+#
+# Lastly, link compute_ctl into zenith_ctl while we're at it,
+# so that we don't need to put this in another layer.
 RUN apt update &&  \
    apt install --no-install-recommends -y \
        libreadline8 \
@@ -208,7 +188,14 @@ RUN apt update &&  \
        libgdal28 \
        libproj19 \
        libprotobuf-c1 && \
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
+    echo "Installing GLIBC 2.34" && \
+    echo "deb http://ftp.debian.org/debian testing main" >> /etc/apt/sources.list && \
+    echo "APT::Default-Release \"stable\";" > /etc/apt/apt.conf.d/default-release && \
+    apt update && \
+    apt install -y --no-install-recommends -t testing libc6 && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
+    ln /usr/local/bin/compute_ctl /usr/local/bin/zenith_ctl

 USER postgres
 ENTRYPOINT ["/usr/local/bin/compute_ctl"]
--- a/Dockerfile.compute-node-v15
+++ b/Dockerfile.compute-node-v15
@@ -4,39 +4,44 @@
 #

 ARG TAG=pinned
+# apparently, ARGs don't get replaced in RUN commands in kaniko
+# ARG POSTGIS_VERSION=3.3.1
+# ARG PLV8_VERSION=3.1.4
+# ARG PG_VERSION=v15

-#########################################################################################
 #
 # Layer "build-deps"
 #
-#########################################################################################
 FROM debian:bullseye-slim AS build-deps
+RUN echo "deb http://ftp.debian.org/debian testing main" >> /etc/apt/sources.list && \
+    echo "APT::Default-Release \"stable\";" > /etc/apt/apt.conf.d/default-release && \
+    apt update
 RUN apt update &&  \
-    apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev \
-    zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libssl-dev
+    apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \
+    libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libglib2.0-dev

-#########################################################################################
 #
 # Layer "pg-build"
 # Build Postgres from the neon postgres repository.
 #
-#########################################################################################
 FROM build-deps AS pg-build
 COPY vendor/postgres-v15 postgres
 RUN cd postgres && \
-    ./configure CFLAGS='-O2 -g3' --enable-debug --with-openssl --with-uuid=ossp && \
+    ./configure CFLAGS='-O2 -g3' --enable-debug --with-uuid=ossp && \
    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s install && \
    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C contrib/ install && \
    # Install headers
    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/include install && \
    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/interfaces/libpq install

-#########################################################################################
 #
 # Layer "postgis-build"
 # Build PostGIS from the upstream PostGIS mirror.
 #
-#########################################################################################
+# PostGIS compiles against neon postgres sources without changes. Perhaps we
+# could even use the upstream binaries, compiled against vanilla Postgres, but
+# it would require some investigation to check that it works, and also keeps
+# working in the future. So for now, we compile our own binaries.
 FROM build-deps AS postgis-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
 RUN apt update && \
@@ -57,29 +62,19 @@ RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.1.tar.gz && \
    echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_tiger_geocoder.control && \
    echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_topology.control

-#########################################################################################
 #
 # Layer "plv8-build"
 # Build plv8
 #
-#########################################################################################
 FROM build-deps AS plv8-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
 RUN apt update && \
-    apt install -y ninja-build python3-dev libc++-dev libc++abi-dev libncurses5 binutils
+    apt install -y ninja-build python3-dev libc++-dev libc++abi-dev libncurses5

-# https://github.com/plv8/plv8/issues/475:
-#   v8 uses gold for linking and sets `--thread-count=4` which breaks
-#   gold version <= 1.35 (https://sourceware.org/bugzilla/show_bug.cgi?id=23607)
-# Install newer gold version manually as debian-testing binutils version updates
-# libc version, which in turn breaks other extension built against non-testing libc.
-RUN wget https://ftp.gnu.org/gnu/binutils/binutils-2.38.tar.gz && \
-    tar xvzf binutils-2.38.tar.gz && \
-    cd binutils-2.38 && \
-    cd libiberty && ./configure && make -j $(getconf _NPROCESSORS_ONLN) && \
-    cd ../bfd && ./configure && make bfdver.h && \
-    cd ../gold && ./configure && make -j $(getconf _NPROCESSORS_ONLN) && make install && \
-    cp /usr/local/bin/ld.gold /usr/bin/gold
+# https://github.com/plv8/plv8/issues/475
+# Debian bullseye provides binutils 2.35 when >= 2.38 is necessary
+RUN apt update && \
+    apt install -y --no-install-recommends -t testing binutils

 # Sed is used to patch for https://github.com/plv8/plv8/issues/503
 RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.4.tar.gz && \
@@ -87,25 +82,21 @@ RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.4.tar.gz && \
    cd plv8-3.1.4 && \
    export PATH="/usr/local/pgsql/bin:$PATH" && \
    sed -i 's/MemoryContextAlloc(/MemoryContextAllocZero(/' plv8.cc && \
-    make DOCKER=1 -j $(getconf _NPROCESSORS_ONLN) install && \
+    make -j $(getconf _NPROCESSORS_ONLN) && \
+    make -j $(getconf _NPROCESSORS_ONLN) install && \
    rm -rf /plv8-* && \
    echo 'trusted = true' >> /usr/local/pgsql/share/extension/plv8.control

-#########################################################################################
 #
 # Layer "h3-pg-build"
 # Build h3_pg
 #
-#########################################################################################
 FROM build-deps AS h3-pg-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/

 # packaged cmake is too old
-RUN wget https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2-linux-x86_64.sh \
-      -q -O /tmp/cmake-install.sh \
-      && chmod u+x /tmp/cmake-install.sh \
-      && /tmp/cmake-install.sh --skip-license --prefix=/usr/local/ \
-      && rm /tmp/cmake-install.sh
+RUN apt update && \
+    apt install -y --no-install-recommends -t testing cmake

 RUN wget https://github.com/uber/h3/archive/refs/tags/v4.0.1.tar.gz -O h3.tgz && \
    tar xvzf h3.tgz  && \
@@ -124,15 +115,12 @@ RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.0.1.tar.gz -O h3
    export PATH="/usr/local/pgsql/bin:$PATH" && \
    make -j $(getconf _NPROCESSORS_ONLN) && \
    make -j $(getconf _NPROCESSORS_ONLN) install && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3_postgis.control
+    echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3.control

-#########################################################################################
 #
 # Layer "neon-pg-ext-build"
 # compile neon extensions
 #
-#########################################################################################
 FROM build-deps AS neon-pg-ext-build
 COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/
@@ -145,22 +133,16 @@ RUN make -j $(getconf _NPROCESSORS_ONLN) \
        -C pgxn/neon \
        -s install

-#########################################################################################
-#
 # Compile and run the Neon-specific `compute_ctl` binary
-#
-#########################################################################################
 FROM 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:$TAG AS compute-tools
 USER nonroot
 # Copy entire project to get Cargo.* files with proper dependencies for the whole project
 COPY --chown=nonroot . .
 RUN cd compute_tools && cargo build --locked --profile release-line-debug-size-lto

-#########################################################################################
 #
 # Clean up postgres folder before inclusion
 #
-#########################################################################################
 FROM neon-pg-ext-build AS postgres-cleanup-layer
 COPY --from=neon-pg-ext-build /usr/local/pgsql /usr/local/pgsql

@@ -178,12 +160,10 @@ RUN rm -r /usr/local/pgsql/lib/pgxs/src
 # if they were to be used by other libraries.
 RUN rm /usr/local/pgsql/lib/lib*.a

-#########################################################################################
 #
 # Final layer
 # Put it all together into the final image
 #
-#########################################################################################
 FROM debian:bullseye-slim
 # Add user postgres
 RUN mkdir /var/db && useradd -m -d /var/db/postgres postgres && \
@@ -200,6 +180,11 @@ COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-deb
 # libreadline8 for psql
 # libossp-uuid16 for extension ossp-uuid
 # libgeos, libgdal, libproj and libprotobuf-c1 for PostGIS
+# GLIBC 2.34 for plv8.
+#     Debian bullseye provides GLIBC 2.31, so we install the library from testing
+#
+# Lastly, link compute_ctl into zenith_ctl while we're at it,
+# so that we don't need to put this in another layer.
 RUN apt update &&  \
    apt install --no-install-recommends -y \
        libreadline8 \
@@ -208,7 +193,14 @@ RUN apt update &&  \
        libgdal28 \
        libproj19 \
        libprotobuf-c1 && \
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
+    echo "Installing GLIBC 2.34" && \
+    echo "deb http://ftp.debian.org/debian testing main" >> /etc/apt/sources.list && \
+    echo "APT::Default-Release \"stable\";" > /etc/apt/apt.conf.d/default-release && \
+    apt update && \
+    apt install -y --no-install-recommends -t testing libc6 && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
+    ln /usr/local/bin/compute_ctl /usr/local/bin/zenith_ctl

 USER postgres
 ENTRYPOINT ["/usr/local/bin/compute_ctl"]
--- a/Dockerfile.compute-node.legacy
+++ b/Dockerfile.compute-node.legacy
@@ -0,0 +1,88 @@
+#
+# Legacy version of the Dockerfile for the compute node.
+# Used by e2e CI. Building Dockerfile.compute-node will take
+# unreasonable ammount of time without v2 runners.
+#
+# TODO: remove once cloud repo CI is moved to v2 runners.
+#
+
+
+# Allow specifiyng different compute-tools tag and image repo, so we are
+# able to use different images
+ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com
+ARG IMAGE=compute-tools
+ARG TAG=latest
+
+#
+# Image with pre-built tools
+#
+FROM $REPOSITORY/$IMAGE:$TAG AS compute-deps
+# Only to get ready compute_ctl binary as deppendency
+
+#
+# Image with Postgres build deps
+#
+FROM debian:bullseye-slim AS build-deps
+
+RUN apt-get update && apt-get -yq install automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \
+                                          libcurl4-openssl-dev libossp-uuid-dev
+
+#
+# Image with built Postgres
+#
+FROM build-deps AS pg-build
+
+# Add user postgres
+RUN adduser postgres
+RUN mkdir /pg && chown postgres:postgres /pg
+
+# Copy source files
+# version 14 is default for now
+COPY ./vendor/postgres-v14 /pg/
+COPY ./pgxn /pg/
+
+# Build and install Postgres locally
+RUN mkdir /pg/compute_build && cd /pg/compute_build && \
+    ../configure CFLAGS='-O2 -g3' --prefix=$(pwd)/postgres_bin --enable-debug --with-uuid=ossp && \
+    # Install main binaries and contribs
+    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s install && \
+    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C contrib/ install && \
+    # Install headers
+    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/include install
+
+# Install neon contrib
+RUN make MAKELEVEL=0 PG_CONFIG=/pg/compute_build/postgres_bin/bin/pg_config -j $(getconf _NPROCESSORS_ONLN) -C /pg/neon install
+
+USER postgres
+WORKDIR /pg
+
+#
+# Final compute node image to be exported
+#
+FROM debian:bullseye-slim
+
+# libreadline-dev is required to run psql
+RUN apt-get update && apt-get -yq install libreadline-dev libossp-uuid-dev
+
+# Add user postgres
+RUN mkdir /var/db && useradd -m -d /var/db/postgres postgres && \
+    echo "postgres:test_console_pass" | chpasswd && \
+    mkdir /var/db/postgres/compute && mkdir /var/db/postgres/specs && \
+    chown -R postgres:postgres /var/db/postgres && \
+    chmod 0750 /var/db/postgres/compute
+
+# Copy ready Postgres binaries
+COPY --from=pg-build /pg/compute_build/postgres_bin /usr/local
+
+# Copy binaries from compute-tools
+COPY --from=compute-deps /usr/local/bin/compute_ctl /usr/local/bin/compute_ctl
+
+# XXX: temporary symlink for compatibility with old control-plane
+RUN ln -s /usr/local/bin/compute_ctl /usr/local/bin/zenith_ctl
+
+# Add postgres shared objects to the search path
+RUN echo '/usr/local/lib' >> /etc/ld.so.conf && /sbin/ldconfig
+
+USER postgres
+
+ENTRYPOINT ["/usr/local/bin/compute_ctl"]
--- a/38
+++ b/38
@@ -20,18 +20,18 @@ else
 	$(error Bad build type '$(BUILD_TYPE)', see Makefile for options)
 endif

+# Seccomp BPF is only available for Linux
 UNAME_S := $(shell uname -s)
 ifeq ($(UNAME_S),Linux)
-	# Seccomp BPF is only available for Linux
 	PG_CONFIGURE_OPTS += --with-libseccomp
-else ifeq ($(UNAME_S),Darwin)
-	# macOS with brew-installed openssl requires explicit paths
-	# It can be configured with OPENSSL_PREFIX variable
-	OPENSSL_PREFIX ?= $(shell brew --prefix openssl@3)
-	PG_CONFIGURE_OPTS += --with-includes=$(OPENSSL_PREFIX)/include --with-libraries=$(OPENSSL_PREFIX)/lib
-	# macOS already has bison and flex in the system, but they are old and result in postgres-v14 target failure
-	# brew formulae are keg-only and not symlinked into HOMEBREW_PREFIX, force their usage
-	EXTRA_PATH_OVERRIDES += $(shell brew --prefix bison)/bin/:$(shell brew --prefix flex)/bin/:
+endif
+
+# macOS with brew-installed openssl requires explicit paths
+# It can be configured with OPENSSL_PREFIX variable
+UNAME_S := $(shell uname -s)
+ifeq ($(UNAME_S),Darwin)
+    OPENSSL_PREFIX ?= $(shell brew --prefix openssl@3)
+    PG_CONFIGURE_OPTS += --with-includes=$(OPENSSL_PREFIX)/include --with-libraries=$(OPENSSL_PREFIX)/lib
 endif

 # Use -C option so that when PostgreSQL "make install" installs the
@@ -73,8 +73,7 @@ $(POSTGRES_INSTALL_DIR)/build/v14/config.status:
 	+@echo "Configuring Postgres v14 build"
 	mkdir -p $(POSTGRES_INSTALL_DIR)/build/v14
 	(cd $(POSTGRES_INSTALL_DIR)/build/v14 && \
-	env PATH="$(EXTRA_PATH_OVERRIDES):$$PATH" $(ROOT_PROJECT_DIR)/vendor/postgres-v14/configure \
-		CFLAGS='$(PG_CFLAGS)' \
+	$(ROOT_PROJECT_DIR)/vendor/postgres-v14/configure CFLAGS='$(PG_CFLAGS)' \
 		$(PG_CONFIGURE_OPTS) \
 		--prefix=$(abspath $(POSTGRES_INSTALL_DIR))/v14 > configure.log)

@@ -82,8 +81,7 @@ $(POSTGRES_INSTALL_DIR)/build/v15/config.status:
 	+@echo "Configuring Postgres v15 build"
 	mkdir -p $(POSTGRES_INSTALL_DIR)/build/v15
 	(cd $(POSTGRES_INSTALL_DIR)/build/v15 && \
-	env PATH="$(EXTRA_PATH_OVERRIDES):$$PATH" $(ROOT_PROJECT_DIR)/vendor/postgres-v15/configure \
-		CFLAGS='$(PG_CFLAGS)' \
+	$(ROOT_PROJECT_DIR)/vendor/postgres-v15/configure CFLAGS='$(PG_CFLAGS)' \
 		$(PG_CONFIGURE_OPTS) \
 		--prefix=$(abspath $(POSTGRES_INSTALL_DIR))/v15 > configure.log)

@@ -113,8 +111,6 @@ postgres-v14: postgres-v14-configure \
 	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14 MAKELEVEL=0 install
 	+@echo "Compiling libpq v14"
 	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/src/interfaces/libpq install
-	+@echo "Compiling pg_prewarm v14"
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pg_prewarm install
 	+@echo "Compiling pg_buffercache v14"
 	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pg_buffercache install
 	+@echo "Compiling pageinspect v14"
@@ -127,8 +123,6 @@ postgres-v15: postgres-v15-configure \
 	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15 MAKELEVEL=0 install
 	+@echo "Compiling libpq v15"
 	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/src/interfaces/libpq install
-	+@echo "Compiling pg_prewarm v15"
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/contrib/pg_prewarm install
 	+@echo "Compiling pg_buffercache v15"
 	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/contrib/pg_buffercache install
 	+@echo "Compiling pageinspect v15"
@@ -157,11 +151,6 @@ neon-pg-ext-v14: postgres-v14
 	(cd $(POSTGRES_INSTALL_DIR)/build/neon-v14 && \
 	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v14/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \
 		-f $(ROOT_PROJECT_DIR)/pgxn/neon/Makefile install)
-	+@echo "Compiling neon_walredo v14"
-	mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-walredo-v14
-	(cd $(POSTGRES_INSTALL_DIR)/build/neon-walredo-v14 && \
-	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v14/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \
-		-f $(ROOT_PROJECT_DIR)/pgxn/neon_walredo/Makefile install)
 	+@echo "Compiling neon_test_utils" v14
 	mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-v14
 	(cd $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-v14 && \
@@ -174,11 +163,6 @@ neon-pg-ext-v15: postgres-v15
 	(cd $(POSTGRES_INSTALL_DIR)/build/neon-v15 && \
 	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v15/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \
 		-f $(ROOT_PROJECT_DIR)/pgxn/neon/Makefile install)
-	+@echo "Compiling neon_walredo v15"
-	mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-walredo-v15
-	(cd $(POSTGRES_INSTALL_DIR)/build/neon-walredo-v15 && \
-	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v15/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \
-		-f $(ROOT_PROJECT_DIR)/pgxn/neon_walredo/Makefile install)
 	+@echo "Compiling neon_test_utils" v15
 	mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-v15
 	(cd $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-v15 && \
--- a/README.md
+++ b/README.md
@@ -35,12 +35,12 @@ Pageserver consists of:
 * On Ubuntu or Debian, this set of packages should be sufficient to build the code:
 ```bash
 apt install build-essential libtool libreadline-dev zlib1g-dev flex bison libseccomp-dev \
-libssl-dev clang pkg-config libpq-dev etcd cmake postgresql-client protobuf-compiler
+libssl-dev clang pkg-config libpq-dev etcd cmake postgresql-client
 ```
 * On Fedora, these packages are needed:
 ```bash
 dnf install flex bison readline-devel zlib-devel openssl-devel \
-  libseccomp-devel perl clang cmake etcd postgresql postgresql-contrib protobuf-compiler
+  libseccomp-devel perl clang cmake etcd postgresql postgresql-contrib
 ```

 2. [Install Rust](https://www.rust-lang.org/tools/install)
@@ -53,7 +53,7 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
 1. Install XCode and dependencies
 ```
 xcode-select --install
-brew install protobuf etcd openssl flex bison
+brew install protobuf etcd openssl
 ```

 2. [Install Rust](https://www.rust-lang.org/tools/install)
@@ -125,23 +125,24 @@ Python (3.9 or higher), and install python3 packages using `./scripts/pysync` (r
 # Create repository in .neon with proper paths to binaries and data
 # Later that would be responsibility of a package install script
 > ./target/debug/neon_local init
-Starting pageserver at '127.0.0.1:64000' in '.neon'.
-pageserver started, pid: 2545906
-Successfully initialized timeline de200bd42b49cc1814412c7e592dd6e9
-Stopped pageserver 1 process with pid 2545906
+Starting pageserver at '127.0.0.1:64000' in '.neon'
+
+Pageserver started
+Successfully initialized timeline 7dd0907914ac399ff3be45fb252bfdb7
+Stopping pageserver gracefully...done!

 # start pageserver and safekeeper
 > ./target/debug/neon_local start
-Starting etcd broker using "/usr/bin/etcd"
-etcd started, pid: 2545996
-Starting pageserver at '127.0.0.1:64000' in '.neon'.
-pageserver started, pid: 2546005
-Starting safekeeper at '127.0.0.1:5454' in '.neon/safekeepers/sk1'.
-safekeeper 1 started, pid: 2546041
+Starting etcd broker using /usr/bin/etcd
+Starting pageserver at '127.0.0.1:64000' in '.neon'
+
+Pageserver started
+Starting safekeeper at '127.0.0.1:5454' in '.neon/safekeepers/sk1'
+Safekeeper started

 # start postgres compute node
 > ./target/debug/neon_local pg start main
-Starting new postgres (v14) main on timeline de200bd42b49cc1814412c7e592dd6e9 ...
+Starting new postgres main on timeline de200bd42b49cc1814412c7e592dd6e9 ...
 Extracting base backup to create postgres instance: path=.neon/pgdatadirs/tenants/9ef87a5bf0d92544f6fafeeb3239695c/main port=55432
 Starting postgres node at 'host=127.0.0.1 port=55432 user=cloud_admin dbname=postgres'

@@ -222,7 +223,10 @@ Ensure your dependencies are installed as described [here](https://github.com/ne
 ```sh
 git clone --recursive https://github.com/neondatabase/neon.git

+# either:
 CARGO_BUILD_FLAGS="--features=testing" make
+# or:
+make debug

 ./scripts/pytest
 ```
--- a/cli-v2-story.md
+++ b/cli-v2-story.md
@@ -0,0 +1,188 @@
+Create a new Zenith repository in the current directory:
+
+    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli init
+    The files belonging to this database system will be owned by user "heikki".
+    This user must also own the server process.
+    
+    The database cluster will be initialized with locale "en_GB.UTF-8".
+    The default database encoding has accordingly been set to "UTF8".
+    The default text search configuration will be set to "english".
+    
+    Data page checksums are disabled.
+    
+    creating directory tmp ... ok
+    creating subdirectories ... ok
+    selecting dynamic shared memory implementation ... posix
+    selecting default max_connections ... 100
+    selecting default shared_buffers ... 128MB
+    selecting default time zone ... Europe/Helsinki
+    creating configuration files ... ok
+    running bootstrap script ... ok
+    performing post-bootstrap initialization ... ok
+    syncing data to disk ... ok
+    
+    initdb: warning: enabling "trust" authentication for local connections
+    You can change this by editing pg_hba.conf or using the option -A, or
+    --auth-local and --auth-host, the next time you run initdb.
+    new zenith repository was created in .zenith
+
+Initially, there is only one branch:
+
+    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli branch
+      main
+
+Start a local Postgres instance on the branch:
+
+    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli start main
+    Creating data directory from snapshot at 0/15FFB08...
+    waiting for server to start....2021-04-13 09:27:43.919 EEST [984664] LOG:  starting PostgreSQL 14devel on x86_64-pc-linux-gnu, compiled by gcc (Debian 10.2.1-6) 10.2.1 20210110, 64-bit
+    2021-04-13 09:27:43.920 EEST [984664] LOG:  listening on IPv6 address "::1", port 5432
+    2021-04-13 09:27:43.920 EEST [984664] LOG:  listening on IPv4 address "127.0.0.1", port 5432
+    2021-04-13 09:27:43.927 EEST [984664] LOG:  listening on Unix socket "/tmp/.s.PGSQL.5432"
+    2021-04-13 09:27:43.939 EEST [984665] LOG:  database system was interrupted; last known up at 2021-04-13 09:27:33 EEST
+    2021-04-13 09:27:43.939 EEST [984665] LOG:  creating missing WAL directory "pg_wal/archive_status"
+    2021-04-13 09:27:44.189 EEST [984665] LOG:  database system was not properly shut down; automatic recovery in progress
+    2021-04-13 09:27:44.195 EEST [984665] LOG:  invalid record length at 0/15FFB80: wanted 24, got 0
+    2021-04-13 09:27:44.195 EEST [984665] LOG:  redo is not required
+    2021-04-13 09:27:44.225 EEST [984664] LOG:  database system is ready to accept connections
+     done
+    server started
+
+Run some commands against it:
+
+    ~/git-sandbox/zenith (cli-v2)$ psql postgres -c "create table foo (t text);" 
+    CREATE TABLE
+    ~/git-sandbox/zenith (cli-v2)$ psql postgres -c "insert into foo values ('inserted on the main branch');" 
+    INSERT 0 1
+    ~/git-sandbox/zenith (cli-v2)$ psql postgres -c "select * from foo" 
+                  t              
+    -----------------------------
+     inserted on the main branch
+    (1 row)
+
+Create a new branch called 'experimental'. We create it from the
+current end of the 'main' branch, but you could specify a different
+LSN as the start point instead.
+
+    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli branch experimental main
+    branching at end of WAL: 0/161F478
+    
+    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli branch 
+      experimental
+      main
+
+Start another Postgres instance off the 'experimental' branch:
+
+    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli start experimental -- -o -p5433
+    Creating data directory from snapshot at 0/15FFB08...
+    waiting for server to start....2021-04-13 09:28:41.874 EEST [984766] LOG:  starting PostgreSQL 14devel on x86_64-pc-linux-gnu, compiled by gcc (Debian 10.2.1-6) 10.2.1 20210110, 64-bit
+    2021-04-13 09:28:41.875 EEST [984766] LOG:  listening on IPv6 address "::1", port 5433
+    2021-04-13 09:28:41.875 EEST [984766] LOG:  listening on IPv4 address "127.0.0.1", port 5433
+    2021-04-13 09:28:41.883 EEST [984766] LOG:  listening on Unix socket "/tmp/.s.PGSQL.5433"
+    2021-04-13 09:28:41.896 EEST [984767] LOG:  database system was interrupted; last known up at 2021-04-13 09:27:33 EEST
+    2021-04-13 09:28:42.265 EEST [984767] LOG:  database system was not properly shut down; automatic recovery in progress
+    2021-04-13 09:28:42.269 EEST [984767] LOG:  redo starts at 0/15FFB80
+    2021-04-13 09:28:42.272 EEST [984767] LOG:  invalid record length at 0/161F4B0: wanted 24, got 0
+    2021-04-13 09:28:42.272 EEST [984767] LOG:  redo done at 0/161F478 system usage: CPU: user: 0.00 s, system: 0.00 s, elapsed: 0.00 s
+    2021-04-13 09:28:42.321 EEST [984766] LOG:  database system is ready to accept connections
+     done
+    server started
+
+Insert some a row on the 'experimental' branch:
+
+    ~/git-sandbox/zenith (cli-v2)$ psql postgres -p5433 -c "select * from foo" 
+                  t              
+    -----------------------------
+     inserted on the main branch
+    (1 row)
+    
+    ~/git-sandbox/zenith (cli-v2)$ psql postgres -p5433 -c "insert into foo values ('inserted on experimental')" 
+    INSERT 0 1
+    ~/git-sandbox/zenith (cli-v2)$ psql postgres -p5433 -c "select * from foo" 
+                  t              
+    -----------------------------
+     inserted on the main branch
+     inserted on experimental
+    (2 rows)
+    
+See that the other Postgres instance is still running on 'main' branch on port 5432:
+
+
+    ~/git-sandbox/zenith (cli-v2)$ psql postgres -p5432 -c "select * from foo" 
+                  t              
+    -----------------------------
+     inserted on the main branch
+    (1 row)
+
+
+
+
+Everything is stored in the .zenith directory:
+
+    ~/git-sandbox/zenith (cli-v2)$ ls -l .zenith/
+    total 12
+    drwxr-xr-x 4 heikki heikki 4096 Apr 13 09:28 datadirs
+    drwxr-xr-x 4 heikki heikki 4096 Apr 13 09:27 refs
+    drwxr-xr-x 4 heikki heikki 4096 Apr 13 09:28 timelines
+
+The 'datadirs' directory contains the datadirs of the running instances:
+
+    ~/git-sandbox/zenith (cli-v2)$ ls -l .zenith/datadirs/
+    total 8
+    drwx------ 18 heikki heikki 4096 Apr 13 09:27 3c0c634c1674079b2c6d4edf7c91523e
+    drwx------ 18 heikki heikki 4096 Apr 13 09:28 697e3c103d4b1763cd6e82e4ff361d76
+    ~/git-sandbox/zenith (cli-v2)$ ls -l .zenith/datadirs/3c0c634c1674079b2c6d4edf7c91523e/
+    total 124
+    drwxr-xr-x 5 heikki heikki  4096 Apr 13 09:27 base
+    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 global
+    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_commit_ts
+    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_dynshmem
+    -rw------- 1 heikki heikki  4760 Apr 13 09:27 pg_hba.conf
+    -rw------- 1 heikki heikki  1636 Apr 13 09:27 pg_ident.conf
+    drwxr-xr-x 4 heikki heikki  4096 Apr 13 09:32 pg_logical
+    drwxr-xr-x 4 heikki heikki  4096 Apr 13 09:27 pg_multixact
+    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_notify
+    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_replslot
+    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_serial
+    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_snapshots
+    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_stat
+    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:34 pg_stat_tmp
+    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_subtrans
+    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_tblspc
+    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_twophase
+    -rw------- 1 heikki heikki     3 Apr 13 09:27 PG_VERSION
+    lrwxrwxrwx 1 heikki heikki    52 Apr 13 09:27 pg_wal -> ../../timelines/3c0c634c1674079b2c6d4edf7c91523e/wal
+    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_xact
+    -rw------- 1 heikki heikki    88 Apr 13 09:27 postgresql.auto.conf
+    -rw------- 1 heikki heikki 28688 Apr 13 09:27 postgresql.conf
+    -rw------- 1 heikki heikki    96 Apr 13 09:27 postmaster.opts
+    -rw------- 1 heikki heikki   149 Apr 13 09:27 postmaster.pid
+
+Note how 'pg_wal' is just a symlink to the 'timelines' directory. The
+datadir is ephemeral, you can delete it at any time, and it can be reconstructed
+from the snapshots and WAL stored in the 'timelines' directory. So if you push/pull
+the repository, the 'datadirs' are not included. (They are like git working trees)
+
+    ~/git-sandbox/zenith (cli-v2)$ killall -9 postgres
+    ~/git-sandbox/zenith (cli-v2)$ rm -rf .zenith/datadirs/*
+    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli start experimental -- -o -p5433
+    Creating data directory from snapshot at 0/15FFB08...
+    waiting for server to start....2021-04-13 09:37:05.476 EEST [985340] LOG:  starting PostgreSQL 14devel on x86_64-pc-linux-gnu, compiled by gcc (Debian 10.2.1-6) 10.2.1 20210110, 64-bit
+    2021-04-13 09:37:05.477 EEST [985340] LOG:  listening on IPv6 address "::1", port 5433
+    2021-04-13 09:37:05.477 EEST [985340] LOG:  listening on IPv4 address "127.0.0.1", port 5433
+    2021-04-13 09:37:05.487 EEST [985340] LOG:  listening on Unix socket "/tmp/.s.PGSQL.5433"
+    2021-04-13 09:37:05.498 EEST [985341] LOG:  database system was interrupted; last known up at 2021-04-13 09:27:33 EEST
+    2021-04-13 09:37:05.808 EEST [985341] LOG:  database system was not properly shut down; automatic recovery in progress
+    2021-04-13 09:37:05.813 EEST [985341] LOG:  redo starts at 0/15FFB80
+    2021-04-13 09:37:05.815 EEST [985341] LOG:  invalid record length at 0/161F770: wanted 24, got 0
+    2021-04-13 09:37:05.815 EEST [985341] LOG:  redo done at 0/161F738 system usage: CPU: user: 0.00 s, system: 0.00 s, elapsed: 0.00 s
+    2021-04-13 09:37:05.866 EEST [985340] LOG:  database system is ready to accept connections
+     done
+    server started
+    ~/git-sandbox/zenith (cli-v2)$ psql postgres -p5433 -c "select * from foo" 
+                  t              
+    -----------------------------
+     inserted on the main branch
+     inserted on experimental
+    (2 rows)
+
--- a/compute_tools/src/pg_helpers.rs
+++ b/compute_tools/src/pg_helpers.rs
@@ -65,7 +65,7 @@ impl GenericOption {
            let name = match self.name.as_str() {
                "safekeepers" => "neon.safekeepers",
                "wal_acceptor_reconnect" => "neon.safekeeper_reconnect_timeout",
-                "wal_acceptor_connection_timeout" => "neon.safekeeper_connection_timeout",
+                "wal_acceptor_connect_timeout" => "neon.safekeeper_connect_timeout",
                it => it,
            };

--- a/compute_tools/src/spec.rs
+++ b/compute_tools/src/spec.rs
@@ -424,7 +424,7 @@ pub fn handle_grants(node: &ComputeNode, client: &mut Client) -> Result<()> {
        db_client.simple_query(&alter_query)?;

        // Explicitly grant CREATE ON SCHEMA PUBLIC to the web_access user.
-        // This is needed because since postgres 15 this privilege is removed by default.
+        // This is needed since postgres 15, where this privilege is removed by default.
        let grant_query = "DO $$\n\
                BEGIN\n\
                    IF EXISTS(\n\
@@ -432,7 +432,7 @@ pub fn handle_grants(node: &ComputeNode, client: &mut Client) -> Result<()> {
                        FROM pg_catalog.pg_namespace\n\
                        WHERE nspname = 'public'\n\
                    ) AND\n\
-                    current_setting('server_version_num')::int/10000 >= 15\n\
+                    version() LIKE 'PostgreSQL 15%'\n\
                    THEN\n\
                        IF EXISTS(\n\
                            SELECT rolname\n\
--- a/control_plane/Cargo.toml
+++ b/control_plane/Cargo.toml
@@ -4,26 +4,24 @@ version = "0.1.0"
 edition = "2021"

 [dependencies]
-anyhow = "1.0"
 clap = "4.0"
 comfy-table = "6.1"
 git-version = "0.3.5"
-nix = "0.25"
-once_cell = "1.13.0"
+tar = "0.4.38"
 postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev = "d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-regex = "1"
-reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "rustls-tls"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_with = "2.0"
-tar = "0.4.38"
-thiserror = "1"
 toml = "0.5"
-url = "2.2.2"
+once_cell = "1.13.0"
+regex = "1"
+anyhow = "1.0"
+thiserror = "1"
+nix = "0.25"
+reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "rustls-tls"] }

 # Note: Do not directly depend on pageserver or safekeeper; use pageserver_api or safekeeper_api
 # instead, so that recompile times are better.
 pageserver_api = { path = "../libs/pageserver_api" }
-postgres_connection = { path = "../libs/postgres_connection" }
 safekeeper_api = { path = "../libs/safekeeper_api" }
 utils = { path = "../libs/utils" }
 workspace_hack = { version = "0.1", path = "../workspace_hack" }
--- a/control_plane/src/background_process.rs
+++ b/control_plane/src/background_process.rs
@@ -1,285 +0,0 @@
-//! Spawns and kills background processes that are needed by Neon CLI.
-//! Applies common set-up such as log and pid files (if needed) to every process.
-//!
-//! Neon CLI does not run in background, so it needs to store the information about
-//! spawned processes, which it does in this module.
-//! We do that by storing the pid of the process in the "${process_name}.pid" file.
-//! The pid file can be created by the process itself
-//! (Neon storage binaries do that and also ensure that a lock is taken onto that file)
-//! or we create such file after starting the process
-//! (non-Neon binaries don't necessarily follow our pidfile conventions).
-//! The pid stored in the file is later used to stop the service.
-//!
-//! See [`lock_file`] module for more info.
-
-use std::ffi::OsStr;
-use std::io::Write;
-use std::path::Path;
-use std::process::{Child, Command};
-use std::time::Duration;
-use std::{fs, io, thread};
-
-use anyhow::{anyhow, bail, Context, Result};
-use nix::errno::Errno;
-use nix::sys::signal::{kill, Signal};
-use nix::unistd::Pid;
-
-use utils::lock_file;
-
-// These constants control the loop used to poll for process start / stop.
-//
-// The loop waits for at most 10 seconds, polling every 100 ms.
-// Once a second, it prints a dot ("."), to give the user an indication that
-// it's waiting. If the process hasn't started/stopped after 5 seconds,
-// it prints a notice that it's taking long, but keeps waiting.
-//
-const RETRY_UNTIL_SECS: u64 = 10;
-const RETRIES: u64 = (RETRY_UNTIL_SECS * 1000) / RETRY_INTERVAL_MILLIS;
-const RETRY_INTERVAL_MILLIS: u64 = 100;
-const DOT_EVERY_RETRIES: u64 = 10;
-const NOTICE_AFTER_RETRIES: u64 = 50;
-
-/// Argument to `start_process`, to indicate whether it should create pidfile or if the process creates
-/// it itself.
-pub enum InitialPidFile<'t> {
-    /// Create a pidfile, to allow future CLI invocations to manipulate the process.
-    Create(&'t Path),
-    /// The process will create the pidfile itself, need to wait for that event.
-    Expect(&'t Path),
-}
-
-/// Start a background child process using the parameters given.
-pub fn start_process<
-    F,
-    S: AsRef<OsStr>,
-    EI: IntoIterator<Item = (String, String)>, // Not generic AsRef<OsStr>, otherwise empty `envs` prevents type inference
->(
-    process_name: &str,
-    datadir: &Path,
-    command: &Path,
-    args: &[S],
-    envs: EI,
-    initial_pid_file: InitialPidFile,
-    process_status_check: F,
-) -> anyhow::Result<Child>
-where
-    F: Fn() -> anyhow::Result<bool>,
-{
-    let log_path = datadir.join(format!("{process_name}.log"));
-    let process_log_file = fs::OpenOptions::new()
-        .create(true)
-        .write(true)
-        .append(true)
-        .open(&log_path)
-        .with_context(|| {
-            format!("Could not open {process_name} log file {log_path:?} for writing")
-        })?;
-    let same_file_for_stderr = process_log_file.try_clone().with_context(|| {
-        format!("Could not reuse {process_name} log file {log_path:?} for writing stderr")
-    })?;
-
-    let mut command = Command::new(command);
-    let background_command = command
-        .stdout(process_log_file)
-        .stderr(same_file_for_stderr)
-        .args(args);
-    let filled_cmd = fill_aws_secrets_vars(fill_rust_env_vars(background_command));
-    filled_cmd.envs(envs);
-
-    let mut spawned_process = filled_cmd.spawn().with_context(|| {
-        format!("Could not spawn {process_name}, see console output and log files for details.")
-    })?;
-    let pid = spawned_process.id();
-    let pid = Pid::from_raw(
-        i32::try_from(pid)
-            .with_context(|| format!("Subprocess {process_name} has invalid pid {pid}"))?,
-    );
-
-    let pid_file_to_check = match initial_pid_file {
-        InitialPidFile::Create(target_pid_file_path) => {
-            match lock_file::create_lock_file(target_pid_file_path, pid.to_string()) {
-                lock_file::LockCreationResult::Created { .. } => {
-                    // We use "lock" file here only to create the pid file. The lock on the pidfile will be dropped as soon
-                    // as this CLI invocation exits, so it's a bit useless, but doesn't any harm either.
-                }
-                lock_file::LockCreationResult::AlreadyLocked { .. } => {
-                    anyhow::bail!("Cannot write pid file for {process_name} at path {target_pid_file_path:?}: file is already locked by another process")
-                }
-                lock_file::LockCreationResult::CreationFailed(e) => {
-                    return Err(e.context(format!(
-                    "Failed to create pid file for {process_name} at path {target_pid_file_path:?}"
-                )))
-                }
-            }
-            None
-        }
-        InitialPidFile::Expect(pid_file_path) => Some(pid_file_path),
-    };
-
-    for retries in 0..RETRIES {
-        match process_started(pid, pid_file_to_check, &process_status_check) {
-            Ok(true) => {
-                println!("\n{process_name} started, pid: {pid}");
-                return Ok(spawned_process);
-            }
-            Ok(false) => {
-                if retries == NOTICE_AFTER_RETRIES {
-                    // The process is taking a long time to start up. Keep waiting, but
-                    // print a message
-                    print!("\n{process_name} has not started yet, continuing to wait");
-                }
-                if retries % DOT_EVERY_RETRIES == 0 {
-                    print!(".");
-                    io::stdout().flush().unwrap();
-                }
-                thread::sleep(Duration::from_millis(RETRY_INTERVAL_MILLIS));
-            }
-            Err(e) => {
-                println!("{process_name} failed to start: {e:#}");
-                if let Err(e) = spawned_process.kill() {
-                    println!("Could not stop {process_name} subprocess: {e:#}")
-                };
-                return Err(e);
-            }
-        }
-    }
-    println!();
-    anyhow::bail!("{process_name} did not start in {RETRY_UNTIL_SECS} seconds");
-}
-
-/// Stops the process, using the pid file given. Returns Ok also if the process is already not running.
-pub fn stop_process(immediate: bool, process_name: &str, pid_file: &Path) -> anyhow::Result<()> {
-    if !pid_file.exists() {
-        println!("{process_name} is already stopped: no pid file {pid_file:?} is present");
-        return Ok(());
-    }
-    let pid = read_pidfile(pid_file)?;
-
-    let sig = if immediate {
-        print!("Stopping {process_name} with pid {pid} immediately..");
-        Signal::SIGQUIT
-    } else {
-        print!("Stopping {process_name} with pid {pid} gracefully..");
-        Signal::SIGTERM
-    };
-    io::stdout().flush().unwrap();
-    match kill(pid, sig) {
-        Ok(()) => (),
-        Err(Errno::ESRCH) => {
-            println!(
-                "{process_name} with pid {pid} does not exist, but a pid file {pid_file:?} was found"
-            );
-            return Ok(());
-        }
-        Err(e) => anyhow::bail!("Failed to send signal to {process_name} with pid {pid}: {e}"),
-    }
-
-    // Wait until process is gone
-    for retries in 0..RETRIES {
-        match process_has_stopped(pid) {
-            Ok(true) => {
-                println!("\n{process_name} stopped");
-                if let Err(e) = fs::remove_file(pid_file) {
-                    if e.kind() != io::ErrorKind::NotFound {
-                        eprintln!("Failed to remove pid file {pid_file:?} after stopping the process: {e:#}");
-                    }
-                }
-                return Ok(());
-            }
-            Ok(false) => {
-                if retries == NOTICE_AFTER_RETRIES {
-                    // The process is taking a long time to start up. Keep waiting, but
-                    // print a message
-                    print!("\n{process_name} has not stopped yet, continuing to wait");
-                }
-                if retries % DOT_EVERY_RETRIES == 0 {
-                    print!(".");
-                    io::stdout().flush().unwrap();
-                }
-                thread::sleep(Duration::from_millis(RETRY_INTERVAL_MILLIS));
-            }
-            Err(e) => {
-                println!("{process_name} with pid {pid} failed to stop: {e:#}");
-                return Err(e);
-            }
-        }
-    }
-    println!();
-    anyhow::bail!("{process_name} with pid {pid} did not stop in {RETRY_UNTIL_SECS} seconds");
-}
-
-fn fill_rust_env_vars(cmd: &mut Command) -> &mut Command {
-    let mut filled_cmd = cmd.env_clear().env("RUST_BACKTRACE", "1");
-
-    // Pass through these environment variables to the command
-    for var in ["LLVM_PROFILE_FILE", "FAILPOINTS", "RUST_LOG"] {
-        if let Some(val) = std::env::var_os(var) {
-            filled_cmd = filled_cmd.env(var, val);
-        }
-    }
-
-    filled_cmd
-}
-
-fn fill_aws_secrets_vars(mut cmd: &mut Command) -> &mut Command {
-    for env_key in [
-        "AWS_ACCESS_KEY_ID",
-        "AWS_SECRET_ACCESS_KEY",
-        "AWS_SESSION_TOKEN",
-    ] {
-        if let Ok(value) = std::env::var(env_key) {
-            cmd = cmd.env(env_key, value);
-        }
-    }
-    cmd
-}
-
-fn process_started<F>(
-    pid: Pid,
-    pid_file_to_check: Option<&Path>,
-    status_check: &F,
-) -> anyhow::Result<bool>
-where
-    F: Fn() -> anyhow::Result<bool>,
-{
-    match status_check() {
-        Ok(true) => match pid_file_to_check {
-            Some(pid_file_path) => {
-                if pid_file_path.exists() {
-                    let pid_in_file = read_pidfile(pid_file_path)?;
-                    Ok(pid_in_file == pid)
-                } else {
-                    Ok(false)
-                }
-            }
-            None => Ok(true),
-        },
-        Ok(false) => Ok(false),
-        Err(e) => anyhow::bail!("process failed to start: {e}"),
-    }
-}
-
-/// Read a PID file
-///
-/// We expect a file that contains a single integer.
-fn read_pidfile(pidfile: &Path) -> Result<Pid> {
-    let pid_str = fs::read_to_string(pidfile)
-        .with_context(|| format!("failed to read pidfile {pidfile:?}"))?;
-    let pid: i32 = pid_str
-        .parse()
-        .map_err(|_| anyhow!("failed to parse pidfile {pidfile:?}"))?;
-    if pid < 1 {
-        bail!("pidfile {pidfile:?} contained bad value '{pid}'");
-    }
-    Ok(Pid::from_raw(pid))
-}
-
-fn process_has_stopped(pid: Pid) -> anyhow::Result<bool> {
-    match kill(pid, None) {
-        // Process exists, keep waiting
-        Ok(_) => Ok(false),
-        // Process not found, we're done
-        Err(Errno::ESRCH) => Ok(true),
-        Err(err) => anyhow::bail!("Failed to send signal to process with pid {pid}: {err}"),
-    }
-}
--- a/control_plane/src/bin/neon_local.rs
+++ b/control_plane/src/bin/neon_local.rs
@@ -9,8 +9,8 @@ use anyhow::{anyhow, bail, Context, Result};
 use clap::{value_parser, Arg, ArgAction, ArgMatches, Command};
 use control_plane::compute::ComputeControlPlane;
 use control_plane::local_env::{EtcdBroker, LocalEnv};
-use control_plane::pageserver::PageServerNode;
 use control_plane::safekeeper::SafekeeperNode;
+use control_plane::storage::PageServerNode;
 use control_plane::{etcd, local_env};
 use pageserver_api::models::TimelineInfo;
 use pageserver_api::{
--- a/control_plane/src/compute.rs
+++ b/control_plane/src/compute.rs
@@ -12,14 +12,15 @@ use std::time::Duration;

 use anyhow::{Context, Result};
 use utils::{
+    connstring::connection_host_port,
    id::{TenantId, TimelineId},
    lsn::Lsn,
    postgres_backend::AuthType,
 };

 use crate::local_env::{LocalEnv, DEFAULT_PG_VERSION};
-use crate::pageserver::PageServerNode;
 use crate::postgresql_conf::PostgresConf;
+use crate::storage::PageServerNode;

 //
 // ComputeControlPlane
@@ -281,7 +282,9 @@ impl PostgresNode {
    fn setup_pg_conf(&self, auth_type: AuthType) -> Result<()> {
        let mut conf = PostgresConf::new();
        conf.append("max_wal_senders", "10");
-        conf.append("wal_log_hints", "off");
+        // wal_log_hints is mandatory when running against pageserver (see gh issue#192)
+        // TODO: is it possible to check wal_log_hints at pageserver side via XLOG_PARAMETER_CHANGE?
+        conf.append("wal_log_hints", "on");
        conf.append("max_replication_slots", "10");
        conf.append("hot_standby", "on");
        conf.append("shared_buffers", "1MB");
@@ -299,8 +302,7 @@ impl PostgresNode {

        // Configure the node to fetch pages from pageserver
        let pageserver_connstr = {
-            let config = &self.pageserver.pg_connection_config;
-            let (host, port) = (config.host(), config.port());
+            let (host, port) = connection_host_port(&self.pageserver.pg_connection_config);

            // Set up authentication
            //
@@ -322,9 +324,6 @@ impl PostgresNode {
        conf.append("shared_preload_libraries", "neon");
        conf.append_line("");
        conf.append("neon.pageserver_connstring", &pageserver_connstr);
-        if let AuthType::NeonJWT = auth_type {
-            conf.append("neon.safekeeper_token_env", "$ZENITH_AUTH_TOKEN");
-        }
        conf.append("neon.tenant_id", &self.tenant_id.to_string());
        conf.append("neon.timeline_id", &self.timeline_id.to_string());
        if let Some(lsn) = self.lsn {
@@ -346,7 +345,7 @@ impl PostgresNode {
        //   To be able to restore database in case of pageserver node crash, safekeeper should not
        //   remove WAL beyond this point. Too large lag can cause space exhaustion in safekeepers
        //   (if they are not able to upload WAL to S3).
-        conf.append("max_replication_write_lag", "15MB");
+        conf.append("max_replication_write_lag", "500MB");
        conf.append("max_replication_flush_lag", "10GB");

        if !self.env.safekeepers.is_empty() {
--- a/control_plane/src/etcd.rs
+++ b/control_plane/src/etcd.rs
@@ -1,76 +1,95 @@
-use std::{fs, path::PathBuf};
+use std::{
+    fs,
+    path::PathBuf,
+    process::{Command, Stdio},
+};

 use anyhow::Context;
+use nix::{
+    sys::signal::{kill, Signal},
+    unistd::Pid,
+};

-use crate::{background_process, local_env};
+use crate::{local_env, read_pidfile};

 pub fn start_etcd_process(env: &local_env::LocalEnv) -> anyhow::Result<()> {
    let etcd_broker = &env.etcd_broker;
-    print!(
-        "Starting etcd broker using {:?}",
-        etcd_broker.etcd_binary_path
+    println!(
+        "Starting etcd broker using {}",
+        etcd_broker.etcd_binary_path.display()
    );

    let etcd_data_dir = env.base_data_dir.join("etcd");
-    fs::create_dir_all(&etcd_data_dir)
-        .with_context(|| format!("Failed to create etcd data dir {etcd_data_dir:?}"))?;
+    fs::create_dir_all(&etcd_data_dir).with_context(|| {
+        format!(
+            "Failed to create etcd data dir: {}",
+            etcd_data_dir.display()
+        )
+    })?;

+    let etcd_stdout_file =
+        fs::File::create(etcd_data_dir.join("etcd.stdout.log")).with_context(|| {
+            format!(
+                "Failed to create etcd stout file in directory {}",
+                etcd_data_dir.display()
+            )
+        })?;
+    let etcd_stderr_file =
+        fs::File::create(etcd_data_dir.join("etcd.stderr.log")).with_context(|| {
+            format!(
+                "Failed to create etcd stderr file in directory {}",
+                etcd_data_dir.display()
+            )
+        })?;
    let client_urls = etcd_broker.comma_separated_endpoints();
-    let args = [
-        format!("--data-dir={}", etcd_data_dir.display()),
-        format!("--listen-client-urls={client_urls}"),
-        format!("--advertise-client-urls={client_urls}"),
-        // Set --quota-backend-bytes to keep the etcd virtual memory
-        // size smaller. Our test etcd clusters are very small.
-        // See https://github.com/etcd-io/etcd/issues/7910
-        "--quota-backend-bytes=100000000".to_string(),
-        // etcd doesn't compact (vacuum) with default settings,
-        // enable it to prevent space exhaustion.
-        "--auto-compaction-mode=revision".to_string(),
-        "--auto-compaction-retention=1".to_string(),
-    ];

-    let pid_file_path = etcd_pid_file_path(env);
+    let etcd_process = Command::new(&etcd_broker.etcd_binary_path)
+        .args(&[
+            format!("--data-dir={}", etcd_data_dir.display()),
+            format!("--listen-client-urls={client_urls}"),
+            format!("--advertise-client-urls={client_urls}"),
+            // Set --quota-backend-bytes to keep the etcd virtual memory
+            // size smaller. Our test etcd clusters are very small.
+            // See https://github.com/etcd-io/etcd/issues/7910
+            "--quota-backend-bytes=100000000".to_string(),
+        ])
+        .stdout(Stdio::from(etcd_stdout_file))
+        .stderr(Stdio::from(etcd_stderr_file))
+        .spawn()
+        .context("Failed to spawn etcd subprocess")?;
+    let pid = etcd_process.id();

-    let client = reqwest::blocking::Client::new();
-
-    background_process::start_process(
-        "etcd",
-        &etcd_data_dir,
-        &etcd_broker.etcd_binary_path,
-        &args,
-        [],
-        background_process::InitialPidFile::Create(&pid_file_path),
-        || {
-            for broker_endpoint in &etcd_broker.broker_endpoints {
-                let request = broker_endpoint
-                    .join("health")
-                    .with_context(|| {
-                        format!(
-                            "Failed to append /health path to broker endopint {}",
-                            broker_endpoint
-                        )
-                    })
-                    .and_then(|url| {
-                        client.get(&url.to_string()).build().with_context(|| {
-                            format!("Failed to construct request to etcd endpoint {url}")
-                        })
-                    })?;
-                if client.execute(request).is_ok() {
-                    return Ok(true);
-                }
-            }
-
-            Ok(false)
-        },
-    )
-    .context("Failed to spawn etcd subprocess")?;
+    let etcd_pid_file_path = etcd_pid_file_path(env);
+    fs::write(&etcd_pid_file_path, pid.to_string()).with_context(|| {
+        format!(
+            "Failed to create etcd pid file at {}",
+            etcd_pid_file_path.display()
+        )
+    })?;

    Ok(())
 }

 pub fn stop_etcd_process(env: &local_env::LocalEnv) -> anyhow::Result<()> {
-    background_process::stop_process(true, "etcd", &etcd_pid_file_path(env))
+    let etcd_path = &env.etcd_broker.etcd_binary_path;
+    println!("Stopping etcd broker at {}", etcd_path.display());
+
+    let etcd_pid_file_path = etcd_pid_file_path(env);
+    let pid = Pid::from_raw(read_pidfile(&etcd_pid_file_path).with_context(|| {
+        format!(
+            "Failed to read etcd pid file at {}",
+            etcd_pid_file_path.display()
+        )
+    })?);
+
+    kill(pid, Signal::SIGTERM).with_context(|| {
+        format!(
+            "Failed to stop etcd with pid {pid} at {}",
+            etcd_pid_file_path.display()
+        )
+    })?;
+
+    Ok(())
 }

 fn etcd_pid_file_path(env: &local_env::LocalEnv) -> PathBuf {
--- a/control_plane/src/lib.rs
+++ b/control_plane/src/lib.rs
@@ -6,11 +6,59 @@
 // Intended to be used in integration tests and in CLI tools for
 // local installations.
 //
+use anyhow::{anyhow, bail, Context, Result};
+use std::fs;
+use std::path::Path;
+use std::process::Command;

-mod background_process;
 pub mod compute;
 pub mod etcd;
 pub mod local_env;
-pub mod pageserver;
 pub mod postgresql_conf;
 pub mod safekeeper;
+pub mod storage;
+
+/// Read a PID file
+///
+/// We expect a file that contains a single integer.
+/// We return an i32 for compatibility with libc and nix.
+pub fn read_pidfile(pidfile: &Path) -> Result<i32> {
+    let pid_str = fs::read_to_string(pidfile)
+        .with_context(|| format!("failed to read pidfile {:?}", pidfile))?;
+    let pid: i32 = pid_str
+        .parse()
+        .map_err(|_| anyhow!("failed to parse pidfile {:?}", pidfile))?;
+    if pid < 1 {
+        bail!("pidfile {:?} contained bad value '{}'", pidfile, pid);
+    }
+    Ok(pid)
+}
+
+fn fill_rust_env_vars(cmd: &mut Command) -> &mut Command {
+    let cmd = cmd.env_clear().env("RUST_BACKTRACE", "1");
+
+    let var = "LLVM_PROFILE_FILE";
+    if let Some(val) = std::env::var_os(var) {
+        cmd.env(var, val);
+    }
+
+    const RUST_LOG_KEY: &str = "RUST_LOG";
+    if let Ok(rust_log_value) = std::env::var(RUST_LOG_KEY) {
+        cmd.env(RUST_LOG_KEY, rust_log_value)
+    } else {
+        cmd
+    }
+}
+
+fn fill_aws_secrets_vars(mut cmd: &mut Command) -> &mut Command {
+    for env_key in [
+        "AWS_ACCESS_KEY_ID",
+        "AWS_SECRET_ACCESS_KEY",
+        "AWS_SESSION_TOKEN",
+    ] {
+        if let Ok(value) = std::env::var(env_key) {
+            cmd = cmd.env(env_key, value);
+        }
+    }
+    cmd
+}
--- a/control_plane/src/local_env.rs
+++ b/control_plane/src/local_env.rs
@@ -226,12 +226,12 @@ impl LocalEnv {
        }
    }

-    pub fn pageserver_bin(&self) -> PathBuf {
-        self.neon_distrib_dir.join("pageserver")
+    pub fn pageserver_bin(&self) -> anyhow::Result<PathBuf> {
+        Ok(self.neon_distrib_dir.join("pageserver"))
    }

-    pub fn safekeeper_bin(&self) -> PathBuf {
-        self.neon_distrib_dir.join("safekeeper")
+    pub fn safekeeper_bin(&self) -> anyhow::Result<PathBuf> {
+        Ok(self.neon_distrib_dir.join("safekeeper"))
    }

    pub fn pg_data_dirs_path(&self) -> PathBuf {
--- a/control_plane/src/safekeeper.rs
+++ b/control_plane/src/safekeeper.rs
@@ -1,21 +1,23 @@
 use std::io::Write;
 use std::path::PathBuf;
-use std::process::Child;
+use std::process::Command;
 use std::sync::Arc;
-use std::{io, result};
+use std::time::Duration;
+use std::{io, result, thread};

-use anyhow::Context;
-use postgres_connection::PgConnectionConfig;
+use anyhow::bail;
+use nix::errno::Errno;
+use nix::sys::signal::{kill, Signal};
+use nix::unistd::Pid;
+use postgres::Config;
 use reqwest::blocking::{Client, RequestBuilder, Response};
 use reqwest::{IntoUrl, Method};
 use thiserror::Error;
-use utils::{http::error::HttpErrorBody, id::NodeId};
+use utils::{connstring::connection_address, http::error::HttpErrorBody, id::NodeId};

-use crate::pageserver::PageServerNode;
-use crate::{
-    background_process,
-    local_env::{LocalEnv, SafekeeperConf},
-};
+use crate::local_env::{LocalEnv, SafekeeperConf};
+use crate::storage::PageServerNode;
+use crate::{fill_aws_secrets_vars, fill_rust_env_vars, read_pidfile};

 #[derive(Error, Debug)]
 pub enum SafekeeperHttpError {
@@ -61,7 +63,7 @@ pub struct SafekeeperNode {

    pub conf: SafekeeperConf,

-    pub pg_connection_config: PgConnectionConfig,
+    pub pg_connection_config: Config,
    pub env: LocalEnv,
    pub http_client: Client,
    pub http_base_url: String,
@@ -85,12 +87,15 @@ impl SafekeeperNode {
    }

    /// Construct libpq connection string for connecting to this safekeeper.
-    fn safekeeper_connection_config(port: u16) -> PgConnectionConfig {
-        PgConnectionConfig::new_host_port(url::Host::parse("127.0.0.1").unwrap(), port)
+    fn safekeeper_connection_config(port: u16) -> Config {
+        // TODO safekeeper authentication not implemented yet
+        format!("postgresql://no_user@127.0.0.1:{}/no_db", port)
+            .parse()
+            .unwrap()
    }

    pub fn datadir_path_by_id(env: &LocalEnv, sk_id: NodeId) -> PathBuf {
-        env.safekeeper_data_dir(&format!("sk{sk_id}"))
+        env.safekeeper_data_dir(format!("sk{}", sk_id).as_ref())
    }

    pub fn datadir_path(&self) -> PathBuf {
@@ -101,79 +106,92 @@ impl SafekeeperNode {
        self.datadir_path().join("safekeeper.pid")
    }

-    pub fn start(&self) -> anyhow::Result<Child> {
+    pub fn start(&self) -> anyhow::Result<()> {
        print!(
            "Starting safekeeper at '{}' in '{}'",
-            self.pg_connection_config.raw_address(),
+            connection_address(&self.pg_connection_config),
            self.datadir_path().display()
        );
        io::stdout().flush().unwrap();

        let listen_pg = format!("127.0.0.1:{}", self.conf.pg_port);
        let listen_http = format!("127.0.0.1:{}", self.conf.http_port);
-        let id = self.id;
-        let datadir = self.datadir_path();

-        let id_string = id.to_string();
-        let mut args = vec![
-            "-D",
-            datadir.to_str().with_context(|| {
-                format!("Datadir path {datadir:?} cannot be represented as a unicode string")
-            })?,
-            "--id",
-            &id_string,
-            "--listen-pg",
-            &listen_pg,
-            "--listen-http",
-            &listen_http,
-        ];
+        let mut cmd = Command::new(self.env.safekeeper_bin()?);
+        fill_rust_env_vars(
+            cmd.args(&["-D", self.datadir_path().to_str().unwrap()])
+                .args(&["--id", self.id.to_string().as_ref()])
+                .args(&["--listen-pg", &listen_pg])
+                .args(&["--listen-http", &listen_http])
+                .args(&["--recall", "1 second"])
+                .arg("--daemonize"),
+        );
        if !self.conf.sync {
-            args.push("--no-sync");
+            cmd.arg("--no-sync");
        }

        let comma_separated_endpoints = self.env.etcd_broker.comma_separated_endpoints();
        if !comma_separated_endpoints.is_empty() {
-            args.extend(["--broker-endpoints", &comma_separated_endpoints]);
+            cmd.args(&["--broker-endpoints", &comma_separated_endpoints]);
        }
        if let Some(prefix) = self.env.etcd_broker.broker_etcd_prefix.as_deref() {
-            args.extend(["--broker-etcd-prefix", prefix]);
+            cmd.args(&["--broker-etcd-prefix", prefix]);
        }
-
-        let mut backup_threads = String::new();
        if let Some(threads) = self.conf.backup_threads {
-            backup_threads = threads.to_string();
-            args.extend(["--backup-threads", &backup_threads]);
-        } else {
-            drop(backup_threads);
+            cmd.args(&["--backup-threads", threads.to_string().as_ref()]);
        }
-
        if let Some(ref remote_storage) = self.conf.remote_storage {
-            args.extend(["--remote-storage", remote_storage]);
+            cmd.args(&["--remote-storage", remote_storage]);
        }
-
-        let key_path = self.env.base_data_dir.join("auth_public_key.pem");
        if self.conf.auth_enabled {
-            args.extend([
-                "--auth-validation-public-key-path",
-                key_path.to_str().with_context(|| {
-                    format!("Key path {key_path:?} cannot be represented as a unicode string")
-                })?,
-            ]);
+            cmd.arg("--auth-validation-public-key-path");
+            // PathBuf is better be passed as is, not via `String`.
+            cmd.arg(self.env.base_data_dir.join("auth_public_key.pem"));
        }

-        background_process::start_process(
-            &format!("safekeeper {id}"),
-            &datadir,
-            &self.env.safekeeper_bin(),
-            &args,
-            [],
-            background_process::InitialPidFile::Expect(&self.pid_file()),
-            || match self.check_status() {
-                Ok(()) => Ok(true),
-                Err(SafekeeperHttpError::Transport(_)) => Ok(false),
-                Err(e) => Err(anyhow::anyhow!("Failed to check node status: {e}")),
-            },
-        )
+        fill_aws_secrets_vars(&mut cmd);
+
+        if !cmd.status()?.success() {
+            bail!(
+                "Safekeeper failed to start. See '{}' for details.",
+                self.datadir_path().join("safekeeper.log").display()
+            );
+        }
+
+        // It takes a while for the safekeeper to start up. Wait until it is
+        // open for business.
+        const RETRIES: i8 = 15;
+        for retries in 1..RETRIES {
+            match self.check_status() {
+                Ok(_) => {
+                    println!("\nSafekeeper started");
+                    return Ok(());
+                }
+                Err(err) => {
+                    match err {
+                        SafekeeperHttpError::Transport(err) => {
+                            if err.is_connect() && retries < 5 {
+                                print!(".");
+                                io::stdout().flush().unwrap();
+                            } else {
+                                if retries == 5 {
+                                    println!() // put a line break after dots for second message
+                                }
+                                println!(
+                                    "Safekeeper not responding yet, err {} retrying ({})...",
+                                    err, retries
+                                );
+                            }
+                        }
+                        SafekeeperHttpError::Response(msg) => {
+                            bail!("safekeeper failed to start: {} ", msg)
+                        }
+                    }
+                    thread::sleep(Duration::from_secs(1));
+                }
+            }
+        }
+        bail!("safekeeper failed to start in {} seconds", RETRIES);
    }

    ///
@@ -185,11 +203,63 @@ impl SafekeeperNode {
    /// If the server is not running, returns success
    ///
    pub fn stop(&self, immediate: bool) -> anyhow::Result<()> {
-        background_process::stop_process(
-            immediate,
-            &format!("safekeeper {}", self.id),
-            &self.pid_file(),
-        )
+        let pid_file = self.pid_file();
+        if !pid_file.exists() {
+            println!("Safekeeper {} is already stopped", self.id);
+            return Ok(());
+        }
+        let pid = read_pidfile(&pid_file)?;
+        let pid = Pid::from_raw(pid);
+
+        let sig = if immediate {
+            print!("Stopping safekeeper {} immediately..", self.id);
+            Signal::SIGQUIT
+        } else {
+            print!("Stopping safekeeper {} gracefully..", self.id);
+            Signal::SIGTERM
+        };
+        io::stdout().flush().unwrap();
+        match kill(pid, sig) {
+            Ok(_) => (),
+            Err(Errno::ESRCH) => {
+                println!(
+                    "Safekeeper with pid {} does not exist, but a PID file was found",
+                    pid
+                );
+                return Ok(());
+            }
+            Err(err) => bail!(
+                "Failed to send signal to safekeeper with pid {}: {}",
+                pid,
+                err.desc()
+            ),
+        }
+
+        // Wait until process is gone
+        for i in 0..600 {
+            let signal = None; // Send no signal, just get the error code
+            match kill(pid, signal) {
+                Ok(_) => (), // Process exists, keep waiting
+                Err(Errno::ESRCH) => {
+                    // Process not found, we're done
+                    println!("done!");
+                    return Ok(());
+                }
+                Err(err) => bail!(
+                    "Failed to send signal to pageserver with pid {}: {}",
+                    pid,
+                    err.desc()
+                ),
+            };
+
+            if i % 10 == 0 {
+                print!(".");
+                io::stdout().flush().unwrap();
+            }
+            thread::sleep(Duration::from_millis(100));
+        }
+
+        bail!("Failed to stop safekeeper with pid {}", pid);
    }

    fn http_request<U: IntoUrl>(&self, method: Method, url: U) -> RequestBuilder {
--- a/control_plane/src/pageserver.rs
+++ b/control_plane/src/pageserver.rs
@@ -1,28 +1,33 @@
 use std::collections::HashMap;
-use std::fs::{self, File};
+use std::fs::File;
 use std::io::{BufReader, Write};
 use std::num::NonZeroU64;
 use std::path::{Path, PathBuf};
-use std::process::Child;
-use std::{io, result};
+use std::process::Command;
+use std::time::Duration;
+use std::{io, result, thread};

 use anyhow::{bail, Context};
+use nix::errno::Errno;
+use nix::sys::signal::{kill, Signal};
+use nix::unistd::Pid;
 use pageserver_api::models::{
    TenantConfigRequest, TenantCreateRequest, TenantInfo, TimelineCreateRequest, TimelineInfo,
 };
-use postgres_connection::{parse_host_port, PgConnectionConfig};
+use postgres::{Config, NoTls};
 use reqwest::blocking::{Client, RequestBuilder, Response};
 use reqwest::{IntoUrl, Method};
 use thiserror::Error;
-use utils::auth::{Claims, Scope};
 use utils::{
+    connstring::connection_address,
    http::error::HttpErrorBody,
    id::{TenantId, TimelineId},
    lsn::Lsn,
    postgres_backend::AuthType,
 };

-use crate::{background_process, local_env::LocalEnv};
+use crate::local_env::LocalEnv;
+use crate::{fill_aws_secrets_vars, fill_rust_env_vars, read_pidfile};

 #[derive(Error, Debug)]
 pub enum PageserverHttpError {
@@ -70,7 +75,7 @@ impl ResponseErrorMessageExt for Response {
 //
 #[derive(Debug)]
 pub struct PageServerNode {
-    pub pg_connection_config: PgConnectionConfig,
+    pub pg_connection_config: Config,
    pub env: LocalEnv,
    pub http_client: Client,
    pub http_base_url: String,
@@ -78,24 +83,30 @@ pub struct PageServerNode {

 impl PageServerNode {
    pub fn from_env(env: &LocalEnv) -> PageServerNode {
-        let (host, port) = parse_host_port(&env.pageserver.listen_pg_addr)
-            .expect("Unable to parse listen_pg_addr");
-        let port = port.unwrap_or(5432);
        let password = if env.pageserver.auth_type == AuthType::NeonJWT {
-            Some(env.pageserver.auth_token.clone())
+            &env.pageserver.auth_token
        } else {
-            None
+            ""
        };

        Self {
-            pg_connection_config: PgConnectionConfig::new_host_port(host, port)
-                .set_password(password),
+            pg_connection_config: Self::pageserver_connection_config(
+                password,
+                &env.pageserver.listen_pg_addr,
+            ),
            env: env.clone(),
            http_client: Client::new(),
            http_base_url: format!("http://{}/v1", env.pageserver.listen_http_addr),
        }
    }

+    /// Construct libpq connection string for connecting to the pageserver.
+    fn pageserver_connection_config(password: &str, listen_addr: &str) -> Config {
+        format!("postgresql://no_user:{password}@{listen_addr}/no_db")
+            .parse()
+            .unwrap()
+    }
+
    pub fn initialize(
        &self,
        create_tenant: Option<TenantId>,
@@ -150,15 +161,7 @@ impl PageServerNode {
            init_config_overrides.push("auth_validation_public_key_path='auth_public_key.pem'");
        }

-        let mut pageserver_process = self
-            .start_node(&init_config_overrides, &self.env.base_data_dir, true)
-            .with_context(|| {
-                format!(
-                    "Failed to start a process for pageserver {}",
-                    self.env.pageserver.id,
-                )
-            })?;
-
+        self.start_node(&init_config_overrides, &self.env.base_data_dir, true)?;
        let init_result = self
            .try_init_timeline(create_tenant, initial_timeline_id, pg_version)
            .context("Failed to create initial tenant and timeline for pageserver");
@@ -168,29 +171,7 @@ impl PageServerNode {
            }
            Err(e) => eprintln!("{e:#}"),
        }
-        match pageserver_process.kill() {
-            Err(e) => {
-                eprintln!(
-                    "Failed to stop pageserver {} process with pid {}: {e:#}",
-                    self.env.pageserver.id,
-                    pageserver_process.id(),
-                )
-            }
-            Ok(()) => {
-                println!(
-                    "Stopped pageserver {} process with pid {}",
-                    self.env.pageserver.id,
-                    pageserver_process.id(),
-                );
-                // cleanup after pageserver startup, since we do not call regular `stop_process` during init
-                let pid_file = self.pid_file();
-                if let Err(e) = fs::remove_file(&pid_file) {
-                    if e.kind() != io::ErrorKind::NotFound {
-                        eprintln!("Failed to remove pid file {pid_file:?} after stopping the process: {e:#}");
-                    }
-                }
-            }
-        }
+        self.stop(false)?;
        init_result
    }

@@ -215,14 +196,11 @@ impl PageServerNode {
        self.env.pageserver_data_dir()
    }

-    /// The pid file is created by the pageserver process, with its pid stored inside.
-    /// Other pageservers cannot lock the same file and overwrite it for as long as the current
-    /// pageserver runs. (Unless someone removes the file manually; never do that!)
-    fn pid_file(&self) -> PathBuf {
+    pub fn pid_file(&self) -> PathBuf {
        self.repo_path().join("pageserver.pid")
    }

-    pub fn start(&self, config_overrides: &[&str]) -> anyhow::Result<Child> {
+    pub fn start(&self, config_overrides: &[&str]) -> anyhow::Result<()> {
        self.start_node(config_overrides, &self.repo_path(), false)
    }

@@ -231,10 +209,10 @@ impl PageServerNode {
        config_overrides: &[&str],
        datadir: &Path,
        update_config: bool,
-    ) -> anyhow::Result<Child> {
-        print!(
+    ) -> anyhow::Result<()> {
+        println!(
            "Starting pageserver at '{}' in '{}'",
-            self.pg_connection_config.raw_address(),
+            connection_address(&self.pg_connection_config),
            datadir.display()
        );
        io::stdout().flush()?;
@@ -242,7 +220,10 @@ impl PageServerNode {
        let mut args = vec![
            "-D",
            datadir.to_str().with_context(|| {
-                format!("Datadir path {datadir:?} cannot be represented as a unicode string")
+                format!(
+                    "Datadir path '{}' cannot be represented as a unicode string",
+                    datadir.display()
+                )
            })?,
        ];

@@ -254,28 +235,48 @@ impl PageServerNode {
            args.extend(["-c", config_override]);
        }

-        let envs = if self.env.pageserver.auth_type != AuthType::Trust {
-            // Generate a token to connect from the pageserver to a safekeeper
-            let token = self
-                .env
-                .generate_auth_token(&Claims::new(None, Scope::SafekeeperData))?;
-            vec![("ZENITH_AUTH_TOKEN".to_owned(), token)]
-        } else {
-            vec![]
-        };
-        background_process::start_process(
-            "pageserver",
-            datadir,
-            &self.env.pageserver_bin(),
-            &args,
-            envs,
-            background_process::InitialPidFile::Expect(&self.pid_file()),
-            || match self.check_status() {
-                Ok(()) => Ok(true),
-                Err(PageserverHttpError::Transport(_)) => Ok(false),
-                Err(e) => Err(anyhow::anyhow!("Failed to check node status: {e}")),
-            },
-        )
+        let mut cmd = Command::new(self.env.pageserver_bin()?);
+        let mut filled_cmd = fill_rust_env_vars(cmd.args(&args).arg("--daemonize"));
+        filled_cmd = fill_aws_secrets_vars(filled_cmd);
+
+        if !filled_cmd.status()?.success() {
+            bail!(
+                "Pageserver failed to start. See console output and '{}' for details.",
+                datadir.join("pageserver.log").display()
+            );
+        }
+
+        // It takes a while for the page server to start up. Wait until it is
+        // open for business.
+        const RETRIES: i8 = 15;
+        for retries in 1..RETRIES {
+            match self.check_status() {
+                Ok(()) => {
+                    println!("\nPageserver started");
+                    return Ok(());
+                }
+                Err(err) => {
+                    match err {
+                        PageserverHttpError::Transport(err) => {
+                            if err.is_connect() && retries < 5 {
+                                print!(".");
+                                io::stdout().flush().unwrap();
+                            } else {
+                                if retries == 5 {
+                                    println!() // put a line break after dots for second message
+                                }
+                                println!("Pageserver not responding yet, err {err} retrying ({retries})...");
+                            }
+                        }
+                        PageserverHttpError::Response(msg) => {
+                            bail!("pageserver failed to start: {msg} ")
+                        }
+                    }
+                    thread::sleep(Duration::from_secs(1));
+                }
+            }
+        }
+        bail!("pageserver failed to start in {RETRIES} seconds");
    }

    ///
@@ -287,18 +288,69 @@ impl PageServerNode {
    /// If the server is not running, returns success
    ///
    pub fn stop(&self, immediate: bool) -> anyhow::Result<()> {
-        background_process::stop_process(immediate, "pageserver", &self.pid_file())
+        let pid_file = self.pid_file();
+        if !pid_file.exists() {
+            println!("Pageserver is already stopped");
+            return Ok(());
+        }
+        let pid = Pid::from_raw(read_pidfile(&pid_file)?);
+
+        let sig = if immediate {
+            print!("Stopping pageserver immediately..");
+            Signal::SIGQUIT
+        } else {
+            print!("Stopping pageserver gracefully..");
+            Signal::SIGTERM
+        };
+        io::stdout().flush().unwrap();
+        match kill(pid, sig) {
+            Ok(_) => (),
+            Err(Errno::ESRCH) => {
+                println!("Pageserver with pid {pid} does not exist, but a PID file was found");
+                return Ok(());
+            }
+            Err(err) => bail!(
+                "Failed to send signal to pageserver with pid {pid}: {}",
+                err.desc()
+            ),
+        }
+
+        // Wait until process is gone
+        for i in 0..600 {
+            let signal = None; // Send no signal, just get the error code
+            match kill(pid, signal) {
+                Ok(_) => (), // Process exists, keep waiting
+                Err(Errno::ESRCH) => {
+                    // Process not found, we're done
+                    println!("done!");
+                    return Ok(());
+                }
+                Err(err) => bail!(
+                    "Failed to send signal to pageserver with pid {}: {}",
+                    pid,
+                    err.desc()
+                ),
+            };
+
+            if i % 10 == 0 {
+                print!(".");
+                io::stdout().flush().unwrap();
+            }
+            thread::sleep(Duration::from_millis(100));
+        }
+
+        bail!("Failed to stop pageserver with pid {pid}");
    }

    pub fn page_server_psql(&self, sql: &str) -> Vec<postgres::SimpleQueryMessage> {
-        let mut client = self.pg_connection_config.connect_no_tls().unwrap();
+        let mut client = self.pg_connection_config.connect(NoTls).unwrap();

        println!("Pageserver query: '{sql}'");
        client.simple_query(sql).unwrap()
    }

    pub fn page_server_psql_client(&self) -> result::Result<postgres::Client, postgres::Error> {
-        self.pg_connection_config.connect_no_tls()
+        self.pg_connection_config.connect(NoTls)
    }

    fn http_request<U: IntoUrl>(&self, method: Method, url: U) -> RequestBuilder {
@@ -367,11 +419,6 @@ impl PageServerNode {
                .map(|x| x.parse::<NonZeroU64>())
                .transpose()
                .context("Failed to parse 'max_lsn_wal_lag' as non zero integer")?,
-            trace_read_requests: settings
-                .remove("trace_read_requests")
-                .map(|x| x.parse::<bool>())
-                .transpose()
-                .context("Failed to parse 'trace_read_requests' as bool")?,
        };
        if !settings.is_empty() {
            bail!("Unrecognized tenant settings: {settings:?}")
@@ -434,11 +481,6 @@ impl PageServerNode {
                    .map(|x| x.parse::<NonZeroU64>())
                    .transpose()
                    .context("Failed to parse 'max_lsn_wal_lag' as non zero integer")?,
-                trace_read_requests: settings
-                    .get("trace_read_requests")
-                    .map(|x| x.parse::<bool>())
-                    .transpose()
-                    .context("Failed to parse 'trace_read_requests' as bool")?,
            })
            .send()?
            .error_from_body()?;
@@ -507,7 +549,7 @@ impl PageServerNode {
        pg_wal: Option<(Lsn, PathBuf)>,
        pg_version: u32,
    ) -> anyhow::Result<()> {
-        let mut client = self.pg_connection_config.connect_no_tls().unwrap();
+        let mut client = self.pg_connection_config.connect(NoTls).unwrap();

        // Init base reader
        let (start_lsn, base_tarfile_path) = base;
--- a/docker-compose/compute_wrapper/Dockerfile
+++ b/docker-compose/compute_wrapper/Dockerfile
@@ -1,13 +0,0 @@
-ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com
-ARG COMPUTE_IMAGE=compute-node-v14
-ARG TAG=latest
-
-FROM $REPOSITORY/${COMPUTE_IMAGE}:$TAG
-
-USER root
-RUN apt-get update &&       \
-    apt-get install -y curl \
-                       jq   \
-                       netcat
-
-USER postgres
--- a/docker-compose/compute_wrapper/shell/compute.sh
+++ b/docker-compose/compute_wrapper/shell/compute.sh
@@ -1,48 +0,0 @@
-#!/bin/bash
-set -eux
-
-PG_VERSION=${PG_VERSION:-14}
-
-SPEC_FILE_ORG=/var/db/postgres/specs/spec.json
-SPEC_FILE=/tmp/spec.json
-
-echo "Waiting pageserver become ready."
-while ! nc -z pageserver 6400; do
-     sleep 1;
-done
-echo "Page server is ready."
-
-echo "Create a tenant and timeline"
-PARAMS=(
-     -sb 
-     -X POST
-     -H "Content-Type: application/json"
-     -d "{}"
-     http://pageserver:9898/v1/tenant/
-)
-tenant_id=$(curl "${PARAMS[@]}" | sed 's/"//g')
-
-PARAMS=(
-     -sb 
-     -X POST
-     -H "Content-Type: application/json"
-     -d "{\"tenant_id\":\"${tenant_id}\", \"pg_version\": ${PG_VERSION}}"
-     "http://pageserver:9898/v1/tenant/${tenant_id}/timeline/"
-)
-result=$(curl "${PARAMS[@]}")
-echo $result | jq .
-
-echo "Overwrite tenant id and timeline id in spec file"
-tenant_id=$(echo ${result} | jq -r .tenant_id)
-timeline_id=$(echo ${result} | jq -r .timeline_id)
-
-sed "s/TENANT_ID/${tenant_id}/" ${SPEC_FILE_ORG} > ${SPEC_FILE}
-sed -i "s/TIMELINE_ID/${timeline_id}/" ${SPEC_FILE}
-
-cat ${SPEC_FILE}
-
-echo "Start compute node"
-/usr/local/bin/compute_ctl --pgdata /var/db/postgres/compute \
-     -C "postgresql://cloud_admin@localhost:55433/postgres"  \
-     -b /usr/local/bin/postgres                              \
-     -S ${SPEC_FILE}
--- a/docker-compose/compute_wrapper/var/db/postgres/specs/spec.json
+++ b/docker-compose/compute_wrapper/var/db/postgres/specs/spec.json
@@ -1,141 +0,0 @@
-{
-    "format_version": 1.0,
-
-    "timestamp": "2022-10-12T18:00:00.000Z",
-    "operation_uuid": "0f657b36-4b0f-4a2d-9c2e-1dcd615e7d8c",
-
-    "cluster": {
-        "cluster_id": "docker_compose",
-        "name": "docker_compose_test",
-        "state": "restarted",
-        "roles": [
-            {
-                "name": "cloud_admin",
-                "encrypted_password": "b093c0d3b281ba6da1eacc608620abd8",
-                "options": null
-            }
-        ],
-        "databases": [
-        ],
-        "settings": [
-            {
-                "name": "fsync",
-                "value": "off",
-                "vartype": "bool"
-            },
-            {
-                "name": "wal_level",
-                "value": "replica",
-                "vartype": "enum"
-            },
-            {
-                "name": "hot_standby",
-                "value": "on",
-                "vartype": "bool"
-            },
-            {
-                "name": "wal_log_hints",
-                "value": "on",
-                "vartype": "bool"
-            },
-            {
-                "name": "log_connections",
-                "value": "on",
-                "vartype": "bool"
-            },
-            {
-                "name": "port",
-                "value": "55433",
-                "vartype": "integer"
-            },
-            {
-                "name": "shared_buffers",
-                "value": "1MB",
-                "vartype": "string"
-            },
-            {
-                "name": "max_connections",
-                "value": "100",
-                "vartype": "integer"
-            },
-            {
-                "name": "listen_addresses",
-                "value": "0.0.0.0",
-                "vartype": "string"
-            },
-            {
-                "name": "max_wal_senders",
-                "value": "10",
-                "vartype": "integer"
-            },
-            {
-                "name": "max_replication_slots",
-                "value": "10",
-                "vartype": "integer"
-            },
-            {
-                "name": "wal_sender_timeout",
-                "value": "5s",
-                "vartype": "string"
-            },
-            {
-                "name": "wal_keep_size",
-                "value": "0",
-                "vartype": "integer"
-            },
-            {
-                "name": "password_encryption",
-                "value": "md5",
-                "vartype": "enum"
-            },
-            {
-                "name": "restart_after_crash",
-                "value": "off",
-                "vartype": "bool"
-            },
-            {
-                "name": "synchronous_standby_names",
-                "value": "walproposer",
-                "vartype": "string"
-            },
-            {
-                "name": "shared_preload_libraries",
-                "value": "neon",
-                "vartype": "string"
-            },
-            {
-                "name": "neon.safekeepers",
-                "value": "safekeeper1:5454,safekeeper2:5454,safekeeper3:5454",
-                "vartype": "string"
-            },
-            {
-                "name": "neon.timeline_id",
-                "value": "TIMELINE_ID",
-                "vartype": "string"
-            },
-            {
-                "name": "neon.tenant_id",
-                "value": "TENANT_ID",
-                "vartype": "string"
-            },
-            {
-                "name": "neon.pageserver_connstring",
-                "value": "host=pageserver port=6400",
-                "vartype": "string"
-            },
-            {
-                "name": "max_replication_write_lag",
-                "value": "500MB",
-                "vartype": "string"
-            },
-            {
-                "name": "max_replication_flush_lag",
-                "value": "10GB",
-                "vartype": "string"
-            }
-        ]
-    },
-
-    "delta_operations": [
-    ]
-}
--- a/docker-compose/docker-compose.yml
+++ b/docker-compose/docker-compose.yml
@@ -1,209 +0,0 @@
-version: '3'
-
-services:
-  etcd:
-    restart: always
-    image: quay.io/coreos/etcd:v3.5.4
-    ports:
-      - 2379:2379
-      - 2380:2380
-    environment:
-      # This signifficantly speeds up etcd and we anyway don't data persistency there.
-      ETCD_UNSAFE_NO_FSYNC: "1"
-    command:
-      - "etcd"
-      - "--auto-compaction-mode=revision"
-      - "--auto-compaction-retention=1"
-      - "--name=etcd-cluster"
-      - "--initial-cluster-state=new"
-      - "--initial-cluster-token=etcd-cluster-1"
-      - "--initial-cluster=etcd-cluster=http://etcd:2380"
-      - "--initial-advertise-peer-urls=http://etcd:2380"
-      - "--advertise-client-urls=http://etcd:2379"
-      - "--listen-client-urls=http://0.0.0.0:2379"
-      - "--listen-peer-urls=http://0.0.0.0:2380"
-      - "--quota-backend-bytes=134217728" # 128 MB
-
-  minio:
-    restart: always
-    image: quay.io/minio/minio:RELEASE.2022-10-20T00-55-09Z
-    ports:
-      - 9000:9000
-      - 9001:9001
-    environment:
-      - MINIO_ROOT_USER=minio
-      - MINIO_ROOT_PASSWORD=password
-    command: server /data --address :9000 --console-address ":9001"
-
-  minio_create_buckets:
-    image: minio/mc
-    environment:
-      - MINIO_ROOT_USER=minio
-      - MINIO_ROOT_PASSWORD=password
-    entrypoint:
-      - "/bin/sh"
-      - "-c"
-    command:
-      - "until (/usr/bin/mc alias set minio http://minio:9000 $$MINIO_ROOT_USER $$MINIO_ROOT_PASSWORD) do
-             echo 'Waiting to start minio...' && sleep 1;
-         done;
-         /usr/bin/mc mb minio/neon --region=eu-north-1;
-         exit 0;"
-    depends_on:
-      - minio
-
-  pageserver:
-    restart: always
-    image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
-    environment:
-      - BROKER_ENDPOINT='http://etcd:2379'
-      - AWS_ACCESS_KEY_ID=minio
-      - AWS_SECRET_ACCESS_KEY=password
-      #- RUST_BACKTRACE=1
-    ports:
-       #- 6400:6400  # pg protocol handler
-       - 9898:9898 # http endpoints
-    entrypoint:
-      - "/bin/sh"
-      - "-c"
-    command:
-      - "/usr/local/bin/pageserver -D /data/.neon/
-                                   -c \"broker_endpoints=[$$BROKER_ENDPOINT]\"
-                                   -c \"listen_pg_addr='0.0.0.0:6400'\"
-                                   -c \"listen_http_addr='0.0.0.0:9898'\"
-                                   -c \"remote_storage={endpoint='http://minio:9000',
-                                                        bucket_name='neon',
-                                                        bucket_region='eu-north-1',
-                                                        prefix_in_bucket='/pageserver/'}\""
-    depends_on:
-      - etcd
-      - minio_create_buckets
-
-  safekeeper1:
-    restart: always
-    image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
-    environment:
-      - SAFEKEEPER_ADVERTISE_URL=safekeeper1:5454
-      - SAFEKEEPER_ID=1
-      - BROKER_ENDPOINT=http://etcd:2379
-      - AWS_ACCESS_KEY_ID=minio
-      - AWS_SECRET_ACCESS_KEY=password
-      #- RUST_BACKTRACE=1
-    ports:
-      #- 5454:5454 # pg protocol handler
-      - 7676:7676 # http endpoints
-    entrypoint:
-      - "/bin/sh"
-      - "-c"
-    command:
-      - "safekeeper --listen-pg=$$SAFEKEEPER_ADVERTISE_URL
-                    --listen-http='0.0.0.0:7676'
-                    --id=$$SAFEKEEPER_ID
-                    --broker-endpoints=$$BROKER_ENDPOINT
-                    -D /data
-                    --remote-storage=\"{endpoint='http://minio:9000',
-                                        bucket_name='neon',
-                                        bucket_region='eu-north-1',
-                                        prefix_in_bucket='/safekeeper/'}\""
-    depends_on:
-      - etcd
-      - minio_create_buckets
-
-  safekeeper2:
-    restart: always
-    image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
-    environment:
-      - SAFEKEEPER_ADVERTISE_URL=safekeeper2:5454
-      - SAFEKEEPER_ID=2
-      - BROKER_ENDPOINT=http://etcd:2379
-      - AWS_ACCESS_KEY_ID=minio
-      - AWS_SECRET_ACCESS_KEY=password
-      #- RUST_BACKTRACE=1
-    ports:
-      #- 5454:5454 # pg protocol handler
-      - 7677:7676 # http endpoints
-    entrypoint:
-      - "/bin/sh"
-      - "-c"
-    command:
-      - "safekeeper --listen-pg=$$SAFEKEEPER_ADVERTISE_URL
-                    --listen-http='0.0.0.0:7676'
-                    --id=$$SAFEKEEPER_ID
-                    --broker-endpoints=$$BROKER_ENDPOINT
-                    -D /data
-                    --remote-storage=\"{endpoint='http://minio:9000',
-                                        bucket_name='neon',
-                                        bucket_region='eu-north-1',
-                                        prefix_in_bucket='/safekeeper/'}\""
-    depends_on:
-      - etcd
-      - minio_create_buckets
-
-  safekeeper3:
-    restart: always
-    image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
-    environment:
-      - SAFEKEEPER_ADVERTISE_URL=safekeeper3:5454
-      - SAFEKEEPER_ID=3
-      - BROKER_ENDPOINT=http://etcd:2379
-      - AWS_ACCESS_KEY_ID=minio
-      - AWS_SECRET_ACCESS_KEY=password
-      #- RUST_BACKTRACE=1
-    ports:
-      #- 5454:5454 # pg protocol handler
-      - 7678:7676 # http endpoints
-    entrypoint:
-      - "/bin/sh"
-      - "-c"
-    command:
-      - "safekeeper --listen-pg=$$SAFEKEEPER_ADVERTISE_URL
-                    --listen-http='0.0.0.0:7676'
-                    --id=$$SAFEKEEPER_ID
-                    --broker-endpoints=$$BROKER_ENDPOINT
-                    -D /data
-                    --remote-storage=\"{endpoint='http://minio:9000',
-                                        bucket_name='neon',
-                                        bucket_region='eu-north-1',
-                                        prefix_in_bucket='/safekeeper/'}\""
-    depends_on:
-      - etcd
-      - minio_create_buckets
-
-  compute:
-    restart: always
-    build:
-      context: ./compute_wrapper/
-      args:
-        - COMPUTE_IMAGE=compute-node-v${PG_VERSION:-14}
-        - TAG=${TAG:-latest}
-        - http_proxy=$http_proxy
-        - https_proxy=$https_proxy
-    environment:
-      - PG_VERSION=${PG_VERSION:-14}
-      #- RUST_BACKTRACE=1
-    # Mount the test files directly, for faster editing cycle.
-    volumes:
-      - ./compute_wrapper/var/db/postgres/specs/:/var/db/postgres/specs/
-      - ./compute_wrapper/shell/:/shell/
-    ports:
-      - 55433:55433 # pg protocol handler
-      - 3080:3080 # http endpoints
-    entrypoint:
-      - "/shell/compute.sh"
-    depends_on:
-      - safekeeper1
-      - safekeeper2
-      - safekeeper3
-      - pageserver
-
-  compute_is_ready:
-    image: postgres:latest
-    entrypoint:
-      - "/bin/bash"
-      - "-c"
-    command:
-      - "until pg_isready -h compute -p 55433 ; do
-            echo 'Waiting to start compute...' && sleep 1;
-         done"
-    depends_on:
-      - compute
--- a/docker-compose/docker_compose_test.sh
+++ b/docker-compose/docker_compose_test.sh
@@ -1,60 +0,0 @@
-#!/bin/bash
-
-# A basic test to ensure Docker images are built correctly.
-# Build a wrapper around the compute, start all services and runs a simple SQL query.
-# Repeats the process for all currenly supported Postgres versions.
-
-# Implicitly accepts `REPOSITORY` and `TAG` env vars that are passed into the compose file
-# Their defaults point at DockerHub `neondatabase/neon:latest` image.`,
-# to verify custom image builds (e.g pre-published ones).
-
-# XXX: Current does not work on M1 macs due to x86_64 Docker images compiled only, and no seccomp support in M1 Docker emulation layer.
-
-set -eux -o pipefail
-
-SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
-COMPOSE_FILE=$SCRIPT_DIR/docker-compose.yml
-
-COMPUTE_CONTAINER_NAME=docker-compose-compute-1
-SQL="CREATE TABLE t(key int primary key, value text); insert into t values(1,1); select * from t;"
-PSQL_OPTION="-h localhost -U cloud_admin -p 55433 -c '$SQL' postgres"
-
-cleanup() {
-    echo "show container information"
-    docker ps
-    docker compose -f $COMPOSE_FILE logs
-    echo "stop containers..."
-    docker compose -f $COMPOSE_FILE down
-}
-
-echo "clean up containers if exists"
-cleanup
-
-for pg_version in 14 15; do
-    echo "start containers (pg_version=$pg_version)."
-    PG_VERSION=$pg_version docker compose -f $COMPOSE_FILE up --build -d
-
-    echo "wait until the compute is ready. timeout after 60s. "
-    cnt=0
-    while sleep 1; do
-        # check timeout
-        cnt=`expr $cnt + 1`
-        if [ $cnt -gt 60 ]; then
-            echo "timeout before the compute is ready."
-            cleanup
-            exit 1
-        fi
-
-        # check if the compute is ready
-        set +o pipefail
-        result=`docker compose -f $COMPOSE_FILE logs "compute_is_ready" | grep "accepting connections" | wc -l`
-        set -o pipefail
-        if [ $result -eq 1 ]; then
-            echo "OK. The compute is ready to connect."
-            echo "execute simple queries."
-            docker exec $COMPUTE_CONTAINER_NAME /bin/bash -c "psql $PSQL_OPTION"
-            cleanup
-            break
-        fi
-    done
-done
--- a/docs/SUMMARY.md
+++ b/docs/SUMMARY.md
@@ -37,7 +37,7 @@

 - [Source view](./sourcetree.md)
  - [docker.md](./docker.md) — Docker images and building pipeline.
-  - [Error handling and logging](./error-handling.md)
+  - [Error handling and logging]()
  - [Testing]()
    - [Unit testing]()
    - [Integration testing]()
--- a/docs/authentication.md
+++ b/docs/authentication.md
@@ -1,154 +1,30 @@
 ## Authentication

 ### Overview
-We use JWT tokens in communication between almost all components (compute, pageserver, safekeeper, CLI) regardless of the protocol used (HTTP/PostgreSQL).
-Etcd currently has no authentication.
-Authentication is optional and is disabled by default for easier debugging.
-It is used in some tests, though.
-Note that we do not cover authentication with `pg.neon.tech` here.

-For HTTP connections we use the Bearer authentication scheme.
-For PostgreSQL connections we expect the token to be passed as a password.
-There is a caveat for `psql`: it silently truncates passwords to 100 symbols, so to correctly pass JWT via `psql` you have to either use `PGPASSWORD` environment variable, or store password in `psql`'s config file.
-
-Current token scopes are described in `utils::auth::Scope`.
-There are no expiration or rotation schemes.
-
-_TODO_: some scopes allow both access to server management API and to the data.
-These probably should be split into multiple scopes.
-
-Tokens should not occur in logs.
-They may sometimes occur in configuration files, although this is discouraged
-because configs may be parsed and dumped into logs.
-
-#### Tokens generation and validation
-JWT tokens are signed using a private key.
-Compute/pageserver/safekeeper use the private key's public counterpart to validate JWT tokens.
-These components should not have access to the private key and may only get tokens from their configuration or external clients. 
-
-The key pair is generated once for an installation of compute/pageserver/safekeeper, e.g. by `neon_local init`.
-There is currently no way to rotate the key without bringing down all components.
-
-### CLI
-CLI generates a key pair during call to `neon_local init` with the following commands:
+Current state of authentication includes usage of JWT tokens in communication between compute and pageserver and between CLI and pageserver. JWT token is signed using RSA keys. CLI generates a key pair during call to `neon_local init`. Using following openssl commands:

 ```bash
-openssl genrsa -out auth_private_key.pem 2048
-openssl rsa -in auth_private_key.pem -pubout -outform PEM -out auth_public_key.pem
+openssl genrsa -out private_key.pem 2048
+openssl rsa -in private_key.pem -pubout -outform PEM -out public_key.pem
 ```

-Configuration files for all components point to `public_key.pem` for JWT validation.
-However, authentication is disabled by default.
-There is no way to automatically enable it everywhere, you have to configure each component individually.
+CLI also generates signed token and saves it in the config for later access to pageserver. Now authentication is optional. Pageserver has two variables in config: `auth_validation_public_key_path` and `auth_type`, so when auth type present and set to `NeonJWT` pageserver will require authentication for connections. Actual JWT is passed in password field of connection string. There is a caveat for psql, it silently truncates passwords to 100 symbols, so to correctly pass JWT via psql you have to either use PGPASSWORD environment variable, or store password in psql config file.

-CLI also generates signed token (full access to Pageserver) and saves it in
-the CLI's `config` file under `pageserver.auth_token`.
-Note that pageserver's config does not have any similar parameter.
-CLI is the only component which accesses that token.
-Technically it could generate it from the private key on each run,
-but it does not do that for some reason (_TODO_).
+Currently there is no authentication between compute and safekeepers, because this communication layer is under heavy refactoring. After this refactoring support for authentication will be added there too. Now safekeeper supports "hardcoded" token passed via environment variable to be able to use callmemaybe command in pageserver.

-### Compute
-#### Overview
-Compute is a per-timeline PostgreSQL instance, so it should not have
-any access to data of other tenants.
-All tokens used by a compute are restricted to a specific tenant.
-There is no auth isolation from other timelines of the same tenant,
-but a non-rogue client never accesses another timeline even by an accident:
-timeline IDs are random and hard to guess.
+Compute uses token passed via environment variable to communicate to pageserver and in the future to the safekeeper too.

-#### Incoming connections
-All incoming connections are from PostgreSQL clients.
-Their authentication is just plain PostgreSQL authentication and out of scope for this document.
+JWT authentication now supports two scopes: tenant and pageserverapi. Tenant scope is intended for use in tenant related api calls, e.g. create_branch. Compute launched for particular tenant also uses this scope. Scope pageserver api is intended to be used by console to manage pageserver. For now we have only one management operation - create tenant.

-There is no administrative API except those provided by PostgreSQL.
-
-#### Outgoing connections
-Compute connects to Pageserver for getting pages.
-The connection string is configured by the `neon.pageserver_connstring` PostgreSQL GUC, e.g. `postgresql://no_user:$ZENITH_AUTH_TOKEN@localhost:15028`.
-The environment variable inside the connection string is substituted with
-the JWT token.
-
-Compute connects to Safekeepers to write and commit data.
-The token is the same for all safekeepers.
-It's stored in an environment variable, whose name is configured
-by the `neon.safekeeper_token_env` PostgreSQL GUC.
-If the GUC is unset, no token is passed.
-
-Note that both tokens can be (and typically are) the same;
-the scope is the tenant and the token is usually passed through the
-`$ZENITH_AUTH_TOKEN` environment variable.
-
-### Pageserver
-#### Overview
-Pageserver keeps track of multiple tenants, each having multiple timelines.
-For each timeline, it connects to the corresponding Safekeeper.
-Information about "corresponding Safekeeper" is published by Safekeepers
-in the Etcd, but they do not publish access tokens, otherwise what is
-the point of authentication.
-
-Pageserver keeps a connection to some set of Safekeepers, which
-may or may not correspond to active Computes.
-Hence, we cannot obtain a per-timeline access token from a Compute.
-E.g. if the timeline's Compute terminates before all WAL is
-consumed by the Pageserver, the Pageserver continues consuming WAL.
-
-Pageserver replicas' authentication is the same as the main's.
-
-#### Incoming connections
-Pageserver listens for connections from computes.
-Each compute should present a token valid for the timeline's tenant.
-
-Pageserver also has HTTP API: some parts are per-tenant,
-some parts are server-wide, these are different scopes.
-
-The `auth_type` configuration variable in Pageserver's config may have
-either of three values:
-
-* `Trust` removes all authentication. The outdated `MD5` value does likewise
-* `NeonJWT` enables JWT validation.
-   Tokens are validated using the public key which lies in a PEM file
-   specified in the `auth_validation_public_key_path` config.
-
-#### Outgoing connections
-Pageserver makes a connection to a Safekeeper for each active timeline.
-As Pageserver may want to access any timeline it has on the disk,
-it is given a blanket JWT token to access any data on any Safekeeper.
-This token is passed through an environment variable called `ZENITH_AUTH_TOKEN`
-(non-configurable as of writing this text).
-
-A better way _may be_ to store JWT token for each timeline next to it,
-but may be not.
-
-### Safekeeper
-#### Overview
-Safekeeper keeps track of multiple tenants, each having multiple timelines.
-
-#### Incoming connections
-Safekeeper accepts connections from Compute/Pageserver, each
-connection corresponds to a specific timeline and requires
-a corresponding JWT token.
-
-Safekeeper also has HTTP API: some parts are per-tenant,
-some parts are server-wide, these are different scopes.
-
-The `auth-validation-public-key-path` command line options controls
-the authentication mode:
-
-* If the option is missing, there is no authentication or JWT token validation.
-* If the option is present, it should be a path to the public key PEM file used for JWT token validation.
-
-#### Outgoing connections
-No connections are initiated by a Safekeeper.
-
-### In the source code
-Tests do not use authentication by default.
-If you need it, you can enable it by configuring the test's environment:
+Examples for token generation in python:

 ```python
-neon_env_builder.auth_enabled = True
+# generate pageserverapi token
+management_token = jwt.encode({"scope": "pageserverapi"}, auth_keys.priv, algorithm="RS256")
+
+# generate tenant token
+tenant_token = jwt.encode({"scope": "tenant", "tenant_id": ps.initial_tenant}, auth_keys.priv, algorithm="RS256")
 ```

-You will have to generate tokens if you want to access components inside the test directly,
-use `AuthKeys.generate_*_token` methods for that.
-If you create a new scope, please create a new method to prevent mistypes in scope's name.
+Utility functions to work with jwts in rust are located in libs/utils/src/auth.rs
--- a/docs/docker.md
+++ b/docs/docker.md
@@ -18,67 +18,3 @@ We build all images after a successful `release` tests run and push automaticall
 1. `neondatabase/compute-tools` and `neondatabase/compute-node`

 2. `neondatabase/neon`
-
-## Docker Compose example
-
-You can see a [docker compose](https://docs.docker.com/compose/) example to create a neon cluster in [/docker-compose/docker-compose.yml](/docker-compose/docker-compose.yml). It creates the following conatainers.
-
- etcd x 1
- pageserver x 1
- safekeeper x 3
- compute x 1
- MinIO x 1        # This is Amazon S3 compatible object storage
-
-### How to use
-
-1. create containers
-
-You can specify version of neon cluster using following environment values.
- PG_VERSION: postgres version for compute (default is 14)
- TAG: the tag version of [docker image](https://registry.hub.docker.com/r/neondatabase/neon/tags) (default is latest), which is tagged in [CI test](/.github/workflows/build_and_test.yml)
-```
-$ cd docker-compose/docker-compose.yml
-$ docker-compose down   # remove the conainers if exists
-$ PG_VERSION=15 TAG=2221 docker-compose up --build -d  # You can specify the postgres and image version
-Creating network "dockercompose_default" with the default driver
-Creating dockercompose_etcd3_1 ...
-(...omit...)
-```
-
-2. connect compute node
-```
-$ echo "localhost:55433:postgres:cloud_admin:cloud_admin" >> ~/.pgpass
-$ psql -h localhost -p 55433 -U cloud_admin
-postgres=# CREATE TABLE t(key int primary key, value text);
-CREATE TABLE
-postgres=# insert into t values(1,1);
-INSERT 0 1
-postgres=# select * from t;
- key | value
-----+-------
-   1 | 1
-(1 row)
-```
-
-3. If you want to see the log, you can use `docker-compose logs` command.
-```
-# check the container name you want to see
-$ docker ps
-CONTAINER ID   IMAGE                                              COMMAND                  CREATED         STATUS         PORTS                                                                                                                                  NAMES
-d6968a5ae912   dockercompose_compute                              "/shell/compute.sh"      5 minutes ago   Up 5 minutes   0.0.0.0:3080->3080/tcp, 0.0.0.0:55433->55433/tcp                                                                                       dockercompose_compute_1
-(...omit...)
-
-$ docker logs -f dockercompose_compute_1
-2022-10-21 06:15:48.757 GMT [56] LOG:  connection authorized: user=cloud_admin database=postgres application_name=psql
-2022-10-21 06:17:00.307 GMT [56] LOG:  [NEON_SMGR] libpagestore: connected to 'host=pageserver port=6400'
-(...omit...)
-```
-
-4. If you want to see durable data in MinIO which is s3 compatible storage
-
-Access http://localhost:9001 and sign in.
-
- Username: `minio`
- Password: `password`
-
-You can see durable pages and WAL data in `neon` bucket.
--- a/docs/error-handling.md
+++ b/docs/error-handling.md
@@ -1,198 +0,0 @@
-# Error handling and logging
-
-## Logging errors
-
-The principle is that errors are logged when they are handled. If you
-just propagate an error to the caller in a function, you don't need to
-log it; the caller will. But if you consume an error in a function,
-you *must* log it (if it needs to be logged at all).
-
-For example:
-
-```rust
-fn read_motd_file() -> std::io::Result<String> {
-    let mut f = File::open("/etc/motd")?;
-    let mut result = String::new();
-    f.read_to_string(&mut result)?;
-    result
-}
-```
-
-Opening or reading the file could fail, but there is no need to log
-the error here. The function merely propagates the error to the
-caller, and it is up to the caller to log the error or propagate it
-further, if the failure is not expected. But if, for example, it is
-normal that the "/etc/motd" file doesn't exist, the caller can choose
-to silently ignore the error, or log it as an INFO or DEBUG level
-message:
-
-```rust
-fn get_message_of_the_day() -> String {
-    // Get the motd from /etc/motd, or return the default proverb
-    match read_motd_file() {
-        Ok(motd) => motd,
-        Err(err)  => {
-            // It's normal that /etc/motd doesn't exist, but if we fail to
-            // read it for some other reason, that's unexpected. The message
-            // of the day isn't very important though, so we just WARN and
-            // continue with the default in any case.
-            if err.kind() != std::io::ErrorKind::NotFound {
-                 tracing::warn!("could not read \"/etc/motd\": {err:?}");
-            }
-            "An old error is always more popular than a new truth. - German proverb"
-        }
-    }
-}
-```
-
-## Error types
-
-We use the `anyhow` crate widely. It contains many convenient macros
-like `bail!` and `ensure!` to construct and return errors, and to
-propagate many kinds of low-level errors, wrapped in `anyhow::Error`.
-
-A downside of `anyhow::Error` is that the caller cannot distinguish
-between different error cases. Most errors are propagated all the way
-to the mgmt API handler function, or the main loop that handles a
-connection with the compute node, and they are all handled the same
-way: the error is logged and returned to the client as an HTTP or
-libpq error.
-
-But in some cases, we need to distinguish between errors and handle
-them differently. For example, attaching a tenant to the pageserver
-could fail either because the tenant has already been attached, or
-because we could not load its metadata from cloud storage. The first
-case is more or less expected. The console sends the Attach request to
-the pageserver, and the pageserver completes the operation, but the
-network connection might be lost before the console receives the
-response. The console will retry the operation in that case, but the
-tenant has already been attached. It is important that the pagserver
-responds with the HTTP 403 Already Exists error in that case, rather
-than a generic HTTP 500 Internal Server Error.
-
-If you need to distinguish between different kinds of errors, create a
-new `Error` type. The `thiserror` crate is useful for that. But in
-most cases `anyhow::Error` is good enough.
-
-## Panics
-
-Depending on where a panic happens, it can cause the whole pageserver
-or safekeeper to restart, or just a single tenant. In either case,
-that is pretty bad and causes an outage. Avoid panics. Never use
-`unwrap()` or other calls that might panic, to verify inputs from the
-network or from disk.
-
-It is acceptable to use functions that might panic, like `unwrap()`, if
-it is obvious that it cannot panic. For example, if you have just
-checked that a variable is not None, it is OK to call `unwrap()` on it,
-but it is still preferable to use `expect("reason")` instead to explain
-why the function cannot fail.
-
-`assert!` and `panic!` are reserved for checking clear invariants and
-very obvious "can't happen" cases. When in doubt, use anyhow `ensure!`
-or `bail!` instead.
-
-## Error levels
-
-`tracing::Level` doesn't provide very clear guidelines on what the
-different levels mean, or when to use which level. Here is how we use
-them:
-
-### Error
-
-Examples:
- could not open file "foobar"
- invalid tenant id
-
-Errors are not expected to happen during normal operation. Incorrect
-inputs from client can cause ERRORs. For example, if a client tries to
-call a mgmt API that doesn't exist, or if a compute node sends passes
-an LSN that has already been garbage collected away.
-
-These should *not* happen during normal operations. "Normal
-operations" is not a very precise concept. But for example, disk
-errors are not expected to happen when the system is working, so those
-count as Errors. However, if a TCP connection to a compute node is
-lost, that is not considered an Error, because it doesn't affect the
-pageserver's or safekeeper's operation in any way, and happens fairly
-frequently when compute nodes are shut down, or are killed abruptly
-because of errors in the compute.
-
-**Errors are monitored, and always need human investigation to determine
-the cause.**
-
-Whether something should be logged at ERROR, WARNING or INFO level can
-depend on the callers and clients. For example, it might be unexpected
-and a sign of a serious issue if the console calls the
-"timeline_detail" mgmt API for a timeline that doesn't exist. ERROR
-would be appropriate in that case. But if the console routinely calls
-the API after deleting a timeline, to check if the deletion has
-completed, then it would be totally normal and an INFO or DEBUG level
-message would be more appropriate. If a message is logged as an ERROR,
-but it in fact happens frequently in production and never requires any
-action, it should probably be demoted to an INFO level message.
-
-### Warn
-
-Examples:
- could not remove temporary file "foobar.temp"
- unrecognized file "foobar" in timeline directory
-
-Warnings are similar to Errors, in that they should not happen
-when the system is operating normally. The difference between Error and
-Warning is that an Error means that the operation failed, whereas Warning
-means that something unexpected happened, but the operation continued anyway.
-For example, if deleting a file fails because the file already didn't exist,
-it should be logged as Warning.
-
-> **Note:** The python regression tests, under `test_regress`, check the
-> pageserver log after each test for any ERROR and WARN lines. If there are
-> any ERRORs or WARNs that have not been explicitly listed in the test as
-> allowed, the test is marked a failed. This is to catch unexpected errors
-> e.g. in background operations, that don't cause immediate misbehaviour in
-> the tested functionality.
-
-### Info
-
-Info level is used to log useful information when the system is
-operating normally. Info level is appropriate e.g. for logging state
-changes, background operations, and network connections.
-
-Examples:
- "system is shutting down"
- "tenant was created"
- "retrying S3 upload"
-
-### Debug & Trace
-
-Debug and Trace level messages are not printed to the log in our normal
-production configuration, but could be enabled for a specific server or
-tenant, to aid debugging. (Although we don't actually have that
-capability as of this writing).
-
-## Context
-
-We use logging "spans" to hold context information about the current
-operation. Almost every operation happens on a particular tenant and
-timeline, so we enter a span with the "tenant_id" and "timeline_id"
-very early when processing an incoming API request, for example. All
-background operations should also run in a span containing at least
-those two fields, and any other parameters or information that might
-be useful when debugging an error that might happen when performing
-the operation.
-
-TODO: Spans are not captured in the Error when it is created, but when
-the error is logged. It would be more useful to capture them at Error
-creation. We should consider using `tracing_error::SpanTrace` to do
-that.
-
-## Error message style
-
-PostgreSQL has a style guide for writing error messages:
-
-https://www.postgresql.org/docs/current/error-style-guide.html
-
-Follow that guide when writing error messages in the PostgreSQL
-extension. We don't follow it strictly in the pageserver and
-safekeeper, but the advice in the PostgreSQL style guide is generally
-good, and you can't go wrong by following it.
--- a/docs/rfcs/020-pageserver-s3-coordination.md
+++ b/docs/rfcs/020-pageserver-s3-coordination.md
@@ -1,246 +0,0 @@
-# Coordinating access of multiple pageservers to the same s3 data
-
-## Motivation
-
-There are some blind spots around coordinating access of multiple pageservers
-to the same s3 data. Currently this is applicable only to tenant relocation
-case, but in the future we'll need to solve similar problems for
-replica/standby pageservers.
-
-## Impacted components (e.g. pageserver, safekeeper, console, etc)
-
-Pageserver
-
-## The problem
-
-### Relocation
-
-During relocation both pageservers can write to s3. This should be ok for all
-data except the `index_part.json`. For index part it causes problems during
-compaction/gc because they remove files from index/s3.
-
-Imagine this case:
-
-```mermaid
-sequenceDiagram
-    autonumber
-    participant PS1
-    participant S3
-    participant PS2
-
-    PS1->>S3: Uploads L1, L2 <br/> Index contains L1 L2
-    PS2->>S3: Attach called, sees L1, L2
-    PS1->>S3: Compaction comes <br/> Removes L1, adds L3
-    note over S3: Index now L2, L3
-    PS2->>S3: Uploads new layer L4 <br/> (added to previous view of the index)
-    note over S3: Index now L1, L2, L4
-```
-
-At this point it is not possible to restore from index, it contains L2 which
-is no longer available in s3 and doesnt contain L3 added by compaction by the
-first pageserver. So if any of the pageservers restart initial sync will fail
-(or in on-demand world it will fail a bit later during page request from
-missing layer)
-
-### Standby pageserver
-
-Another related case is standby pageserver. In this case second pageserver can
-be used as a replica to scale reads and serve as a failover target in case
-first one fails.
-
-In this mode second pageserver needs to have the same picture of s3 files to
-be able to load layers on-demand. To accomplish that second pageserver
-cannot run gc/compaction jobs. Instead it needs to receive updates for index
-contents. (There is no need to run walreceiver on the second pageserver then).
-
-## Observations
-
- If both pageservers ingest wal then their layer set diverges, because layer
-  file generation is not deterministic
- If one of the pageservers does not ingest wal (and just picks up layer
-  updates) then it lags behind and cannot really answer queries in the same
-  pace as the primary one
- Can compaction help make layers deterministic? E g we do not upload level
-  zero layers and construction of higher levels should be deterministic.
-  This way we can guarantee that layer creation by timeout wont mess things up.
-  This way one pageserver uploads data and second one can just ingest it.
-  But we still need some form of election
-
-## Solutions
-
-### Manual orchestration
-
-One possible solution for relocation case is to orchestrate background jobs
-from outside. The oracle who runs migration can turn off background jobs on
-PS1 before migration and then run migration -> enable them on PS2. The problem
-comes if migration fails. In this case in order to resume background jobs
-oracle needs to guarantee that PS2 doesnt run background jobs and if it doesnt
-respond then PS1 is stuck unable to run compaction/gc. This cannot be solved
-without human ensuring that no upload from PS2 can happen. In order to be able
-to resolve this automatically CAS is required on S3 side so pageserver can
-avoid overwriting index part if it is no longer the leading one
-
-Note that flag that disables background jobs needs to be persistent, because
-otherwise pageserver restart will clean it
-
-### Avoid index_part.json
-
-Index part consists of two parts, list of layers and metadata. List of layers
-can be easily obtained by `ListObjects` S3 API method. But what to do with
-metadata? Create metadata instance for each checkpoint and add some counter
-to the file name?
-
-Back to potentially long s3 ls.
-
-### Coordination based approach
-
-Do it like safekeepers chose leader for WAL upload. Ping each other and decide
-based on some heuristics e g smallest node id. During relocation PS1 sends
-"resign" ping message so others can start election without waiting for a timeout.
-
-This still leaves metadata question open and non deterministic layers are a
-problem as well
-
-### Avoid metadata file
-
-One way to eliminate metadata file is to store it in layer files under some
-special key. This may resonate with intention to keep all relation sizes in
-some special segment to avoid initial download during size calculation.
-Maybe with that we can even store pre calculated value.
-
-As a downside each checkpoint gets 512 bytes larger.
-
-If we entirely avoid metadata file this opens up many approaches
-
-* * *
-
-During discussion it seems that we converged on the approach consisting of:
-
- index files stored per pageserver in the same timeline directory. With that
-  index file name starts to look like: `<pageserver_node_id>_index_part.json`.
-  In such set up there are no concurrent overwrites of index file by different
-  pageservers.
- For replica pageservers the solution would be for primary to broadcast index
-  changes to any followers with an ability to check index files in s3 and
-  restore the full state. To properly merge changes with index files we can use
-  a counter that is persisted in an index file, is incremented on every change
-  to it and passed along with broadcasted change. This way we can determine
-  whether we need to apply change to the index state or not.
- Responsibility for running background jobs is assigned externally. Pageserver
-  keeps locally persistent flag for each tenant that indicates whether this
-  pageserver is considered as primary one or not. TODO what happends if we
-  crash and cannot start for some extended period of time? Control plane can
-  assign ownership to some other pageserver. Pageserver needs some way to check
-  if its still the blessed one. Maybe by explicit request to control plane on
-  start.
-
-Requirement for deterministic layer generation was considered overly strict
-because of two reasons:
-
- It can limit possible optimizations e g when pageserver wants to reshuffle
-  some data locally and doesnt want to coordinate this
- The deterministic algorithm itself can change so during deployments for some
-  time there will be two different version running at the same time which can
-  cause non determinism
-
-### External elections
-
-The above case with lost state in this schema with externally managed
-leadership is represented like this:
-
-Note that here we keep objects list in the index file.
-
-```mermaid
-sequenceDiagram
-    autonumber
-    participant PS1
-    participant CP as Control Plane
-    participant S3
-    participant PS2
-
-    note over PS1,PS2: PS1 starts up and still a leader
-    PS1->>CP: Am I still the leader for Tenant X?
-    activate CP
-    CP->>PS1: Yes
-    deactivate CP
-    PS1->>S3: Fetch PS1 index.
-    note over PS1: Continue operations, start backround jobs
-    note over PS1,PS2: PS1 starts up and still and is not a leader anymore
-    PS1->>CP: Am I still the leader for Tenant X?
-    CP->>PS1: No
-    PS1->>PS2: Subscribe to index changes
-    PS1->>S3: Fetch PS1 and PS2 indexes
-    note over PS1: Combine index file to include layers <br> from both indexes to be able <br> to see newer files from leader (PS2)
-    note over PS1: Continue operations, do not start background jobs
-```
-
-### Internal elections
-
-To manage leadership internally we can use broker to exchange pings so nodes
-can decide on the leader roles. In case multiple pageservers are active leader
-is the one with lowest node id.
-
-Operations with internally managed elections:
-
-```mermaid
-sequenceDiagram
-    autonumber
-    participant PS1
-    participant S3
-
-    note over PS1: Starts up
-    note over PS1: Subscribes to changes, waits for two ping <br> timeouts to see if there is a leader
-    PS1->>S3: Fetch indexes from s3
-    alt there is a leader
-        note over PS1: do not start background jobs, <br> continue applying index updates
-    else there is no leader
-        note over PS1: start background jobs, <br> broadcast index changes
-    end
-
-    note over PS1,S3: Then the picture is similar to external elections <br> the difference is that follower can become a leader <br> if there are no pings after some timeout new leader gets elected
-```
-
-### Eviction
-
-When two pageservers operate on a tenant for extended period of time follower
-doesnt perform write operations in s3. When layer is evicted follower relies
-on updates from primary to get info about layers it needs to cover range for
-evicted layer.
-
-Note that it wont match evicted layer exactly, so layers will overlap and
-lookup code needs to correctly handle that.
-
-### Relocation flow
-
-Actions become:
-
- Attach tenant to new pageserver
- New pageserver becomes follower since previous one is still leading
- New pageserver starts replicating from safekeepers but does not upload layers
- Detach is called on the old one
- New pageserver becomes leader after it realizes that old one disappeared
-
-### Index File
-
-Using `s3 ls` on startup simplifies things, but we still need metadata, so we
-need to fetch index files anyway. If they contain list of files we can combine
-them and avoid costly `s3 ls`
-
-### Remaining issues
-
- More than one remote consistent lsn for safekeepers to know
-
-Anything else?
-
-### Proposed solution
-
-To recap. On meeting we converged on approach with external elections but I
-think it will be overall harder to manage and will introduce a dependency on
-control plane for pageserver. Using separate index files for each pageserver
-consisting of log of operations and a metadata snapshot should be enough.
-
-### What we need to get there?
-
- Change index file structure to contain log of changes instead of just the
-  file list
- Implement pinging/elections for pageservers
--- a/docs/sourcetree.md
+++ b/docs/sourcetree.md
@@ -2,11 +2,6 @@

 Below you will find a brief overview of each subdir in the source tree in alphabetical order.

-`storage_broker`:
-
-Neon storage broker, providing messaging between safekeepers and pageservers.
-[storage_broker.md](./storage_broker.md)
-
 `/control_plane`:

 Local control plane.
@@ -57,10 +52,6 @@ PostgreSQL extension that implements storage manager API and network communicati

 PostgreSQL extension that contains functions needed for testing and debugging.

-`/pgxn/neon_walredo`:
-
-Library to run Postgres as a "WAL redo process" in the pageserver.
-
 `/safekeeper`:

 The neon WAL service that receives WAL from a primary compute nodes and streams it to the pageserver.
@@ -88,16 +79,6 @@ A subject for future modularization.
 `/libs/metrics`:
 Helpers for exposing Prometheus metrics from the server.

-### Adding dependencies
-When you add a Cargo dependency, you should update hakari manifest by running commands below and committing the updated `Cargo.lock` and `workspace_hack/`. There may be no changes, that's fine.
-
-```bash
-cargo hakari generate
-cargo hakari manage-deps
-```
-
-If you don't have hakari installed (`error: no such subcommand: hakari`), install it by running `cargo install cargo-hakari`.
-
 ## Using Python
 Note that Debian/Ubuntu Python packages are stale, as it commonly happens,
 so manual installation of dependencies is not recommended.
--- a/docs/storage_broker.md
+++ b/docs/storage_broker.md
@@ -1,27 +0,0 @@
-# Storage broker
-
-Storage broker targets two issues:
- Allowing safekeepers and pageservers learn which nodes also hold their
-  timelines, and timeline statuses there.
- Avoiding O(n^2) connections between storage nodes while doing so.
-
-This is used
- By pageservers to determine the most advanced and alive safekeeper to pull WAL from.
- By safekeepers to synchronize on the timeline: advance
-  `remote_consistent_lsn`, `backup_lsn`, choose who offloads WAL to s3.
-
-Technically, it is a simple stateless pub-sub message broker based on tonic
-(grpc) making multiplexing easy. Since it is stateless, fault tolerance can be
-provided by k8s; there is no built in replication support, though it is not hard
-to add.
-
-Currently, the only message is `SafekeeperTimelineInfo`. Each safekeeper, for
-each active timeline, once in a while pushes timeline status to the broker.
-Other nodes subscribe and receive this info, using it per above.
-
-Broker serves /metrics on the same port as grpc service. 
-
-grpcurl can be used to check which values are currently being pushed:
-```
-grpcurl -proto broker/proto/broker.proto -d '{"all":{}}' -plaintext localhost:50051 storage_broker.BrokerService/SubscribeSafekeeperInfo
-```
--- a/libs/etcd_broker/src/subscription_value.rs
+++ b/libs/etcd_broker/src/subscription_value.rs
@@ -29,9 +29,6 @@ pub struct SkTimelineInfo {
    #[serde_as(as = "Option<DisplayFromStr>")]
    #[serde(default)]
    pub peer_horizon_lsn: Option<Lsn>,
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    #[serde(default)]
-    pub local_start_lsn: Option<Lsn>,
    /// A connection string to use for WAL receiving.
    #[serde(default)]
    pub safekeeper_connstr: Option<String>,
--- a/libs/pageserver_api/Cargo.toml
+++ b/libs/pageserver_api/Cargo.toml
@@ -7,10 +7,6 @@ edition = "2021"
 serde = { version = "1.0", features = ["derive"] }
 serde_with = "2.0"
 const_format = "0.2.21"
-anyhow = { version = "1.0", features = ["backtrace"] }
-bytes = "1.0.1"
-byteorder = "1.4.3"

 utils = { path = "../utils" }
-postgres_ffi = { path = "../postgres_ffi" }
 workspace_hack = { version = "0.1", path = "../../workspace_hack" }
--- a/libs/pageserver_api/src/lib.rs
+++ b/libs/pageserver_api/src/lib.rs
@@ -2,7 +2,6 @@ use const_format::formatcp;

 /// Public API types
 pub mod models;
-pub mod reltag;

 pub const DEFAULT_PG_LISTEN_PORT: u16 = 64000;
 pub const DEFAULT_PG_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_PG_LISTEN_PORT}");
--- a/libs/pageserver_api/src/models.rs
+++ b/libs/pageserver_api/src/models.rs
@@ -1,6 +1,5 @@
 use std::num::NonZeroU64;

-use byteorder::{BigEndian, ReadBytesExt};
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
 use utils::{
@@ -8,54 +7,15 @@ use utils::{
    lsn::Lsn,
 };

-use crate::reltag::RelTag;
-use anyhow::bail;
-use bytes::{BufMut, Bytes, BytesMut};
-
 /// A state of a tenant in pageserver's memory.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 pub enum TenantState {
-    // This tenant is being loaded from local disk
-    Loading,
-    // This tenant is being downloaded from cloud storage.
-    Attaching,
-    /// Tenant is fully operational
-    Active,
-    /// A tenant is recognized by pageserver, but it is being detached or the
-    /// system is being shut down.
+    /// Tenant is fully operational, its background jobs might be running or not.
+    Active { background_jobs_running: bool },
+    /// A tenant is recognized by pageserver, but not yet ready to operate:
+    /// e.g. not present locally and being downloaded or being read into memory from the file system.
    Paused,
-    /// A tenant is recognized by the pageserver, but can no longer be used for
-    /// any operations, because it failed to be activated.
-    Broken,
-}
-
-impl TenantState {
-    pub fn has_in_progress_downloads(&self) -> bool {
-        match self {
-            Self::Loading => true,
-            Self::Attaching => true,
-            Self::Active => false,
-            Self::Paused => false,
-            Self::Broken => false,
-        }
-    }
-}
-
-/// A state of a timeline in pageserver's memory.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
-pub enum TimelineState {
-    /// Timeline is fully operational. If the containing Tenant is Active, the timeline's
-    /// background jobs are running otherwise they will be launched when the tenant is activated.
-    Active,
-    /// A timeline is recognized by pageserver, but not yet ready to operate.
-    /// The status indicates, that the timeline could eventually go back to Active automatically:
-    /// for example, if the owning tenant goes back to Active again.
-    Suspended,
-    /// A timeline is recognized by pageserver, but not yet ready to operate and not allowed to
-    /// automatically become Active after certain events: only a management call can change this status.
-    Paused,
-    /// A timeline is recognized by the pageserver, but can no longer be used for
-    /// any operations, because it failed to be activated.
+    /// A tenant is recognized by the pageserver, but no longer used for any operations, as failed to get activated.
    Broken,
 }

@@ -92,7 +52,6 @@ pub struct TenantCreateRequest {
    pub walreceiver_connect_timeout: Option<String>,
    pub lagging_wal_timeout: Option<String>,
    pub max_lsn_wal_lag: Option<NonZeroU64>,
-    pub trace_read_requests: Option<bool>,
 }

 #[serde_as]
@@ -132,7 +91,6 @@ pub struct TenantConfigRequest {
    pub walreceiver_connect_timeout: Option<String>,
    pub lagging_wal_timeout: Option<String>,
    pub max_lsn_wal_lag: Option<NonZeroU64>,
-    pub trace_read_requests: Option<bool>,
 }

 impl TenantConfigRequest {
@@ -151,7 +109,6 @@ impl TenantConfigRequest {
            walreceiver_connect_timeout: None,
            lagging_wal_timeout: None,
            max_lsn_wal_lag: None,
-            trace_read_requests: None,
        }
    }
 }
@@ -187,8 +144,6 @@ pub struct TimelineInfo {
    pub latest_gc_cutoff_lsn: Lsn,
    #[serde_as(as = "DisplayFromStr")]
    pub disk_consistent_lsn: Lsn,
-    #[serde_as(as = "DisplayFromStr")]
-    pub remote_consistent_lsn: Lsn,
    pub current_logical_size: Option<u64>, // is None when timeline is Unloaded
    pub current_physical_size: Option<u64>, // is None when timeline is Unloaded
    pub current_logical_size_non_incremental: Option<u64>,
@@ -201,10 +156,10 @@ pub struct TimelineInfo {
    pub last_received_msg_ts: Option<u128>,
    pub pg_version: u32,

+    #[serde_as(as = "Option<DisplayFromStr>")]
+    pub remote_consistent_lsn: Option<Lsn>,
    pub awaits_download: bool,

-    pub state: TimelineState,
-
    // Some of the above fields are duplicated in 'local' and 'remote', for backwards-
    // compatility with older clients.
    pub local: LocalTimelineInfo,
@@ -246,262 +201,3 @@ pub struct FailpointConfig {
 pub struct TimelineGcRequest {
    pub gc_horizon: Option<u64>,
 }
-
-// Wrapped in libpq CopyData
-#[derive(PartialEq, Eq)]
-pub enum PagestreamFeMessage {
-    Exists(PagestreamExistsRequest),
-    Nblocks(PagestreamNblocksRequest),
-    GetPage(PagestreamGetPageRequest),
-    DbSize(PagestreamDbSizeRequest),
-}
-
-// Wrapped in libpq CopyData
-pub enum PagestreamBeMessage {
-    Exists(PagestreamExistsResponse),
-    Nblocks(PagestreamNblocksResponse),
-    GetPage(PagestreamGetPageResponse),
-    Error(PagestreamErrorResponse),
-    DbSize(PagestreamDbSizeResponse),
-}
-
-#[derive(Debug, PartialEq, Eq)]
-pub struct PagestreamExistsRequest {
-    pub latest: bool,
-    pub lsn: Lsn,
-    pub rel: RelTag,
-}
-
-#[derive(Debug, PartialEq, Eq)]
-pub struct PagestreamNblocksRequest {
-    pub latest: bool,
-    pub lsn: Lsn,
-    pub rel: RelTag,
-}
-
-#[derive(Debug, PartialEq, Eq)]
-pub struct PagestreamGetPageRequest {
-    pub latest: bool,
-    pub lsn: Lsn,
-    pub rel: RelTag,
-    pub blkno: u32,
-}
-
-#[derive(Debug, PartialEq, Eq)]
-pub struct PagestreamDbSizeRequest {
-    pub latest: bool,
-    pub lsn: Lsn,
-    pub dbnode: u32,
-}
-
-#[derive(Debug)]
-pub struct PagestreamExistsResponse {
-    pub exists: bool,
-}
-
-#[derive(Debug)]
-pub struct PagestreamNblocksResponse {
-    pub n_blocks: u32,
-}
-
-#[derive(Debug)]
-pub struct PagestreamGetPageResponse {
-    pub page: Bytes,
-}
-
-#[derive(Debug)]
-pub struct PagestreamErrorResponse {
-    pub message: String,
-}
-
-#[derive(Debug)]
-pub struct PagestreamDbSizeResponse {
-    pub db_size: i64,
-}
-
-impl PagestreamFeMessage {
-    pub fn serialize(&self) -> Bytes {
-        let mut bytes = BytesMut::new();
-
-        match self {
-            Self::Exists(req) => {
-                bytes.put_u8(0);
-                bytes.put_u8(if req.latest { 1 } else { 0 });
-                bytes.put_u64(req.lsn.0);
-                bytes.put_u32(req.rel.spcnode);
-                bytes.put_u32(req.rel.dbnode);
-                bytes.put_u32(req.rel.relnode);
-                bytes.put_u8(req.rel.forknum);
-            }
-
-            Self::Nblocks(req) => {
-                bytes.put_u8(1);
-                bytes.put_u8(if req.latest { 1 } else { 0 });
-                bytes.put_u64(req.lsn.0);
-                bytes.put_u32(req.rel.spcnode);
-                bytes.put_u32(req.rel.dbnode);
-                bytes.put_u32(req.rel.relnode);
-                bytes.put_u8(req.rel.forknum);
-            }
-
-            Self::GetPage(req) => {
-                bytes.put_u8(2);
-                bytes.put_u8(if req.latest { 1 } else { 0 });
-                bytes.put_u64(req.lsn.0);
-                bytes.put_u32(req.rel.spcnode);
-                bytes.put_u32(req.rel.dbnode);
-                bytes.put_u32(req.rel.relnode);
-                bytes.put_u8(req.rel.forknum);
-                bytes.put_u32(req.blkno);
-            }
-
-            Self::DbSize(req) => {
-                bytes.put_u8(3);
-                bytes.put_u8(if req.latest { 1 } else { 0 });
-                bytes.put_u64(req.lsn.0);
-                bytes.put_u32(req.dbnode);
-            }
-        }
-
-        bytes.into()
-    }
-
-    pub fn parse<R: std::io::Read>(body: &mut R) -> anyhow::Result<PagestreamFeMessage> {
-        // TODO these gets can fail
-
-        // these correspond to the NeonMessageTag enum in pagestore_client.h
-        //
-        // TODO: consider using protobuf or serde bincode for less error prone
-        // serialization.
-        let msg_tag = body.read_u8()?;
-        match msg_tag {
-            0 => Ok(PagestreamFeMessage::Exists(PagestreamExistsRequest {
-                latest: body.read_u8()? != 0,
-                lsn: Lsn::from(body.read_u64::<BigEndian>()?),
-                rel: RelTag {
-                    spcnode: body.read_u32::<BigEndian>()?,
-                    dbnode: body.read_u32::<BigEndian>()?,
-                    relnode: body.read_u32::<BigEndian>()?,
-                    forknum: body.read_u8()?,
-                },
-            })),
-            1 => Ok(PagestreamFeMessage::Nblocks(PagestreamNblocksRequest {
-                latest: body.read_u8()? != 0,
-                lsn: Lsn::from(body.read_u64::<BigEndian>()?),
-                rel: RelTag {
-                    spcnode: body.read_u32::<BigEndian>()?,
-                    dbnode: body.read_u32::<BigEndian>()?,
-                    relnode: body.read_u32::<BigEndian>()?,
-                    forknum: body.read_u8()?,
-                },
-            })),
-            2 => Ok(PagestreamFeMessage::GetPage(PagestreamGetPageRequest {
-                latest: body.read_u8()? != 0,
-                lsn: Lsn::from(body.read_u64::<BigEndian>()?),
-                rel: RelTag {
-                    spcnode: body.read_u32::<BigEndian>()?,
-                    dbnode: body.read_u32::<BigEndian>()?,
-                    relnode: body.read_u32::<BigEndian>()?,
-                    forknum: body.read_u8()?,
-                },
-                blkno: body.read_u32::<BigEndian>()?,
-            })),
-            3 => Ok(PagestreamFeMessage::DbSize(PagestreamDbSizeRequest {
-                latest: body.read_u8()? != 0,
-                lsn: Lsn::from(body.read_u64::<BigEndian>()?),
-                dbnode: body.read_u32::<BigEndian>()?,
-            })),
-            _ => bail!("unknown smgr message tag: {:?}", msg_tag),
-        }
-    }
-}
-
-impl PagestreamBeMessage {
-    pub fn serialize(&self) -> Bytes {
-        let mut bytes = BytesMut::new();
-
-        match self {
-            Self::Exists(resp) => {
-                bytes.put_u8(100); /* tag from pagestore_client.h */
-                bytes.put_u8(resp.exists as u8);
-            }
-
-            Self::Nblocks(resp) => {
-                bytes.put_u8(101); /* tag from pagestore_client.h */
-                bytes.put_u32(resp.n_blocks);
-            }
-
-            Self::GetPage(resp) => {
-                bytes.put_u8(102); /* tag from pagestore_client.h */
-                bytes.put(&resp.page[..]);
-            }
-
-            Self::Error(resp) => {
-                bytes.put_u8(103); /* tag from pagestore_client.h */
-                bytes.put(resp.message.as_bytes());
-                bytes.put_u8(0); // null terminator
-            }
-            Self::DbSize(resp) => {
-                bytes.put_u8(104); /* tag from pagestore_client.h */
-                bytes.put_i64(resp.db_size);
-            }
-        }
-
-        bytes.into()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use bytes::Buf;
-
-    use super::*;
-
-    #[test]
-    fn test_pagestream() {
-        // Test serialization/deserialization of PagestreamFeMessage
-        let messages = vec![
-            PagestreamFeMessage::Exists(PagestreamExistsRequest {
-                latest: true,
-                lsn: Lsn(4),
-                rel: RelTag {
-                    forknum: 1,
-                    spcnode: 2,
-                    dbnode: 3,
-                    relnode: 4,
-                },
-            }),
-            PagestreamFeMessage::Nblocks(PagestreamNblocksRequest {
-                latest: false,
-                lsn: Lsn(4),
-                rel: RelTag {
-                    forknum: 1,
-                    spcnode: 2,
-                    dbnode: 3,
-                    relnode: 4,
-                },
-            }),
-            PagestreamFeMessage::GetPage(PagestreamGetPageRequest {
-                latest: true,
-                lsn: Lsn(4),
-                rel: RelTag {
-                    forknum: 1,
-                    spcnode: 2,
-                    dbnode: 3,
-                    relnode: 4,
-                },
-                blkno: 7,
-            }),
-            PagestreamFeMessage::DbSize(PagestreamDbSizeRequest {
-                latest: true,
-                lsn: Lsn(4),
-                dbnode: 7,
-            }),
-        ];
-        for msg in messages {
-            let bytes = msg.serialize();
-            let reconstructed = PagestreamFeMessage::parse(&mut bytes.reader()).unwrap();
-            assert!(msg == reconstructed);
-        }
-    }
-}
--- a/libs/postgres_connection/Cargo.toml
+++ b/libs/postgres_connection/Cargo.toml
@@ -1,17 +0,0 @@
-[package]
-name = "postgres_connection"
-version = "0.1.0"
-edition = "2021"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-anyhow = "1.0"
-itertools = "0.10.3"
-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev = "d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-url = "2.2.2"
-workspace_hack = { version = "0.1", path = "../../workspace_hack" }
-
-[dev-dependencies]
-once_cell = "1.13.0"
--- a/libs/postgres_connection/src/lib.rs
+++ b/libs/postgres_connection/src/lib.rs
@@ -1,253 +0,0 @@
-use anyhow::{bail, Context};
-use itertools::Itertools;
-use std::borrow::Cow;
-use std::fmt;
-use url::Host;
-
-/// Parses a string of format either `host:port` or `host` into a corresponding pair.
-/// The `host` part should be a correct `url::Host`, while `port` (if present) should be
-/// a valid decimal u16 of digits only.
-pub fn parse_host_port<S: AsRef<str>>(host_port: S) -> Result<(Host, Option<u16>), anyhow::Error> {
-    let (host, port) = match host_port.as_ref().rsplit_once(':') {
-        Some((host, port)) => (
-            host,
-            // +80 is a valid u16, but not a valid port
-            if port.chars().all(|c| c.is_ascii_digit()) {
-                Some(port.parse::<u16>().context("Unable to parse port")?)
-            } else {
-                bail!("Port contains a non-ascii-digit")
-            },
-        ),
-        None => (host_port.as_ref(), None), // No colons, no port specified
-    };
-    let host = Host::parse(host).context("Unable to parse host")?;
-    Ok((host, port))
-}
-
-#[cfg(test)]
-mod tests_parse_host_port {
-    use crate::parse_host_port;
-    use url::Host;
-
-    #[test]
-    fn test_normal() {
-        let (host, port) = parse_host_port("hello:123").unwrap();
-        assert_eq!(host, Host::Domain("hello".to_owned()));
-        assert_eq!(port, Some(123));
-    }
-
-    #[test]
-    fn test_no_port() {
-        let (host, port) = parse_host_port("hello").unwrap();
-        assert_eq!(host, Host::Domain("hello".to_owned()));
-        assert_eq!(port, None);
-    }
-
-    #[test]
-    fn test_ipv6() {
-        let (host, port) = parse_host_port("[::1]:123").unwrap();
-        assert_eq!(host, Host::<String>::Ipv6(std::net::Ipv6Addr::LOCALHOST));
-        assert_eq!(port, Some(123));
-    }
-
-    #[test]
-    fn test_invalid_host() {
-        assert!(parse_host_port("hello world").is_err());
-    }
-
-    #[test]
-    fn test_invalid_port() {
-        assert!(parse_host_port("hello:+80").is_err());
-    }
-}
-
-#[derive(Clone)]
-pub struct PgConnectionConfig {
-    host: Host,
-    port: u16,
-    password: Option<String>,
-    options: Vec<String>,
-}
-
-/// A simplified PostgreSQL connection configuration. Supports only a subset of possible
-/// settings for simplicity. A password getter or `to_connection_string` methods are not
-/// added by design to avoid accidentally leaking password through logging, command line
-/// arguments to a child process, or likewise.
-impl PgConnectionConfig {
-    pub fn new_host_port(host: Host, port: u16) -> Self {
-        PgConnectionConfig {
-            host,
-            port,
-            password: None,
-            options: vec![],
-        }
-    }
-
-    pub fn host(&self) -> &Host {
-        &self.host
-    }
-
-    pub fn port(&self) -> u16 {
-        self.port
-    }
-
-    pub fn set_host(mut self, h: Host) -> Self {
-        self.host = h;
-        self
-    }
-
-    pub fn set_port(mut self, p: u16) -> Self {
-        self.port = p;
-        self
-    }
-
-    pub fn set_password(mut self, s: Option<String>) -> Self {
-        self.password = s;
-        self
-    }
-
-    pub fn extend_options<I: IntoIterator<Item = S>, S: Into<String>>(mut self, i: I) -> Self {
-        self.options.extend(i.into_iter().map(|s| s.into()));
-        self
-    }
-
-    /// Return a `<host>:<port>` string.
-    pub fn raw_address(&self) -> String {
-        format!("{}:{}", self.host(), self.port())
-    }
-
-    /// Build a client library-specific connection configuration.
-    /// Used for testing and when we need to add some obscure configuration
-    /// elements at the last moment.
-    pub fn to_tokio_postgres_config(&self) -> tokio_postgres::Config {
-        // Use `tokio_postgres::Config` instead of `postgres::Config` because
-        // the former supports more options to fiddle with later.
-        let mut config = tokio_postgres::Config::new();
-        config.host(&self.host().to_string()).port(self.port);
-        if let Some(password) = &self.password {
-            config.password(password);
-        }
-        if !self.options.is_empty() {
-            // These options are command-line options and should be escaped before being passed
-            // as an 'options' connection string parameter, see
-            // https://www.postgresql.org/docs/15/libpq-connect.html#LIBPQ-CONNECT-OPTIONS
-            //
-            // They will be space-separated, so each space inside an option should be escaped,
-            // and all backslashes should be escaped before that. Although we don't expect options
-            // with spaces at the moment, they're supported by PostgreSQL. Hence we support them
-            // in this typesafe interface.
-            //
-            // We use `Cow` to avoid allocations in the best case (no escaping). A fully imperative
-            // solution would require 1-2 allocations in the worst case as well, but it's harder to
-            // implement and this function is hardly a bottleneck. The function is only called around
-            // establishing a new connection.
-            #[allow(unstable_name_collisions)]
-            config.options(
-                &self
-                    .options
-                    .iter()
-                    .map(|s| {
-                        if s.contains(['\\', ' ']) {
-                            Cow::Owned(s.replace('\\', "\\\\").replace(' ', "\\ "))
-                        } else {
-                            Cow::Borrowed(s.as_str())
-                        }
-                    })
-                    .intersperse(Cow::Borrowed(" ")) // TODO: use impl from std once it's stabilized
-                    .collect::<String>(),
-            );
-        }
-        config
-    }
-
-    /// Connect using postgres protocol with TLS disabled.
-    pub fn connect_no_tls(&self) -> Result<postgres::Client, postgres::Error> {
-        postgres::Config::from(self.to_tokio_postgres_config()).connect(postgres::NoTls)
-    }
-}
-
-impl fmt::Debug for PgConnectionConfig {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        // We want `password: Some(REDACTED-STRING)`, not `password: Some("REDACTED-STRING")`
-        // so even if the password is `REDACTED-STRING` (quite unlikely) there is no confusion.
-        // Hence `format_args!()`, it returns a "safe" string which is not escaped by `Debug`.
-        f.debug_struct("PgConnectionConfig")
-            .field("host", &self.host)
-            .field("port", &self.port)
-            .field(
-                "password",
-                &self
-                    .password
-                    .as_ref()
-                    .map(|_| format_args!("REDACTED-STRING")),
-            )
-            .finish()
-    }
-}
-
-#[cfg(test)]
-mod tests_pg_connection_config {
-    use crate::PgConnectionConfig;
-    use once_cell::sync::Lazy;
-    use url::Host;
-
-    static STUB_HOST: Lazy<Host> = Lazy::new(|| Host::Domain("stub.host.example".to_owned()));
-
-    #[test]
-    fn test_no_password() {
-        let cfg = PgConnectionConfig::new_host_port(STUB_HOST.clone(), 123);
-        assert_eq!(cfg.host(), &*STUB_HOST);
-        assert_eq!(cfg.port(), 123);
-        assert_eq!(cfg.raw_address(), "stub.host.example:123");
-        assert_eq!(
-            format!("{:?}", cfg),
-            "PgConnectionConfig { host: Domain(\"stub.host.example\"), port: 123, password: None }"
-        );
-    }
-
-    #[test]
-    fn test_ipv6() {
-        // May be a special case because hostname contains a colon.
-        let cfg = PgConnectionConfig::new_host_port(Host::parse("[::1]").unwrap(), 123);
-        assert_eq!(
-            cfg.host(),
-            &Host::<String>::Ipv6(std::net::Ipv6Addr::LOCALHOST)
-        );
-        assert_eq!(cfg.port(), 123);
-        assert_eq!(cfg.raw_address(), "[::1]:123");
-        assert_eq!(
-            format!("{:?}", cfg),
-            "PgConnectionConfig { host: Ipv6(::1), port: 123, password: None }"
-        );
-    }
-
-    #[test]
-    fn test_with_password() {
-        let cfg = PgConnectionConfig::new_host_port(STUB_HOST.clone(), 123)
-            .set_password(Some("password".to_owned()));
-        assert_eq!(cfg.host(), &*STUB_HOST);
-        assert_eq!(cfg.port(), 123);
-        assert_eq!(cfg.raw_address(), "stub.host.example:123");
-        assert_eq!(
-            format!("{:?}", cfg),
-            "PgConnectionConfig { host: Domain(\"stub.host.example\"), port: 123, password: Some(REDACTED-STRING) }"
-        );
-    }
-
-    #[test]
-    fn test_with_options() {
-        let cfg = PgConnectionConfig::new_host_port(STUB_HOST.clone(), 123).extend_options([
-            "hello",
-            "world",
-            "with space",
-            "and \\ backslashes",
-        ]);
-        assert_eq!(cfg.host(), &*STUB_HOST);
-        assert_eq!(cfg.port(), 123);
-        assert_eq!(cfg.raw_address(), "stub.host.example:123");
-        assert_eq!(
-            cfg.to_tokio_postgres_config().get_options(),
-            Some("hello world with\\ space and\\ \\\\\\ backslashes")
-        );
-    }
-}
--- a/libs/postgres_ffi/src/lib.rs
+++ b/libs/postgres_ffi/src/lib.rs
@@ -163,27 +163,6 @@ pub fn page_set_lsn(pg: &mut [u8], lsn: Lsn) {
    pg[4..8].copy_from_slice(&(lsn.0 as u32).to_le_bytes());
 }

-// This is port of function with the same name from freespace.c.
-// The only difference is that it does not have "level" parameter because XLogRecordPageWithFreeSpace
-// always call it with level=FSM_BOTTOM_LEVEL
-pub fn fsm_logical_to_physical(addr: BlockNumber) -> BlockNumber {
-    let mut leafno = addr;
-    const FSM_TREE_DEPTH: u32 = if pg_constants::SLOTS_PER_FSM_PAGE >= 1626 {
-        3
-    } else {
-        4
-    };
-
-    /* Count upper level nodes required to address the leaf page */
-    let mut pages: BlockNumber = 0;
-    for _l in 0..FSM_TREE_DEPTH {
-        pages += leafno + 1;
-        leafno /= pg_constants::SLOTS_PER_FSM_PAGE;
-    }
-    /* Turn the page count into 0-based block number */
-    pages - 1
-}
-
 pub mod waldecoder {

    use crate::{v14, v15};
--- a/libs/postgres_ffi/src/pg_constants.rs
+++ b/libs/postgres_ffi/src/pg_constants.rs
@@ -197,16 +197,6 @@ pub const XLOG_CHECKPOINT_SHUTDOWN: u8 = 0x00;
 pub const XLOG_CHECKPOINT_ONLINE: u8 = 0x10;
 pub const XLP_LONG_HEADER: u16 = 0x0002;

-/* From fsm_internals.h */
-const FSM_NODES_PER_PAGE: usize = BLCKSZ as usize - SIZEOF_PAGE_HEADER_DATA - 4;
-const FSM_NON_LEAF_NODES_PER_PAGE: usize = BLCKSZ as usize / 2 - 1;
-const FSM_LEAF_NODES_PER_PAGE: usize = FSM_NODES_PER_PAGE - FSM_NON_LEAF_NODES_PER_PAGE;
-pub const SLOTS_PER_FSM_PAGE: u32 = FSM_LEAF_NODES_PER_PAGE as u32;
-
-/* From visibilitymap.c */
-pub const VM_HEAPBLOCKS_PER_PAGE: u32 =
-    (BLCKSZ as usize - SIZEOF_PAGE_HEADER_DATA) as u32 * (8 / 2); // MAPSIZE * (BITS_PER_BYTE / BITS_PER_HEAPBLOCK)
-
 // List of subdirectories inside pgdata.
 // Copied from src/bin/initdb/initdb.c
 pub const PGDATA_SUBDIRS: [&str; 22] = [
--- a/libs/pq_proto/Cargo.toml
+++ b/libs/pq_proto/Cargo.toml
@@ -1,16 +0,0 @@
-[package]
-name = "pq_proto"
-version = "0.1.0"
-edition = "2021"
-
-[dependencies]
-anyhow = "1.0"
-bytes = "1.0.1"
-pin-project-lite = "0.2.7"
-postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-rand = "0.8.3"
-serde = { version = "1.0", features = ["derive"] }
-tokio = { version = "1.17", features = ["macros"] }
-tracing = "0.1"
-
-workspace_hack = { version = "0.1", path = "../../workspace_hack" }
--- a/libs/tenant_size_model/.gitignore
+++ b/libs/tenant_size_model/.gitignore
@@ -1,3 +0,0 @@
-*.dot
-*.png
-*.svg
--- a/libs/tenant_size_model/Cargo.toml
+++ b/libs/tenant_size_model/Cargo.toml
@@ -1,8 +0,0 @@
-[package]
-name = "tenant_size_model"
-version = "0.1.0"
-edition = "2021"
-publish = false
-
-[dependencies]
-workspace_hack = { version = "0.1", path = "../../workspace_hack" }
--- a/libs/tenant_size_model/Makefile
+++ b/libs/tenant_size_model/Makefile
@@ -1,13 +0,0 @@
-all: 1.svg 2.svg 3.svg 4.svg 1.png 2.png 3.png 4.png
-
-../../target/debug/tenant_size_model: Cargo.toml src/main.rs src/lib.rs
-	cargo build --bin tenant_size_model
-
-%.svg: %.dot
-	dot -Tsvg $< > $@
-
-%.png: %.dot
-	dot -Tpng $< > $@
-
-%.dot: ../../target/debug/tenant_size_model
-	../../target/debug/tenant_size_model $* > $@
--- a/libs/tenant_size_model/README.md
+++ b/libs/tenant_size_model/README.md
@@ -1,7 +0,0 @@
-# Logical size + WAL pricing
-
-This is a simulator to calculate the tenant size in different scenarios,
-using the "Logical size + WAL" method. Makefile produces diagrams used in a
-private presentation:
-
-https://docs.google.com/presentation/d/1OapE4k11xmcwMh7I7YvNWGC63yCRLh6udO9bXZ-fZmo/edit?usp=sharing
--- a/libs/tenant_size_model/src/lib.rs
+++ b/libs/tenant_size_model/src/lib.rs
@@ -1,382 +0,0 @@
-use std::borrow::Cow;
-use std::collections::HashMap;
-
-/// Pricing model or history size builder.
-///
-/// Maintains knowledge of the branches and their modifications. Generic over the branch name key
-/// type.
-pub struct Storage<K: 'static> {
-    segments: Vec<Segment>,
-
-    /// Mapping from the branch name to the index of a segment describing it's latest state.
-    branches: HashMap<K, usize>,
-}
-
-/// Snapshot of a branch.
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub struct Segment {
-    /// Previous segment index into ['Storage::segments`], if any.
-    parent: Option<usize>,
-
-    /// Description of how did we get to this state.
-    ///
-    /// Mainly used in the original scenarios 1..=4 with insert, delete and update. Not used when
-    /// modifying a branch directly.
-    pub op: Cow<'static, str>,
-
-    /// LSN before this state
-    start_lsn: u64,
-
-    /// LSN at this state
-    pub end_lsn: u64,
-
-    /// Logical size before this state
-    start_size: u64,
-
-    /// Logical size at this state. Can be None in the last Segment of a branch.
-    pub end_size: Option<u64>,
-
-    /// Indices to [`Storage::segments`]
-    ///
-    /// FIXME: this could be an Option<usize>
-    children_after: Vec<usize>,
-
-    /// Determined by `retention_period` given to [`Storage::calculate`]
-    pub needed: bool,
-}
-
-//
-//
-//
-//
-//                 *-g--*---D--->
-//                /
-//               /
-//              /                 *---b----*-B--->
-//             /                 /
-//            /                 /
-//      -----*--e---*-----f----* C
-//           E                  \
-//                               \
-//                                *--a---*---A-->
-//
-// If A and B need to be retained, is it cheaper to store
-// snapshot at C+a+b, or snapshots at A and B ?
-//
-// If D also needs to be retained, which is cheaper:
-//
-// 1. E+g+e+f+a+b
-// 2. D+C+a+b
-// 3. D+A+B
-
-/// [`Segment`] which has had it's size calculated.
-pub struct SegmentSize {
-    pub seg_id: usize,
-
-    pub method: SegmentMethod,
-
-    this_size: u64,
-
-    pub children: Vec<SegmentSize>,
-}
-
-impl SegmentSize {
-    fn total(&self) -> u64 {
-        self.this_size + self.children.iter().fold(0, |acc, x| acc + x.total())
-    }
-
-    pub fn total_children(&self) -> u64 {
-        if self.method == SnapshotAfter {
-            self.this_size + self.children.iter().fold(0, |acc, x| acc + x.total())
-        } else {
-            self.children.iter().fold(0, |acc, x| acc + x.total())
-        }
-    }
-}
-
-/// Different methods to retain history from a particular state
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub enum SegmentMethod {
-    SnapshotAfter,
-    Wal,
-    WalNeeded,
-    Skipped,
-}
-
-use SegmentMethod::*;
-
-impl<K: std::hash::Hash + Eq + 'static> Storage<K> {
-    /// Creates a new storage with the given default branch name.
-    pub fn new(initial_branch: K) -> Storage<K> {
-        let init_segment = Segment {
-            op: "".into(),
-            needed: false,
-            parent: None,
-            start_lsn: 0,
-            end_lsn: 0,
-            start_size: 0,
-            end_size: Some(0),
-            children_after: Vec::new(),
-        };
-
-        Storage {
-            segments: vec![init_segment],
-            branches: HashMap::from([(initial_branch, 0)]),
-        }
-    }
-
-    /// Advances the branch with a new point, at given LSN.
-    pub fn insert_point<Q: ?Sized>(
-        &mut self,
-        branch: &Q,
-        op: Cow<'static, str>,
-        lsn: u64,
-        size: Option<u64>,
-    ) where
-        K: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq,
-    {
-        let lastseg_id = *self.branches.get(branch).unwrap();
-        let newseg_id = self.segments.len();
-        let lastseg = &mut self.segments[lastseg_id];
-
-        assert!(lsn > lastseg.end_lsn);
-
-        let newseg = Segment {
-            op,
-            parent: Some(lastseg_id),
-            start_lsn: lastseg.end_lsn,
-            end_lsn: lsn,
-            start_size: lastseg.end_size.unwrap(),
-            end_size: size,
-            children_after: Vec::new(),
-            needed: false,
-        };
-        lastseg.children_after.push(newseg_id);
-
-        self.segments.push(newseg);
-        *self.branches.get_mut(branch).expect("read already") = newseg_id;
-    }
-
-    /// Advances the branch with the named operation, by the relative LSN and logical size bytes.
-    pub fn modify_branch<Q: ?Sized>(
-        &mut self,
-        branch: &Q,
-        op: Cow<'static, str>,
-        lsn_bytes: u64,
-        size_bytes: i64,
-    ) where
-        K: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq,
-    {
-        let lastseg_id = *self.branches.get(branch).unwrap();
-        let newseg_id = self.segments.len();
-        let lastseg = &mut self.segments[lastseg_id];
-
-        let newseg = Segment {
-            op,
-            parent: Some(lastseg_id),
-            start_lsn: lastseg.end_lsn,
-            end_lsn: lastseg.end_lsn + lsn_bytes,
-            start_size: lastseg.end_size.unwrap(),
-            end_size: Some((lastseg.end_size.unwrap() as i64 + size_bytes) as u64),
-            children_after: Vec::new(),
-            needed: false,
-        };
-        lastseg.children_after.push(newseg_id);
-
-        self.segments.push(newseg);
-        *self.branches.get_mut(branch).expect("read already") = newseg_id;
-    }
-
-    pub fn insert<Q: ?Sized>(&mut self, branch: &Q, bytes: u64)
-    where
-        K: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq,
-    {
-        self.modify_branch(branch, "insert".into(), bytes, bytes as i64);
-    }
-
-    pub fn update<Q: ?Sized>(&mut self, branch: &Q, bytes: u64)
-    where
-        K: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq,
-    {
-        self.modify_branch(branch, "update".into(), bytes, 0i64);
-    }
-
-    pub fn delete<Q: ?Sized>(&mut self, branch: &Q, bytes: u64)
-    where
-        K: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq,
-    {
-        self.modify_branch(branch, "delete".into(), bytes, -(bytes as i64));
-    }
-
-    /// Panics if the parent branch cannot be found.
-    pub fn branch<Q: ?Sized>(&mut self, parent: &Q, name: K)
-    where
-        K: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq,
-    {
-        // Find the right segment
-        let branchseg_id = *self
-            .branches
-            .get(parent)
-            .expect("should had found the parent by key");
-        let _branchseg = &mut self.segments[branchseg_id];
-
-        // Create branch name for it
-        self.branches.insert(name, branchseg_id);
-    }
-
-    pub fn calculate(&mut self, retention_period: u64) -> SegmentSize {
-        // Phase 1: Mark all the segments that need to be retained
-        for (_branch, &last_seg_id) in self.branches.iter() {
-            let last_seg = &self.segments[last_seg_id];
-            let cutoff_lsn = last_seg.start_lsn.saturating_sub(retention_period);
-            let mut seg_id = last_seg_id;
-            loop {
-                let seg = &mut self.segments[seg_id];
-                if seg.end_lsn < cutoff_lsn {
-                    break;
-                }
-                seg.needed = true;
-                if let Some(prev_seg_id) = seg.parent {
-                    seg_id = prev_seg_id;
-                } else {
-                    break;
-                }
-            }
-        }
-
-        // Phase 2: For each oldest segment in a chain that needs to be retained,
-        // calculate if we should store snapshot or WAL
-        self.size_from_snapshot_later(0)
-    }
-
-    fn size_from_wal(&self, seg_id: usize) -> SegmentSize {
-        let seg = &self.segments[seg_id];
-
-        let this_size = seg.end_lsn - seg.start_lsn;
-
-        let mut children = Vec::new();
-
-        // try both ways
-        for &child_id in seg.children_after.iter() {
-            // try each child both ways
-            let child = &self.segments[child_id];
-            let p1 = self.size_from_wal(child_id);
-
-            let p = if !child.needed {
-                let p2 = self.size_from_snapshot_later(child_id);
-                if p1.total() < p2.total() {
-                    p1
-                } else {
-                    p2
-                }
-            } else {
-                p1
-            };
-            children.push(p);
-        }
-        SegmentSize {
-            seg_id,
-            method: if seg.needed { WalNeeded } else { Wal },
-            this_size,
-            children,
-        }
-    }
-
-    fn size_from_snapshot_later(&self, seg_id: usize) -> SegmentSize {
-        // If this is needed, then it's time to do the snapshot and continue
-        // with wal method.
-        let seg = &self.segments[seg_id];
-        //eprintln!("snap: seg{}: {} needed: {}", seg_id, seg.children_after.len(), seg.needed);
-        if seg.needed {
-            let mut children = Vec::new();
-
-            for &child_id in seg.children_after.iter() {
-                // try each child both ways
-                let child = &self.segments[child_id];
-                let p1 = self.size_from_wal(child_id);
-
-                let p = if !child.needed {
-                    let p2 = self.size_from_snapshot_later(child_id);
-                    if p1.total() < p2.total() {
-                        p1
-                    } else {
-                        p2
-                    }
-                } else {
-                    p1
-                };
-                children.push(p);
-            }
-            SegmentSize {
-                seg_id,
-                method: WalNeeded,
-                this_size: seg.start_size,
-                children,
-            }
-        } else {
-            // If any of the direct children are "needed", need to be able to reconstruct here
-            let mut children_needed = false;
-            for &child in seg.children_after.iter() {
-                let seg = &self.segments[child];
-                if seg.needed {
-                    children_needed = true;
-                    break;
-                }
-            }
-
-            let method1 = if !children_needed {
-                let mut children = Vec::new();
-                for child in seg.children_after.iter() {
-                    children.push(self.size_from_snapshot_later(*child));
-                }
-                Some(SegmentSize {
-                    seg_id,
-                    method: Skipped,
-                    this_size: 0,
-                    children,
-                })
-            } else {
-                None
-            };
-
-            // If this a junction, consider snapshotting here
-            let method2 = if children_needed || seg.children_after.len() >= 2 {
-                let mut children = Vec::new();
-                for child in seg.children_after.iter() {
-                    children.push(self.size_from_wal(*child));
-                }
-                Some(SegmentSize {
-                    seg_id,
-                    method: SnapshotAfter,
-                    this_size: seg.end_size.unwrap(),
-                    children,
-                })
-            } else {
-                None
-            };
-
-            match (method1, method2) {
-                (None, None) => panic!(),
-                (Some(method), None) => method,
-                (None, Some(method)) => method,
-                (Some(method1), Some(method2)) => {
-                    if method1.total() < method2.total() {
-                        method1
-                    } else {
-                        method2
-                    }
-                }
-            }
-        }
-    }
-
-    pub fn into_segments(self) -> Vec<Segment> {
-        self.segments
-    }
-}
--- a/libs/tenant_size_model/src/main.rs
+++ b/libs/tenant_size_model/src/main.rs
@@ -1,268 +0,0 @@
-//! Tenant size model testing ground.
-//!
-//! Has a number of scenarios and a `main` for invoking these by number, calculating the history
-//! size, outputs graphviz graph. Makefile in directory shows how to use graphviz to turn scenarios
-//! into pngs.
-
-use tenant_size_model::{Segment, SegmentSize, Storage};
-
-// Main branch only. Some updates on it.
-fn scenario_1() -> (Vec<Segment>, SegmentSize) {
-    // Create main branch
-    let mut storage = Storage::new("main");
-
-    // Bulk load 5 GB of data to it
-    storage.insert("main", 5_000);
-
-    // Stream of updates
-    for _ in 0..5 {
-        storage.update("main", 1_000);
-    }
-
-    let size = storage.calculate(1000);
-
-    (storage.into_segments(), size)
-}
-
-// Main branch only. Some updates on it.
-fn scenario_2() -> (Vec<Segment>, SegmentSize) {
-    // Create main branch
-    let mut storage = Storage::new("main");
-
-    // Bulk load 5 GB of data to it
-    storage.insert("main", 5_000);
-
-    // Stream of updates
-    for _ in 0..5 {
-        storage.update("main", 1_000);
-    }
-
-    // Branch
-    storage.branch("main", "child");
-    storage.update("child", 1_000);
-
-    // More updates on parent
-    storage.update("main", 1_000);
-
-    let size = storage.calculate(1000);
-
-    (storage.into_segments(), size)
-}
-
-// Like 2, but more updates on main
-fn scenario_3() -> (Vec<Segment>, SegmentSize) {
-    // Create main branch
-    let mut storage = Storage::new("main");
-
-    // Bulk load 5 GB of data to it
-    storage.insert("main", 5_000);
-
-    // Stream of updates
-    for _ in 0..5 {
-        storage.update("main", 1_000);
-    }
-
-    // Branch
-    storage.branch("main", "child");
-    storage.update("child", 1_000);
-
-    // More updates on parent
-    for _ in 0..5 {
-        storage.update("main", 1_000);
-    }
-
-    let size = storage.calculate(1000);
-
-    (storage.into_segments(), size)
-}
-
-// Diverged branches
-fn scenario_4() -> (Vec<Segment>, SegmentSize) {
-    // Create main branch
-    let mut storage = Storage::new("main");
-
-    // Bulk load 5 GB of data to it
-    storage.insert("main", 5_000);
-
-    // Stream of updates
-    for _ in 0..5 {
-        storage.update("main", 1_000);
-    }
-
-    // Branch
-    storage.branch("main", "child");
-    storage.update("child", 1_000);
-
-    // More updates on parent
-    for _ in 0..8 {
-        storage.update("main", 1_000);
-    }
-
-    let size = storage.calculate(1000);
-
-    (storage.into_segments(), size)
-}
-
-fn scenario_5() -> (Vec<Segment>, SegmentSize) {
-    let mut storage = Storage::new("a");
-    storage.insert("a", 5000);
-    storage.branch("a", "b");
-    storage.update("b", 4000);
-    storage.update("a", 2000);
-    storage.branch("a", "c");
-    storage.insert("c", 4000);
-    storage.insert("a", 2000);
-
-    let size = storage.calculate(5000);
-
-    (storage.into_segments(), size)
-}
-
-fn scenario_6() -> (Vec<Segment>, SegmentSize) {
-    use std::borrow::Cow;
-
-    const NO_OP: Cow<'static, str> = Cow::Borrowed("");
-
-    let branches = [
-        Some(0x7ff1edab8182025f15ae33482edb590a_u128),
-        Some(0xb1719e044db05401a05a2ed588a3ad3f),
-        Some(0xb68d6691c895ad0a70809470020929ef),
-    ];
-
-    // compared to other scenarios, this one uses bytes instead of kB
-
-    let mut storage = Storage::new(None);
-
-    storage.branch(&None, branches[0]); // at 0
-    storage.modify_branch(&branches[0], NO_OP, 108951064, 43696128); // at 108951064
-    storage.branch(&branches[0], branches[1]); // at 108951064
-    storage.modify_branch(&branches[1], NO_OP, 15560408, -1851392); // at 124511472
-    storage.modify_branch(&branches[0], NO_OP, 174464360, -1531904); // at 283415424
-    storage.branch(&branches[0], branches[2]); // at 283415424
-    storage.modify_branch(&branches[2], NO_OP, 15906192, 8192); // at 299321616
-    storage.modify_branch(&branches[0], NO_OP, 18909976, 32768); // at 302325400
-
-    let size = storage.calculate(100_000);
-
-    (storage.into_segments(), size)
-}
-
-fn main() {
-    let args: Vec<String> = std::env::args().collect();
-
-    let scenario = if args.len() < 2 { "1" } else { &args[1] };
-
-    let (segments, size) = match scenario {
-        "1" => scenario_1(),
-        "2" => scenario_2(),
-        "3" => scenario_3(),
-        "4" => scenario_4(),
-        "5" => scenario_5(),
-        "6" => scenario_6(),
-        other => {
-            eprintln!("invalid scenario {}", other);
-            std::process::exit(1);
-        }
-    };
-
-    graphviz_tree(&segments, &size);
-}
-
-fn graphviz_recurse(segments: &[Segment], node: &SegmentSize) {
-    use tenant_size_model::SegmentMethod::*;
-
-    let seg_id = node.seg_id;
-    let seg = segments.get(seg_id).unwrap();
-    let lsn = seg.end_lsn;
-    let size = seg.end_size.unwrap_or(0);
-    let method = node.method;
-
-    println!("  {{");
-    println!("    node [width=0.1 height=0.1 shape=oval]");
-
-    let tenant_size = node.total_children();
-
-    let penwidth = if seg.needed { 6 } else { 3 };
-    let x = match method {
-        SnapshotAfter =>
-            format!("label=\"lsn: {lsn}\\nsize: {size}\\ntenant_size: {tenant_size}\" style=filled penwidth={penwidth}"),
-        Wal =>
-            format!("label=\"lsn: {lsn}\\nsize: {size}\\ntenant_size: {tenant_size}\" color=\"black\" penwidth={penwidth}"),
-        WalNeeded =>
-            format!("label=\"lsn: {lsn}\\nsize: {size}\\ntenant_size: {tenant_size}\" color=\"black\" penwidth={penwidth}"),
-        Skipped =>
-            format!("label=\"lsn: {lsn}\\nsize: {size}\\ntenant_size: {tenant_size}\" color=\"gray\" penwidth={penwidth}"),
-    };
-
-    println!("    \"seg{seg_id}\" [{x}]");
-    println!("  }}");
-
-    // Recurse. Much of the data is actually on the edge
-    for child in node.children.iter() {
-        let child_id = child.seg_id;
-        graphviz_recurse(segments, child);
-
-        let edge_color = match child.method {
-            SnapshotAfter => "gray",
-            Wal => "black",
-            WalNeeded => "black",
-            Skipped => "gray",
-        };
-
-        println!("  {{");
-        println!("    edge [] ");
-        print!("    \"seg{seg_id}\" -> \"seg{child_id}\" [");
-        print!("color={edge_color}");
-        if child.method == WalNeeded {
-            print!(" penwidth=6");
-        }
-        if child.method == Wal {
-            print!(" penwidth=3");
-        }
-
-        let next = segments.get(child_id).unwrap();
-
-        if next.op.is_empty() {
-            print!(
-                " label=\"{} / {}\"",
-                next.end_lsn - seg.end_lsn,
-                (next.end_size.unwrap_or(0) as i128 - seg.end_size.unwrap_or(0) as i128)
-            );
-        } else {
-            print!(" label=\"{}: {}\"", next.op, next.end_lsn - seg.end_lsn);
-        }
-        println!("]");
-        println!("  }}");
-    }
-}
-
-fn graphviz_tree(segments: &[Segment], tree: &SegmentSize) {
-    println!("digraph G {{");
-    println!("  fontname=\"Helvetica,Arial,sans-serif\"");
-    println!("  node [fontname=\"Helvetica,Arial,sans-serif\"]");
-    println!("  edge [fontname=\"Helvetica,Arial,sans-serif\"]");
-    println!("  graph [center=1 rankdir=LR]");
-    println!("  edge [dir=none]");
-
-    graphviz_recurse(segments, tree);
-
-    println!("}}");
-}
-
-#[test]
-fn scenarios_return_same_size() {
-    type ScenarioFn = fn() -> (Vec<Segment>, SegmentSize);
-    let truths: &[(u32, ScenarioFn, _)] = &[
-        (line!(), scenario_1, 8000),
-        (line!(), scenario_2, 9000),
-        (line!(), scenario_3, 13000),
-        (line!(), scenario_4, 16000),
-        (line!(), scenario_5, 17000),
-        (line!(), scenario_6, 333_792_000),
-    ];
-
-    for (line, scenario, expected) in truths {
-        let (_, size) = scenario();
-        assert_eq!(*expected, size.total_children(), "scenario on line {line}");
-    }
-}
--- a/libs/utils/Cargo.toml
+++ b/libs/utils/Cargo.toml
@@ -9,6 +9,9 @@ anyhow = "1.0"
 bincode = "1.3"
 bytes = "1.0.1"
 hyper = { version = "0.14.7", features = ["full"] }
+pin-project-lite = "0.2.7"
+postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
 routerify = "3"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
@@ -16,7 +19,7 @@ thiserror = "1.0"
 tokio = { version = "1.17", features = ["macros"]}
 tokio-rustls = "0.23"
 tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 nix = "0.25"
 signal-hook = "0.3.10"
 rand = "0.8.3"
@@ -27,11 +30,9 @@ rustls-split = "0.3.0"
 git-version = "0.3.5"
 serde_with = "2.0"
 once_cell = "1.13.0"
-strum = "0.24"
-strum_macros = "0.24"
+

 metrics = { path = "../metrics" }
-pq_proto = { path = "../pq_proto" }
 workspace_hack = { version = "0.1", path = "../../workspace_hack" }

 [dev-dependencies]
--- a/libs/utils/src/auth.rs
+++ b/libs/utils/src/auth.rs
@@ -7,7 +7,7 @@ use serde;
 use std::fs;
 use std::path::Path;

-use anyhow::Result;
+use anyhow::{bail, Result};
 use jsonwebtoken::{
    decode, encode, Algorithm, DecodingKey, EncodingKey, Header, TokenData, Validation,
 };
@@ -21,16 +21,8 @@ const JWT_ALGORITHM: Algorithm = Algorithm::RS256;
 #[derive(Debug, Serialize, Deserialize, Clone)]
 #[serde(rename_all = "lowercase")]
 pub enum Scope {
-    // Provides access to all data for a specific tenant (specified in `struct Claims` below)
-    // TODO: join these two?
    Tenant,
-    // Provides blanket access to all tenants on the pageserver plus pageserver-wide APIs.
-    // Should only be used e.g. for status check/tenant creation/list.
    PageServerApi,
-    // Provides blanket access to all data on the safekeeper plus safekeeper-wide APIs.
-    // Should only be used e.g. for status check.
-    // Currently also used for connection from any pageserver to any safekeeper.
-    SafekeeperData,
 }

 #[serde_as]
@@ -48,6 +40,22 @@ impl Claims {
    }
 }

+pub fn check_permission(claims: &Claims, tenant_id: Option<TenantId>) -> Result<()> {
+    match (&claims.scope, tenant_id) {
+        (Scope::Tenant, None) => {
+            bail!("Attempt to access management api with tenant scope. Permission denied")
+        }
+        (Scope::Tenant, Some(tenant_id)) => {
+            if claims.tenant_id.unwrap() != tenant_id {
+                bail!("Tenant id mismatch. Permission denied")
+            }
+            Ok(())
+        }
+        (Scope::PageServerApi, None) => Ok(()), // access to management api for PageServerApi scope
+        (Scope::PageServerApi, Some(_)) => Ok(()), // access to tenant api using PageServerApi scope
+    }
+}
+
 pub struct JwtAuth {
    decoding_key: DecodingKey,
    validation: Validation,
--- a/libs/utils/src/connstring.rs
+++ b/libs/utils/src/connstring.rs
@@ -0,0 +1,52 @@
+use postgres::Config;
+
+pub fn connection_host_port(config: &Config) -> (String, u16) {
+    assert_eq!(
+        config.get_hosts().len(),
+        1,
+        "only one pair of host and port is supported in connection string"
+    );
+    assert_eq!(
+        config.get_ports().len(),
+        1,
+        "only one pair of host and port is supported in connection string"
+    );
+    let host = match &config.get_hosts()[0] {
+        postgres::config::Host::Tcp(host) => host.as_ref(),
+        postgres::config::Host::Unix(host) => host.to_str().unwrap(),
+    };
+    (host.to_owned(), config.get_ports()[0])
+}
+
+pub fn connection_address(config: &Config) -> String {
+    let (host, port) = connection_host_port(config);
+    format!("{}:{}", host, port)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_connection_host_port() {
+        let config: Config = "postgresql://no_user@localhost:64000/no_db"
+            .parse()
+            .unwrap();
+        assert_eq!(
+            connection_host_port(&config),
+            ("localhost".to_owned(), 64000)
+        );
+    }
+
+    #[test]
+    #[should_panic(expected = "only one pair of host and port is supported in connection string")]
+    fn test_connection_host_port_multiple_ports() {
+        let config: Config = "postgresql://no_user@localhost:64000,localhost:64001/no_db"
+            .parse()
+            .unwrap();
+        assert_eq!(
+            connection_host_port(&config),
+            ("localhost".to_owned(), 64000)
+        );
+    }
+}
--- a/libs/utils/src/fs_ext.rs
+++ b/libs/utils/src/fs_ext.rs
@@ -1,45 +0,0 @@
-/// Extensions to `std::fs` types.
-use std::{fs, io, path::Path};
-
-pub trait PathExt {
-    /// Returns an error if `self` is not a directory.
-    fn is_empty_dir(&self) -> io::Result<bool>;
-}
-
-impl<P> PathExt for P
-where
-    P: AsRef<Path>,
-{
-    fn is_empty_dir(&self) -> io::Result<bool> {
-        Ok(fs::read_dir(self)?.into_iter().next().is_none())
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use std::path::PathBuf;
-
-    #[test]
-    fn is_empty_dir() {
-        use super::PathExt;
-
-        let dir = tempfile::tempdir().unwrap();
-        let dir_path = dir.path();
-
-        // test positive case
-        assert!(
-            dir_path.is_empty_dir().expect("test failure"),
-            "new tempdir should be empty"
-        );
-
-        // invoke on a file to ensure it returns an error
-        let file_path: PathBuf = dir_path.join("testfile");
-        let f = std::fs::File::create(&file_path).unwrap();
-        drop(f);
-        assert!(file_path.is_empty_dir().is_err());
-
-        // do it again on a path, we know to be nonexistent
-        std::fs::remove_file(&file_path).unwrap();
-        assert!(file_path.is_empty_dir().is_err());
-    }
-}
--- a/libs/utils/src/http/endpoint.rs
+++ b/libs/utils/src/http/endpoint.rs
@@ -1,5 +1,6 @@
-use crate::auth::{Claims, JwtAuth};
+use crate::auth::{self, Claims, JwtAuth};
 use crate::http::error;
+use crate::id::TenantId;
 use anyhow::anyhow;
 use hyper::header::AUTHORIZATION;
 use hyper::{header::CONTENT_TYPE, Body, Request, Response, Server};
@@ -143,14 +144,10 @@ pub fn auth_middleware<B: hyper::body::HttpBody + Send + Sync + 'static>(
    })
 }

-pub fn check_permission_with(
-    req: &Request<Body>,
-    check_permission: impl Fn(&Claims) -> Result<(), anyhow::Error>,
-) -> Result<(), ApiError> {
+pub fn check_permission(req: &Request<Body>, tenant_id: Option<TenantId>) -> Result<(), ApiError> {
    match req.context::<Claims>() {
-        Some(claims) => {
-            Ok(check_permission(&claims).map_err(|err| ApiError::Forbidden(err.to_string()))?)
-        }
+        Some(claims) => Ok(auth::check_permission(&claims, tenant_id)
+            .map_err(|err| ApiError::Forbidden(err.to_string()))?),
        None => Ok(()), // claims is None because auth is disabled
    }
 }
--- a/libs/utils/src/id.rs
+++ b/libs/utils/src/id.rs
@@ -3,13 +3,6 @@ use std::{fmt, str::FromStr};
 use hex::FromHex;
 use rand::Rng;
 use serde::{Deserialize, Serialize};
-use thiserror::Error;
-
-#[derive(Error, Debug)]
-pub enum IdError {
-    #[error("invalid id length {0}")]
-    SliceParseError(usize),
-}

 /// Neon ID is a 128-bit random ID.
 /// Used to represent various identifiers. Provides handy utility methods and impls.
@@ -29,15 +22,6 @@ impl Id {
        Id::from(arr)
    }

-    pub fn from_slice(src: &[u8]) -> Result<Id, IdError> {
-        if src.len() != 16 {
-            return Err(IdError::SliceParseError(src.len()));
-        }
-        let mut id_array = [0u8; 16];
-        id_array.copy_from_slice(src);
-        Ok(id_array.into())
-    }
-
    pub fn as_arr(&self) -> [u8; 16] {
        self.0
    }
@@ -91,12 +75,6 @@ impl From<[u8; 16]> for Id {
    }
 }

-impl From<Id> for u128 {
-    fn from(id: Id) -> Self {
-        u128::from_le_bytes(id.0)
-    }
-}
-
 impl fmt::Display for Id {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str(&self.hex_encode())
@@ -116,10 +94,6 @@ macro_rules! id_newtype {
                $t(Id::get_from_buf(buf))
            }

-            pub fn from_slice(src: &[u8]) -> Result<$t, IdError> {
-                Ok($t(Id::from_slice(src)?))
-            }
-
            pub fn as_arr(&self) -> [u8; 16] {
                self.0.as_arr()
            }
@@ -162,12 +136,6 @@ macro_rules! id_newtype {
            }
        }

-        impl From<$t> for u128 {
-            fn from(id: $t) -> Self {
-                u128::from(id.0)
-            }
-        }
-
        impl fmt::Display for $t {
            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
                self.0.fmt(f)
@@ -224,17 +192,6 @@ pub struct TenantId(Id);

 id_newtype!(TenantId);

-/// Neon Connection Id identifies long-lived connections (for example a pagestream
-/// connection with the page_service). Is used for better logging and tracing
-///
-/// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look
-/// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`.
-/// See [`Id`] for alternative ways to serialize it.
-#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)]
-pub struct ConnectionId(Id);
-
-id_newtype!(ConnectionId);
-
 // A pair uniquely identifying Neon instance.
 #[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct TenantTimelineId {
--- a/libs/utils/src/lib.rs
+++ b/libs/utils/src/lib.rs
@@ -1,6 +1,8 @@
 //! `utils` is intended to be a place to put code that is shared
 //! between other crates in this repository.

+#![allow(clippy::manual_range_contains)]
+
 /// `Lsn` type implements common tasks on Log Sequence Numbers
 pub mod lsn;
 /// SeqWait allows waiting for a future sequence number to arrive
@@ -15,6 +17,10 @@ pub mod vec_map;
 pub mod bin_ser;
 pub mod postgres_backend;
 pub mod postgres_backend_async;
+pub mod pq_proto;
+
+// dealing with connstring parsing and handy access to it's parts
+pub mod connstring;

 // helper functions for creating and fsyncing
 pub mod crashsafe;
@@ -33,12 +39,13 @@ pub mod sock_split;
 // common log initialisation routine
 pub mod logging;

-pub mod lock_file;
-
 // Misc
 pub mod accum;
 pub mod shutdown;

+// Tools for calling certain async methods in sync contexts
+pub mod sync;
+
 // Utility for binding TcpListeners with proper socket options.
 pub mod tcp_listener;

@@ -48,27 +55,6 @@ pub mod nonblock;
 // Default signal handling
 pub mod signals;

-pub mod fs_ext;
-
-/// use with fail::cfg("$name", "return(2000)")
-#[macro_export]
-macro_rules! failpoint_sleep_millis_async {
-    ($name:literal) => {{
-        let should_sleep: Option<std::time::Duration> = (|| {
-            fail::fail_point!($name, |v: Option<_>| {
-                let millis = v.unwrap().parse::<u64>().unwrap();
-                Some(Duration::from_millis(millis))
-            });
-            None
-        })();
-        if let Some(d) = should_sleep {
-            tracing::info!("failpoint {:?}: sleeping for {:?}", $name, d);
-            tokio::time::sleep(d).await;
-            tracing::info!("failpoint {:?}: sleep done", $name);
-        }
-    }};
-}
-
 /// This is a shortcut to embed git sha into binaries and avoid copying the same build script to all packages
 ///
 /// we have several cases:
--- a/libs/utils/src/lock_file.rs
+++ b/libs/utils/src/lock_file.rs
@@ -1,81 +0,0 @@
-//! A module to create and read lock files. A lock file ensures that only one
-//! process is running at a time, in a particular directory.
-//!
-//! File locking is done using [`fcntl::flock`], which means that holding the
-//! lock on file only prevents acquiring another lock on it; all other
-//! operations are still possible on files. Other process can still open, read,
-//! write, or remove the file, for example.
-//! If the file is removed while a process is holding a lock on it,
-//! the process that holds the lock does not get any error or notification.
-//! Furthermore, you can create a new file with the same name and lock the new file,
-//! while the old process is still running.
-//! Deleting the lock file while the locking process is still running is a bad idea!
-
-use std::{fs, os::unix::prelude::AsRawFd, path::Path};
-
-use anyhow::Context;
-use nix::fcntl;
-
-use crate::crashsafe;
-
-pub enum LockCreationResult {
-    Created {
-        new_lock_contents: String,
-        file: fs::File,
-    },
-    AlreadyLocked {
-        existing_lock_contents: String,
-    },
-    CreationFailed(anyhow::Error),
-}
-
-/// Creates a lock file in the path given and writes the given contents into the file.
-/// Note: The lock is automatically released when the file closed. You might want to use Box::leak to make sure it lives until the end of the program.
-pub fn create_lock_file(lock_file_path: &Path, contents: String) -> LockCreationResult {
-    let lock_file = match fs::OpenOptions::new()
-        .create(true) // O_CREAT
-        .write(true)
-        .open(lock_file_path)
-        .context("Failed to open lock file")
-    {
-        Ok(file) => file,
-        Err(e) => return LockCreationResult::CreationFailed(e),
-    };
-
-    match fcntl::flock(
-        lock_file.as_raw_fd(),
-        fcntl::FlockArg::LockExclusiveNonblock,
-    ) {
-        Ok(()) => {
-            match lock_file
-                .set_len(0)
-                .context("Failed to truncate lockfile")
-                .and_then(|()| {
-                    fs::write(lock_file_path, &contents).with_context(|| {
-                        format!("Failed to write '{contents}' contents into lockfile")
-                    })
-                })
-                .and_then(|()| {
-                    crashsafe::fsync_file_and_parent(lock_file_path)
-                        .context("Failed to fsync lockfile")
-                }) {
-                Ok(()) => LockCreationResult::Created {
-                    new_lock_contents: contents,
-                    file: lock_file,
-                },
-                Err(e) => LockCreationResult::CreationFailed(e),
-            }
-        }
-        Err(nix::errno::Errno::EAGAIN) => {
-            match fs::read_to_string(lock_file_path).context("Failed to read lockfile contents") {
-                Ok(existing_lock_contents) => LockCreationResult::AlreadyLocked {
-                    existing_lock_contents,
-                },
-                Err(e) => LockCreationResult::CreationFailed(e),
-            }
-        }
-        Err(e) => {
-            LockCreationResult::CreationFailed(anyhow::anyhow!("Failed to lock lockfile: {e}"))
-        }
-    }
-}
--- a/libs/utils/src/logging.rs
+++ b/libs/utils/src/logging.rs
@@ -1,28 +1,19 @@
-use std::str::FromStr;
+use std::{
+    fs::{File, OpenOptions},
+    path::Path,
+};

-use anyhow::Context;
-use strum_macros::{EnumString, EnumVariantNames};
+use anyhow::{Context, Result};

-#[derive(EnumString, EnumVariantNames, Eq, PartialEq, Debug, Clone, Copy)]
-#[strum(serialize_all = "snake_case")]
-pub enum LogFormat {
-    Plain,
-    Json,
-}
+pub fn init(log_filename: impl AsRef<Path>, daemonize: bool) -> Result<File> {
+    // Don't open the same file for output multiple times;
+    // the different fds could overwrite each other's output.
+    let log_file = OpenOptions::new()
+        .create(true)
+        .append(true)
+        .open(&log_filename)
+        .with_context(|| format!("failed to open {:?}", log_filename.as_ref()))?;

-impl LogFormat {
-    pub fn from_config(s: &str) -> anyhow::Result<LogFormat> {
-        use strum::VariantNames;
-        LogFormat::from_str(s).with_context(|| {
-            format!(
-                "Unrecognized log format. Please specify one of: {:?}",
-                LogFormat::VARIANTS
-            )
-        })
-    }
-}
-
-pub fn init(log_format: LogFormat) -> anyhow::Result<()> {
    let default_filter_str = "info";

    // We fall back to printing all spans at info-level or above if
@@ -32,14 +23,20 @@ pub fn init(log_format: LogFormat) -> anyhow::Result<()> {

    let base_logger = tracing_subscriber::fmt()
        .with_env_filter(env_filter)
-        .with_target(false)
-        .with_ansi(false)
-        .with_writer(std::io::stdout);
+        .with_target(false) // don't include event targets
+        .with_ansi(false); // don't use colors in log file;

-    match log_format {
-        LogFormat::Json => base_logger.json().init(),
-        LogFormat::Plain => base_logger.init(),
+    // we are cloning and returning log file in order to allow redirecting daemonized stdout and stderr to it
+    // if we do not use daemonization (e.g. in docker) it is better to log to stdout directly
+    // for example to be in line with docker log command which expects logs comimg from stdout
+    if daemonize {
+        let x = log_file.try_clone().unwrap();
+        base_logger
+            .with_writer(move || x.try_clone().unwrap())
+            .init();
+    } else {
+        base_logger.init();
    }

-    Ok(())
+    Ok(log_file)
 }
--- a/libs/utils/src/lsn.rs
+++ b/libs/utils/src/lsn.rs
@@ -13,7 +13,7 @@ use crate::seqwait::MonotonicCounter;
 pub const XLOG_BLCKSZ: u32 = 8192;

 /// A Postgres LSN (Log Sequence Number), also known as an XLogRecPtr
-#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Hash, Serialize, Deserialize)]
+#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
 #[serde(transparent)]
 pub struct Lsn(pub u64);

@@ -138,7 +138,7 @@ impl FromStr for Lsn {
    ///
    /// If the input string is missing the '/' character, then use `Lsn::from_hex`
    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        let mut splitter = s.trim().split('/');
+        let mut splitter = s.split('/');
        if let (Some(left), Some(right), None) = (splitter.next(), splitter.next(), splitter.next())
        {
            let left_num = u32::from_str_radix(left, 16).map_err(|_| LsnParseError)?;
@@ -270,11 +270,6 @@ mod tests {
        );
        assert_eq!(Lsn::from_hex("0"), Ok(Lsn(0)));
        assert_eq!(Lsn::from_hex("F12345678AAAA5555"), Err(LsnParseError));
-
-        let expected_lsn = Lsn(0x3C490F8);
-        assert_eq!(" 0/3C490F8".parse(), Ok(expected_lsn));
-        assert_eq!("0/3C490F8 ".parse(), Ok(expected_lsn));
-        assert_eq!(" 0/3C490F8 ".parse(), Ok(expected_lsn));
    }

    #[test]
--- a/libs/utils/src/postgres_backend.rs
+++ b/libs/utils/src/postgres_backend.rs
@@ -3,10 +3,10 @@
 //! implementation determining how to process the queries. Currently its API
 //! is rather narrow, but we can extend it once required.

+use crate::pq_proto::{BeMessage, BeParameterStatusMessage, FeMessage, FeStartupPacket};
 use crate::sock_split::{BidiStream, ReadStream, WriteStream};
 use anyhow::{bail, ensure, Context, Result};
 use bytes::{Bytes, BytesMut};
-use pq_proto::{BeMessage, BeParameterStatusMessage, FeMessage, FeStartupPacket};
 use rand::Rng;
 use serde::{Deserialize, Serialize};
 use std::fmt;
--- a/libs/utils/src/postgres_backend_async.rs
+++ b/libs/utils/src/postgres_backend_async.rs
@@ -4,9 +4,9 @@
 //! is rather narrow, but we can extend it once required.

 use crate::postgres_backend::AuthType;
+use crate::pq_proto::{BeMessage, BeParameterStatusMessage, FeMessage, FeStartupPacket};
 use anyhow::{bail, Context, Result};
 use bytes::{Bytes, BytesMut};
-use pq_proto::{BeMessage, BeParameterStatusMessage, FeMessage, FeStartupPacket};
 use rand::Rng;
 use std::future::Future;
 use std::net::SocketAddr;
--- a/libs/utils/src/pq_proto.rs
+++ b/libs/utils/src/pq_proto.rs
@@ -2,9 +2,7 @@
 //! <https://www.postgresql.org/docs/devel/protocol-message-formats.html>
 //! on message formats.

-// Tools for calling certain async methods in sync contexts.
-pub mod sync;
-
+use crate::sync::{AsyncishRead, SyncFuture};
 use anyhow::{bail, ensure, Context, Result};
 use bytes::{Buf, BufMut, Bytes, BytesMut};
 use postgres_protocol::PG_EPOCH;
@@ -18,7 +16,6 @@ use std::{
    str,
    time::{Duration, SystemTime},
 };
-use sync::{AsyncishRead, SyncFuture};
 use tokio::io::AsyncReadExt;
 use tracing::{trace, warn};

@@ -201,7 +198,7 @@ impl FeMessage {
    ///
    /// ```
    /// # use std::io;
-    /// # use pq_proto::FeMessage;
+    /// # use utils::pq_proto::FeMessage;
    /// #
    /// # fn process_message(msg: FeMessage) -> anyhow::Result<()> {
    /// #     Ok(())
@@ -305,7 +302,6 @@ impl FeStartupPacket {
                Err(e) => return Err(e.into()),
            };

-            #[allow(clippy::manual_range_contains)]
            if len < 4 || len > MAX_STARTUP_PACKET_LENGTH {
                bail!("invalid message length");
            }
--- a/libs/pq_proto/src/sync.rs
+++ b/libs/pq_proto/src/sync.rs
@@ -29,7 +29,7 @@ impl<S, T: Future> SyncFuture<S, T> {
    /// Example:
    ///
    /// ```
-    /// # use pq_proto::sync::SyncFuture;
+    /// # use utils::sync::SyncFuture;
    /// # use std::future::Future;
    /// # use tokio::io::AsyncReadExt;
    /// #
--- a/pageserver/Cargo.toml
+++ b/pageserver/Cargo.toml
@@ -12,63 +12,61 @@ testing = ["fail/failpoints"]
 profiling = ["pprof"]

 [dependencies]
-amplify_num = { git = "https://github.com/hlinnaka/rust-amplify.git", branch = "unsigned-int-perf" }
-anyhow = { version = "1.0", features = ["backtrace"] }
 async-stream = "0.3"
 async-trait = "0.1"
-byteorder = "1.4.3"
-bytes = "1.0.1"
 chrono = "0.4.19"
-clap = { version = "4.0", features = ["string"] }
-close_fds = "0.3.2"
-const_format = "0.2.21"
-crc32c = "0.6.0"
-crossbeam-utils = "0.8.5"
-fail = "0.5.0"
-futures = "0.3.13"
-git-version = "0.3.5"
-hex = "0.4.3"
-humantime = "2.1.0"
-humantime-serde = "1.1.1"
-hyper = "0.14"
-itertools = "0.10.3"
-nix = "0.25"
-num-traits = "0.2.15"
-once_cell = "1.13.0"
-pin-project-lite = "0.2.7"
-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-pprof = { git = "https://github.com/neondatabase/pprof-rs.git", branch = "wallclock-profiling", features = ["flamegraph"], optional = true }
 rand = "0.8.3"
 regex = "1.4.5"
-rstar = "0.9.3"
-scopeguard = "1.1.0"
+bytes = "1.0.1"
+byteorder = "1.4.3"
+futures = "0.3.13"
+hex = "0.4.3"
+hyper = "0.14"
+itertools = "0.10.3"
+clap = { version = "4.0", features = ["string"] }
+daemonize = "0.4.1"
+tokio = { version = "1.17", features = ["process", "sync", "macros", "fs", "rt", "io-util", "time"] }
+tokio-util = { version = "0.7.3", features = ["io", "io-util"] }
+postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+anyhow = { version = "1.0", features = ["backtrace"] }
+crc32c = "0.6.0"
+thiserror = "1.0"
+tar = "0.4.33"
+humantime = "2.1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
 serde_with = "2.0"
-signal-hook = "0.3.10"
-svg_fmt = "0.4.1"
-tar = "0.4.33"
-thiserror = "1.0"
-tokio = { version = "1.17", features = ["process", "sync", "macros", "fs", "rt", "io-util", "time"] }
-tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-tokio-util = { version = "0.7.3", features = ["io", "io-util"] }
-toml_edit = { version = "0.14", features = ["easy"] }
-tracing = "0.1.36"
-url = "2"
-walkdir = "2.3.2"
+humantime-serde = "1.1.1"

+pprof = { git = "https://github.com/neondatabase/pprof-rs.git", branch = "wallclock-profiling", features = ["flamegraph"], optional = true }
+
+toml_edit = { version = "0.14", features = ["easy"] }
+scopeguard = "1.1.0"
+const_format = "0.2.21"
+tracing = "0.1.36"
+signal-hook = "0.3.10"
+url = "2"
+nix = "0.25"
+once_cell = "1.13.0"
+crossbeam-utils = "0.8.5"
+fail = "0.5.0"
+git-version = "0.3.5"
+rstar = "0.9.3"
+num-traits = "0.2.15"
+amplify_num = { git = "https://github.com/hlinnaka/rust-amplify.git", branch = "unsigned-int-perf" }
+
+pageserver_api = { path = "../libs/pageserver_api" }
+postgres_ffi = { path = "../libs/postgres_ffi" }
 etcd_broker = { path = "../libs/etcd_broker" }
 metrics = { path = "../libs/metrics" }
-pageserver_api = { path = "../libs/pageserver_api" }
-postgres_connection = { path = "../libs/postgres_connection" }
-postgres_ffi = { path = "../libs/postgres_ffi" }
-pq_proto = { path = "../libs/pq_proto" }
-remote_storage = { path = "../libs/remote_storage" }
-tenant_size_model = { path = "../libs/tenant_size_model" }
 utils = { path = "../libs/utils" }
+remote_storage = { path = "../libs/remote_storage" }
 workspace_hack = { version = "0.1", path = "../workspace_hack" }
+close_fds = "0.3.2"
+walkdir = "2.3.2"

 [dev-dependencies]
 criterion = "0.4"
@@ -78,7 +76,3 @@ tempfile = "3.2"
 [[bench]]
 name = "bench_layer_map"
 harness = false
-
-[[bench]]
-name = "bench_walredo"
-harness = false
--- a/pageserver/benches/bench_walredo.rs
+++ b/pageserver/benches/bench_walredo.rs
--- a/pageserver/src/auth.rs
+++ b/pageserver/src/auth.rs
@@ -1,22 +0,0 @@
-use anyhow::{bail, Result};
-use utils::auth::{Claims, Scope};
-use utils::id::TenantId;
-
-pub fn check_permission(claims: &Claims, tenant_id: Option<TenantId>) -> Result<()> {
-    match (&claims.scope, tenant_id) {
-        (Scope::Tenant, None) => {
-            bail!("Attempt to access management api with tenant scope. Permission denied")
-        }
-        (Scope::Tenant, Some(tenant_id)) => {
-            if claims.tenant_id.unwrap() != tenant_id {
-                bail!("Tenant id mismatch. Permission denied")
-            }
-            Ok(())
-        }
-        (Scope::PageServerApi, None) => Ok(()), // access to management api for PageServerApi scope
-        (Scope::PageServerApi, Some(_)) => Ok(()), // access to tenant api using PageServerApi scope
-        (Scope::SafekeeperData, _) => {
-            bail!("SafekeeperData scope makes no sense for Pageserver")
-        }
-    }
-}
--- a/pageserver/src/basebackup.rs
+++ b/pageserver/src/basebackup.rs
@@ -22,8 +22,8 @@ use std::time::SystemTime;
 use tar::{Builder, EntryType, Header};
 use tracing::*;

+use crate::reltag::{RelTag, SlruKind};
 use crate::tenant::Timeline;
-use pageserver_api::reltag::{RelTag, SlruKind};

 use postgres_ffi::pg_constants::{DEFAULTTABLESPACE_OID, GLOBALTABLESPACE_OID};
 use postgres_ffi::pg_constants::{PGDATA_SPECIAL_FILES, PGDATA_SUBDIRS, PG_HBA};
--- a/pageserver/src/bin/draw_timeline_dir.rs
+++ b/pageserver/src/bin/draw_timeline_dir.rs
@@ -1,150 +0,0 @@
-//! A tool for visualizing the arrangement of layerfiles within a timeline.
-//!
-//! It reads filenames from stdin and prints a svg on stdout. The image is a plot in
-//! page-lsn space, where every delta layer is a rectangle and every image layer is a
-//! thick line. Legend:
-//! - The x axis (left to right) represents page index.
-//! - The y axis represents LSN, growing upwards.
-//!
-//! Coordinates in both axis are compressed for better readability.
-//! (see https://medium.com/algorithms-digest/coordinate-compression-2fff95326fb)
-//!
-//! Example use:
-//! ```
-//! $ cd test_output/test_pgbench\[neon-45-684\]/repo/tenants/$TENANT/timelines/$TIMELINE
-//! $ ls | grep "__" | cargo run --release --bin draw_timeline_dir > out.svg
-//! $ firefox out.svg
-//! ```
-//!
-//! This API was chosen so that we can easily work with filenames extracted from ssh,
-//! or from pageserver log files.
-//!
-//! TODO Consider shipping this as a grafana panel plugin:
-//!      https://grafana.com/tutorials/build-a-panel-plugin/
-use anyhow::Result;
-use pageserver::repository::Key;
-use std::cmp::Ordering;
-use std::io::{self, BufRead};
-use std::{
-    collections::{BTreeMap, BTreeSet},
-    ops::Range,
-};
-use svg_fmt::{rectangle, rgb, BeginSvg, EndSvg, Fill, Stroke};
-use utils::{lsn::Lsn, project_git_version};
-
-project_git_version!(GIT_VERSION);
-
-// Map values to their compressed coordinate - the index the value
-// would have in a sorted and deduplicated list of all values.
-fn build_coordinate_compression_map<T: Ord + Copy>(coords: Vec<T>) -> BTreeMap<T, usize> {
-    let set: BTreeSet<T> = coords.into_iter().collect();
-
-    let mut map: BTreeMap<T, usize> = BTreeMap::new();
-    for (i, e) in set.iter().enumerate() {
-        map.insert(*e, i);
-    }
-
-    map
-}
-
-fn parse_filename(name: &str) -> (Range<Key>, Range<Lsn>) {
-    let split: Vec<&str> = name.split("__").collect();
-    let keys: Vec<&str> = split[0].split('-').collect();
-    let mut lsns: Vec<&str> = split[1].split('-').collect();
-    if lsns.len() == 1 {
-        lsns.push(lsns[0]);
-    }
-
-    let keys = Key::from_hex(keys[0]).unwrap()..Key::from_hex(keys[1]).unwrap();
-    let lsns = Lsn::from_hex(lsns[0]).unwrap()..Lsn::from_hex(lsns[1]).unwrap();
-    (keys, lsns)
-}
-
-fn main() -> Result<()> {
-    // Parse layer filenames from stdin
-    let mut ranges: Vec<(Range<Key>, Range<Lsn>)> = vec![];
-    let stdin = io::stdin();
-    for line in stdin.lock().lines() {
-        let range = parse_filename(&line.unwrap());
-        ranges.push(range);
-    }
-
-    // Collect all coordinates
-    let mut keys: Vec<Key> = vec![];
-    let mut lsns: Vec<Lsn> = vec![];
-    for (keyr, lsnr) in &ranges {
-        keys.push(keyr.start);
-        keys.push(keyr.end);
-        lsns.push(lsnr.start);
-        lsns.push(lsnr.end);
-    }
-
-    // Analyze
-    let key_map = build_coordinate_compression_map(keys);
-    let lsn_map = build_coordinate_compression_map(lsns);
-
-    // Initialize stats
-    let mut num_deltas = 0;
-    let mut num_images = 0;
-
-    // Draw
-    let stretch = 3.0; // Stretch out vertically for better visibility
-    println!(
-        "{}",
-        BeginSvg {
-            w: key_map.len() as f32,
-            h: stretch * lsn_map.len() as f32
-        }
-    );
-    for (keyr, lsnr) in &ranges {
-        let key_start = *key_map.get(&keyr.start).unwrap();
-        let key_end = *key_map.get(&keyr.end).unwrap();
-        let key_diff = key_end - key_start;
-        let lsn_max = lsn_map.len();
-
-        if key_start >= key_end {
-            panic!("Invalid key range {}-{}", key_start, key_end);
-        }
-
-        let lsn_start = *lsn_map.get(&lsnr.start).unwrap();
-        let lsn_end = *lsn_map.get(&lsnr.end).unwrap();
-
-        let mut lsn_diff = (lsn_end - lsn_start) as f32;
-        let mut fill = Fill::None;
-        let mut margin = 0.05 * lsn_diff; // Height-dependent margin to disambiguate overlapping deltas
-        let mut lsn_offset = 0.0;
-
-        // Fill in and thicken rectangle if it's an
-        // image layer so that we can see it.
-        match lsn_start.cmp(&lsn_end) {
-            Ordering::Less => num_deltas += 1,
-            Ordering::Equal => {
-                num_images += 1;
-                lsn_diff = 0.3;
-                lsn_offset = -lsn_diff / 2.0;
-                margin = 0.05;
-                fill = Fill::Color(rgb(0, 0, 0));
-            }
-            Ordering::Greater => panic!("Invalid lsn range {}-{}", lsn_start, lsn_end),
-        }
-
-        println!(
-            "    {}",
-            rectangle(
-                key_start as f32 + stretch * margin,
-                stretch * (lsn_max as f32 - (lsn_end as f32 - margin - lsn_offset)),
-                key_diff as f32 - stretch * 2.0 * margin,
-                stretch * (lsn_diff - 2.0 * margin)
-            )
-            .fill(fill)
-            .stroke(Stroke::Color(rgb(0, 0, 0), 0.1))
-            .border_radius(0.4)
-        );
-    }
-    println!("{}", EndSvg);
-
-    eprintln!("num_images: {}", num_images);
-    eprintln!("num_deltas: {}", num_deltas);
-
-    Ok(())
-}
--- a/pageserver/src/bin/pageserver.rs
+++ b/pageserver/src/bin/pageserver.rs
@@ -1,16 +1,17 @@
 //! Main entry point for the Page Server executable.

-use std::env::{var, VarError};
-use std::sync::Arc;
+use remote_storage::GenericRemoteStorage;
 use std::{env, ops::ControlFlow, path::Path, str::FromStr};
-
-use anyhow::{anyhow, Context};
-use clap::{Arg, ArgAction, Command};
-use fail::FailScenario;
-use nix::unistd::Pid;
 use tracing::*;

+use anyhow::{anyhow, bail, Context, Result};
+
+use clap::{Arg, ArgAction, Command};
+use daemonize::Daemonize;
+
+use fail::FailScenario;
 use metrics::set_build_info_metric;
+
 use pageserver::{
    config::{defaults::*, PageServerConf},
    http, page_cache, page_service, profiling, task_mgr,
@@ -18,22 +19,20 @@ use pageserver::{
    task_mgr::{
        BACKGROUND_RUNTIME, COMPUTE_REQUEST_RUNTIME, MGMT_REQUEST_RUNTIME, WALRECEIVER_RUNTIME,
    },
-    tenant_mgr, virtual_file,
+    tenant_mgr, virtual_file, LOG_FILE_NAME,
 };
-use remote_storage::GenericRemoteStorage;
 use utils::{
    auth::JwtAuth,
-    lock_file, logging,
+    logging,
    postgres_backend::AuthType,
    project_git_version,
+    shutdown::exit_now,
    signals::{self, Signal},
    tcp_listener,
 };

 project_git_version!(GIT_VERSION);

-const PID_FILE_NAME: &str = "pageserver.pid";
-
 const FEATURES: &[&str] = &[
    #[cfg(feature = "testing")]
    "testing",
@@ -66,7 +65,6 @@ fn main() -> anyhow::Result<()> {
    let workdir = workdir
        .canonicalize()
        .with_context(|| format!("Error opening workdir '{}'", workdir.display()))?;
-
    let cfg_file_path = workdir.join("pageserver.toml");

    // Set CWD to workdir for non-daemon modes
@@ -77,6 +75,8 @@ fn main() -> anyhow::Result<()> {
        )
    })?;

+    let daemonize = arg_matches.get_flag("daemonize");
+
    let conf = match initialize_config(&cfg_file_path, arg_matches, &workdir)? {
        ControlFlow::Continue(conf) => conf,
        ControlFlow::Break(()) => {
@@ -102,7 +102,7 @@ fn main() -> anyhow::Result<()> {
    virtual_file::init(conf.max_file_descriptors);
    page_cache::init(conf.page_cache_size);

-    start_pageserver(conf).context("Failed to start pageserver")?;
+    start_pageserver(conf, daemonize).context("Failed to start pageserver")?;

    scenario.teardown();
    Ok(())
@@ -197,48 +197,12 @@ fn initialize_config(
    })
 }

-fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
-    logging::init(conf.log_format)?;
+fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()> {
+    // Initialize logger
+    let log_file = logging::init(LOG_FILE_NAME, daemonize)?;
+
    info!("version: {}", version());

-    // If any failpoints were set from FAILPOINTS environment variable,
-    // print them to the log for debugging purposes
-    let failpoints = fail::list();
-    if !failpoints.is_empty() {
-        info!(
-            "started with failpoints: {}",
-            failpoints
-                .iter()
-                .map(|(name, actions)| format!("{name}={actions}"))
-                .collect::<Vec<String>>()
-                .join(";")
-        )
-    }
-
-    let lock_file_path = conf.workdir.join(PID_FILE_NAME);
-    let lock_file = match lock_file::create_lock_file(&lock_file_path, Pid::this().to_string()) {
-        lock_file::LockCreationResult::Created {
-            new_lock_contents,
-            file,
-        } => {
-            info!("Created lock file at {lock_file_path:?} with contenst {new_lock_contents}");
-            file
-        }
-        lock_file::LockCreationResult::AlreadyLocked {
-            existing_lock_contents,
-        } => anyhow::bail!(
-            "Could not lock pid file; pageserver is already running in {:?} with PID {}",
-            conf.workdir,
-            existing_lock_contents
-        ),
-        lock_file::LockCreationResult::CreationFailed(e) => {
-            return Err(e.context(format!("Failed to create lock file at {lock_file_path:?}")))
-        }
-    };
-    // ensure that the lock file is held even if the main thread of the process is panics
-    // we need to release the lock file only when the current process is gone
-    let _ = Box::leak(Box::new(lock_file));
-
    // TODO: Check that it looks like a valid repository before going further

    // bind sockets before daemonizing so we report errors early and do not return until we are listening
@@ -254,6 +218,33 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
    );
    let pageserver_listener = tcp_listener::bind(conf.listen_pg_addr.clone())?;

+    // NB: Don't spawn any threads before daemonizing!
+    if daemonize {
+        info!("daemonizing...");
+
+        // There shouldn't be any logging to stdin/stdout. Redirect it to the main log so
+        // that we will see any accidental manual fprintf's or backtraces.
+        let stdout = log_file
+            .try_clone()
+            .with_context(|| format!("Failed to clone log file '{:?}'", log_file))?;
+        let stderr = log_file;
+
+        let daemonize = Daemonize::new()
+            .pid_file("pageserver.pid")
+            .working_directory(".")
+            .stdout(stdout)
+            .stderr(stderr);
+
+        // XXX: The parent process should exit abruptly right after
+        // it has spawned a child to prevent coverage machinery from
+        // dumping stats into a `profraw` file now owned by the child.
+        // Otherwise, the coverage data will be damaged.
+        match daemonize.exit_action(|| exit_now(0)).start() {
+            Ok(_) => info!("Success, daemonized"),
+            Err(err) => bail!("{err}. could not daemonize. bailing."),
+        }
+    }
+
    let signals = signals::install_shutdown_handlers()?;

    // start profiler (if enabled)
@@ -272,23 +263,6 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
    };
    info!("Using auth: {:#?}", conf.auth_type);

-    match var("ZENITH_AUTH_TOKEN") {
-        Ok(v) => {
-            info!("Loaded JWT token for authentication with Safekeeper");
-            pageserver::config::SAFEKEEPER_AUTH_TOKEN
-                .set(Arc::new(v))
-                .map_err(|_| anyhow!("Could not initialize SAFEKEEPER_AUTH_TOKEN"))?;
-        }
-        Err(VarError::NotPresent) => {
-            info!("No JWT token for authentication with Safekeeper detected");
-        }
-        Err(e) => {
-            return Err(e).with_context(|| {
-                "Failed to either load to detect non-present ZENITH_AUTH_TOKEN environment variable"
-            })
-        }
-    };
-
    let remote_storage = conf
        .remote_storage_config
        .as_ref()
@@ -297,7 +271,7 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
        })
        .transpose()
        .context("Failed to init generic remote storage")?;
-    {
+    let remote_index = {
        let _rt_guard = BACKGROUND_RUNTIME.enter();
        tenant_mgr::init_tenant_mgr(conf, remote_storage.clone())?
    };
@@ -309,7 +283,7 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
    {
        let _rt_guard = MGMT_REQUEST_RUNTIME.enter();

-        let router = http::make_router(conf, auth.clone(), remote_storage)?;
+        let router = http::make_router(conf, auth.clone(), remote_index, remote_storage)?;
        let service =
            utils::http::RouterService::new(router.build().map_err(|err| anyhow!(err))?).unwrap();
        let server = hyper::Server::from_tcp(http_listener)?
@@ -373,6 +347,14 @@ fn cli() -> Command {
    Command::new("Neon page server")
        .about("Materializes WAL stream to pages and serves them to the postgres")
        .version(version())
+        .arg(
+
+            Arg::new("daemonize")
+                .short('d')
+                .long("daemonize")
+                .action(ArgAction::SetTrue)
+                .help("Run in the background"),
+        )
        .arg(
            Arg::new("init")
                .long("init")
--- a/pageserver/src/config.rs
+++ b/pageserver/src/config.rs
@@ -8,26 +8,25 @@ use anyhow::{anyhow, bail, ensure, Context, Result};
 use remote_storage::RemoteStorageConfig;
 use std::env;
 use utils::crashsafe::path_with_suffix_extension;
-use utils::id::ConnectionId;

-use once_cell::sync::OnceCell;
-use std::num::NonZeroUsize;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
-use std::sync::Arc;
 use std::time::Duration;
 use toml_edit;
 use toml_edit::{Document, Item};
 use url::Url;
 use utils::{
    id::{NodeId, TenantId, TimelineId},
-    logging::LogFormat,
    postgres_backend::AuthType,
 };

-use crate::tenant::{TENANT_ATTACHING_MARKER_FILENAME, TIMELINES_SEGMENT_NAME};
+use crate::tenant::TIMELINES_SEGMENT_NAME;
 use crate::tenant_config::{TenantConf, TenantConfOpt};
-use crate::{METADATA_FILE_NAME, TENANT_CONFIG_NAME, TIMELINE_UNINIT_MARK_SUFFIX};
+
+/// The name of the metadata file pageserver creates per timeline.
+pub const METADATA_FILE_NAME: &str = "metadata";
+pub const TIMELINE_UNINIT_MARK_SUFFIX: &str = "___uninit";
+const TENANT_CONFIG_NAME: &str = "config";

 pub mod defaults {
    use crate::tenant_config::defaults::*;
@@ -46,11 +45,6 @@ pub mod defaults {
    pub const DEFAULT_PAGE_CACHE_SIZE: usize = 8192;
    pub const DEFAULT_MAX_FILE_DESCRIPTORS: usize = 100;

-    pub const DEFAULT_LOG_FORMAT: &str = "plain";
-
-    pub const DEFAULT_CONCURRENT_TENANT_SIZE_LOGICAL_SIZE_QUERIES: usize =
-        super::ConfigurableSemaphore::DEFAULT_INITIAL.get();
-
    ///
    /// Default built-in configuration file.
    ///
@@ -69,10 +63,6 @@ pub mod defaults {
 # initial superuser role name to use when creating a new tenant
 #initial_superuser_name = '{DEFAULT_SUPERUSER}'

-#log_format = '{DEFAULT_LOG_FORMAT}'
-
-#concurrent_tenant_size_logical_size_queries = '{DEFAULT_CONCURRENT_TENANT_SIZE_LOGICAL_SIZE_QUERIES}'
-
 # [tenant_config]
 #checkpoint_distance = {DEFAULT_CHECKPOINT_DISTANCE} # in bytes
 #checkpoint_timeout = {DEFAULT_CHECKPOINT_TIMEOUT}
@@ -136,22 +126,8 @@ pub struct PageServerConf {

    /// Etcd broker endpoints to connect to.
    pub broker_endpoints: Vec<Url>,
-
-    pub log_format: LogFormat,
-
-    /// Number of concurrent [`Tenant::gather_size_inputs`] allowed.
-    pub concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore,
 }

-/// We do not want to store this in a PageServerConf because the latter may be logged
-/// and/or serialized at a whim, while the token is secret. Currently this token is the
-/// same for accessing all tenants/timelines, but may become per-tenant/per-timeline in
-/// the future, more tokens and auth may arrive for etcd and/or its rewrite (see
-/// https://github.com/neondatabase/neon/issues/2394), completely changing the logic.
-/// Hence, we resort to a global variable for now instead of passing the token from the
-/// startup code to the connection code through a dozen layers.
-pub static SAFEKEEPER_AUTH_TOKEN: OnceCell<Arc<String>> = OnceCell::new();
-
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum ProfilingConfig {
    Disabled,
@@ -216,10 +192,6 @@ struct PageServerConfigBuilder {
    profiling: BuilderValue<ProfilingConfig>,
    broker_etcd_prefix: BuilderValue<String>,
    broker_endpoints: BuilderValue<Vec<Url>>,
-
-    log_format: BuilderValue<LogFormat>,
-
-    concurrent_tenant_size_logical_size_queries: BuilderValue<ConfigurableSemaphore>,
 }

 impl Default for PageServerConfigBuilder {
@@ -247,9 +219,6 @@ impl Default for PageServerConfigBuilder {
            profiling: Set(ProfilingConfig::Disabled),
            broker_etcd_prefix: Set(etcd_broker::DEFAULT_NEON_BROKER_ETCD_PREFIX.to_string()),
            broker_endpoints: Set(Vec::new()),
-            log_format: Set(LogFormat::from_str(DEFAULT_LOG_FORMAT).unwrap()),
-
-            concurrent_tenant_size_logical_size_queries: Set(ConfigurableSemaphore::default()),
        }
    }
 }
@@ -322,14 +291,6 @@ impl PageServerConfigBuilder {
        self.profiling = BuilderValue::Set(profiling)
    }

-    pub fn log_format(&mut self, log_format: LogFormat) {
-        self.log_format = BuilderValue::Set(log_format)
-    }
-
-    pub fn concurrent_tenant_size_logical_size_queries(&mut self, u: ConfigurableSemaphore) {
-        self.concurrent_tenant_size_logical_size_queries = BuilderValue::Set(u);
-    }
-
    pub fn build(self) -> anyhow::Result<PageServerConf> {
        let broker_endpoints = self
            .broker_endpoints
@@ -374,12 +335,6 @@ impl PageServerConfigBuilder {
            broker_etcd_prefix: self
                .broker_etcd_prefix
                .ok_or(anyhow!("missing broker_etcd_prefix"))?,
-            log_format: self.log_format.ok_or(anyhow!("missing log_format"))?,
-            concurrent_tenant_size_logical_size_queries: self
-                .concurrent_tenant_size_logical_size_queries
-                .ok_or(anyhow!(
-                    "missing concurrent_tenant_size_logical_size_queries"
-                ))?,
        })
    }
 }
@@ -397,11 +352,6 @@ impl PageServerConf {
        self.tenants_path().join(tenant_id.to_string())
    }

-    pub fn tenant_attaching_mark_file_path(&self, tenant_id: &TenantId) -> PathBuf {
-        self.tenant_path(tenant_id)
-            .join(TENANT_ATTACHING_MARKER_FILENAME)
-    }
-
    /// Points to a place in pageserver's local directory,
    /// where certain tenant's tenantconf file should be located.
    pub fn tenant_config_path(&self, tenant_id: TenantId) -> PathBuf {
@@ -427,22 +377,6 @@ impl PageServerConf {
        )
    }

-    pub fn traces_path(&self) -> PathBuf {
-        self.workdir.join("traces")
-    }
-
-    pub fn trace_path(
-        &self,
-        tenant_id: &TenantId,
-        timeline_id: &TimelineId,
-        connection_id: &ConnectionId,
-    ) -> PathBuf {
-        self.traces_path()
-            .join(tenant_id.to_string())
-            .join(timeline_id.to_string())
-            .join(connection_id.to_string())
-    }
-
    /// Points to a place in pageserver's local directory,
    /// where certain timeline's metadata file should be located.
    pub fn metadata_path(&self, timeline_id: TimelineId, tenant_id: TenantId) -> PathBuf {
@@ -525,15 +459,6 @@ impl PageServerConf {
                        })
                        .collect::<anyhow::Result<_>>()?,
                ),
-                "log_format" => builder.log_format(
-                    LogFormat::from_config(&parse_toml_string(key, item)?)?
-                ),
-                "concurrent_tenant_size_logical_size_queries" => builder.concurrent_tenant_size_logical_size_queries({
-                    let input = parse_toml_string(key, item)?;
-                    let permits = input.parse::<usize>().context("expected a number of initial permits, not {s:?}")?;
-                    let permits = NonZeroUsize::new(permits).context("initial semaphore permits out of range: 0, use other configuration to disable a feature")?;
-                    ConfigurableSemaphore::new(permits)
-                }),
                _ => bail!("unrecognized pageserver option '{key}'"),
            }
        }
@@ -626,9 +551,8 @@ impl PageServerConf {
        PathBuf::from(format!("../tmp_check/test_{test_name}"))
    }

+    #[cfg(test)]
    pub fn dummy_conf(repo_dir: PathBuf) -> Self {
-        let pg_distrib_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../pg_install");
-
        PageServerConf {
            id: NodeId(0),
            wait_lsn_timeout: Duration::from_secs(60),
@@ -639,7 +563,7 @@ impl PageServerConf {
            listen_http_addr: defaults::DEFAULT_HTTP_LISTEN_ADDR.to_string(),
            superuser: "cloud_admin".to_string(),
            workdir: repo_dir,
-            pg_distrib_dir,
+            pg_distrib_dir: PathBuf::new(),
            auth_type: AuthType::Trust,
            auth_validation_public_key_path: None,
            remote_storage_config: None,
@@ -647,8 +571,6 @@ impl PageServerConf {
            default_tenant_conf: TenantConf::dummy_conf(),
            broker_endpoints: Vec::new(),
            broker_etcd_prefix: etcd_broker::DEFAULT_NEON_BROKER_ETCD_PREFIX.to_string(),
-            log_format: LogFormat::from_str(defaults::DEFAULT_LOG_FORMAT).unwrap(),
-            concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::default(),
        }
    }
 }
@@ -714,58 +636,6 @@ fn parse_toml_array(name: &str, item: &Item) -> anyhow::Result<Vec<String>> {
        .collect()
 }

-/// Configurable semaphore permits setting.
-///
-/// Does not allow semaphore permits to be zero, because at runtime initially zero permits and empty
-/// semaphore cannot be distinguished, leading any feature using these to await forever (or until
-/// new permits are added).
-#[derive(Debug, Clone)]
-pub struct ConfigurableSemaphore {
-    initial_permits: NonZeroUsize,
-    inner: std::sync::Arc<tokio::sync::Semaphore>,
-}
-
-impl ConfigurableSemaphore {
-    pub const DEFAULT_INITIAL: NonZeroUsize = match NonZeroUsize::new(1) {
-        Some(x) => x,
-        None => panic!("const unwrap is not yet stable"),
-    };
-
-    /// Initializse using a non-zero amount of permits.
-    ///
-    /// Require a non-zero initial permits, because using permits == 0 is a crude way to disable a
-    /// feature such as [`Tenant::gather_size_inputs`]. Otherwise any semaphore using future will
-    /// behave like [`futures::future::pending`], just waiting until new permits are added.
-    pub fn new(initial_permits: NonZeroUsize) -> Self {
-        ConfigurableSemaphore {
-            initial_permits,
-            inner: std::sync::Arc::new(tokio::sync::Semaphore::new(initial_permits.get())),
-        }
-    }
-}
-
-impl Default for ConfigurableSemaphore {
-    fn default() -> Self {
-        Self::new(Self::DEFAULT_INITIAL)
-    }
-}
-
-impl PartialEq for ConfigurableSemaphore {
-    fn eq(&self, other: &Self) -> bool {
-        // the number of permits can be increased at runtime, so we cannot really fulfill the
-        // PartialEq value equality otherwise
-        self.initial_permits == other.initial_permits
-    }
-}
-
-impl Eq for ConfigurableSemaphore {}
-
-impl ConfigurableSemaphore {
-    pub fn inner(&self) -> &std::sync::Arc<tokio::sync::Semaphore> {
-        &self.inner
-    }
-}
-
 #[cfg(test)]
 mod tests {
    use std::{
@@ -795,8 +665,6 @@ max_file_descriptors = 333
 initial_superuser_name = 'zzzz'
 id = 10

-log_format = 'json'
-
 "#;

    #[test]
@@ -836,8 +704,6 @@ log_format = 'json'
                    .parse()
                    .expect("Failed to parse a valid broker endpoint URL")],
                broker_etcd_prefix: etcd_broker::DEFAULT_NEON_BROKER_ETCD_PREFIX.to_string(),
-                log_format: LogFormat::from_str(defaults::DEFAULT_LOG_FORMAT).unwrap(),
-                concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::default(),
            },
            "Correct defaults should be used when no config values are provided"
        );
@@ -882,8 +748,6 @@ log_format = 'json'
                    .parse()
                    .expect("Failed to parse a valid broker endpoint URL")],
                broker_etcd_prefix: etcd_broker::DEFAULT_NEON_BROKER_ETCD_PREFIX.to_string(),
-                log_format: LogFormat::Json,
-                concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::default(),
            },
            "Should be able to parse all basic config values correctly"
        );
--- a/pageserver/src/http/openapi_spec.yml
+++ b/pageserver/src/http/openapi_spec.yml
@@ -354,54 +354,6 @@ paths:
              schema:
                $ref: "#/components/schemas/Error"

-  /v1/tenant/{tenant_id}/size:
-    parameters:
-      - name: tenant_id
-        in: path
-        required: true
-        schema:
-          type: string
-          format: hex
-    get:
-      description: |
-        Calculate tenant's size, which is a mixture of WAL (bytes) and logical_size (bytes).
-      responses:
-        "200":
-          description: OK,
-          content:
-            application/json:
-              schema:
-                type: object
-                required:
-                  - id
-                  - size
-                properties:
-                  id:
-                    type: string
-                    format: hex
-                  size:
-                    type: integer
-                    description: |
-                      Size metric in bytes.
-        "401":
-          description: Unauthorized Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/UnauthorizedError"
-        "403":
-          description: Forbidden Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ForbiddenError"
-        "500":
-          description: Generic operation error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-
  /v1/tenant/{tenant_id}/timeline/:
    parameters:
      - name: tenant_id
@@ -604,7 +556,13 @@ components:
        id:
          type: string
        state:
-          type: string
+            oneOf:
+              - type: string
+              - type: object
+                properties:
+                  background_jobs_running:
+                    type: boolean
+
        current_physical_size:
          type: integer
        has_in_progress_downloads:
@@ -660,8 +618,6 @@ components:
        - last_record_lsn
        - disk_consistent_lsn
        - awaits_download
-        - state
-        - latest_gc_cutoff_lsn
      properties:
        timeline_id:
          type: string
@@ -704,11 +660,6 @@ components:
          type: integer
        awaits_download:
          type: boolean
-        state:
-          type: string
-        latest_gc_cutoff_lsn:
-          type: string
-          format: hex

        # These 'local' and 'remote' fields just duplicate some of the fields
        # above. They are kept for backwards-compatibility. They can be removed,
--- a/pageserver/src/http/routes.rs
+++ b/pageserver/src/http/routes.rs
@@ -3,29 +3,31 @@ use std::sync::Arc;
 use anyhow::{anyhow, Context, Result};
 use hyper::StatusCode;
 use hyper::{Body, Request, Response, Uri};
-use pageserver_api::models::TenantState;
 use remote_storage::GenericRemoteStorage;
 use tokio::task::JoinError;
 use tracing::*;

+use super::models::{LocalTimelineInfo, RemoteTimelineInfo, TimelineInfo};
 use super::models::{
-    LocalTimelineInfo, RemoteTimelineInfo, StatusResponse, TenantConfigRequest,
-    TenantCreateRequest, TenantCreateResponse, TenantInfo, TimelineCreateRequest, TimelineInfo,
+    StatusResponse, TenantConfigRequest, TenantCreateRequest, TenantCreateResponse, TenantInfo,
+    TimelineCreateRequest,
 };
 use crate::pgdatadir_mapping::LsnForTimestamp;
-use crate::tenant::Timeline;
+use crate::storage_sync;
+use crate::storage_sync::index::{RemoteIndex, RemoteTimeline};
+use crate::tenant::{TenantState, Timeline};
 use crate::tenant_config::TenantConfOpt;
 use crate::{config::PageServerConf, tenant_mgr};
 use utils::{
    auth::JwtAuth,
    http::{
-        endpoint::{self, attach_openapi_ui, auth_middleware, check_permission_with},
+        endpoint::{self, attach_openapi_ui, auth_middleware, check_permission},
        error::{ApiError, HttpErrorBody},
        json::{json_request, json_response},
        request::parse_request_param,
        RequestExt, RouterBuilder,
    },
-    id::{TenantId, TimelineId},
+    id::{TenantId, TenantTimelineId, TimelineId},
    lsn::Lsn,
 };

@@ -38,6 +40,7 @@ use crate::CheckpointConfig;
 struct State {
    conf: &'static PageServerConf,
    auth: Option<Arc<JwtAuth>>,
+    remote_index: RemoteIndex,
    allowlist_routes: Vec<Uri>,
    remote_storage: Option<GenericRemoteStorage>,
 }
@@ -46,6 +49,7 @@ impl State {
    fn new(
        conf: &'static PageServerConf,
        auth: Option<Arc<JwtAuth>>,
+        remote_index: RemoteIndex,
        remote_storage: Option<GenericRemoteStorage>,
    ) -> anyhow::Result<Self> {
        let allowlist_routes = ["/v1/status", "/v1/doc", "/swagger.yml"]
@@ -56,6 +60,7 @@ impl State {
            conf,
            auth,
            allowlist_routes,
+            remote_index,
            remote_storage,
        })
    }
@@ -74,41 +79,19 @@ fn get_config(request: &Request<Body>) -> &'static PageServerConf {
    get_state(request).conf
 }

-fn check_permission(request: &Request<Body>, tenant_id: Option<TenantId>) -> Result<(), ApiError> {
-    check_permission_with(request, |claims| {
-        crate::auth::check_permission(claims, tenant_id)
-    })
-}
-
 // Helper function to construct a TimelineInfo struct for a timeline
-fn build_timeline_info(
-    tenant_state: TenantState,
+async fn build_timeline_info(
+    state: &State,
    timeline: &Arc<Timeline>,
    include_non_incremental_logical_size: bool,
    include_non_incremental_physical_size: bool,
-) -> anyhow::Result<TimelineInfo> {
-    let mut info = build_timeline_info_common(tenant_state, timeline)?;
-    if include_non_incremental_logical_size {
-        info.current_logical_size_non_incremental =
-            Some(timeline.get_current_logical_size_non_incremental(info.last_record_lsn)?);
-    }
-    if include_non_incremental_physical_size {
-        info.current_physical_size_non_incremental =
-            Some(timeline.get_physical_size_non_incremental()?)
-    }
-    Ok(info)
-}
-
-fn build_timeline_info_common(
-    tenant_state: TenantState,
-    timeline: &Arc<Timeline>,
 ) -> anyhow::Result<TimelineInfo> {
    let last_record_lsn = timeline.get_last_record_lsn();
    let (wal_source_connstr, last_received_msg_lsn, last_received_msg_ts) = {
        let guard = timeline.last_received_wal.lock().unwrap();
        if let Some(info) = guard.as_ref() {
            (
-                Some(format!("{:?}", info.wal_source_connconf)), // Password is hidden, but it's for statistics only.
+                Some(info.wal_source_connstr.clone()),
                Some(info.last_received_msg_lsn),
                Some(info.last_received_msg_ts),
            )
@@ -117,6 +100,22 @@ fn build_timeline_info_common(
        }
    };

+    let (remote_consistent_lsn, awaits_download) = if let Some(remote_entry) = state
+        .remote_index
+        .read()
+        .await
+        .timeline_entry(&TenantTimelineId {
+            tenant_id: timeline.tenant_id,
+            timeline_id: timeline.timeline_id,
+        }) {
+        (
+            Some(remote_entry.metadata.disk_consistent_lsn()),
+            remote_entry.awaits_download,
+        )
+    } else {
+        (None, false)
+    };
+
    let ancestor_timeline_id = timeline.get_ancestor_timeline_id();
    let ancestor_lsn = match timeline.get_ancestor_lsn() {
        Lsn(0) => None,
@@ -130,8 +129,6 @@ fn build_timeline_info_common(
        }
    };
    let current_physical_size = Some(timeline.get_physical_size());
-    let state = timeline.current_state();
-    let remote_consistent_lsn = timeline.get_remote_consistent_lsn().unwrap_or(Lsn(0));

    let info = TimelineInfo {
        tenant_id: timeline.tenant_id,
@@ -139,24 +136,28 @@ fn build_timeline_info_common(
        ancestor_timeline_id,
        ancestor_lsn,
        disk_consistent_lsn: timeline.get_disk_consistent_lsn(),
-        remote_consistent_lsn,
        last_record_lsn,
        prev_record_lsn: Some(timeline.get_prev_record_lsn()),
        latest_gc_cutoff_lsn: *timeline.get_latest_gc_cutoff_lsn(),
        current_logical_size,
        current_physical_size,
-        current_logical_size_non_incremental: None,
-        current_physical_size_non_incremental: None,
+        current_logical_size_non_incremental: if include_non_incremental_logical_size {
+            Some(timeline.get_current_logical_size_non_incremental(last_record_lsn)?)
+        } else {
+            None
+        },
+        current_physical_size_non_incremental: if include_non_incremental_physical_size {
+            Some(timeline.get_physical_size_non_incremental()?)
+        } else {
+            None
+        },
        wal_source_connstr,
        last_received_msg_lsn,
        last_received_msg_ts,
        pg_version: timeline.pg_version,

-        state,
-
-        // XXX bring back tracking of downloads per timeline, or, introduce
-        // an 'Attaching' state for the timeline and get rid of this field.
-        awaits_download: tenant_state == TenantState::Attaching,
+        remote_consistent_lsn,
+        awaits_download,

        // Duplicate some fields in 'local' and 'remote' fields, for backwards-compatility
        // with the control plane.
@@ -167,7 +168,7 @@ fn build_timeline_info_common(
            current_physical_size,
        },
        remote: RemoteTimelineInfo {
-            remote_consistent_lsn: Some(remote_consistent_lsn),
+            remote_consistent_lsn,
        },
    };
    Ok(info)
@@ -175,7 +176,6 @@ fn build_timeline_info_common(

 // healthcheck handler
 async fn status_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    check_permission(&request, None)?;
    let config = get_config(&request);
    json_response(StatusCode::OK, StatusResponse { id: config.id })
 }
@@ -185,28 +185,34 @@ async fn timeline_create_handler(mut request: Request<Body>) -> Result<Response<
    let request_data: TimelineCreateRequest = json_request(&mut request).await?;
    check_permission(&request, Some(tenant_id))?;

-    let new_timeline_id = request_data
-        .new_timeline_id
-        .unwrap_or_else(TimelineId::generate);
+    let state = get_state(&request);

    let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
-    match tenant.create_timeline(
-        new_timeline_id,
-        request_data.ancestor_timeline_id.map(TimelineId::from),
-        request_data.ancestor_start_lsn,
-        request_data.pg_version.unwrap_or(crate::DEFAULT_PG_VERSION)
-    )
-    .instrument(info_span!("timeline_create", tenant = %tenant_id, new_timeline = ?request_data.new_timeline_id, timeline_id = %new_timeline_id, lsn=?request_data.ancestor_start_lsn, pg_version=?request_data.pg_version))
-    .await {
-        Ok(Some(new_timeline)) => {
-            // Created. Construct a TimelineInfo for it.
-            let timeline_info = build_timeline_info_common(tenant.current_state(), &new_timeline)
-                .map_err(ApiError::InternalServerError)?;
-            json_response(StatusCode::CREATED, timeline_info)
+    let new_timeline_info = async {
+        match tenant.create_timeline(
+            request_data.new_timeline_id.map(TimelineId::from),
+            request_data.ancestor_timeline_id.map(TimelineId::from),
+            request_data.ancestor_start_lsn,
+            request_data.pg_version.unwrap_or(crate::DEFAULT_PG_VERSION)
+        ).await {
+            Ok(Some(new_timeline)) => {
+                // Created. Construct a TimelineInfo for it.
+                let timeline_info = build_timeline_info(state, &new_timeline, false, false)
+                    .await
+                    .map_err(ApiError::InternalServerError)?;
+                Ok(Some(timeline_info))
+            }
+            Ok(None) => Ok(None), // timeline already exists
+            Err(err) => Err(ApiError::InternalServerError(err)),
        }
-        Ok(None) => json_response(StatusCode::CONFLICT, ()), // timeline already exists
-        Err(err) => Err(ApiError::InternalServerError(err)),
    }
+    .instrument(info_span!("timeline_create", tenant = %tenant_id, new_timeline = ?request_data.new_timeline_id, lsn=?request_data.ancestor_start_lsn, pg_version=?request_data.pg_version))
+        .await?;
+
+    Ok(match new_timeline_info {
+        Some(info) => json_response(StatusCode::CREATED, info)?,
+        None => json_response(StatusCode::CONFLICT, ())?,
+    })
 }

 async fn timeline_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -217,21 +223,25 @@ async fn timeline_list_handler(request: Request<Body>) -> Result<Response<Body>,
        query_param_present(&request, "include-non-incremental-physical-size");
    check_permission(&request, Some(tenant_id))?;

-    let _entered = info_span!("timeline_list", tenant = %tenant_id).entered();
+    let state = get_state(&request);

-    let (tenant_state, timelines) = {
+    let timelines = tokio::task::spawn_blocking(move || {
+        let _enter = info_span!("timeline_list", tenant = %tenant_id).entered();
        let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
-        (tenant.current_state(), tenant.list_timelines())
-    };
+        Ok(tenant.list_timelines())
+    })
+    .await
+    .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??;

    let mut response_data = Vec::with_capacity(timelines.len());
    for timeline in timelines {
        let timeline_info = build_timeline_info(
-            tenant_state,
+            state,
            &timeline,
            include_non_incremental_logical_size,
            include_non_incremental_physical_size,
        )
+        .await
        .context("Failed to convert tenant timeline {timeline_id} into the local one: {e:?}")
        .map_err(ApiError::InternalServerError)?;

@@ -280,25 +290,24 @@ async fn timeline_detail_handler(request: Request<Body>) -> Result<Response<Body
        query_param_present(&request, "include-non-incremental-physical-size");
    check_permission(&request, Some(tenant_id))?;

+    let state = get_state(&request);
+
    let timeline_info = async {
-        let (tenant_state, timeline) = tokio::task::spawn_blocking(move || {
-            let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
-            Ok((
-                tenant.current_state(),
-                tenant.get_timeline(timeline_id, false),
-            ))
+        let timeline = tokio::task::spawn_blocking(move || {
+            tenant_mgr::get_tenant(tenant_id, true)?.get_timeline(timeline_id)
        })
        .await
-        .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??;
+        .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))?;

        let timeline = timeline.map_err(ApiError::NotFound)?;

        let timeline_info = build_timeline_info(
-            tenant_state,
+            state,
            &timeline,
            include_non_incremental_logical_size,
            include_non_incremental_physical_size,
        )
+        .await
        .context("Failed to get local timeline info: {e:#}")
        .map_err(ApiError::InternalServerError)?;

@@ -322,13 +331,14 @@ async fn get_lsn_by_timestamp_handler(request: Request<Body>) -> Result<Response
    let timestamp_pg = postgres_ffi::to_pg_timestamp(timestamp);

    let timeline = tenant_mgr::get_tenant(tenant_id, true)
-        .and_then(|tenant| tenant.get_timeline(timeline_id, true))
+        .and_then(|tenant| tenant.get_timeline(timeline_id))
+        .with_context(|| format!("No timeline {timeline_id} in repository for tenant {tenant_id}"))
        .map_err(ApiError::NotFound)?;
    let result = match timeline
        .find_lsn_for_timestamp(timestamp_pg)
        .map_err(ApiError::InternalServerError)?
    {
-        LsnForTimestamp::Present(lsn) => format!("{lsn}"),
+        LsnForTimestamp::Present(lsn) => format!("{}", lsn),
        LsnForTimestamp::Future(_lsn) => "future".into(),
        LsnForTimestamp::Past(_lsn) => "past".into(),
        LsnForTimestamp::NoData(_lsn) => "nodata".into(),
@@ -343,28 +353,117 @@ async fn tenant_attach_handler(request: Request<Body>) -> Result<Response<Body>,

    info!("Handling tenant attach {tenant_id}");

-    let state = get_state(&request);
+    tokio::task::spawn_blocking(move || match tenant_mgr::get_tenant(tenant_id, false) {
+        Ok(tenant) => {
+            if tenant.list_timelines().is_empty() {
+                info!("Attaching to tenant {tenant_id} with zero timelines");
+                Ok(())
+            } else {
+                Err(ApiError::Conflict(
+                    "Tenant is already present locally".to_owned(),
+                ))
+            }
+        }
+        Err(_) => Ok(()),
+    })
+    .await
+    .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??;

-    if let Some(remote_storage) = &state.remote_storage {
-        // FIXME: distinguish between "Tenant already exists" and other errors
-        tenant_mgr::attach_tenant(state.conf, tenant_id, remote_storage)
-            .instrument(info_span!("tenant_attach", tenant = %tenant_id))
+    let state = get_state(&request);
+    let remote_index = &state.remote_index;
+
+    let mut index_accessor = remote_index.write().await;
+    if let Some(tenant_entry) = index_accessor.tenant_entry_mut(&tenant_id) {
+        if tenant_entry.has_in_progress_downloads() {
+            return Err(ApiError::Conflict(
+                "Tenant download is already in progress".to_string(),
+            ));
+        }
+
+        for (timeline_id, remote_timeline) in tenant_entry.iter_mut() {
+            storage_sync::schedule_layer_download(tenant_id, *timeline_id);
+            remote_timeline.awaits_download = true;
+        }
+        return json_response(StatusCode::ACCEPTED, ());
+    }
+    // no tenant in the index, release the lock to make the potentially lengthy download operation
+    drop(index_accessor);
+
+    // download index parts for every tenant timeline
+    let remote_timelines = match gather_tenant_timelines_index_parts(state, tenant_id).await {
+        Ok(Some(remote_timelines)) => remote_timelines,
+        Ok(None) => return Err(ApiError::NotFound(anyhow!("Unknown remote tenant"))),
+        Err(e) => {
+            error!("Failed to retrieve remote tenant data: {:?}", e);
+            return Err(ApiError::NotFound(anyhow!(
+                "Failed to retrieve remote tenant"
+            )));
+        }
+    };
+
+    // recheck that download is not in progress because
+    // we've released the lock to avoid holding it during the download
+    let mut index_accessor = remote_index.write().await;
+    let tenant_entry = match index_accessor.tenant_entry_mut(&tenant_id) {
+        Some(tenant_entry) => {
+            if tenant_entry.has_in_progress_downloads() {
+                return Err(ApiError::Conflict(
+                    "Tenant download is already in progress".to_string(),
+                ));
+            }
+            tenant_entry
+        }
+        None => index_accessor.add_tenant_entry(tenant_id),
+    };
+
+    // populate remote index with the data from index part and create directories on the local filesystem
+    for (timeline_id, mut remote_timeline) in remote_timelines {
+        tokio::fs::create_dir_all(state.conf.timeline_path(&timeline_id, &tenant_id))
            .await
+            .context("Failed to create new timeline directory")
            .map_err(ApiError::InternalServerError)?;
-    } else {
-        return Err(ApiError::BadRequest(anyhow!(
-            "attach_tenant is possible because pageserver was configured without remote storage"
-        )));
+
+        remote_timeline.awaits_download = true;
+        tenant_entry.insert(timeline_id, remote_timeline);
+        // schedule actual download
+        storage_sync::schedule_layer_download(tenant_id, timeline_id);
    }

    json_response(StatusCode::ACCEPTED, ())
 }

+/// Note: is expensive from s3 access perspective,
+/// for details see comment to `storage_sync::gather_tenant_timelines_index_parts`
+async fn gather_tenant_timelines_index_parts(
+    state: &State,
+    tenant_id: TenantId,
+) -> anyhow::Result<Option<Vec<(TimelineId, RemoteTimeline)>>> {
+    let index_parts = match state.remote_storage.as_ref() {
+        Some(storage) => {
+            storage_sync::gather_tenant_timelines_index_parts(state.conf, storage, tenant_id).await
+        }
+        None => return Ok(None),
+    }
+    .with_context(|| format!("Failed to download index parts for tenant {tenant_id}"))?;
+
+    let mut remote_timelines = Vec::with_capacity(index_parts.len());
+    for (timeline_id, index_part) in index_parts {
+        let timeline_path = state.conf.timeline_path(&timeline_id, &tenant_id);
+        let remote_timeline = RemoteTimeline::from_index_part(&timeline_path, index_part)
+            .with_context(|| {
+                format!("Failed to convert index part into remote timeline for timeline {tenant_id}/{timeline_id}")
+            })?;
+        remote_timelines.push((timeline_id, remote_timeline));
+    }
+    Ok(Some(remote_timelines))
+}
+
 async fn timeline_delete_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
    let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
    let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    check_permission(&request, Some(tenant_id))?;

+    let state = get_state(&request);
    tenant_mgr::delete_timeline(tenant_id, timeline_id)
        .instrument(info_span!("timeline_delete", tenant = %tenant_id, timeline = %timeline_id))
        .await
@@ -373,6 +472,12 @@ async fn timeline_delete_handler(request: Request<Body>) -> Result<Response<Body
        // it.
        .map_err(ApiError::InternalServerError)?;

+    let mut remote_index = state.remote_index.write().await;
+    remote_index.remove_timeline_entry(TenantTimelineId {
+        tenant_id,
+        timeline_id,
+    });
+
    json_response(StatusCode::OK, ())
 }

@@ -389,23 +494,22 @@ async fn tenant_detach_handler(request: Request<Body>) -> Result<Response<Body>,
        // Replace this with better handling once the error type permits it.
        .map_err(ApiError::InternalServerError)?;

+    let mut remote_index = state.remote_index.write().await;
+    remote_index.remove_tenant_entry(&tenant_id);
+
    json_response(StatusCode::OK, ())
 }

 async fn tenant_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
    check_permission(&request, None)?;

+    let state = get_state(&request);
+    // clone to avoid holding the lock while awaiting for blocking task
+    let remote_index = state.remote_index.read().await.clone();
+
    let response_data = tokio::task::spawn_blocking(move || {
        let _enter = info_span!("tenant_list").entered();
-        tenant_mgr::list_tenants()
-            .iter()
-            .map(|(id, state)| TenantInfo {
-                id: *id,
-                state: *state,
-                current_physical_size: None,
-                has_in_progress_downloads: Some(state.has_in_progress_downloads()),
-            })
-            .collect::<Vec<TenantInfo>>()
+        crate::tenant_mgr::list_tenant_info(&remote_index)
    })
    .await
    .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))?;
@@ -417,67 +521,51 @@ async fn tenant_status(request: Request<Body>) -> Result<Response<Body>, ApiErro
    let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
    check_permission(&request, Some(tenant_id))?;

-    let tenant_info = tokio::task::spawn_blocking(move || {
-        let _enter = info_span!("tenant_status_handler", tenant = %tenant_id).entered();
-        let tenant = tenant_mgr::get_tenant(tenant_id, false)?;
-
-        // Calculate total physical size of all timelines
-        let mut current_physical_size = 0;
-        for timeline in tenant.list_timelines().iter() {
-            current_physical_size += timeline.get_physical_size();
-        }
-
-        let state = tenant.current_state();
-        let tenant_info = TenantInfo {
-            id: tenant_id,
-            state,
-            current_physical_size: Some(current_physical_size),
-            has_in_progress_downloads: Some(state.has_in_progress_downloads()),
-        };
-
-        Ok::<_, anyhow::Error>(tenant_info)
-    })
-    .await
-    .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))?
-    .map_err(ApiError::InternalServerError)?;
-
-    json_response(StatusCode::OK, tenant_info)
-}
-
-async fn tenant_size_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
-    check_permission(&request, Some(tenant_id))?;
-
-    let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::InternalServerError)?;
-
-    // this can be long operation, it currently is not backed by any request coalescing or similar
-    let inputs = tenant
-        .gather_size_inputs()
+    // if tenant is in progress of downloading it can be absent in global tenant map
+    let tenant = tokio::task::spawn_blocking(move || tenant_mgr::get_tenant(tenant_id, false))
        .await
-        .map_err(ApiError::InternalServerError)?;
+        .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))?;

-    let size = inputs.calculate().map_err(ApiError::InternalServerError)?;
+    let state = get_state(&request);
+    let remote_index = &state.remote_index;

-    /// Private response type with the additional "unstable" `inputs` field.
-    ///
-    /// The type is described with `id` and `size` in the openapi_spec file, but the `inputs` is
-    /// intentionally left out. The type resides in the pageserver not to expose `ModelInputs`.
-    #[serde_with::serde_as]
-    #[derive(serde::Serialize)]
-    struct TenantHistorySize {
-        #[serde_as(as = "serde_with::DisplayFromStr")]
-        id: TenantId,
-        /// Size is a mixture of WAL and logical size, so the unit is bytes.
-        size: u64,
-        inputs: crate::tenant::size::ModelInputs,
-    }
+    let index_accessor = remote_index.read().await;
+    let has_in_progress_downloads = index_accessor
+        .tenant_entry(&tenant_id)
+        .map(|t| t.has_in_progress_downloads())
+        .unwrap_or_else(|| {
+            info!("Tenant {tenant_id} not found in remote index");
+            false
+        });
+
+    let (tenant_state, current_physical_size) = match tenant {
+        Ok(tenant) => {
+            let timelines = tenant.list_timelines();
+            // Calculate total physical size of all timelines
+            let mut current_physical_size = 0;
+            for timeline in timelines {
+                current_physical_size += timeline.get_physical_size();
+            }
+
+            (tenant.current_state(), Some(current_physical_size))
+        }
+        Err(e) => {
+            error!("Failed to get local tenant state: {e:#}");
+            if has_in_progress_downloads {
+                (TenantState::Paused, None)
+            } else {
+                (TenantState::Broken, None)
+            }
+        }
+    };

    json_response(
        StatusCode::OK,
-        TenantHistorySize {
+        TenantInfo {
            id: tenant_id,
-            size,
-            inputs,
+            state: tenant_state,
+            current_physical_size,
+            has_in_progress_downloads: Some(has_in_progress_downloads),
        },
    )
 }
@@ -496,6 +584,7 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
    check_permission(&request, None)?;

    let request_data: TenantCreateRequest = json_request(&mut request).await?;
+    let remote_index = get_state(&request).remote_index.clone();

    let mut tenant_conf = TenantConfOpt::default();
    if let Some(gc_period) = request_data.gc_period {
@@ -536,9 +625,6 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
    if let Some(max_lsn_wal_lag) = request_data.max_lsn_wal_lag {
        tenant_conf.max_lsn_wal_lag = Some(max_lsn_wal_lag);
    }
-    if let Some(trace_read_requests) = request_data.trace_read_requests {
-        tenant_conf.trace_read_requests = Some(trace_read_requests);
-    }

    tenant_conf.checkpoint_distance = request_data.checkpoint_distance;
    if let Some(checkpoint_timeout) = request_data.checkpoint_timeout {
@@ -565,42 +651,20 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
        .map(TenantId::from)
        .unwrap_or_else(TenantId::generate);

-    let new_tenant = tokio::task::spawn_blocking(move || {
+    let new_tenant_id = tokio::task::spawn_blocking(move || {
        let _enter = info_span!("tenant_create", tenant = ?target_tenant_id).entered();
-        let state = get_state(&request);
+        let conf = get_config(&request);

-        tenant_mgr::create_tenant(
-            state.conf,
-            tenant_conf,
-            target_tenant_id,
-            state.remote_storage.clone(),
-        )
-        // FIXME: `create_tenant` can fail from both user and internal errors. Replace this
-        // with better error handling once the type permits it
-        .map_err(ApiError::InternalServerError)
+        tenant_mgr::create_tenant(conf, tenant_conf, target_tenant_id, remote_index)
+            // FIXME: `create_tenant` can fail from both user and internal errors. Replace this
+            // with better error handling once the type permits it
+            .map_err(ApiError::InternalServerError)
    })
    .await
    .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??;

-    Ok(match new_tenant {
-        Some(tenant) => {
-            // We created the tenant. Existing API semantics are that the tenant
-            // is Active when this function returns.
-            if let res @ Err(_) = tenant.wait_to_become_active().await {
-                // This shouldn't happen because we just created the tenant directory
-                // in tenant_mgr::create_tenant, and there aren't any remote timelines
-                // to load, so, nothing can really fail during load.
-                // Don't do cleanup because we don't know how we got here.
-                // The tenant will likely be in `Broken` state and subsequent
-                // calls will fail.
-                res.context("created tenant failed to become active")
-                    .map_err(ApiError::InternalServerError)?;
-            }
-            json_response(
-                StatusCode::CREATED,
-                TenantCreateResponse(tenant.tenant_id()),
-            )?
-        }
+    Ok(match new_tenant_id {
+        Some(id) => json_response(StatusCode::CREATED, TenantCreateResponse(id))?,
        None => json_response(StatusCode::CONFLICT, ())?,
    })
 }
@@ -648,9 +712,6 @@ async fn tenant_config_handler(mut request: Request<Body>) -> Result<Response<Bo
    if let Some(max_lsn_wal_lag) = request_data.max_lsn_wal_lag {
        tenant_conf.max_lsn_wal_lag = Some(max_lsn_wal_lag);
    }
-    if let Some(trace_read_requests) = request_data.trace_read_requests {
-        tenant_conf.trace_read_requests = Some(trace_read_requests);
-    }

    tenant_conf.checkpoint_distance = request_data.checkpoint_distance;
    if let Some(checkpoint_timeout) = request_data.checkpoint_timeout {
@@ -726,16 +787,22 @@ async fn timeline_gc_handler(mut request: Request<Body>) -> Result<Response<Body
    let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    check_permission(&request, Some(tenant_id))?;

+    // FIXME: currently this will return a 500 error on bad tenant id; it should be 4XX
+    let repo = tenant_mgr::get_tenant(tenant_id, false).map_err(ApiError::NotFound)?;
    let gc_req: TimelineGcRequest = json_request(&mut request).await?;

-    let wait_task_done = tenant_mgr::immediate_gc(tenant_id, timeline_id, gc_req)?;
-    let gc_result = wait_task_done
-        .await
-        .context("wait for gc task")
-        .map_err(ApiError::InternalServerError)?
-        .map_err(ApiError::InternalServerError)?;
+    let _span_guard =
+        info_span!("manual_gc", tenant = %tenant_id, timeline = %timeline_id).entered();
+    let gc_horizon = gc_req.gc_horizon.unwrap_or_else(|| repo.get_gc_horizon());

-    json_response(StatusCode::OK, gc_result)
+    // Use tenant's pitr setting
+    let pitr = repo.get_pitr_interval();
+    let result = repo
+        .gc_iteration(Some(timeline_id), gc_horizon, pitr, true)
+        // FIXME: `gc_iteration` can return an error for multiple reasons; we should handle it
+        // better once the types support it.
+        .map_err(ApiError::InternalServerError)?;
+    json_response(StatusCode::OK, result)
 }

 // Run compaction immediately on given timeline.
@@ -745,14 +812,12 @@ async fn timeline_compact_handler(request: Request<Body>) -> Result<Response<Bod
    let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    check_permission(&request, Some(tenant_id))?;

-    let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
-    let timeline = tenant
-        .get_timeline(timeline_id, true)
+    let repo = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
+    let timeline = repo
+        .get_timeline(timeline_id)
+        .with_context(|| format!("No timeline {timeline_id} in repository for tenant {tenant_id}"))
        .map_err(ApiError::NotFound)?;
-    timeline
-        .compact()
-        .await
-        .map_err(ApiError::InternalServerError)?;
+    timeline.compact().map_err(ApiError::InternalServerError)?;

    json_response(StatusCode::OK, ())
 }
@@ -764,13 +829,13 @@ async fn timeline_checkpoint_handler(request: Request<Body>) -> Result<Response<
    let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    check_permission(&request, Some(tenant_id))?;

-    let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
-    let timeline = tenant
-        .get_timeline(timeline_id, true)
+    let repo = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
+    let timeline = repo
+        .get_timeline(timeline_id)
+        .with_context(|| format!("No timeline {timeline_id} in repository for tenant {tenant_id}"))
        .map_err(ApiError::NotFound)?;
    timeline
        .checkpoint(CheckpointConfig::Forced)
-        .await
        .map_err(ApiError::InternalServerError)?;

    json_response(StatusCode::OK, ())
@@ -786,6 +851,7 @@ async fn handler_404(_: Request<Body>) -> Result<Response<Body>, ApiError> {
 pub fn make_router(
    conf: &'static PageServerConf,
    auth: Option<Arc<JwtAuth>>,
+    remote_index: RemoteIndex,
    remote_storage: Option<GenericRemoteStorage>,
 ) -> anyhow::Result<RouterBuilder<hyper::Body, ApiError>> {
    let spec = include_bytes!("openapi_spec.yml");
@@ -822,7 +888,8 @@ pub fn make_router(

    Ok(router
        .data(Arc::new(
-            State::new(conf, auth, remote_storage).context("Failed to initialize router state")?,
+            State::new(conf, auth, remote_index, remote_storage)
+                .context("Failed to initialize router state")?,
        ))
        .get("/v1/status", status_handler)
        .put(
@@ -832,7 +899,6 @@ pub fn make_router(
        .get("/v1/tenant", tenant_list_handler)
        .post("/v1/tenant", tenant_create_handler)
        .get("/v1/tenant/:tenant_id", tenant_status)
-        .get("/v1/tenant/:tenant_id/size", tenant_size_handler)
        .put("/v1/tenant/config", tenant_config_handler)
        .get("/v1/tenant/:tenant_id/timeline", timeline_list_handler)
        .post("/v1/tenant/:tenant_id/timeline", timeline_create_handler)
--- a/pageserver/src/import_datadir.rs
+++ b/pageserver/src/import_datadir.rs
@@ -12,10 +12,10 @@ use tracing::*;
 use walkdir::WalkDir;

 use crate::pgdatadir_mapping::*;
+use crate::reltag::{RelTag, SlruKind};
 use crate::tenant::Timeline;
 use crate::walingest::WalIngest;
 use crate::walrecord::DecodedWALRecord;
-use pageserver_api::reltag::{RelTag, SlruKind};
 use postgres_ffi::pg_constants;
 use postgres_ffi::relfile_utils::*;
 use postgres_ffi::waldecoder::WalStreamDecoder;
--- a/pageserver/src/lib.rs
+++ b/pageserver/src/lib.rs
@@ -1,14 +1,14 @@
-mod auth;
 pub mod basebackup;
 pub mod config;
 pub mod http;
 pub mod import_datadir;
 pub mod keyspace;
-pub(crate) mod metrics;
+pub mod metrics;
 pub mod page_cache;
 pub mod page_service;
 pub mod pgdatadir_mapping;
 pub mod profiling;
+pub mod reltag;
 pub mod repository;
 pub mod storage_sync;
 pub mod task_mgr;
@@ -16,16 +16,16 @@ pub mod tenant;
 pub mod tenant_config;
 pub mod tenant_mgr;
 pub mod tenant_tasks;
-pub mod trace;
 pub mod virtual_file;
 pub mod walingest;
 pub mod walreceiver;
 pub mod walrecord;
 pub mod walredo;

-use std::path::Path;
+use std::collections::HashMap;

 use tracing::info;
+use utils::id::{TenantId, TimelineId};

 use crate::task_mgr::TaskKind;

@@ -44,6 +44,8 @@ pub const DEFAULT_PG_VERSION: u32 = 14;
 pub const IMAGE_FILE_MAGIC: u16 = 0x5A60;
 pub const DELTA_FILE_MAGIC: u16 = 0x5A61;

+pub const LOG_FILE_NAME: &str = "pageserver.log";
+
 static ZERO_PAGE: bytes::Bytes = bytes::Bytes::from_static(&[0u8; 8192]);

 /// Config for the Repository checkpointer
@@ -71,7 +73,7 @@ pub async fn shutdown_pageserver(exit_code: i32) {
    //
    // FIXME: Does this wait for the sync tasks to finish syncing what's queued up?
    // Should it?
-    task_mgr::shutdown_tasks(Some(TaskKind::RemoteUploadTask), None, None).await;
+    task_mgr::shutdown_tasks(Some(TaskKind::StorageSync), None, None).await;

    // Shut down the HTTP endpoint last, so that you can still check the server's
    // status while it's shutting down.
@@ -80,6 +82,7 @@ pub async fn shutdown_pageserver(exit_code: i32) {

    // There should be nothing left, but let's be sure
    task_mgr::shutdown_tasks(None, None, None).await;
+
    info!("Shut down successfully completed");
    std::process::exit(exit_code);
 }
@@ -106,39 +109,23 @@ fn exponential_backoff_duration_seconds(n: u32, base_increment: f64, max_seconds
    }
 }

-/// The name of the metadata file pageserver creates per timeline.
-/// Full path: `tenants/<tenant_id>/timelines/<timeline_id>/metadata`.
-pub const METADATA_FILE_NAME: &str = "metadata";
+/// A newtype to store arbitrary data grouped by tenant and timeline ids.
+/// One could use [`utils::id::TenantTimelineId`] for grouping, but that would
+/// not include the cases where a certain tenant has zero timelines.
+/// This is sometimes important: a tenant could be registered during initial load from FS,
+/// even if he has no timelines on disk.
+#[derive(Debug)]
+pub struct TenantTimelineValues<T>(HashMap<TenantId, HashMap<TimelineId, T>>);

-/// Per-tenant configuration file.
-/// Full path: `tenants/<tenant_id>/config`.
-pub const TENANT_CONFIG_NAME: &str = "config";
-
-/// A suffix used for various temporary files. Any temporary files found in the
-/// data directory at pageserver startup can be automatically removed.
-pub const TEMP_FILE_SUFFIX: &str = "___temp";
-
-/// A marker file to mark that a timeline directory was not fully initialized.
-/// If a timeline directory with this marker is encountered at pageserver startup,
-/// the timeline directory and the marker file are both removed.
-/// Full path: `tenants/<tenant_id>/timelines/<timeline_id>___uninit`.
-pub const TIMELINE_UNINIT_MARK_SUFFIX: &str = "___uninit";
-
-pub fn is_temporary(path: &Path) -> bool {
-    match path.file_name() {
-        Some(name) => name.to_string_lossy().ends_with(TEMP_FILE_SUFFIX),
-        None => false,
+impl<T> TenantTimelineValues<T> {
+    fn new() -> Self {
+        Self(HashMap::new())
    }
 }

-pub fn is_uninit_mark(path: &Path) -> bool {
-    match path.file_name() {
-        Some(name) => name
-            .to_string_lossy()
-            .ends_with(TIMELINE_UNINIT_MARK_SUFFIX),
-        None => false,
-    }
-}
+/// A suffix to be used during file sync from the remote storage,
+/// to ensure that we do not leave corrupted files that pretend to be layers.
+const TEMP_FILE_SUFFIX: &str = "___temp";

 #[cfg(test)]
 mod backoff_defaults_tests {
--- a/pageserver/src/metrics.rs
+++ b/pageserver/src/metrics.rs
@@ -1,8 +1,9 @@
 use metrics::core::{AtomicU64, GenericCounter};
 use metrics::{
-    register_histogram, register_histogram_vec, register_int_counter, register_int_counter_vec,
-    register_int_gauge, register_int_gauge_vec, register_uint_gauge_vec, Histogram, HistogramVec,
-    IntCounter, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
+    register_gauge_vec, register_histogram, register_histogram_vec, register_int_counter,
+    register_int_counter_vec, register_int_gauge, register_int_gauge_vec, register_uint_gauge_vec,
+    GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge,
+    UIntGaugeVec,
 };
 use once_cell::sync::Lazy;
 use utils::id::{TenantId, TimelineId};
@@ -30,7 +31,6 @@ const STORAGE_TIME_OPERATIONS: &[&str] = &[
    "compact",
    "create images",
    "init logical size",
-    "logical size",
    "load layer map",
    "gc",
 ];
@@ -199,59 +199,63 @@ pub static NUM_ONDISK_LAYERS: Lazy<IntGauge> = Lazy::new(|| {
        .expect("failed to define a metric")
 });

-// remote storage metrics
-
-pub static REMOTE_UPLOAD_QUEUE_UNFINISHED_TASKS: Lazy<IntGaugeVec> = Lazy::new(|| {
-    register_int_gauge_vec!(
-        "pageserver_remote_upload_queue_unfinished_tasks",
-        "Number of tasks in the upload queue that are not finished yet.",
-        &["tenant_id", "timeline_id", "file_kind", "op_kind"],
+pub static REMAINING_SYNC_ITEMS: Lazy<IntGauge> = Lazy::new(|| {
+    register_int_gauge!(
+        "pageserver_remote_storage_remaining_sync_items",
+        "Number of storage sync items left in the queue"
    )
-    .expect("failed to define a metric")
+    .expect("failed to register pageserver remote storage remaining sync items int gauge")
 });

-#[derive(Debug, Clone, Copy)]
-pub enum RemoteOpKind {
-    Upload,
-    Download,
-    Delete,
-}
-impl RemoteOpKind {
-    pub fn as_str(&self) -> &str {
-        match self {
-            Self::Upload => "upload",
-            Self::Download => "download",
-            Self::Delete => "delete",
-        }
-    }
-}
-
-#[derive(Debug, Clone, Copy)]
-pub enum RemoteOpFileKind {
-    Layer,
-    Index,
-}
-impl RemoteOpFileKind {
-    pub fn as_str(&self) -> &str {
-        match self {
-            Self::Layer => "layer",
-            Self::Index => "index",
-        }
-    }
-}
-
-pub static REMOTE_OPERATION_KINDS: &[&str] = &["upload", "download", "delete"];
-pub static REMOTE_OPERATION_FILE_KINDS: &[&str] = &["layer", "index"];
-pub static REMOTE_OPERATION_STATUSES: &[&str] = &["success", "failure"];
-
-pub static REMOTE_OPERATION_TIME: Lazy<HistogramVec> = Lazy::new(|| {
-    register_histogram_vec!(
-        "pageserver_remote_operation_seconds",
-        "Time spent on remote storage operations. \
-        Grouped by tenant, timeline, operation_kind and status",
-        &["tenant_id", "timeline_id", "file_kind", "op_kind", "status"]
+pub static IMAGE_SYNC_TIME: Lazy<GaugeVec> = Lazy::new(|| {
+    register_gauge_vec!(
+        "pageserver_remote_storage_image_sync_duration",
+        "Time spent to synchronize (up/download) a whole pageserver image",
+        &["tenant_id", "timeline_id"],
    )
-    .expect("failed to define a metric")
+    .expect("failed to register per-timeline pageserver image sync time vec")
+});
+
+pub static IMAGE_SYNC_OPERATION_KINDS: &[&str] = &["upload", "download", "delete"];
+pub static IMAGE_SYNC_STATUS: &[&str] = &["success", "failure", "abort"];
+
+pub static IMAGE_SYNC_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
+    register_int_counter_vec!(
+        "pageserver_remote_storage_image_sync_count",
+        "Number of synchronization operations executed for pageserver images. \
+        Grouped by tenant, timeline, operation_kind and status",
+        &["tenant_id", "timeline_id", "operation_kind", "status"]
+    )
+    .expect("failed to register pageserver image sync count vec")
+});
+
+pub static IMAGE_SYNC_TIME_HISTOGRAM: Lazy<HistogramVec> = Lazy::new(|| {
+    register_histogram_vec!(
+        "pageserver_remote_storage_image_sync_seconds",
+        "Time took to synchronize (download or upload) a whole pageserver image. \
+        Grouped by operation_kind and status",
+        &["operation_kind", "status"],
+        vec![0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 3.0, 10.0, 20.0]
+    )
+    .expect("failed to register pageserver image sync time histogram vec")
+});
+
+pub static REMOTE_INDEX_UPLOAD: Lazy<IntCounterVec> = Lazy::new(|| {
+    register_int_counter_vec!(
+        "pageserver_remote_storage_remote_index_uploads_total",
+        "Number of remote index uploads",
+        &["tenant_id", "timeline_id"],
+    )
+    .expect("failed to register pageserver remote index upload vec")
+});
+
+pub static NO_LAYERS_UPLOAD: Lazy<IntCounterVec> = Lazy::new(|| {
+    register_int_counter_vec!(
+        "pageserver_remote_storage_no_layers_uploads_total",
+        "Number of skipped uploads due to no layers",
+        &["tenant_id", "timeline_id"],
+    )
+    .expect("failed to register pageserver no layers upload vec")
 });

 pub static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
@@ -361,7 +365,6 @@ pub struct TimelineMetrics {
    pub compact_time_histo: Histogram,
    pub create_images_time_histo: Histogram,
    pub init_logical_size_histo: Histogram,
-    pub logical_size_histo: Histogram,
    pub load_layer_map_histo: Histogram,
    pub last_record_gauge: IntGauge,
    pub wait_lsn_time_histo: Histogram,
@@ -394,9 +397,6 @@ impl TimelineMetrics {
        let init_logical_size_histo = STORAGE_TIME
            .get_metric_with_label_values(&["init logical size", &tenant_id, &timeline_id])
            .unwrap();
-        let logical_size_histo = STORAGE_TIME
-            .get_metric_with_label_values(&["logical size", &tenant_id, &timeline_id])
-            .unwrap();
        let load_layer_map_histo = STORAGE_TIME
            .get_metric_with_label_values(&["load layer map", &tenant_id, &timeline_id])
            .unwrap();
@@ -428,7 +428,6 @@ impl TimelineMetrics {
            compact_time_histo,
            create_images_time_histo,
            init_logical_size_histo,
-            logical_size_histo,
            load_layer_map_histo,
            last_record_gauge,
            wait_lsn_time_histo,
@@ -468,90 +467,16 @@ impl Drop for TimelineMetrics {
            let _ = SMGR_QUERY_TIME.remove_label_values(&[op, tenant_id, timeline_id]);
        }

-        let _ = REMOTE_UPLOAD_QUEUE_UNFINISHED_TASKS.remove_label_values(&[tenant_id, timeline_id]);
-        for file_kind in REMOTE_OPERATION_FILE_KINDS {
-            for op in REMOTE_OPERATION_KINDS {
-                for status in REMOTE_OPERATION_STATUSES {
-                    let _ = REMOTE_OPERATION_TIME.remove_label_values(&[
-                        tenant_id,
-                        timeline_id,
-                        file_kind,
-                        op,
-                        status,
-                    ]);
-                }
+        for op in IMAGE_SYNC_OPERATION_KINDS {
+            for status in IMAGE_SYNC_STATUS {
+                let _ = IMAGE_SYNC_COUNT.remove_label_values(&[tenant_id, timeline_id, op, status]);
            }
        }
+
+        let _ = IMAGE_SYNC_TIME.remove_label_values(&[tenant_id, timeline_id]);
    }
 }

 pub fn remove_tenant_metrics(tenant_id: &TenantId) {
    let _ = STORAGE_TIME.remove_label_values(&["gc", &tenant_id.to_string(), "-"]);
 }
-
-use futures::Future;
-use pin_project_lite::pin_project;
-use std::pin::Pin;
-use std::task::{Context, Poll};
-use std::time::Instant;
-
-/// Wrapper future that measures the time spent by a remote storage operation,
-/// and records the time and success/failure as a prometheus metric.
-pub trait MeasureRemoteOp: Sized {
-    fn measure_remote_op(
-        self,
-        tenant_id: TenantId,
-        timeline_id: TimelineId,
-        file_kind: RemoteOpFileKind,
-        op: RemoteOpKind,
-    ) -> MeasuredRemoteOp<Self> {
-        let start = Instant::now();
-        MeasuredRemoteOp {
-            inner: self,
-            tenant_id,
-            timeline_id,
-            file_kind,
-            op,
-            start,
-        }
-    }
-}
-
-impl<T: Sized> MeasureRemoteOp for T {}
-
-pin_project! {
-    pub struct MeasuredRemoteOp<F>
-    {
-        #[pin]
-        inner: F,
-        tenant_id: TenantId,
-        timeline_id: TimelineId,
-        file_kind: RemoteOpFileKind,
-        op: RemoteOpKind,
-        start: Instant,
-    }
-}
-
-impl<F: Future<Output = Result<O, E>>, O, E> Future for MeasuredRemoteOp<F> {
-    type Output = Result<O, E>;
-
-    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        let this = self.project();
-        let poll_result = this.inner.poll(cx);
-        if let Poll::Ready(ref res) = poll_result {
-            let duration = this.start.elapsed();
-            let status = if res.is_ok() { &"success" } else { &"failure" };
-            REMOTE_OPERATION_TIME
-                .get_metric_with_label_values(&[
-                    &this.tenant_id.to_string(),
-                    &this.timeline_id.to_string(),
-                    this.file_kind.as_str(),
-                    this.op.as_str(),
-                    status,
-                ])
-                .unwrap()
-                .observe(duration.as_secs_f64());
-        }
-        poll_result
-    }
-}
--- a/pageserver/src/page_service.rs
+++ b/pageserver/src/page_service.rs
@@ -10,52 +10,198 @@
 //

 use anyhow::{bail, ensure, Context, Result};
-use bytes::Buf;
-use bytes::Bytes;
+use bytes::{Buf, BufMut, Bytes, BytesMut};
 use futures::{Stream, StreamExt};
-use pageserver_api::models::{
-    PagestreamBeMessage, PagestreamDbSizeRequest, PagestreamDbSizeResponse,
-    PagestreamErrorResponse, PagestreamExistsRequest, PagestreamExistsResponse,
-    PagestreamFeMessage, PagestreamGetPageRequest, PagestreamGetPageResponse,
-    PagestreamNblocksRequest, PagestreamNblocksResponse,
-};
-use pq_proto::{BeMessage, FeMessage, RowDescriptor};
 use std::io;
 use std::net::TcpListener;
 use std::str;
 use std::str::FromStr;
 use std::sync::Arc;
-use std::time::Duration;
-use tokio::pin;
 use tokio_util::io::StreamReader;
 use tokio_util::io::SyncIoBridge;
 use tracing::*;
-use utils::id::ConnectionId;
 use utils::{
-    auth::{Claims, JwtAuth, Scope},
+    auth::{self, Claims, JwtAuth, Scope},
    id::{TenantId, TimelineId},
    lsn::Lsn,
    postgres_backend::AuthType,
    postgres_backend_async::{self, PostgresBackend},
+    pq_proto::{BeMessage, FeMessage, RowDescriptor},
    simple_rcu::RcuReadGuard,
 };

-use crate::auth::check_permission;
 use crate::basebackup;
 use crate::config::{PageServerConf, ProfilingConfig};
 use crate::import_datadir::import_wal_from_tar;
 use crate::metrics::{LIVE_CONNECTIONS_COUNT, SMGR_QUERY_TIME};
 use crate::profiling::profpoint_start;
+use crate::reltag::RelTag;
 use crate::task_mgr;
 use crate::task_mgr::TaskKind;
-use crate::tenant::{Tenant, Timeline};
+use crate::tenant::Timeline;
 use crate::tenant_mgr;
-use crate::trace::Tracer;
 use crate::CheckpointConfig;

 use postgres_ffi::pg_constants::DEFAULTTABLESPACE_OID;
 use postgres_ffi::BLCKSZ;

+// Wrapped in libpq CopyData
+enum PagestreamFeMessage {
+    Exists(PagestreamExistsRequest),
+    Nblocks(PagestreamNblocksRequest),
+    GetPage(PagestreamGetPageRequest),
+    DbSize(PagestreamDbSizeRequest),
+}
+
+// Wrapped in libpq CopyData
+enum PagestreamBeMessage {
+    Exists(PagestreamExistsResponse),
+    Nblocks(PagestreamNblocksResponse),
+    GetPage(PagestreamGetPageResponse),
+    Error(PagestreamErrorResponse),
+    DbSize(PagestreamDbSizeResponse),
+}
+
+#[derive(Debug)]
+struct PagestreamExistsRequest {
+    latest: bool,
+    lsn: Lsn,
+    rel: RelTag,
+}
+
+#[derive(Debug)]
+struct PagestreamNblocksRequest {
+    latest: bool,
+    lsn: Lsn,
+    rel: RelTag,
+}
+
+#[derive(Debug)]
+struct PagestreamGetPageRequest {
+    latest: bool,
+    lsn: Lsn,
+    rel: RelTag,
+    blkno: u32,
+}
+
+#[derive(Debug)]
+struct PagestreamDbSizeRequest {
+    latest: bool,
+    lsn: Lsn,
+    dbnode: u32,
+}
+
+#[derive(Debug)]
+struct PagestreamExistsResponse {
+    exists: bool,
+}
+
+#[derive(Debug)]
+struct PagestreamNblocksResponse {
+    n_blocks: u32,
+}
+
+#[derive(Debug)]
+struct PagestreamGetPageResponse {
+    page: Bytes,
+}
+
+#[derive(Debug)]
+struct PagestreamErrorResponse {
+    message: String,
+}
+
+#[derive(Debug)]
+struct PagestreamDbSizeResponse {
+    db_size: i64,
+}
+
+impl PagestreamFeMessage {
+    fn parse(mut body: Bytes) -> anyhow::Result<PagestreamFeMessage> {
+        // TODO these gets can fail
+
+        // these correspond to the NeonMessageTag enum in pagestore_client.h
+        //
+        // TODO: consider using protobuf or serde bincode for less error prone
+        // serialization.
+        let msg_tag = body.get_u8();
+        match msg_tag {
+            0 => Ok(PagestreamFeMessage::Exists(PagestreamExistsRequest {
+                latest: body.get_u8() != 0,
+                lsn: Lsn::from(body.get_u64()),
+                rel: RelTag {
+                    spcnode: body.get_u32(),
+                    dbnode: body.get_u32(),
+                    relnode: body.get_u32(),
+                    forknum: body.get_u8(),
+                },
+            })),
+            1 => Ok(PagestreamFeMessage::Nblocks(PagestreamNblocksRequest {
+                latest: body.get_u8() != 0,
+                lsn: Lsn::from(body.get_u64()),
+                rel: RelTag {
+                    spcnode: body.get_u32(),
+                    dbnode: body.get_u32(),
+                    relnode: body.get_u32(),
+                    forknum: body.get_u8(),
+                },
+            })),
+            2 => Ok(PagestreamFeMessage::GetPage(PagestreamGetPageRequest {
+                latest: body.get_u8() != 0,
+                lsn: Lsn::from(body.get_u64()),
+                rel: RelTag {
+                    spcnode: body.get_u32(),
+                    dbnode: body.get_u32(),
+                    relnode: body.get_u32(),
+                    forknum: body.get_u8(),
+                },
+                blkno: body.get_u32(),
+            })),
+            3 => Ok(PagestreamFeMessage::DbSize(PagestreamDbSizeRequest {
+                latest: body.get_u8() != 0,
+                lsn: Lsn::from(body.get_u64()),
+                dbnode: body.get_u32(),
+            })),
+            _ => bail!("unknown smgr message tag: {},'{:?}'", msg_tag, body),
+        }
+    }
+}
+
+impl PagestreamBeMessage {
+    fn serialize(&self) -> Bytes {
+        let mut bytes = BytesMut::new();
+
+        match self {
+            Self::Exists(resp) => {
+                bytes.put_u8(100); /* tag from pagestore_client.h */
+                bytes.put_u8(resp.exists as u8);
+            }
+
+            Self::Nblocks(resp) => {
+                bytes.put_u8(101); /* tag from pagestore_client.h */
+                bytes.put_u32(resp.n_blocks);
+            }
+
+            Self::GetPage(resp) => {
+                bytes.put_u8(102); /* tag from pagestore_client.h */
+                bytes.put(&resp.page[..]);
+            }
+
+            Self::Error(resp) => {
+                bytes.put_u8(103); /* tag from pagestore_client.h */
+                bytes.put(resp.message.as_bytes());
+                bytes.put_u8(0); // null terminator
+            }
+            Self::DbSize(resp) => {
+                bytes.put_u8(104); /* tag from pagestore_client.h */
+                bytes.put_i64(resp.db_size);
+            }
+        }
+
+        bytes.into()
+    }
+}
+
 fn copyin_stream(pgb: &mut PostgresBackend) -> impl Stream<Item = io::Result<Bytes>> + '_ {
    async_stream::try_stream! {
        loop {
@@ -78,12 +224,6 @@ fn copyin_stream(pgb: &mut PostgresBackend) -> impl Stream<Item = io::Result<Byt
                        FeMessage::CopyData(bytes) => bytes,
                        FeMessage::CopyDone => { break },
                        FeMessage::Sync => continue,
-                        FeMessage::Terminate => {
-                            let msg = format!("client terminated connection with Terminate message during COPY");
-                            pgb.write_message(&BeMessage::ErrorResponse(&msg))?;
-                            Err(io::Error::new(io::ErrorKind::ConnectionReset, msg))?;
-                            break;
-                        }
                        m => {
                            let msg = format!("unexpected message {:?}", m);
                            pgb.write_message(&BeMessage::ErrorResponse(&msg))?;
@@ -95,10 +235,10 @@ fn copyin_stream(pgb: &mut PostgresBackend) -> impl Stream<Item = io::Result<Byt
                    yield copy_data_bytes;
                }
                Ok(None) => {
-                    let msg = "client closed connection during COPY";
+                    let msg = "client closed connection";
                    pgb.write_message(&BeMessage::ErrorResponse(msg))?;
                    pgb.flush().await?;
-                    Err(io::Error::new(io::ErrorKind::ConnectionReset, msg))?;
+                    Err(io::Error::new(io::ErrorKind::Other, msg))?;
                }
                Err(e) => {
                    Err(io::Error::new(io::ErrorKind::Other, e))?;
@@ -279,20 +419,8 @@ impl PageServerHandler {
        //       so there is no need to reset the association
        task_mgr::associate_with(Some(tenant_id), Some(timeline_id));

-        // Make request tracer if needed
-        let tenant = get_active_tenant_with_timeout(tenant_id).await?;
-        let mut tracer = if tenant.get_trace_read_requests() {
-            let connection_id = ConnectionId::generate();
-            let path = tenant
-                .conf
-                .trace_path(&tenant_id, &timeline_id, &connection_id);
-            Some(Tracer::new(path))
-        } else {
-            None
-        };
-
        // Check that the timeline exists
-        let timeline = tenant.get_timeline(timeline_id, true)?;
+        let timeline = get_local_timeline(tenant_id, timeline_id)?;

        // switch client to COPYBOTH
        pgb.write_message(&BeMessage::CopyBothResponse)?;
@@ -323,12 +451,7 @@ impl PageServerHandler {

            trace!("query: {copy_data_bytes:?}");

-            // Trace request if needed
-            if let Some(t) = tracer.as_mut() {
-                t.trace(&copy_data_bytes)
-            }
-
-            let neon_fe_msg = PagestreamFeMessage::parse(&mut copy_data_bytes.reader())?;
+            let neon_fe_msg = PagestreamFeMessage::parse(copy_data_bytes)?;

            let response = match neon_fe_msg {
                PagestreamFeMessage::Exists(req) => {
@@ -377,7 +500,7 @@ impl PageServerHandler {
        task_mgr::associate_with(Some(tenant_id), Some(timeline_id));
        // Create empty timeline
        info!("creating new timeline");
-        let tenant = get_active_tenant_with_timeout(tenant_id).await?;
+        let tenant = tenant_mgr::get_tenant(tenant_id, true)?;
        let timeline = tenant.create_empty_timeline(timeline_id, base_lsn, pg_version)?;

        // TODO mark timeline as not ready until it reaches end_lsn.
@@ -395,12 +518,14 @@ impl PageServerHandler {
        pgb.write_message(&BeMessage::CopyInResponse)?;
        pgb.flush().await?;

-        let copyin_stream = copyin_stream(pgb);
-        pin!(copyin_stream);
-
-        timeline
-            .import_basebackup_from_tar(&mut copyin_stream, base_lsn)
-            .await?;
+        // import_basebackup_from_tar() is not async, mainly because the Tar crate
+        // it uses is not async. So we need to jump through some hoops:
+        // - convert the input from client connection to a synchronous Read
+        // - use block_in_place()
+        let mut copyin_stream = Box::pin(copyin_stream(pgb));
+        let reader = SyncIoBridge::new(StreamReader::new(&mut copyin_stream));
+        tokio::task::block_in_place(|| timeline.import_basebackup_from_tar(reader, base_lsn))?;
+        timeline.initialize()?;

        // Drain the rest of the Copy data
        let mut bytes_after_tar = 0;
@@ -432,7 +557,7 @@ impl PageServerHandler {
    ) -> anyhow::Result<()> {
        task_mgr::associate_with(Some(tenant_id), Some(timeline_id));

-        let timeline = get_active_timeline_with_timeout(tenant_id, timeline_id).await?;
+        let timeline = get_local_timeline(tenant_id, timeline_id)?;
        ensure!(timeline.get_last_record_lsn() == start_lsn);

        // TODO leave clean state on error. For now you can use detach to clean
@@ -465,7 +590,7 @@ impl PageServerHandler {
        // We only want to persist the data, and it doesn't matter if it's in the
        // shape of deltas or images.
        info!("flushing layers");
-        timeline.checkpoint(CheckpointConfig::Flush).await?;
+        timeline.checkpoint(CheckpointConfig::Flush)?;

        info!("done");
        Ok(())
@@ -625,7 +750,7 @@ impl PageServerHandler {
        full_backup: bool,
    ) -> anyhow::Result<()> {
        // check that the timeline exists
-        let timeline = get_active_timeline_with_timeout(tenant_id, timeline_id).await?;
+        let timeline = get_local_timeline(tenant_id, timeline_id)?;
        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
        if let Some(lsn) = lsn {
            // Backup was requested at a particular LSN. Wait for it to arrive.
@@ -672,7 +797,7 @@ impl PageServerHandler {
            .claims
            .as_ref()
            .expect("claims presence already checked");
-        check_permission(claims, tenant_id)
+        auth::check_permission(claims, tenant_id)
    }
 }

@@ -767,7 +892,7 @@ impl postgres_backend_async::Handler for PageServerHandler {
            let timeline_id = TimelineId::from_str(params[1])?;

            self.check_permission(Some(tenant_id))?;
-            let timeline = get_active_timeline_with_timeout(tenant_id, timeline_id).await?;
+            let timeline = get_local_timeline(tenant_id, timeline_id)?;

            let end_of_timeline = timeline.get_last_record_rlsn();

@@ -890,7 +1015,7 @@ impl postgres_backend_async::Handler for PageServerHandler {

            self.check_permission(Some(tenant_id))?;

-            let tenant = get_active_tenant_with_timeout(tenant_id).await?;
+            let tenant = tenant_mgr::get_tenant(tenant_id, true)?;
            pgb.write_message(&BeMessage::RowDescription(&[
                RowDescriptor::int8_col(b"checkpoint_distance"),
                RowDescriptor::int8_col(b"checkpoint_timeout"),
@@ -934,29 +1059,8 @@ impl postgres_backend_async::Handler for PageServerHandler {
    }
 }

-/// Get active tenant.
-///
-/// If the tenant is Loading, waits for it to become Active, for up to 30 s. That
-/// ensures that queries don't fail immediately after pageserver startup, because
-/// all tenants are still loading.
-async fn get_active_tenant_with_timeout(tenant_id: TenantId) -> Result<Arc<Tenant>> {
-    let tenant = tenant_mgr::get_tenant(tenant_id, false)?;
-    match tokio::time::timeout(Duration::from_secs(30), tenant.wait_to_become_active()).await {
-        Ok(wait_result) => wait_result
-            // no .context(), the error message is good enough and some tests depend on it
-            .map(move |()| tenant),
-        Err(_) => anyhow::bail!("Timeout waiting for tenant {tenant_id} to become Active"),
-    }
-}
-
-/// Shorthand for getting a reference to a Timeline of an Active tenant.
-async fn get_active_timeline_with_timeout(
-    tenant_id: TenantId,
-    timeline_id: TimelineId,
-) -> Result<Arc<Timeline>> {
-    get_active_tenant_with_timeout(tenant_id)
-        .await
-        .and_then(|tenant| tenant.get_timeline(timeline_id, true))
+fn get_local_timeline(tenant_id: TenantId, timeline_id: TimelineId) -> Result<Arc<Timeline>> {
+    tenant_mgr::get_tenant(tenant_id, true).and_then(|tenant| tenant.get_timeline(timeline_id))
 }

 ///
--- a/pageserver/src/pgdatadir_mapping.rs
+++ b/pageserver/src/pgdatadir_mapping.rs
@@ -7,12 +7,12 @@
 //! Clarify that)
 //!
 use crate::keyspace::{KeySpace, KeySpaceAccum};
+use crate::reltag::{RelTag, SlruKind};
 use crate::repository::*;
 use crate::tenant::Timeline;
 use crate::walrecord::NeonWalRecord;
 use anyhow::{bail, ensure, Result};
 use bytes::{Buf, Bytes};
-use pageserver_api::reltag::{RelTag, SlruKind};
 use postgres_ffi::relfile_utils::{FSM_FORKNUM, VISIBILITYMAP_FORKNUM};
 use postgres_ffi::BLCKSZ;
 use postgres_ffi::{Oid, TimestampTz, TransactionId};
--- a/libs/pageserver_api/src/reltag.rs
+++ b/libs/pageserver_api/src/reltag.rs
--- a/Show More
+++ b/Show More