Use B-Tree instead of R-Tree

.github/PULL_REQUEST_TEMPLATE/release-pr.md

@@ -10,7 +10,7 @@
 <!-- List everything that should be done **before** release, any issues / setting changes / etc -->
 
 ### Checklist after release
-- [ ] Based on the merged commits write release notes and open a PR into `website` repo ([example](https://github.com/neondatabase/website/pull/219/files))
+- [ ] Based on the merged commits write release notes and open a PR into `website` repo ([example](https://github.com/neondatabase/website/pull/120/files))
 - [ ] Check [#dev-production-stream](https://neondb.slack.com/archives/C03F5SM1N02) Slack channel
 - [ ] Check [stuck projects page](https://console.neon.tech/admin/projects?sort=last_active&order=desc&stuck=true)
 - [ ] Check [recent operation failures](https://console.neon.tech/admin/operations?action=create_timeline%2Cstart_compute%2Cstop_compute%2Csuspend_compute%2Capply_config%2Cdelete_timeline%2Cdelete_tenant%2Ccreate_branch%2Ccheck_availability&sort=updated_at&order=desc&had_retries=some)

.github/actions/allure-report/action.yml

@@ -47,7 +47,7 @@ runs:
         else
           key=branch-$(echo ${GITHUB_REF#refs/heads/} | tr -c "[:alnum:]._-" "-")
         fi
-        echo "KEY=${key}" >> $GITHUB_OUTPUT
+        echo "::set-output name=KEY::${key}"
 
     - uses: actions/setup-java@v3
       if: ${{ inputs.action == 'generate' }}
@@ -186,7 +186,7 @@ runs:
         aws s3 cp --only-show-errors ./index.html "s3://${BUCKET}/${REPORT_PREFIX}/latest/index.html"
 
         echo "[Allure Report](${REPORT_URL})" >> ${GITHUB_STEP_SUMMARY}
-        echo "report-url=${REPORT_URL}" >> $GITHUB_OUTPUT
+        echo "::set-output name=report-url::${REPORT_URL}"
 
     - name: Release Allure lock
       if: ${{ inputs.action == 'generate' && always() }}

.github/actions/download/action.yml

@@ -34,7 +34,7 @@ runs:
         S3_KEY=$(aws s3api list-objects-v2 --bucket ${BUCKET} --prefix ${PREFIX%$GITHUB_RUN_ATTEMPT} | jq -r '.Contents[].Key' | grep ${FILENAME} | sort --version-sort | tail -1 || true)
         if [ -z "${S3_KEY}" ]; then
           if [ "${SKIP_IF_DOES_NOT_EXIST}" = "true" ]; then
-            echo 'SKIPPED=true' >> $GITHUB_OUTPUT
+            echo '::set-output name=SKIPPED::true'
             exit 0
           else
             echo 2>&1 "Neither s3://${BUCKET}/${PREFIX}/${FILENAME} nor its version from previous attempts exist"
@@ -42,7 +42,7 @@ runs:
           fi
         fi
 
-        echo 'SKIPPED=false' >> $GITHUB_OUTPUT
+        echo '::set-output name=SKIPPED::false'
 
         mkdir -p $(dirname $ARCHIVE)
         time aws s3 cp --only-show-errors s3://${BUCKET}/${S3_KEY} ${ARCHIVE}

.github/actions/neon-project-create/action.yml

@@ -41,8 +41,8 @@ runs:
             ;;
         esac
 
-        echo "api_host=${API_HOST}" >> $GITHUB_OUTPUT
-        echo "region_id=${REGION_ID}" >> $GITHUB_OUTPUT
+        echo "::set-output name=api_host::${API_HOST}"
+        echo "::set-output name=region_id::${REGION_ID}"
       env:
         ENVIRONMENT: ${{ inputs.environment }}
         REGION_ID: ${{ inputs.region_id }}
@@ -72,10 +72,10 @@ runs:
 
         dsn=$(echo $project | jq --raw-output '.roles[] | select(.name != "web_access") | .dsn')/main
         echo "::add-mask::${dsn}"
-        echo "dsn=${dsn}" >> $GITHUB_OUTPUT
+        echo "::set-output name=dsn::${dsn}"
 
         project_id=$(echo $project | jq --raw-output '.id')
-        echo "project_id=${project_id}" >> $GITHUB_OUTPUT
+        echo "::set-output name=project_id::${project_id}"
       env:
         API_KEY: ${{ inputs.api_key }}
         API_HOST: ${{ steps.parse-input.outputs.api_host }}

.github/actions/neon-project-delete/action.yml

@@ -32,7 +32,7 @@ runs:
             ;;
         esac
 
-        echo "api_host=${API_HOST}" >> $GITHUB_OUTPUT
+        echo "::set-output name=api_host::${API_HOST}"
       env:
         ENVIRONMENT: ${{ inputs.environment }}
 

.github/actions/run-python-test-set/action.yml

@@ -55,22 +55,6 @@ runs:
         name: neon-${{ runner.os }}-${{ inputs.build_type }}-artifact
         path: /tmp/neon
 
-    - name: Download Neon binaries for the previous release
-      if: inputs.build_type != 'remote'
-      uses: ./.github/actions/download
-      with:
-        name: neon-${{ runner.os }}-${{ inputs.build_type }}-artifact
-        path: /tmp/neon-previous
-        prefix: latest
-
-    - name: Download compatibility snapshot for Postgres 14
-      if: inputs.build_type != 'remote'
-      uses: ./.github/actions/download
-      with:
-        name: compatibility-snapshot-${{ inputs.build_type }}-pg14
-        path: /tmp/compatibility_snapshot_pg14
-        prefix: latest
-
     - name: Checkout
       if: inputs.needs_postgres_source == 'true'
       uses: actions/checkout@v3
@@ -92,15 +76,10 @@ runs:
     - name: Run pytest
       env:
         NEON_BIN: /tmp/neon/bin
-        COMPATIBILITY_NEON_BIN: /tmp/neon-previous/bin
-        COMPATIBILITY_POSTGRES_DISTRIB_DIR: /tmp/neon-previous/pg_install
         TEST_OUTPUT: /tmp/test_output
         BUILD_TYPE: ${{ inputs.build_type }}
         AWS_ACCESS_KEY_ID: ${{ inputs.real_s3_access_key_id }}
         AWS_SECRET_ACCESS_KEY: ${{ inputs.real_s3_secret_access_key }}
-        COMPATIBILITY_SNAPSHOT_DIR: /tmp/compatibility_snapshot_pg14
-        ALLOW_BACKWARD_COMPATIBILITY_BREAKAGE: contains(github.event.pull_request.labels.*.name, 'backward compatibility breakage')
-        ALLOW_FORWARD_COMPATIBILITY_BREAKAGE: contains(github.event.pull_request.labels.*.name, 'forward compatibility breakage')
       shell: bash -euxo pipefail {0}
       run: |
         # PLATFORM will be embedded in the perf test report
@@ -123,12 +102,7 @@ runs:
           exit 1
         fi
         if [[ "${{ inputs.run_in_parallel }}" == "true" ]]; then
-          # -n4 uses four processes to run tests via pytest-xdist
           EXTRA_PARAMS="-n4 $EXTRA_PARAMS"
-
-          # --dist=loadgroup points tests marked with @pytest.mark.xdist_group
-          # to the same worker to make @pytest.mark.order work with xdist
-          EXTRA_PARAMS="--dist=loadgroup $EXTRA_PARAMS"
         fi
 
         if [[ "${{ inputs.run_with_real_s3 }}" == "true" ]]; then
@@ -163,9 +137,9 @@ runs:
         # --verbose prints name of each test (helpful when there are
         # multiple tests in one file)
         # -rA prints summary in the end
+        # -n4 uses four processes to run tests via pytest-xdist
         # -s is not used to prevent pytest from capturing output, because tests are running
         # in parallel and logs are mixed between different tests
-        #
         mkdir -p $TEST_OUTPUT/allure/results
         "${cov_prefix[@]}" ./scripts/pytest \
           --junitxml=$TEST_OUTPUT/junit.xml \
@@ -180,17 +154,8 @@ runs:
           scripts/generate_and_push_perf_report.sh
         fi
 
-    - name: Upload compatibility snapshot for Postgres 14
-      if: github.ref_name == 'release'
-      uses: ./.github/actions/upload
-      with:
-        name: compatibility-snapshot-${{ inputs.build_type }}-pg14-${{ github.run_id }}
-        # The path includes a test name (test_create_snapshot) and directory that the test creates (compatibility_snapshot_pg14), keep the path in sync with the test
-        path: /tmp/test_output/test_create_snapshot/compatibility_snapshot_pg14/
-        prefix: latest
-
     - name: Create Allure report
-      if: success() || failure()
+      if: always()
       uses: ./.github/actions/allure-report
       with:
         action: store

.github/ansible/.gitignore

@@ -2,6 +2,3 @@ zenith_install.tar.gz
 .zenith_current_version
 neon_install.tar.gz
 .neon_current_version
-
-collections/*
-!collections/.keep

.github/ansible/ansible.cfg

@@ -3,7 +3,6 @@
 localhost_warning = False
 host_key_checking = False
 timeout = 30
-collections_paths = ./collections
 
 [ssh_connection]
 ssh_args   = -F ./ansible.ssh.cfg

.github/ansible/deploy.yaml

@@ -1,7 +1,7 @@
 - name: Upload Neon binaries
   hosts: storage
   gather_facts: False
-  remote_user: "{{ remote_user }}"
+  remote_user: admin
 
   tasks:
 
@@ -14,8 +14,7 @@
       - safekeeper
 
     - name: inform about versions
-      debug:
-        msg: "Version to deploy - {{ current_version }}"
+      debug: msg="Version to deploy - {{ current_version }}"
       tags:
       - pageserver
       - safekeeper
@@ -36,7 +35,7 @@
 - name: Deploy pageserver
   hosts: pageservers
   gather_facts: False
-  remote_user: "{{ remote_user }}"
+  remote_user: admin
 
   tasks:
 
@@ -64,29 +63,15 @@
       tags:
       - pageserver
 
-    - name: read the existing remote pageserver config
-      ansible.builtin.slurp:
-        src: /storage/pageserver/data/pageserver.toml
-      register: _remote_ps_config
-      tags:
-      - pageserver
-
-    - name: parse the existing pageserver configuration
-      ansible.builtin.set_fact:
-        _existing_ps_config: "{{ _remote_ps_config['content'] | b64decode | sivel.toiletwater.from_toml }}"
-      tags:
-      - pageserver
-
-    - name: construct the final pageserver configuration dict
-      ansible.builtin.set_fact:
-        pageserver_config: "{{ pageserver_config_stub | combine({'id': _existing_ps_config.id }) }}"
-      tags:
-      - pageserver
-
-    - name: template the pageserver config
-      template:
-        src: templates/pageserver.toml.j2
-        dest: /storage/pageserver/data/pageserver.toml
+    - name: update remote storage (s3) config
+      lineinfile:
+        path: /storage/pageserver/data/pageserver.toml
+        line: "{{ item }}"
+      loop:
+        - "[remote_storage]"
+        - "bucket_name = '{{ bucket_name }}'"
+        - "bucket_region = '{{ bucket_region }}'"
+        - "prefix_in_bucket = '{{ inventory_hostname }}'"
       become: true
       tags:
       - pageserver
@@ -124,7 +109,7 @@
 - name: Deploy safekeeper
   hosts: safekeepers
   gather_facts: False
-  remote_user: "{{ remote_user }}"
+  remote_user: admin
 
   tasks:
 

.github/ansible/get_binaries.sh

@@ -23,7 +23,6 @@ docker cp ${ID}:/data/postgres_install.tar.gz .
 tar -xzf postgres_install.tar.gz -C neon_install
 mkdir neon_install/bin/
 docker cp ${ID}:/usr/local/bin/pageserver neon_install/bin/
-docker cp ${ID}:/usr/local/bin/pageserver_binutils neon_install/bin/
 docker cp ${ID}:/usr/local/bin/safekeeper neon_install/bin/
 docker cp ${ID}:/usr/local/bin/proxy neon_install/bin/
 docker cp ${ID}:/usr/local/v14/bin/ neon_install/v14/bin/

.github/ansible/neon-stress.hosts

@@ -0,0 +1,20 @@
+[pageservers]
+neon-stress-ps-1 console_region_id=1
+neon-stress-ps-2 console_region_id=1
+
+[safekeepers]
+neon-stress-sk-1 console_region_id=1
+neon-stress-sk-2 console_region_id=1
+neon-stress-sk-3 console_region_id=1
+
+[storage:children]
+pageservers
+safekeepers
+
+[storage:vars]
+env_name = neon-stress
+console_mgmt_base_url = http://neon-stress-console.local
+bucket_name           = neon-storage-ireland
+bucket_region         = eu-west-1
+etcd_endpoints        = etcd-stress.local:2379
+safekeeper_enable_s3_offload = false

.github/ansible/neon-stress.hosts.yaml

@@ -1,31 +0,0 @@
-storage:
-  vars:
-    bucket_name: neon-storage-ireland
-    bucket_region: eu-west-1
-    console_mgmt_base_url: http://neon-stress-console.local
-    etcd_endpoints: neon-stress-etcd.local:2379
-    safekeeper_enable_s3_offload: 'false'
-    pageserver_config_stub:
-      pg_distrib_dir: /usr/local
-      remote_storage:
-        bucket_name: "{{ bucket_name }}"
-        bucket_region: "{{ bucket_region }}"
-        prefix_in_bucket: "{{ inventory_hostname }}"
-    safekeeper_s3_prefix: neon-stress/wal
-    hostname_suffix: ".local"
-    remote_user: admin
-  children:
-    pageservers:
-      hosts:
-        neon-stress-ps-1:
-          console_region_id: aws-eu-west-1
-        neon-stress-ps-2:
-          console_region_id: aws-eu-west-1
-    safekeepers:
-      hosts:
-        neon-stress-sk-1:
-          console_region_id: aws-eu-west-1
-        neon-stress-sk-2:
-          console_region_id: aws-eu-west-1
-        neon-stress-sk-3:
-          console_region_id: aws-eu-west-1

.github/ansible/prod.ap-southeast-1.hosts.yaml

@@ -1,35 +0,0 @@
-storage:
-  vars:
-    bucket_name: neon-prod-storage-ap-southeast-1
-    bucket_region: ap-southeast-1
-    console_mgmt_base_url: http://console-release.local
-    etcd_endpoints: etcd-0.ap-southeast-1.aws.neon.tech:2379
-    pageserver_config_stub:
-      pg_distrib_dir: /usr/local
-      remote_storage:
-        bucket_name: "{{ bucket_name }}"
-        bucket_region: "{{ bucket_region }}"
-        prefix_in_bucket: "pageserver/v1"
-    safekeeper_s3_prefix: safekeeper/v1/wal
-    hostname_suffix: ""
-    remote_user: ssm-user
-    ansible_aws_ssm_region: ap-southeast-1
-    ansible_aws_ssm_bucket_name: neon-prod-storage-ap-southeast-1
-    console_region_id: aws-ap-southeast-1
-
-  children:
-    pageservers:
-      hosts:
-        pageserver-0.ap-southeast-1.aws.neon.tech:
-          ansible_host:  i-064de8ea28bdb495b
-        pageserver-1.ap-southeast-1.aws.neon.tech:
-          ansible_host:  i-0b180defcaeeb6b93
-
-    safekeepers:
-      hosts:
-        safekeeper-0.ap-southeast-1.aws.neon.tech:
-          ansible_host:  i-0d6f1dc5161eef894
-        safekeeper-1.ap-southeast-1.aws.neon.tech:
-          ansible_host:  i-0e338adda8eb2d19f
-        safekeeper-2.ap-southeast-1.aws.neon.tech:
-          ansible_host:  i-04fb63634e4679eb9

.github/ansible/prod.eu-central-1.hosts.yaml

@@ -1,35 +0,0 @@
-storage:
-  vars:
-    bucket_name: neon-prod-storage-eu-central-1
-    bucket_region: eu-central-1
-    console_mgmt_base_url: http://console-release.local
-    etcd_endpoints: etcd-0.eu-central-1.aws.neon.tech:2379
-    pageserver_config_stub:
-      pg_distrib_dir: /usr/local
-      remote_storage:
-        bucket_name: "{{ bucket_name }}"
-        bucket_region: "{{ bucket_region }}"
-        prefix_in_bucket: "pageserver/v1"
-    safekeeper_s3_prefix: safekeeper/v1/wal
-    hostname_suffix: ""
-    remote_user: ssm-user
-    ansible_aws_ssm_region: eu-central-1
-    ansible_aws_ssm_bucket_name: neon-prod-storage-eu-central-1
-    console_region_id: aws-eu-central-1
-
-  children:
-    pageservers:
-      hosts:
-        pageserver-0.eu-central-1.aws.neon.tech:
-          ansible_host:  i-0cd8d316ecbb715be
-        pageserver-1.eu-central-1.aws.neon.tech:
-          ansible_host:  i-090044ed3d383fef0
-
-    safekeepers:
-      hosts:
-        safekeeper-0.eu-central-1.aws.neon.tech:
-          ansible_host:  i-0b238612d2318a050
-        safekeeper-1.eu-central-1.aws.neon.tech:
-          ansible_host:  i-07b9c45e5c2637cd4
-        safekeeper-2.eu-central-1.aws.neon.tech:
-          ansible_host:  i-020257302c3c93d88

.github/ansible/prod.us-east-2.hosts.yaml

@@ -1,36 +0,0 @@
-storage:
-  vars:
-    bucket_name: neon-prod-storage-us-east-2
-    bucket_region: us-east-2
-    console_mgmt_base_url: http://console-release.local
-    etcd_endpoints: etcd-0.us-east-2.aws.neon.tech:2379
-    pageserver_config_stub:
-      pg_distrib_dir: /usr/local
-      remote_storage:
-        bucket_name: "{{ bucket_name }}"
-        bucket_region: "{{ bucket_region }}"
-        prefix_in_bucket: "pageserver/v1"
-    safekeeper_s3_prefix: safekeeper/v1/wal
-    hostname_suffix: ""
-    remote_user: ssm-user
-    ansible_aws_ssm_region: us-east-2
-    ansible_aws_ssm_bucket_name: neon-prod-storage-us-east-2
-    console_region_id: aws-us-east-2
-
-  children:
-    pageservers:
-      hosts:
-        pageserver-0.us-east-2.aws.neon.tech:
-          ansible_host:  i-062227ba7f119eb8c
-        pageserver-1.us-east-2.aws.neon.tech:
-          ansible_host:  i-0b3ec0afab5968938
-
-    safekeepers:
-      hosts:
-        safekeeper-0.us-east-2.aws.neon.tech:
-          ansible_host:  i-0e94224750c57d346
-        safekeeper-1.us-east-2.aws.neon.tech:
-          ansible_host:  i-06d113fb73bfddeb0
-        safekeeper-2.us-east-2.aws.neon.tech:
-          ansible_host:  i-09f66c8e04afff2e8
-          

.github/ansible/production.hosts

@@ -0,0 +1,20 @@
+[pageservers]
+#zenith-1-ps-1 console_region_id=1
+zenith-1-ps-2 console_region_id=1
+zenith-1-ps-3 console_region_id=1
+
+[safekeepers]
+zenith-1-sk-1 console_region_id=1
+zenith-1-sk-2 console_region_id=1
+zenith-1-sk-3 console_region_id=1
+
+[storage:children]
+pageservers
+safekeepers
+
+[storage:vars]
+env_name = prod-1
+console_mgmt_base_url = http://console-release.local
+bucket_name           = zenith-storage-oregon
+bucket_region         = us-west-2
+etcd_endpoints        = zenith-1-etcd.local:2379

.github/ansible/production.hosts.yaml

@@ -1,37 +0,0 @@
----
-storage:
-  vars:
-    console_mgmt_base_url: http://console-release.local
-    bucket_name: zenith-storage-oregon
-    bucket_region: us-west-2
-    etcd_endpoints: zenith-1-etcd.local:2379
-    pageserver_config_stub:
-      pg_distrib_dir: /usr/local
-      remote_storage:
-        bucket_name: "{{ bucket_name }}"
-        bucket_region: "{{ bucket_region }}"
-        prefix_in_bucket: "{{ inventory_hostname }}"
-    safekeeper_s3_prefix: prod-1/wal
-    hostname_suffix: ".local"
-    remote_user: admin
-
-  children:
-    pageservers:
-      hosts:
-        zenith-1-ps-2:
-          console_region_id: aws-us-west-2
-        zenith-1-ps-3:
-          console_region_id: aws-us-west-2
-        zenith-1-ps-4:
-          console_region_id: aws-us-west-2
-        zenith-1-ps-5:
-          console_region_id: aws-us-west-2
-
-    safekeepers:
-      hosts:
-        zenith-1-sk-1:
-          console_region_id: aws-us-west-2
-        zenith-1-sk-2:
-          console_region_id: aws-us-west-2
-        zenith-1-sk-3:
-          console_region_id: aws-us-west-2

.github/ansible/scripts/init_pageserver.sh

@@ -12,19 +12,18 @@ cat <<EOF | tee /tmp/payload
   "version": 1,
   "host": "${HOST}",
   "port": 6400,
-  "region_id": "{{ console_region_id }}",
+  "region_id": {{ console_region_id }},
   "instance_id": "${INSTANCE_ID}",
   "http_host": "${HOST}",
-  "http_port": 9898,
-  "active": false
+  "http_port": 9898
 }
 EOF
 
 # check if pageserver already registered or not
-if ! curl -sf -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/pageservers/${INSTANCE_ID} -o /dev/null; then
+if ! curl -sf -X PATCH -d '{}' {{ console_mgmt_base_url }}/api/v1/pageservers/${INSTANCE_ID} -o /dev/null; then
 
     # not registered, so register it now
-    ID=$(curl -sf -X POST -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/pageservers -d@/tmp/payload | jq -r '.id')
+    ID=$(curl -sf -X POST {{ console_mgmt_base_url }}/api/v1/pageservers -d@/tmp/payload | jq -r '.ID')
 
     # init pageserver
     sudo -u pageserver /usr/local/bin/pageserver -c "id=${ID}" -c "pg_distrib_dir='/usr/local'" --init -D /storage/pageserver/data

.github/ansible/scripts/init_safekeeper.sh

@@ -14,18 +14,18 @@ cat <<EOF | tee /tmp/payload
   "host": "${HOST}",
   "port": 6500,
   "http_port": 7676,
-  "region_id": "{{ console_region_id }}",
+  "region_id": {{ console_region_id }},
   "instance_id": "${INSTANCE_ID}",
-  "availability_zone_id": "${AZ_ID}",
-  "active": false
+  "availability_zone_id": "${AZ_ID}"
 }
 EOF
 
 # check if safekeeper already registered or not
-if ! curl -sf -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/safekeepers/${INSTANCE_ID} -o /dev/null; then
+if ! curl -sf -X PATCH -d '{}' {{ console_mgmt_base_url }}/api/v1/safekeepers/${INSTANCE_ID} -o /dev/null; then
 
     # not registered, so register it now
-    ID=$(curl -sf -X POST -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/safekeepers -d@/tmp/payload | jq -r '.id')
+    ID=$(curl -sf -X POST {{ console_mgmt_base_url }}/api/v1/safekeepers -d@/tmp/payload | jq -r '.ID')
+
     # init safekeeper
     sudo -u safekeeper /usr/local/bin/safekeeper --id ${ID} --init -D /storage/safekeeper/data
 fi

.github/ansible/ssm_config

@@ -1,2 +0,0 @@
-ansible_connection: aws_ssm
-ansible_python_interpreter: /usr/bin/python3

.github/ansible/staging.hosts

@@ -0,0 +1,25 @@
+[pageservers]
+#zenith-us-stage-ps-1 console_region_id=27
+zenith-us-stage-ps-2 console_region_id=27
+zenith-us-stage-ps-3 console_region_id=27
+zenith-us-stage-ps-4 console_region_id=27
+zenith-us-stage-test-ps-1 console_region_id=28
+
+[safekeepers]
+zenith-us-stage-sk-4 console_region_id=27
+zenith-us-stage-sk-5 console_region_id=27
+zenith-us-stage-sk-6 console_region_id=27
+zenith-us-stage-test-sk-1 console_region_id=28
+zenith-us-stage-test-sk-2 console_region_id=28
+zenith-us-stage-test-sk-3 console_region_id=28
+
+[storage:children]
+pageservers
+safekeepers
+
+[storage:vars]
+env_name = us-stage
+console_mgmt_base_url = http://console-staging.local
+bucket_name           = zenith-staging-storage-us-east-1
+bucket_region         = us-east-1
+etcd_endpoints        = zenith-us-stage-etcd.local:2379

.github/ansible/staging.hosts.yaml

@@ -1,34 +0,0 @@
-storage:
-  vars:
-    bucket_name: zenith-staging-storage-us-east-1
-    bucket_region: us-east-1
-    console_mgmt_base_url: http://console-staging.local
-    etcd_endpoints: etcd-0.us-east-2.aws.neon.build:2379
-    pageserver_config_stub:
-      pg_distrib_dir: /usr/local
-      remote_storage:
-        bucket_name: "{{ bucket_name }}"
-        bucket_region: "{{ bucket_region }}"
-        prefix_in_bucket: "{{ inventory_hostname }}"
-    safekeeper_s3_prefix: us-stage/wal
-    hostname_suffix: ".local"
-    remote_user: admin
-
-  children:
-    pageservers:
-      hosts:
-        zenith-us-stage-ps-2:
-          console_region_id: aws-us-east-1
-        zenith-us-stage-ps-3:
-          console_region_id: aws-us-east-1
-        zenith-us-stage-ps-4:
-          console_region_id: aws-us-east-1
-
-    safekeepers:
-      hosts:
-        zenith-us-stage-sk-4:
-          console_region_id: aws-us-east-1
-        zenith-us-stage-sk-5:
-          console_region_id: aws-us-east-1
-        zenith-us-stage-sk-6:
-          console_region_id: aws-us-east-1

.github/ansible/staging.us-east-2.hosts.yaml

@@ -1,33 +0,0 @@
-storage:
-  vars:
-    bucket_name: neon-staging-storage-us-east-2
-    bucket_region: us-east-2
-    console_mgmt_base_url: http://console-staging.local
-    etcd_endpoints: etcd-0.us-east-2.aws.neon.build:2379
-    pageserver_config_stub:
-      pg_distrib_dir: /usr/local
-      remote_storage:
-        bucket_name: "{{ bucket_name }}"
-        bucket_region: "{{ bucket_region }}"
-        prefix_in_bucket: "pageserver/v1"
-    safekeeper_s3_prefix: safekeeper/v1/wal
-    hostname_suffix: ""
-    remote_user: ssm-user
-    ansible_aws_ssm_region: us-east-2
-    ansible_aws_ssm_bucket_name: neon-staging-storage-us-east-2
-    console_region_id: aws-us-east-2
-
-  children:
-    pageservers:
-      hosts:
-        pageserver-0.us-east-2.aws.neon.build:
-          ansible_host: i-0c3e70929edb5d691
-
-    safekeepers:
-      hosts:
-        safekeeper-0.us-east-2.aws.neon.build:
-          ansible_host: i-027662bd552bf5db0
-        safekeeper-1.us-east-2.aws.neon.build:
-          ansible_host: i-0171efc3604a7b907
-        safekeeper-2.us-east-2.aws.neon.build:
-          ansible_host: i-0de0b03a51676a6ce

.github/ansible/systemd/pageserver.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=Neon pageserver
+Description=Zenith pageserver
 After=network.target auditd.service
 
 [Service]

.github/ansible/systemd/safekeeper.service

@@ -1,12 +1,12 @@
 [Unit]
-Description=Neon safekeeper
+Description=Zenith safekeeper
 After=network.target auditd.service
 
 [Service]
 Type=simple
 User=safekeeper
 Environment=RUST_BACKTRACE=1 NEON_REPO_DIR=/storage/safekeeper/data LD_LIBRARY_PATH=/usr/local/v14/lib
-ExecStart=/usr/local/bin/safekeeper -l {{ inventory_hostname }}{{ hostname_suffix }}:6500 --listen-http {{ inventory_hostname }}{{ hostname_suffix }}:7676 -D /storage/safekeeper/data --broker-endpoints={{ etcd_endpoints }} --remote-storage='{bucket_name="{{bucket_name}}", bucket_region="{{bucket_region}}", prefix_in_bucket="{{ safekeeper_s3_prefix }}"}'
+ExecStart=/usr/local/bin/safekeeper -l {{ inventory_hostname }}.local:6500 --listen-http {{ inventory_hostname }}.local:7676 -D /storage/safekeeper/data --broker-endpoints={{ etcd_endpoints }} --remote-storage='{bucket_name="{{bucket_name}}", bucket_region="{{bucket_region}}", prefix_in_bucket="{{ env_name }}/wal"}'
 ExecReload=/bin/kill -HUP $MAINPID
 KillMode=mixed
 KillSignal=SIGINT

.github/ansible/templates/pageserver.toml.j2

@@ -1 +0,0 @@
-{{ pageserver_config | sivel.toiletwater.to_toml }}

.github/helm-values/dev-us-east-2-beta.neon-proxy-scram.yaml

@@ -1,31 +0,0 @@
-# Helm chart values for neon-proxy-scram.
-# This is a YAML-formatted file.
-
-image:
-  repository: neondatabase/neon
-
-settings:
-  authBackend: "console"
-  authEndpoint: "http://console-staging.local/management/api/v2"
-  domain: "*.us-east-2.aws.neon.build"
-
-# -- Additional labels for neon-proxy pods
-podLabels:
-  zenith_service: proxy-scram
-  zenith_env: dev
-  zenith_region: us-east-2
-  zenith_region_slug: us-east-2
-
-exposedService:
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: us-east-2.aws.neon.build
-
-#metrics:
-#  enabled: true
-#  serviceMonitor:
-#    enabled: true
-#    selector:
-#      release: kube-prometheus-stack

.github/helm-values/prod-ap-southeast-1-epsilon.neon-proxy-scram.yaml

@@ -1,31 +0,0 @@
-# Helm chart values for neon-proxy-scram.
-# This is a YAML-formatted file.
-
-image:
-  repository: neondatabase/neon
-
-settings:
-  authBackend: "console"
-  authEndpoint: "http://console-release.local/management/api/v2"
-  domain: "*.ap-southeast-1.aws.neon.tech"
-
-# -- Additional labels for neon-proxy pods
-podLabels:
-  zenith_service: proxy-scram
-  zenith_env: prod
-  zenith_region: ap-southeast-1
-  zenith_region_slug: ap-southeast-1
-
-exposedService:
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: ap-southeast-1.aws.neon.tech
-
-#metrics:
-#  enabled: true
-#  serviceMonitor:
-#    enabled: true
-#    selector:
-#      release: kube-prometheus-stack

.github/helm-values/prod-eu-central-1-gamma.neon-proxy-scram.yaml

@@ -1,31 +0,0 @@
-# Helm chart values for neon-proxy-scram.
-# This is a YAML-formatted file.
-
-image:
-  repository: neondatabase/neon
-
-settings:
-  authBackend: "console"
-  authEndpoint: "http://console-release.local/management/api/v2"
-  domain: "*.eu-central-1.aws.neon.tech"
-
-# -- Additional labels for neon-proxy pods
-podLabels:
-  zenith_service: proxy-scram
-  zenith_env: prod
-  zenith_region: eu-central-1
-  zenith_region_slug: eu-central-1
-
-exposedService:
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: eu-central-1.aws.neon.tech
-
-#metrics:
-#  enabled: true
-#  serviceMonitor:
-#    enabled: true
-#    selector:
-#      release: kube-prometheus-stack

.github/helm-values/prod-us-east-2-delta.neon-proxy-scram.yaml

@@ -1,31 +0,0 @@
-# Helm chart values for neon-proxy-scram.
-# This is a YAML-formatted file.
-
-image:
-  repository: neondatabase/neon
-
-settings:
-  authBackend: "console"
-  authEndpoint: "http://console-release.local/management/api/v2"
-  domain: "*.us-east-2.aws.neon.tech"
-
-# -- Additional labels for neon-proxy pods
-podLabels:
-  zenith_service: proxy-scram
-  zenith_env: prod
-  zenith_region: us-east-2
-  zenith_region_slug: us-east-2
-
-exposedService:
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: us-east-2.aws.neon.tech
-
-#metrics:
-#  enabled: true
-#  serviceMonitor:
-#    enabled: true
-#    selector:
-#      release: kube-prometheus-stack

.github/workflows/benchmarking.yml

@@ -46,7 +46,7 @@ jobs:
     runs-on: [self-hosted, zenith-benchmarker]
 
     env:
-      POSTGRES_DISTRIB_DIR: /usr/pgsql
+      POSTGRES_DISTRIB_DIR: /tmp/pg_install
       DEFAULT_PG_VERSION: 14
 
     steps:
@@ -138,33 +138,24 @@ jobs:
         SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 
   pgbench-compare:
+    env:
+      TEST_PG_BENCH_DURATIONS_MATRIX: "60m"
+      TEST_PG_BENCH_SCALES_MATRIX: "10gb"
+      POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
+      DEFAULT_PG_VERSION: 14
+      TEST_OUTPUT: /tmp/test_output
+      BUILD_TYPE: remote
+      SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref == 'refs/heads/main' ) }}
+
     strategy:
       fail-fast: false
       matrix:
         # neon-captest-new: Run pgbench in a freshly created project
         # neon-captest-reuse: Same, but reusing existing project
         # neon-captest-prefetch: Same, with prefetching enabled (new project)
-        platform: [ neon-captest-new, neon-captest-reuse, neon-captest-prefetch ]
-        db_size: [ 10gb ]
-        include:
-          - platform: neon-captest-new
-            db_size: 50gb
-          - platform: neon-captest-prefetch
-            db_size: 50gb
-          - platform: rds-aurora
-            db_size: 50gb
+        platform: [ neon-captest-new, neon-captest-reuse, neon-captest-prefetch, rds-aurora ]
 
-    env:
-      TEST_PG_BENCH_DURATIONS_MATRIX: "60m"
-      TEST_PG_BENCH_SCALES_MATRIX: ${{ matrix.db_size }}
-      POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
-      DEFAULT_PG_VERSION: 14
-      TEST_OUTPUT: /tmp/test_output
-      BUILD_TYPE: remote
-      SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref == 'refs/heads/main' ) }}
-      PLATFORM: ${{ matrix.platform }}
-
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rustlegacy:pinned
       options: --init
@@ -187,7 +178,7 @@ jobs:
         echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH
 
     - name: Create Neon Project
-      if: contains(fromJson('["neon-captest-new", "neon-captest-prefetch"]'), matrix.platform)
+      if: matrix.platform != 'neon-captest-reuse'
       id: create-neon-project
       uses: ./.github/actions/neon-project-create
       with:
@@ -213,9 +204,11 @@ jobs:
             ;;
         esac
 
-        echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
+        echo "::set-output name=connstr::${CONNSTR}"
 
         psql ${CONNSTR} -c "SELECT version();"
+      env:
+        PLATFORM: ${{ matrix.platform }}
 
     - name: Set database options
       if: matrix.platform == 'neon-captest-prefetch'
@@ -234,6 +227,7 @@ jobs:
         save_perf_report: ${{ env.SAVE_PERF_REPORT }}
         extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_init
       env:
+        PLATFORM: ${{ matrix.platform }}
         BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
         VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
         PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
@@ -247,6 +241,7 @@ jobs:
         save_perf_report: ${{ env.SAVE_PERF_REPORT }}
         extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_simple_update
       env:
+        PLATFORM: ${{ matrix.platform }}
         BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
         VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
         PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
@@ -260,19 +255,20 @@ jobs:
         save_perf_report: ${{ env.SAVE_PERF_REPORT }}
         extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_select_only
       env:
+        PLATFORM: ${{ matrix.platform }}
         BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
         VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
         PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
 
     - name: Create Allure report
-      if: success() || failure()
+      if: always()
       uses: ./.github/actions/allure-report
       with:
         action: generate
         build_type: ${{ env.BUILD_TYPE }}
 
     - name: Delete Neon Project
-      if: ${{ steps.create-neon-project.outputs.project_id && always() }}
+      if: ${{ matrix.platform != 'neon-captest-reuse' && always() }}
       uses: ./.github/actions/neon-project-delete
       with:
         environment: dev

.github/workflows/build_and_test.yml

@@ -18,8 +18,8 @@ env:
 
 jobs:
   tag:
-    runs-on: [ self-hosted, dev, x64 ]
-    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned
+    runs-on: dev
+    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:latest
     outputs:
       build-tag: ${{steps.build-tag.outputs.tag}}
 
@@ -35,18 +35,18 @@ jobs:
           echo ref:$GITHUB_REF_NAME
           echo rev:$(git rev-list --count HEAD)
           if [[ "$GITHUB_REF_NAME" == "main" ]]; then
-            echo "tag=$(git rev-list --count HEAD)" >> $GITHUB_OUTPUT
+            echo "::set-output name=tag::$(git rev-list --count HEAD)"
           elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
-            echo "tag=release-$(git rev-list --count HEAD)" >> $GITHUB_OUTPUT
+            echo "::set-output name=tag::release-$(git rev-list --count HEAD)"
           else
             echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
-            echo "tag=$GITHUB_RUN_ID" >> $GITHUB_OUTPUT
+            echo "::set-output name=tag::$GITHUB_RUN_ID"
           fi
         shell: bash
         id: build-tag
 
   build-neon:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
@@ -78,12 +78,12 @@ jobs:
 
       - name: Set pg 14 revision for caching
         id: pg_v14_rev
-        run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v14) >> $GITHUB_OUTPUT
+        run: echo ::set-output name=pg_rev::$(git rev-parse HEAD:vendor/postgres-v14)
         shell: bash -euxo pipefail {0}
 
       - name: Set pg 15 revision for caching
         id: pg_v15_rev
-        run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v15) >> $GITHUB_OUTPUT
+        run: echo ::set-output name=pg_rev::$(git rev-parse HEAD:vendor/postgres-v15)
         shell: bash -euxo pipefail {0}
 
       # Set some environment variables used by all the steps.
@@ -127,8 +127,8 @@ jobs:
             target/
           # Fall back to older versions of the key, if no cache for current Cargo.lock was found
           key: |
-            v10-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('Cargo.lock') }}
-            v10-${{ runner.os }}-${{ matrix.build_type }}-cargo-
+            v9-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('Cargo.lock') }}
+            v9-${{ runner.os }}-${{ matrix.build_type }}-cargo-
 
       - name: Cache postgres v14 build
         id: cache_pg_14
@@ -236,7 +236,7 @@ jobs:
         uses: ./.github/actions/save-coverage-data
 
   regress-tests:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
@@ -268,8 +268,34 @@ jobs:
         if: matrix.build_type == 'debug'
         uses: ./.github/actions/save-coverage-data
 
+  upload-latest-artifacts:
+    runs-on: dev
+    container:
+      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
+      options: --init
+    needs: [ regress-tests ]
+    if: github.ref_name == 'main'
+    steps:
+      - name: Copy Neon artifact to the latest directory
+        shell: bash -euxo pipefail {0}
+        env:
+          BUCKET: neon-github-public-dev
+          PREFIX: artifacts/${{ github.run_id }}
+        run: |
+          for build_type in debug release; do
+            FILENAME=neon-${{ runner.os }}-${build_type}-artifact.tar.zst
+
+            S3_KEY=$(aws s3api list-objects-v2 --bucket ${BUCKET} --prefix ${PREFIX} | jq -r '.Contents[].Key' | grep ${FILENAME} | sort --version-sort | tail -1 || true)
+            if [ -z "${S3_KEY}" ]; then
+              echo 2>&1 "Neither s3://${BUCKET}/${PREFIX}/${FILENAME} nor its version from previous attempts exist"
+              exit 1
+            fi
+
+            time aws s3 cp --only-show-errors s3://${BUCKET}/${S3_KEY} s3://${BUCKET}/artifacts/latest/${FILENAME}
+          done
+
   benchmarks:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
@@ -300,12 +326,12 @@ jobs:
       # while coverage is currently collected for the debug ones
 
   merge-allure-report:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
     needs: [ regress-tests, benchmarks ]
-    if: success() || failure()
+    if: always()
     strategy:
       fail-fast: false
       matrix:
@@ -338,7 +364,7 @@ jobs:
           DATABASE_URL="$TEST_RESULT_CONNSTR" poetry run python3 scripts/ingest_regress_test_result.py --revision ${SHA} --reference ${GITHUB_REF} --build-type ${BUILD_TYPE} --ingest suites.json
 
   coverage-report:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
@@ -363,7 +389,7 @@ jobs:
             !~/.cargo/registry/src
             ~/.cargo/git/
             target/
-          key: v10-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('Cargo.lock') }}
+          key: v9-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('Cargo.lock') }}
 
       - name: Get Neon artifact
         uses: ./.github/actions/download
@@ -415,19 +441,15 @@ jobs:
         shell: bash -euxo pipefail {0}
 
   trigger-e2e-tests:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned
       options: --init
-    needs: [ push-docker-hub, tag ]
+    needs: [ build-neon ]
     steps:
       - name: Set PR's status to pending and request a remote CI test
         run: |
-          # For pull requests, GH Actions set "github.sha" variable to point at a fake merge commit
-          # but we need to use a real sha of a latest commit in the PR's branch for the e2e job,
-          # to place a job run status update later.
           COMMIT_SHA=${{ github.event.pull_request.head.sha }}
-          # For non-PR kinds of runs, the above will produce an empty variable, pick the original sha value for those
           COMMIT_SHA=${COMMIT_SHA:-${{ github.sha }}}
 
           REMOTE_REPO="${{ github.repository_owner }}/cloud"
@@ -453,15 +475,12 @@ jobs:
               \"inputs\": {
                 \"ci_job_name\": \"neon-cloud-e2e\",
                 \"commit_hash\": \"$COMMIT_SHA\",
-                \"remote_repo\": \"${{ github.repository }}\",
-                \"storage_image_tag\": \"${{ needs.tag.outputs.build-tag }}\",
-                \"compute_image_tag\": \"${{ needs.tag.outputs.build-tag }}\"
+                \"remote_repo\": \"${{ github.repository }}\"
               }
             }"
 
   neon-image:
-    runs-on: [ self-hosted, dev, x64 ]
-    needs: [ tag ]
+    runs-on: dev
     container: gcr.io/kaniko-project/executor:v1.9.0-debug
 
     steps:
@@ -475,11 +494,10 @@ jobs:
         run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json
 
       - name: Kaniko build neon
-        run: /kaniko/executor --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --build-arg GIT_VERSION=${{ github.sha }} --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}}
+        run: /kaniko/executor --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:$GITHUB_RUN_ID
 
   compute-tools-image:
-    runs-on: [ self-hosted, dev, x64 ]
-    needs: [ tag ]
+    runs-on: dev
     container: gcr.io/kaniko-project/executor:v1.9.0-debug
 
     steps:
@@ -490,12 +508,30 @@ jobs:
         run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json
 
       - name: Kaniko build compute tools
-        run: /kaniko/executor --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-tools --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}}
+        run: /kaniko/executor --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --dockerfile Dockerfile.compute-tools --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:$GITHUB_RUN_ID
+
+  compute-node-image:
+    runs-on: dev
+    container: gcr.io/kaniko-project/executor:v1.9.0-debug
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1 # v3 won't work with kaniko
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Configure ECR login
+        run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json
+
+        # compute-node uses postgres 14, which is default now
+        # cloud repo depends on this image name, thus duplicating it
+        # remove compute-node when cloud repo is updated
+      - name: Kaniko build compute node with extensions v14 (compatibility)
+        run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --dockerfile Dockerfile.compute-node-v14 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:$GITHUB_RUN_ID
 
   compute-node-image-v14:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
     container: gcr.io/kaniko-project/executor:v1.9.0-debug
-    needs: [ tag ]
     steps:
       - name: Checkout
         uses: actions/checkout@v1 # v3 won't work with kaniko
@@ -507,12 +543,12 @@ jobs:
         run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json
 
       - name: Kaniko build compute node with extensions v14
-        run: /kaniko/executor --skip-unused-stages  --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache  --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-node-v14 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}}
+        run: /kaniko/executor --skip-unused-stages  --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache  --context . --dockerfile Dockerfile.compute-node-v14 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:$GITHUB_RUN_ID
+
 
   compute-node-image-v15:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
     container: gcr.io/kaniko-project/executor:v1.9.0-debug
-    needs: [ tag ]
     steps:
       - name: Checkout
         uses: actions/checkout@v1 # v3 won't work with kaniko
@@ -524,69 +560,28 @@ jobs:
         run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json
 
       - name: Kaniko build compute node with extensions v15
-        run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-node-v15 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:${{needs.tag.outputs.build-tag}}
-
-  test-images:
-    needs: [ tag, neon-image, compute-node-image-v14, compute-node-image-v15, compute-tools-image ]
-    runs-on: [ self-hosted, dev, x64 ]
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      # `neondatabase/neon` contains multiple binaries, all of them use the same input for the version into the same version formatting library.
-      # Pick pageserver as currently the only binary with extra "version" features printed in the string to verify.
-      # Regular pageserver version string looks like
-      #   Neon page server git-env:32d14403bd6ab4f4520a94cbfd81a6acef7a526c failpoints: true, features: []
-      # Bad versions might loop like:
-      #   Neon page server git-env:local failpoints: true, features: ["testing"]
-      # Ensure that we don't have bad versions.
-      - name: Verify image versions
-        shell: bash # ensure no set -e for better error messages
-        run: |
-          pageserver_version=$(docker run --rm 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}} "/bin/sh" "-c" "/usr/local/bin/pageserver --version")
-
-          echo "Pageserver version string: $pageserver_version"
-
-          if ! echo "$pageserver_version" | grep -qv 'git-env:local' ; then
-            echo "Pageserver version should not be the default Dockerfile one"
-            exit 1
-          fi
-
-          if ! echo "$pageserver_version" | grep -qv '"testing"' ; then
-            echo "Pageserver version should have no testing feature enabled"
-            exit 1
-          fi
-
-      - name: Verify docker-compose example
-        run: env REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com TAG=${{needs.tag.outputs.build-tag}} ./docker-compose/docker_compose_test.sh
-
-      - name: Print logs and clean up
-        if: always()
-        run: |
-          docker compose -f ./docker-compose/docker-compose.yml logs || 0
-          docker compose -f ./docker-compose/docker-compose.yml down
+        run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --dockerfile Dockerfile.compute-node-v15 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:$GITHUB_RUN_ID
 
   promote-images:
-    runs-on: [ self-hosted, dev, x64 ]
-    needs: [ tag, test-images ]
+    runs-on: dev
+    needs: [ neon-image, compute-node-image, compute-node-image-v14, compute-tools-image ]
     if: github.event_name != 'workflow_dispatch'
     container: amazon/aws-cli
     strategy:
       fail-fast: false
       matrix:
-        name: [ neon, compute-node-v14, compute-node-v15, compute-tools ]
+        # compute-node uses postgres 14, which is default now
+        # cloud repo depends on this image name, thus duplicating it
+        # remove compute-node when cloud repo is updated
+        name: [ neon, compute-node, compute-node-v14, compute-tools ]
 
     steps:
       - name: Promote image to latest
-        run: |
-          export MANIFEST=$(aws ecr batch-get-image --repository-name ${{ matrix.name }} --image-ids imageTag=${{needs.tag.outputs.build-tag}} --query 'images[].imageManifest' --output text)
-          aws ecr put-image --repository-name ${{ matrix.name }} --image-tag latest --image-manifest "$MANIFEST"
+        run:
+          MANIFEST=$(aws ecr batch-get-image --repository-name ${{ matrix.name }} --image-ids imageTag=$GITHUB_RUN_ID --query 'images[].imageManifest' --output text) && aws ecr put-image --repository-name ${{ matrix.name }} --image-tag latest --image-manifest "$MANIFEST"
 
   push-docker-hub:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
     needs: [ promote-images, tag ]
     container: golang:1.19-bullseye
 
@@ -602,16 +597,16 @@ jobs:
           echo "{\"credsStore\":\"ecr-login\"}" > /github/home/.docker/config.json
 
       - name: Pull neon image from ECR
-        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}} neon
+        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:latest neon
 
       - name: Pull compute tools image from ECR
-        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} compute-tools
+        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:latest compute-tools
+
+      - name: Pull compute node image from ECR
+        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:latest compute-node
 
       - name: Pull compute node v14 image from ECR
-        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}} compute-node-v14
-
-      - name: Pull compute node v15 image from ECR
-        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:${{needs.tag.outputs.build-tag}} compute-node-v15
+        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:latest compute-node-v14
 
       - name: Pull rust image from ECR
         run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned rust
@@ -621,10 +616,9 @@ jobs:
           (github.ref_name == 'main' || github.ref_name == 'release') &&
           github.event_name != 'workflow_dispatch'
         run: |
-          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/neon:latest
-          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:latest
-          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:latest
-          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:latest
+          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:$GITHUB_RUN_ID 093970136003.dkr.ecr.us-east-2.amazonaws.com/neon:latest
+          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:$GITHUB_RUN_ID 093970136003.dkr.ecr.us-east-2.amazonaws.com/compute-tools:latest
+          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:$GITHUB_RUN_ID 093970136003.dkr.ecr.us-east-2.amazonaws.com/compute-node:latest
 
       - name: Configure Docker Hub login
         run: |
@@ -638,12 +632,12 @@ jobs:
       - name: Push compute tools image to Docker Hub
         run: crane push compute-tools neondatabase/compute-tools:${{needs.tag.outputs.build-tag}}
 
+      - name: Push compute node image to Docker Hub
+        run: crane push compute-node neondatabase/compute-node:${{needs.tag.outputs.build-tag}}
+
       - name: Push compute node v14 image to Docker Hub
         run: crane push compute-node-v14 neondatabase/compute-node-v14:${{needs.tag.outputs.build-tag}}
 
-      - name: Push compute node v15 image to Docker Hub
-        run: crane push compute-node-v15 neondatabase/compute-node-v15:${{needs.tag.outputs.build-tag}}
-
       - name: Push rust image to Docker Hub
         run: crane push rust neondatabase/rust:pinned
 
@@ -654,8 +648,8 @@ jobs:
         run: |
           crane tag neondatabase/neon:${{needs.tag.outputs.build-tag}} latest
           crane tag neondatabase/compute-tools:${{needs.tag.outputs.build-tag}} latest
+          crane tag neondatabase/compute-node:${{needs.tag.outputs.build-tag}} latest
           crane tag neondatabase/compute-node-v14:${{needs.tag.outputs.build-tag}} latest
-          crane tag neondatabase/compute-node-v15:${{needs.tag.outputs.build-tag}} latest
 
   calculate-deploy-targets:
     runs-on: [ self-hosted, Linux, k8s-runner ]
@@ -668,12 +662,12 @@ jobs:
       - id: set-matrix
         run: |
           if [[ "$GITHUB_REF_NAME" == "main" ]]; then
-            STAGING='{"env_name": "staging", "proxy_job": "neon-proxy", "proxy_config": "staging.proxy", "kubeconfig_secret": "STAGING_KUBECONFIG_DATA", "console_api_key_secret": "NEON_STAGING_API_KEY"}'
-            NEON_STRESS='{"env_name": "neon-stress", "proxy_job": "neon-stress-proxy", "proxy_config": "neon-stress.proxy", "kubeconfig_secret": "NEON_STRESS_KUBECONFIG_DATA", "console_api_key_secret": "NEON_CAPTEST_API_KEY"}'
-            echo "include=[$STAGING, $NEON_STRESS]" >> $GITHUB_OUTPUT
+            STAGING='{"env_name": "staging", "proxy_job": "neon-proxy", "proxy_config": "staging.proxy", "kubeconfig_secret": "STAGING_KUBECONFIG_DATA"}'
+            NEON_STRESS='{"env_name": "neon-stress", "proxy_job": "neon-stress-proxy", "proxy_config": "neon-stress.proxy", "kubeconfig_secret": "NEON_STRESS_KUBECONFIG_DATA"}'
+            echo "::set-output name=include::[$STAGING, $NEON_STRESS]"
           elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
-            PRODUCTION='{"env_name": "production", "proxy_job": "neon-proxy", "proxy_config": "production.proxy", "kubeconfig_secret": "PRODUCTION_KUBECONFIG_DATA", "console_api_key_secret": "NEON_PRODUCTION_API_KEY"}'
-            echo "include=[$PRODUCTION]" >> $GITHUB_OUTPUT
+            PRODUCTION='{"env_name": "production", "proxy_job": "neon-proxy", "proxy_config": "production.proxy", "kubeconfig_secret": "PRODUCTION_KUBECONFIG_DATA"}'
+            echo "::set-output name=include::[$PRODUCTION]"
           else
             echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
             exit 1
@@ -709,7 +703,7 @@ jobs:
       - name: Setup ansible
         run: |
           export PATH="/root/.local/bin:$PATH"
-          pip install --progress-bar off --user ansible boto3 toml
+          pip install --progress-bar off --user ansible boto3
 
       - name: Redeploy
         run: |
@@ -731,92 +725,12 @@ jobs:
           chmod 0600 ssh-key
           ssh-add ssh-key
           rm -f ssh-key ssh-key-cert.pub
-          ansible-galaxy collection install sivel.toiletwater
-          ansible-playbook deploy.yaml -i ${{ matrix.env_name }}.hosts.yaml -e CONSOLE_API_TOKEN=${{ secrets[matrix.console_api_key_secret] }}
-          rm -f neon_install.tar.gz .neon_current_version
 
-  deploy-new:
-    runs-on: [ self-hosted, dev, x64 ]
-    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
-    # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version.
-    # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly
-    needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
-    if: |
-      (github.ref_name == 'main') &&
-      github.event_name != 'workflow_dispatch'
-    defaults:
-      run:
-        shell: bash
-    strategy:
-      matrix:
-        target_region: [ us-east-2 ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          fetch-depth: 0
-
-      - name: Redeploy
-        run: |
-          export DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          cd "$(pwd)/.github/ansible"
-
-          if [[ "$GITHUB_REF_NAME" == "main" ]]; then
-            ./get_binaries.sh
-          elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
-            RELEASE=true ./get_binaries.sh
-          else
-            echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
-            exit 1
-          fi
-
-          ansible-galaxy collection install sivel.toiletwater
-          ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_STAGING_API_KEY}}
-          rm -f neon_install.tar.gz .neon_current_version
-
-  deploy-prod-new:
-    runs-on: prod
-    container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
-    # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version.
-    # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly
-    needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
-    if: |
-      (github.ref_name == 'release') &&
-      github.event_name != 'workflow_dispatch'
-    defaults:
-      run:
-        shell: bash
-    strategy:
-      matrix:
-        target_region: [ us-east-2, eu-central-1, ap-southeast-1 ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          fetch-depth: 0
-
-      - name: Redeploy
-        run: |
-          export DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          cd "$(pwd)/.github/ansible"
-
-          if [[ "$GITHUB_REF_NAME" == "main" ]]; then
-            ./get_binaries.sh
-          elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
-            RELEASE=true ./get_binaries.sh
-          else
-            echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
-            exit 1
-          fi
-
-          ansible-galaxy collection install sivel.toiletwater
-          ansible-playbook deploy.yaml -i prod.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_PRODUCTION_API_KEY}}
+          ansible-playbook deploy.yaml -i ${{ matrix.env_name }}.hosts
           rm -f neon_install.tar.gz .neon_current_version
 
   deploy-proxy:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
     container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:latest
     # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
     needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
@@ -854,111 +768,5 @@ jobs:
       - name: Re-deploy proxy
         run: |
           DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade ${{ matrix.proxy_job }}       neondatabase/neon-proxy --namespace neon-proxy --install -f .github/helm-values/${{ matrix.proxy_config }}.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
-          helm upgrade ${{ matrix.proxy_job }}-scram neondatabase/neon-proxy --namespace neon-proxy --install -f .github/helm-values/${{ matrix.proxy_config }}-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
-
-  deploy-proxy-new:
-    runs-on: [ self-hosted, dev, x64 ]
-    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
-    # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
-    needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
-    if: |
-      (github.ref_name == 'main') &&
-      github.event_name != 'workflow_dispatch'
-    defaults:
-      run:
-        shell: bash
-    strategy:
-      matrix:
-        include:
-          - target_region:  us-east-2
-            target_cluster: dev-us-east-2-beta
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          fetch-depth: 0
-
-      - name: Configure environment
-        run: |
-          helm repo add neondatabase https://neondatabase.github.io/helm-charts
-          aws --region ${{ matrix.target_region }} eks update-kubeconfig --name  ${{ matrix.target_cluster }}
-
-      - name: Re-deploy proxy
-        run: |
-          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
-
-  deploy-proxy-prod-new:
-    runs-on: prod
-    container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
-    # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
-    needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
-    if: |
-      (github.ref_name == 'release') &&
-      github.event_name != 'workflow_dispatch'
-    defaults:
-      run:
-        shell: bash
-    strategy:
-      matrix:
-        include:
-          - target_region:  us-east-2
-            target_cluster: prod-us-east-2-delta
-          - target_region: eu-central-1
-            target_cluster: prod-eu-central-1-gamma
-          - target_region: ap-southeast-1
-            target_cluster: prod-ap-southeast-1-epsilon
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          fetch-depth: 0
-
-      - name: Configure environment
-        run: |
-          helm repo add neondatabase https://neondatabase.github.io/helm-charts
-          aws --region ${{ matrix.target_region }} eks update-kubeconfig --name  ${{ matrix.target_cluster }}
-
-      - name: Re-deploy proxy
-        run: |
-          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
-
-  promote-compatibility-data:
-    runs-on: [ self-hosted, dev, x64 ]
-    container:
-      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
-      options: --init
-    needs: [ deploy, deploy-proxy ]
-    if: github.ref_name == 'release' && github.event_name != 'workflow_dispatch'
-    steps:
-      - name: Promote compatibility snapshot for the release
-        shell: bash -euxo pipefail {0}
-        env:
-          BUCKET: neon-github-public-dev
-          PREFIX: artifacts/latest
-        run: |
-          # Update compatibility snapshot for the release
-          for build_type in debug release; do
-            OLD_FILENAME=compatibility-snapshot-${build_type}-pg14-${GITHUB_RUN_ID}.tar.zst
-            NEW_FILENAME=compatibility-snapshot-${build_type}-pg14.tar.zst
-
-            time aws s3 mv --only-show-errors s3://${BUCKET}/${PREFIX}/${OLD_FILENAME} s3://${BUCKET}/${PREFIX}/${NEW_FILENAME}
-          done
-
-          # Update Neon artifact for the release (reuse already uploaded artifact)
-          for build_type in debug release; do
-            OLD_PREFIX=artifacts/${GITHUB_RUN_ID}
-            FILENAME=neon-${{ runner.os }}-${build_type}-artifact.tar.zst
-
-            S3_KEY=$(aws s3api list-objects-v2 --bucket ${BUCKET} --prefix ${OLD_PREFIX} | jq -r '.Contents[].Key' | grep ${FILENAME} | sort --version-sort | tail -1 || true)
-            if [ -z "${S3_KEY}" ]; then
-              echo 2>&1 "Neither s3://${BUCKET}/${OLD_PREFIX}/${FILENAME} nor its version from previous attempts exist"
-              exit 1
-            fi
-
-            time aws s3 cp --only-show-errors s3://${BUCKET}/${S3_KEY} s3://${BUCKET}/${PREFIX}/${FILENAME}
-          done
+          helm upgrade ${{ matrix.proxy_job }}       neondatabase/neon-proxy --namespace default --install -f .github/helm-values/${{ matrix.proxy_config }}.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
+          helm upgrade ${{ matrix.proxy_job }}-scram neondatabase/neon-proxy --namespace default --install -f .github/helm-values/${{ matrix.proxy_config }}-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s

.github/workflows/codestyle.yml

@@ -36,7 +36,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
         with:
           submodules: true
           fetch-depth: 2
@@ -56,12 +56,12 @@ jobs:
 
       - name: Set pg 14 revision for caching
         id: pg_v14_rev
-        run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v14) >> $GITHUB_OUTPUT
+        run: echo ::set-output name=pg_rev::$(git rev-parse HEAD:vendor/postgres-v14)
         shell: bash -euxo pipefail {0}
 
       - name: Set pg 15 revision for caching
         id: pg_v15_rev
-        run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v15) >> $GITHUB_OUTPUT
+        run: echo ::set-output name=pg_rev::$(git rev-parse HEAD:vendor/postgres-v15)
         shell: bash -euxo pipefail {0}
 
       - name: Cache postgres v14 build
@@ -106,7 +106,7 @@ jobs:
             !~/.cargo/registry/src
             ~/.cargo/git
             target
-          key: v6-${{ runner.os }}-cargo-${{ hashFiles('./Cargo.lock') }}-rust
+          key: v5-${{ runner.os }}-cargo-${{ hashFiles('./Cargo.lock') }}-rust
 
       - name: Run cargo clippy
         run: ./run_clippy.sh
@@ -115,7 +115,7 @@ jobs:
         run: cargo build --locked --all --all-targets
 
   check-rust-dependencies:
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: dev
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "vendor/postgres-v14"]
 	path = vendor/postgres-v14
 	url = https://github.com/neondatabase/postgres.git
-	branch = REL_14_STABLE_neon
+	branch = main
 [submodule "vendor/postgres-v15"]
 	path = vendor/postgres-v15
 	url = https://github.com/neondatabase/postgres.git

CODEOWNERS

@@ -1,10 +0,0 @@
-/compute_tools/ @neondatabase/control-plane
-/control_plane/ @neondatabase/compute @neondatabase/storage
-/libs/pageserver_api/ @neondatabase/compute @neondatabase/storage
-/libs/postgres_ffi/ @neondatabase/compute 
-/libs/remote_storage/ @neondatabase/storage 
-/libs/safekeeper_api/ @neondatabase/safekeepers  
-/pageserver/ @neondatabase/compute @neondatabase/storage 
-/pgxn/ @neondatabase/compute
-/proxy/ @neondatabase/control-plane 
-/safekeeper/ @neondatabase/safekeepers

Cargo.toml

@@ -1,14 +1,3 @@
-# 'named-profiles' feature was stabilized in cargo 1.57. This line makes the
-# build work with older cargo versions.
-#
-# We have this because as of this writing, the latest cargo Debian package
-# that's available is 1.56. (Confusingly, the Debian package version number
-# is 0.57, whereas 'cargo --version' says 1.56.)
-#
-# See https://tracker.debian.org/pkg/cargo for the current status of the
-# package. When that gets updated, we can remove this.
-cargo-features = ["named-profiles"]
-
 [workspace]
 members = [
     "compute_tools",
@@ -25,10 +14,6 @@ members = [
 # Besides, debug info should not affect the performance.
 debug = true
 
-# disable debug symbols for all packages except this one to decrease binaries size
-[profile.release.package."*"]
-debug = false
-
 [profile.release-line-debug]
 inherits = "release"
 debug = 1 # true = 2 = all symbols, 1 = line only

Dockerfile

@@ -44,7 +44,7 @@ COPY . .
 # Show build caching stats to check if it was used in the end.
 # Has to be the part of the same RUN since cachepot daemon is killed in the end of this RUN, losing the compilation stats.
 RUN set -e \
-&& mold -run cargo build --bin pageserver --bin pageserver_binutils --bin draw_timeline_dir --bin safekeeper --bin proxy --locked --release \
+&& mold -run cargo build --bin pageserver --bin safekeeper --bin proxy --locked --release \
     && cachepot -s
 
 # Build final image
@@ -63,11 +63,9 @@ RUN set -e \
     && useradd -d /data neon \
     && chown -R neon:neon /data
 
-COPY --from=build --chown=neon:neon /home/nonroot/target/release/pageserver          /usr/local/bin
-COPY --from=build --chown=neon:neon /home/nonroot/target/release/pageserver_binutils /usr/local/bin
-COPY --from=build --chown=neon:neon /home/nonroot/target/release/draw_timeline_dir   /usr/local/bin
-COPY --from=build --chown=neon:neon /home/nonroot/target/release/safekeeper          /usr/local/bin
-COPY --from=build --chown=neon:neon /home/nonroot/target/release/proxy               /usr/local/bin
+COPY --from=build --chown=neon:neon /home/nonroot/target/release/pageserver /usr/local/bin
+COPY --from=build --chown=neon:neon /home/nonroot/target/release/safekeeper /usr/local/bin
+COPY --from=build --chown=neon:neon /home/nonroot/target/release/proxy      /usr/local/bin
 
 COPY --from=pg-build /home/nonroot/pg_install/v14 /usr/local/v14/
 COPY --from=pg-build /home/nonroot/pg_install/v15 /usr/local/v15/
@@ -87,3 +85,4 @@ VOLUME ["/data"]
 USER neon
 EXPOSE 6400
 EXPOSE 9898
+CMD ["/bin/bash"]

Dockerfile.compute-node-v14

@@ -1,50 +1,50 @@
-#
-# This file is identical to the Dockerfile.compute-node-v15 file
-# except for the version of Postgres that is built.
-#
-
 ARG TAG=pinned
+# apparently, ARGs don't get replaced in RUN commands in kaniko
+# ARG POSTGIS_VERSION=3.3.0
+# ARG PLV8_VERSION=3.1.4
+# ARG PG_VERSION=v14
 
-#########################################################################################
 #
 # Layer "build-deps"
 #
-#########################################################################################
 FROM debian:bullseye-slim AS build-deps
+RUN echo "deb http://ftp.debian.org/debian testing main" >> /etc/apt/sources.list && \
+    echo "APT::Default-Release \"stable\";" > /etc/apt/apt.conf.d/default-release && \
+    apt update
 RUN apt update &&  \
-    apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev \
-    zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libssl-dev
+    apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \
+    libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libglib2.0-dev
 
-#########################################################################################
 #
 # Layer "pg-build"
 # Build Postgres from the neon postgres repository.
 #
-#########################################################################################
 FROM build-deps AS pg-build
 COPY vendor/postgres-v14 postgres
 RUN cd postgres && \
-    ./configure CFLAGS='-O2 -g3' --enable-debug --with-openssl --with-uuid=ossp && \
+    ./configure CFLAGS='-O2 -g3' --enable-debug --with-uuid=ossp && \
     make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s install && \
     make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C contrib/ install && \
     # Install headers
     make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/include install && \
     make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/interfaces/libpq install
 
-#########################################################################################
 #
 # Layer "postgis-build"
 # Build PostGIS from the upstream PostGIS mirror.
 #
-#########################################################################################
+# PostGIS compiles against neon postgres sources without changes. Perhaps we
+# could even use the upstream binaries, compiled against vanilla Postgres, but
+# it would require some investigation to check that it works, and also keeps
+# working in the future. So for now, we compile our own binaries.
 FROM build-deps AS postgis-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
 RUN apt update && \
     apt install -y gdal-bin libgdal-dev libprotobuf-c-dev protobuf-c-compiler xsltproc
 
-RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.1.tar.gz && \
-    tar xvzf postgis-3.3.1.tar.gz && \
-    cd postgis-3.3.1 && \
+RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.0.tar.gz && \
+    tar xvzf postgis-3.3.0.tar.gz && \
+    cd postgis-3.3.0 && \
     ./autogen.sh && \
     export PATH="/usr/local/pgsql/bin:$PATH" && \
     ./configure && \
@@ -57,55 +57,39 @@ RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.1.tar.gz && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_tiger_geocoder.control && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_topology.control
 
-#########################################################################################
 #
 # Layer "plv8-build"
 # Build plv8
 #
-#########################################################################################
 FROM build-deps AS plv8-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
 RUN apt update && \
-    apt install -y ninja-build python3-dev libc++-dev libc++abi-dev libncurses5 binutils
+    apt install -y ninja-build python3-dev libc++-dev libc++abi-dev libncurses5
 
-# https://github.com/plv8/plv8/issues/475:
-#   v8 uses gold for linking and sets `--thread-count=4` which breaks
-#   gold version <= 1.35 (https://sourceware.org/bugzilla/show_bug.cgi?id=23607)
-# Install newer gold version manually as debian-testing binutils version updates
-# libc version, which in turn breaks other extension built against non-testing libc.
-RUN wget https://ftp.gnu.org/gnu/binutils/binutils-2.38.tar.gz && \
-    tar xvzf binutils-2.38.tar.gz && \
-    cd binutils-2.38 && \
-    cd libiberty && ./configure && make -j $(getconf _NPROCESSORS_ONLN) && \
-    cd ../bfd && ./configure && make bfdver.h && \
-    cd ../gold && ./configure && make -j $(getconf _NPROCESSORS_ONLN) && make install && \
-    cp /usr/local/bin/ld.gold /usr/bin/gold
+# https://github.com/plv8/plv8/issues/475
+# Debian bullseye provides binutils 2.35 when >= 2.38 is necessary
+RUN apt update && \
+    apt install -y --no-install-recommends -t testing binutils
 
-# Sed is used to patch for https://github.com/plv8/plv8/issues/503
 RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.4.tar.gz && \
     tar xvzf v3.1.4.tar.gz && \
     cd plv8-3.1.4 && \
     export PATH="/usr/local/pgsql/bin:$PATH" && \
-    sed -i 's/MemoryContextAlloc(/MemoryContextAllocZero(/' plv8.cc && \
-    make DOCKER=1 -j $(getconf _NPROCESSORS_ONLN) install && \
+    make -j $(getconf _NPROCESSORS_ONLN) && \
+    make -j $(getconf _NPROCESSORS_ONLN) install && \
     rm -rf /plv8-* && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/plv8.control
 
-#########################################################################################
 #
 # Layer "h3-pg-build"
 # Build h3_pg
 #
-#########################################################################################
 FROM build-deps AS h3-pg-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
 
 # packaged cmake is too old
-RUN wget https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2-linux-x86_64.sh \
-      -q -O /tmp/cmake-install.sh \
-      && chmod u+x /tmp/cmake-install.sh \
-      && /tmp/cmake-install.sh --skip-license --prefix=/usr/local/ \
-      && rm /tmp/cmake-install.sh
+RUN apt update && \
+    apt install -y --no-install-recommends -t testing cmake
 
 RUN wget https://github.com/uber/h3/archive/refs/tags/v4.0.1.tar.gz -O h3.tgz && \
     tar xvzf h3.tgz  && \
@@ -124,18 +108,16 @@ RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.0.1.tar.gz -O h3
     export PATH="/usr/local/pgsql/bin:$PATH" && \
     make -j $(getconf _NPROCESSORS_ONLN) && \
     make -j $(getconf _NPROCESSORS_ONLN) install && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3_postgis.control
+    echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3.control
 
-#########################################################################################
 #
 # Layer "neon-pg-ext-build"
 # compile neon extensions
 #
-#########################################################################################
 FROM build-deps AS neon-pg-ext-build
 COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/
-COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/
+# plv8 still sometimes crashes during the creation
+# COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=h3-pg-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=h3-pg-build /h3/usr /
 COPY pgxn/ pgxn/
@@ -145,22 +127,16 @@ RUN make -j $(getconf _NPROCESSORS_ONLN) \
         -C pgxn/neon \
         -s install
 
-#########################################################################################
-#
 # Compile and run the Neon-specific `compute_ctl` binary
-#
-#########################################################################################
 FROM 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:$TAG AS compute-tools
 USER nonroot
 # Copy entire project to get Cargo.* files with proper dependencies for the whole project
 COPY --chown=nonroot . .
 RUN cd compute_tools && cargo build --locked --profile release-line-debug-size-lto
 
-#########################################################################################
 #
 # Clean up postgres folder before inclusion
 #
-#########################################################################################
 FROM neon-pg-ext-build AS postgres-cleanup-layer
 COPY --from=neon-pg-ext-build /usr/local/pgsql /usr/local/pgsql
 
@@ -178,12 +154,10 @@ RUN rm -r /usr/local/pgsql/lib/pgxs/src
 # if they were to be used by other libraries.
 RUN rm /usr/local/pgsql/lib/lib*.a
 
-#########################################################################################
 #
 # Final layer
 # Put it all together into the final image
 #
-#########################################################################################
 FROM debian:bullseye-slim
 # Add user postgres
 RUN mkdir /var/db && useradd -m -d /var/db/postgres postgres && \
@@ -200,6 +174,8 @@ COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-deb
 # libreadline8 for psql
 # libossp-uuid16 for extension ossp-uuid
 # libgeos, libgdal, libproj and libprotobuf-c1 for PostGIS
+# GLIBC 2.34 for plv8.
+#     Debian bullseye provides GLIBC 2.31, so we install the library from testing
 #
 # Lastly, link compute_ctl into zenith_ctl while we're at it,
 # so that we don't need to put this in another layer.
@@ -212,6 +188,12 @@ RUN apt update &&  \
         libproj19 \
         libprotobuf-c1 && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
+    echo "Installing GLIBC 2.34" && \
+    echo "deb http://ftp.debian.org/debian testing main" >> /etc/apt/sources.list && \
+    echo "APT::Default-Release \"stable\";" > /etc/apt/apt.conf.d/default-release && \
+    apt update && \
+    apt install -y --no-install-recommends -t testing libc6 && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
     ln /usr/local/bin/compute_ctl /usr/local/bin/zenith_ctl
 
 USER postgres

Dockerfile.compute-node-v15

@@ -4,47 +4,49 @@
 #
 
 ARG TAG=pinned
+# apparently, ARGs don't get replaced in RUN commands in kaniko
+# ARG POSTGIS_VERSION=3.3.0
+# ARG PLV8_VERSION=3.1.4
+# ARG PG_VERSION=v15
 
-#########################################################################################
 #
 # Layer "build-deps"
 #
-#########################################################################################
 FROM debian:bullseye-slim AS build-deps
 RUN apt update &&  \
-    apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev \
-    zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libssl-dev
+    apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \
+    libcurl4-openssl-dev libossp-uuid-dev
 
-#########################################################################################
 #
 # Layer "pg-build"
 # Build Postgres from the neon postgres repository.
 #
-#########################################################################################
 FROM build-deps AS pg-build
 COPY vendor/postgres-v15 postgres
 RUN cd postgres && \
-    ./configure CFLAGS='-O2 -g3' --enable-debug --with-openssl --with-uuid=ossp && \
+    ./configure CFLAGS='-O2 -g3' --enable-debug --with-uuid=ossp && \
     make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s install && \
     make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C contrib/ install && \
     # Install headers
     make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/include install && \
     make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/interfaces/libpq install
 
-#########################################################################################
 #
 # Layer "postgis-build"
 # Build PostGIS from the upstream PostGIS mirror.
 #
-#########################################################################################
+# PostGIS compiles against neon postgres sources without changes. Perhaps we
+# could even use the upstream binaries, compiled against vanilla Postgres, but
+# it would require some investigation to check that it works, and also keeps
+# working in the future. So for now, we compile our own binaries.
 FROM build-deps AS postgis-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
 RUN apt update && \
-    apt install -y gdal-bin libgdal-dev libprotobuf-c-dev protobuf-c-compiler xsltproc
+    apt install -y gdal-bin libgdal-dev libprotobuf-c-dev protobuf-c-compiler xsltproc wget
 
-RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.1.tar.gz && \
-    tar xvzf postgis-3.3.1.tar.gz && \
-    cd postgis-3.3.1 && \
+RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.0.tar.gz && \
+    tar xvzf postgis-3.3.0.tar.gz && \
+    cd postgis-3.3.0 && \
     ./autogen.sh && \
     export PATH="/usr/local/pgsql/bin:$PATH" && \
     ./configure && \
@@ -57,87 +59,37 @@ RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.1.tar.gz && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_tiger_geocoder.control && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_topology.control
 
-#########################################################################################
 #
 # Layer "plv8-build"
 # Build plv8
 #
-#########################################################################################
 FROM build-deps AS plv8-build
-COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
+COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/
 RUN apt update && \
-    apt install -y ninja-build python3-dev libc++-dev libc++abi-dev libncurses5 binutils
+    apt install -y git curl wget make ninja-build build-essential libncurses5 python3-dev pkg-config libc++-dev libc++abi-dev libglib2.0-dev
 
-# https://github.com/plv8/plv8/issues/475:
-#   v8 uses gold for linking and sets `--thread-count=4` which breaks
-#   gold version <= 1.35 (https://sourceware.org/bugzilla/show_bug.cgi?id=23607)
-# Install newer gold version manually as debian-testing binutils version updates
-# libc version, which in turn breaks other extension built against non-testing libc.
-RUN wget https://ftp.gnu.org/gnu/binutils/binutils-2.38.tar.gz && \
-    tar xvzf binutils-2.38.tar.gz && \
-    cd binutils-2.38 && \
-    cd libiberty && ./configure && make -j $(getconf _NPROCESSORS_ONLN) && \
-    cd ../bfd && ./configure && make bfdver.h && \
-    cd ../gold && ./configure && make -j $(getconf _NPROCESSORS_ONLN) && make install && \
-    cp /usr/local/bin/ld.gold /usr/bin/gold
+# https://github.com/plv8/plv8/issues/475
+# Debian bullseye provides binutils 2.35 when >= 2.38 is necessary
+RUN echo "deb http://ftp.debian.org/debian testing main" >> /etc/apt/sources.list && \
+    echo "APT::Default-Release \"stable\";" > /etc/apt/apt.conf.d/default-release && \
+    apt update && \
+    apt install -y --no-install-recommends -t testing binutils
 
-# Sed is used to patch for https://github.com/plv8/plv8/issues/503
 RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.4.tar.gz && \
     tar xvzf v3.1.4.tar.gz && \
     cd plv8-3.1.4 && \
     export PATH="/usr/local/pgsql/bin:$PATH" && \
-    sed -i 's/MemoryContextAlloc(/MemoryContextAllocZero(/' plv8.cc && \
-    make DOCKER=1 -j $(getconf _NPROCESSORS_ONLN) install && \
+    make -j $(getconf _NPROCESSORS_ONLN) && \
+    make -j $(getconf _NPROCESSORS_ONLN) install && \
     rm -rf /plv8-* && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/plv8.control
 
-#########################################################################################
-#
-# Layer "h3-pg-build"
-# Build h3_pg
-#
-#########################################################################################
-FROM build-deps AS h3-pg-build
-COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
-
-# packaged cmake is too old
-RUN wget https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2-linux-x86_64.sh \
-      -q -O /tmp/cmake-install.sh \
-      && chmod u+x /tmp/cmake-install.sh \
-      && /tmp/cmake-install.sh --skip-license --prefix=/usr/local/ \
-      && rm /tmp/cmake-install.sh
-
-RUN wget https://github.com/uber/h3/archive/refs/tags/v4.0.1.tar.gz -O h3.tgz && \
-    tar xvzf h3.tgz  && \
-    cd h3-4.0.1 && \
-    mkdir build && \
-    cd build && \
-    cmake .. -DCMAKE_BUILD_TYPE=Release && \
-    make -j $(getconf _NPROCESSORS_ONLN) && \
-    DESTDIR=/h3 make install && \
-    cp -R /h3/usr / && \
-    rm -rf build
-
-RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.0.1.tar.gz -O h3-pg.tgz && \
-    tar xvzf h3-pg.tgz && \
-    cd h3-pg-4.0.1 && \
-    export PATH="/usr/local/pgsql/bin:$PATH" && \
-    make -j $(getconf _NPROCESSORS_ONLN) && \
-    make -j $(getconf _NPROCESSORS_ONLN) install && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3_postgis.control
-
-#########################################################################################
 #
 # Layer "neon-pg-ext-build"
 # compile neon extensions
 #
-#########################################################################################
 FROM build-deps AS neon-pg-ext-build
 COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/
-COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/
-COPY --from=h3-pg-build /usr/local/pgsql/ /usr/local/pgsql/
-COPY --from=h3-pg-build /h3/usr /
 COPY pgxn/ pgxn/
 
 RUN make -j $(getconf _NPROCESSORS_ONLN) \
@@ -145,22 +97,16 @@ RUN make -j $(getconf _NPROCESSORS_ONLN) \
         -C pgxn/neon \
         -s install
 
-#########################################################################################
-#
 # Compile and run the Neon-specific `compute_ctl` binary
-#
-#########################################################################################
 FROM 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:$TAG AS compute-tools
 USER nonroot
 # Copy entire project to get Cargo.* files with proper dependencies for the whole project
 COPY --chown=nonroot . .
 RUN cd compute_tools && cargo build --locked --profile release-line-debug-size-lto
 
-#########################################################################################
 #
 # Clean up postgres folder before inclusion
 #
-#########################################################################################
 FROM neon-pg-ext-build AS postgres-cleanup-layer
 COPY --from=neon-pg-ext-build /usr/local/pgsql /usr/local/pgsql
 
@@ -178,12 +124,10 @@ RUN rm -r /usr/local/pgsql/lib/pgxs/src
 # if they were to be used by other libraries.
 RUN rm /usr/local/pgsql/lib/lib*.a
 
-#########################################################################################
 #
 # Final layer
 # Put it all together into the final image
 #
-#########################################################################################
 FROM debian:bullseye-slim
 # Add user postgres
 RUN mkdir /var/db && useradd -m -d /var/db/postgres postgres && \
@@ -193,6 +137,8 @@ RUN mkdir /var/db && useradd -m -d /var/db/postgres postgres && \
     chmod 0750 /var/db/postgres/compute && \
     echo '/usr/local/lib' >> /etc/ld.so.conf && /sbin/ldconfig
 
+# TODO: Check if we can make the extension setup more modular versus a linear build
+# currently plv8-build copies the output /usr/local/pgsql from postgis-build, etc#
 COPY --from=postgres-cleanup-layer --chown=postgres /usr/local/pgsql /usr/local
 COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-debug-size-lto/compute_ctl /usr/local/bin/compute_ctl
 
@@ -200,6 +146,8 @@ COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-deb
 # libreadline8 for psql
 # libossp-uuid16 for extension ossp-uuid
 # libgeos, libgdal, libproj and libprotobuf-c1 for PostGIS
+# GLIBC 2.34 for plv8.
+#     Debian bullseye provides GLIBC 2.31, so we install the library from testing
 #
 # Lastly, link compute_ctl into zenith_ctl while we're at it,
 # so that we don't need to put this in another layer.
@@ -212,6 +160,12 @@ RUN apt update &&  \
         libproj19 \
         libprotobuf-c1 && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
+    echo "Installing GLIBC 2.34" && \
+    echo "deb http://ftp.debian.org/debian testing main" >> /etc/apt/sources.list && \
+    echo "APT::Default-Release \"stable\";" > /etc/apt/apt.conf.d/default-release && \
+    apt update && \
+    apt install -y --no-install-recommends -t testing libc6 && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
     ln /usr/local/bin/compute_ctl /usr/local/bin/zenith_ctl
 
 USER postgres

Dockerfile.compute-node.legacy

@@ -0,0 +1,88 @@
+#
+# Legacy version of the Dockerfile for the compute node.
+# Used by e2e CI. Building Dockerfile.compute-node will take
+# unreasonable ammount of time without v2 runners.
+#
+# TODO: remove once cloud repo CI is moved to v2 runners.
+#
+
+
+# Allow specifiyng different compute-tools tag and image repo, so we are
+# able to use different images
+ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com
+ARG IMAGE=compute-tools
+ARG TAG=latest
+
+#
+# Image with pre-built tools
+#
+FROM $REPOSITORY/$IMAGE:$TAG AS compute-deps
+# Only to get ready compute_ctl binary as deppendency
+
+#
+# Image with Postgres build deps
+#
+FROM debian:bullseye-slim AS build-deps
+
+RUN apt-get update && apt-get -yq install automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \
+                                          libcurl4-openssl-dev libossp-uuid-dev
+
+#
+# Image with built Postgres
+#
+FROM build-deps AS pg-build
+
+# Add user postgres
+RUN adduser postgres
+RUN mkdir /pg && chown postgres:postgres /pg
+
+# Copy source files
+# version 14 is default for now
+COPY ./vendor/postgres-v14 /pg/
+COPY ./pgxn /pg/
+
+# Build and install Postgres locally
+RUN mkdir /pg/compute_build && cd /pg/compute_build && \
+    ../configure CFLAGS='-O2 -g3' --prefix=$(pwd)/postgres_bin --enable-debug --with-uuid=ossp && \
+    # Install main binaries and contribs
+    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s install && \
+    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C contrib/ install && \
+    # Install headers
+    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/include install
+
+# Install neon contrib
+RUN make MAKELEVEL=0 PG_CONFIG=/pg/compute_build/postgres_bin/bin/pg_config -j $(getconf _NPROCESSORS_ONLN) -C /pg/neon install
+
+USER postgres
+WORKDIR /pg
+
+#
+# Final compute node image to be exported
+#
+FROM debian:bullseye-slim
+
+# libreadline-dev is required to run psql
+RUN apt-get update && apt-get -yq install libreadline-dev libossp-uuid-dev
+
+# Add user postgres
+RUN mkdir /var/db && useradd -m -d /var/db/postgres postgres && \
+    echo "postgres:test_console_pass" | chpasswd && \
+    mkdir /var/db/postgres/compute && mkdir /var/db/postgres/specs && \
+    chown -R postgres:postgres /var/db/postgres && \
+    chmod 0750 /var/db/postgres/compute
+
+# Copy ready Postgres binaries
+COPY --from=pg-build /pg/compute_build/postgres_bin /usr/local
+
+# Copy binaries from compute-tools
+COPY --from=compute-deps /usr/local/bin/compute_ctl /usr/local/bin/compute_ctl
+
+# XXX: temporary symlink for compatibility with old control-plane
+RUN ln -s /usr/local/bin/compute_ctl /usr/local/bin/zenith_ctl
+
+# Add postgres shared objects to the search path
+RUN echo '/usr/local/lib' >> /etc/ld.so.conf && /sbin/ldconfig
+
+USER postgres
+
+ENTRYPOINT ["/usr/local/bin/compute_ctl"]

Makefile

@@ -151,11 +151,6 @@ neon-pg-ext-v14: postgres-v14
 	(cd $(POSTGRES_INSTALL_DIR)/build/neon-v14 && \
 	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v14/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \
 		-f $(ROOT_PROJECT_DIR)/pgxn/neon/Makefile install)
-	+@echo "Compiling neon_walredo v14"
-	mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-walredo-v14
-	(cd $(POSTGRES_INSTALL_DIR)/build/neon-walredo-v14 && \
-	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v14/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \
-		-f $(ROOT_PROJECT_DIR)/pgxn/neon_walredo/Makefile install)
 	+@echo "Compiling neon_test_utils" v14
 	mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-v14
 	(cd $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-v14 && \
@@ -168,11 +163,6 @@ neon-pg-ext-v15: postgres-v15
 	(cd $(POSTGRES_INSTALL_DIR)/build/neon-v15 && \
 	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v15/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \
 		-f $(ROOT_PROJECT_DIR)/pgxn/neon/Makefile install)
-	+@echo "Compiling neon_walredo v15"
-	mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-walredo-v15
-	(cd $(POSTGRES_INSTALL_DIR)/build/neon-walredo-v15 && \
-	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v15/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \
-		-f $(ROOT_PROJECT_DIR)/pgxn/neon_walredo/Makefile install)
 	+@echo "Compiling neon_test_utils" v15
 	mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-v15
 	(cd $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-v15 && \

README.md

@@ -223,7 +223,10 @@ Ensure your dependencies are installed as described [here](https://github.com/ne
 ```sh
 git clone --recursive https://github.com/neondatabase/neon.git
 
+# either:
 CARGO_BUILD_FLAGS="--features=testing" make
+# or:
+make debug
 
 ./scripts/pytest
 ```

compute_tools/Cargo.toml

@@ -6,12 +6,10 @@ edition = "2021"
 [dependencies]
 anyhow = "1.0"
 chrono = "0.4"
-clap = "4.0"
+clap = "3.0"
 env_logger = "0.9"
-futures = "0.3.13"
 hyper = { version = "0.14", features = ["full"] }
 log = { version = "0.4", features = ["std", "serde"] }
-notify = "5.0.0"
 postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
 regex = "1"
 serde = { version = "1.0", features = ["derive"] }

compute_tools/src/bin/compute_ctl.rs

@@ -51,19 +51,53 @@ fn main() -> Result<()> {
     // TODO: re-use `utils::logging` later
     init_logger(DEFAULT_LOG_LEVEL)?;
 
-    let matches = cli().get_matches();
+    // Env variable is set by `cargo`
+    let version: Option<&str> = option_env!("CARGO_PKG_VERSION");
+    let matches = clap::App::new("compute_ctl")
+        .version(version.unwrap_or("unknown"))
+        .arg(
+            Arg::new("connstr")
+                .short('C')
+                .long("connstr")
+                .value_name("DATABASE_URL")
+                .required(true),
+        )
+        .arg(
+            Arg::new("pgdata")
+                .short('D')
+                .long("pgdata")
+                .value_name("DATADIR")
+                .required(true),
+        )
+        .arg(
+            Arg::new("pgbin")
+                .short('b')
+                .long("pgbin")
+                .value_name("POSTGRES_PATH"),
+        )
+        .arg(
+            Arg::new("spec")
+                .short('s')
+                .long("spec")
+                .value_name("SPEC_JSON"),
+        )
+        .arg(
+            Arg::new("spec-path")
+                .short('S')
+                .long("spec-path")
+                .value_name("SPEC_PATH"),
+        )
+        .get_matches();
 
-    let pgdata = matches
-        .get_one::<String>("pgdata")
-        .expect("PGDATA path is required");
+    let pgdata = matches.value_of("pgdata").expect("PGDATA path is required");
     let connstr = matches
-        .get_one::<String>("connstr")
+        .value_of("connstr")
         .expect("Postgres connection string is required");
-    let spec = matches.get_one::<String>("spec");
-    let spec_path = matches.get_one::<String>("spec-path");
+    let spec = matches.value_of("spec");
+    let spec_path = matches.value_of("spec-path");
 
     // Try to use just 'postgres' if no path is provided
-    let pgbin = matches.get_one::<String>("pgbin").unwrap();
+    let pgbin = matches.value_of("pgbin").unwrap_or("postgres");
 
     let spec: ComputeSpec = match spec {
         // First, try to get cluster spec from the cli argument
@@ -139,48 +173,3 @@ fn main() -> Result<()> {
         }
     }
 }
-
-fn cli() -> clap::Command {
-    // Env variable is set by `cargo`
-    let version = option_env!("CARGO_PKG_VERSION").unwrap_or("unknown");
-    clap::Command::new("compute_ctl")
-        .version(version)
-        .arg(
-            Arg::new("connstr")
-                .short('C')
-                .long("connstr")
-                .value_name("DATABASE_URL")
-                .required(true),
-        )
-        .arg(
-            Arg::new("pgdata")
-                .short('D')
-                .long("pgdata")
-                .value_name("DATADIR")
-                .required(true),
-        )
-        .arg(
-            Arg::new("pgbin")
-                .short('b')
-                .long("pgbin")
-                .default_value("postgres")
-                .value_name("POSTGRES_PATH"),
-        )
-        .arg(
-            Arg::new("spec")
-                .short('s')
-                .long("spec")
-                .value_name("SPEC_JSON"),
-        )
-        .arg(
-            Arg::new("spec-path")
-                .short('S')
-                .long("spec-path")
-                .value_name("SPEC_PATH"),
-        )
-}
-
-#[test]
-fn verify_cli() {
-    cli().debug_assert()
-}

compute_tools/src/compute.rs

@@ -178,6 +178,7 @@ impl ComputeNode {
             .args(&["--sync-safekeepers"])
             .env("PGDATA", &self.pgdata) // we cannot use -D in this mode
             .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
             .spawn()
             .expect("postgres --sync-safekeepers failed to start");
 
@@ -190,10 +191,10 @@ impl ComputeNode {
 
         if !sync_output.status.success() {
             anyhow::bail!(
-                "postgres --sync-safekeepers exited with non-zero status: {}. stdout: {}",
+                "postgres --sync-safekeepers exited with non-zero status: {}. stdout: {}, stderr: {}",
                 sync_output.status,
-                String::from_utf8(sync_output.stdout)
-                    .expect("postgres --sync-safekeepers exited, and stdout is not utf-8"),
+                String::from_utf8(sync_output.stdout).expect("postgres --sync-safekeepers exited, and stdout is not utf-8"),
+                String::from_utf8(sync_output.stderr).expect("postgres --sync-safekeepers exited, and stderr is not utf-8"),
             );
         }
 
@@ -257,7 +258,14 @@ impl ComputeNode {
             .spawn()
             .expect("cannot start postgres process");
 
-        wait_for_postgres(&mut pg, pgdata_path)?;
+        // Try default Postgres port if it is not provided
+        let port = self
+            .spec
+            .cluster
+            .settings
+            .find("port")
+            .unwrap_or_else(|| "5432".to_string());
+        wait_for_postgres(&mut pg, &port, pgdata_path)?;
 
         // If connection fails,
         // it may be the old node with `zenith_admin` superuser.

compute_tools/src/pg_helpers.rs

@@ -1,18 +1,18 @@
 use std::fmt::Write;
-use std::fs;
 use std::fs::File;
 use std::io::{BufRead, BufReader};
+use std::net::{SocketAddr, TcpStream};
 use std::os::unix::fs::PermissionsExt;
 use std::path::Path;
 use std::process::Child;
-use std::time::{Duration, Instant};
+use std::str::FromStr;
+use std::{fs, thread, time};
 
 use anyhow::{bail, Result};
-use notify::{RecursiveMode, Watcher};
 use postgres::{Client, Transaction};
 use serde::Deserialize;
 
-const POSTGRES_WAIT_TIMEOUT: Duration = Duration::from_millis(60 * 1000); // milliseconds
+const POSTGRES_WAIT_TIMEOUT: u64 = 60 * 1000; // milliseconds
 
 /// Rust representation of Postgres role info with only those fields
 /// that matter for us.
@@ -65,7 +65,7 @@ impl GenericOption {
             let name = match self.name.as_str() {
                 "safekeepers" => "neon.safekeepers",
                 "wal_acceptor_reconnect" => "neon.safekeeper_reconnect_timeout",
-                "wal_acceptor_connection_timeout" => "neon.safekeeper_connection_timeout",
+                "wal_acceptor_connect_timeout" => "neon.safekeeper_connect_timeout",
                 it => it,
             };
 
@@ -168,7 +168,7 @@ impl Database {
     /// it may require a proper quoting too.
     pub fn to_pg_options(&self) -> String {
         let mut params: String = self.options.as_pg_options();
-        write!(params, " OWNER {}", &self.owner.pg_quote())
+        write!(params, " OWNER {}", &self.owner.quote())
             .expect("String is documented to not to error during write operations");
 
         params
@@ -179,17 +179,18 @@ impl Database {
 /// intended to be used for DB / role names.
 pub type PgIdent = String;
 
-/// Generic trait used to provide quoting / encoding for strings used in the
-/// Postgres SQL queries and DATABASE_URL.
-pub trait Escaping {
-    fn pg_quote(&self) -> String;
+/// Generic trait used to provide quoting for strings used in the
+/// Postgres SQL queries. Currently used only to implement quoting
+/// of identifiers, but could be used for literals in the future.
+pub trait PgQuote {
+    fn quote(&self) -> String;
 }
 
-impl Escaping for PgIdent {
+impl PgQuote for PgIdent {
     /// This is intended to mimic Postgres quote_ident(), but for simplicity it
-    /// always quotes provided string with `""` and escapes every `"`.
-    /// **Not idempotent**, i.e. if string is already escaped it will be escaped again.
-    fn pg_quote(&self) -> String {
+    /// always quotes provided string with `""` and escapes every `"`. Not idempotent,
+    /// i.e. if string is already escaped it will be escaped again.
+    fn quote(&self) -> String {
         let result = format!("\"{}\"", self.replace('"', "\"\""));
         result
     }
@@ -229,112 +230,52 @@ pub fn get_existing_dbs(client: &mut Client) -> Result<Vec<Database>> {
     Ok(postgres_dbs)
 }
 
-/// Wait for Postgres to become ready to accept connections. It's ready to
-/// accept connections when the state-field in `pgdata/postmaster.pid` says
-/// 'ready'.
-pub fn wait_for_postgres(pg: &mut Child, pgdata: &Path) -> Result<()> {
+/// Wait for Postgres to become ready to accept connections:
+/// - state should be `ready` in the `pgdata/postmaster.pid`
+/// - and we should be able to connect to 127.0.0.1:5432
+pub fn wait_for_postgres(pg: &mut Child, port: &str, pgdata: &Path) -> Result<()> {
     let pid_path = pgdata.join("postmaster.pid");
+    let mut slept: u64 = 0; // ms
+    let pause = time::Duration::from_millis(100);
 
-    // PostgreSQL writes line "ready" to the postmaster.pid file, when it has
-    // completed initialization and is ready to accept connections. We want to
-    // react quickly and perform the rest of our initialization as soon as
-    // PostgreSQL starts accepting connections. Use 'notify' to be notified
-    // whenever the PID file is changed, and whenever it changes, read it to
-    // check if it's now "ready".
-    //
-    // You cannot actually watch a file before it exists, so we first watch the
-    // data directory, and once the postmaster.pid file appears, we switch to
-    // watch the file instead. We also wake up every 100 ms to poll, just in
-    // case we miss some events for some reason. Not strictly necessary, but
-    // better safe than sorry.
-    let (tx, rx) = std::sync::mpsc::channel();
-    let (mut watcher, rx): (Box<dyn Watcher>, _) = match notify::recommended_watcher(move |res| {
-        let _ = tx.send(res);
-    }) {
-        Ok(watcher) => (Box::new(watcher), rx),
-        Err(e) => {
-            match e.kind {
-                notify::ErrorKind::Io(os) if os.raw_os_error() == Some(38) => {
-                    // docker on m1 macs does not support recommended_watcher
-                    // but return "Function not implemented (os error 38)"
-                    // see https://github.com/notify-rs/notify/issues/423
-                    let (tx, rx) = std::sync::mpsc::channel();
+    let timeout = time::Duration::from_millis(10);
+    let addr = SocketAddr::from_str(&format!("127.0.0.1:{}", port)).unwrap();
 
-                    // let's poll it faster than what we check the results for (100ms)
-                    let config =
-                        notify::Config::default().with_poll_interval(Duration::from_millis(50));
-
-                    let watcher = notify::PollWatcher::new(
-                        move |res| {
-                            let _ = tx.send(res);
-                        },
-                        config,
-                    )?;
-
-                    (Box::new(watcher), rx)
-                }
-                _ => return Err(e.into()),
-            }
-        }
-    };
-
-    watcher.watch(pgdata, RecursiveMode::NonRecursive)?;
-
-    let started_at = Instant::now();
-    let mut postmaster_pid_seen = false;
     loop {
+        // Sleep POSTGRES_WAIT_TIMEOUT at max (a bit longer actually if consider a TCP timeout,
+        // but postgres starts listening almost immediately, even if it is not really
+        // ready to accept connections).
+        if slept >= POSTGRES_WAIT_TIMEOUT {
+            bail!("timed out while waiting for Postgres to start");
+        }
+
         if let Ok(Some(status)) = pg.try_wait() {
             // Postgres exited, that is not what we expected, bail out earlier.
             let code = status.code().unwrap_or(-1);
             bail!("Postgres exited unexpectedly with code {}", code);
         }
 
-        let res = rx.recv_timeout(Duration::from_millis(100));
-        log::debug!("woken up by notify: {res:?}");
-        // If there are multiple events in the channel already, we only need to be
-        // check once. Swallow the extra events before we go ahead to check the
-        // pid file.
-        while let Ok(res) = rx.try_recv() {
-            log::debug!("swallowing extra event: {res:?}");
-        }
-
         // Check that we can open pid file first.
         if let Ok(file) = File::open(&pid_path) {
-            if !postmaster_pid_seen {
-                log::debug!("postmaster.pid appeared");
-                watcher
-                    .unwatch(pgdata)
-                    .expect("Failed to remove pgdata dir watch");
-                watcher
-                    .watch(&pid_path, RecursiveMode::NonRecursive)
-                    .expect("Failed to add postmaster.pid file watch");
-                postmaster_pid_seen = true;
-            }
-
             let file = BufReader::new(file);
             let last_line = file.lines().last();
 
             // Pid file could be there and we could read it, but it could be empty, for example.
             if let Some(Ok(line)) = last_line {
                 let status = line.trim();
-                log::debug!("last line of postmaster.pid: {status:?}");
+                let can_connect = TcpStream::connect_timeout(&addr, timeout).is_ok();
 
                 // Now Postgres is ready to accept connections
-                if status == "ready" {
+                if status == "ready" && can_connect {
                     break;
                 }
             }
         }
 
-        // Give up after POSTGRES_WAIT_TIMEOUT.
-        let duration = started_at.elapsed();
-        if duration >= POSTGRES_WAIT_TIMEOUT {
-            bail!("timed out while waiting for Postgres to start");
-        }
+        thread::sleep(pause);
+        slept += 100;
     }
 
-    log::info!("PostgreSQL is now running, continuing to configure it");
-
     Ok(())
 }
 

compute_tools/src/spec.rs

@@ -1,9 +1,7 @@
 use std::path::Path;
-use std::str::FromStr;
 
 use anyhow::Result;
 use log::{info, log_enabled, warn, Level};
-use postgres::config::Config;
 use postgres::{Client, NoTls};
 use serde::Deserialize;
 
@@ -117,8 +115,8 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
                     if existing_roles.iter().any(|r| r.name == op.name) {
                         let query: String = format!(
                             "ALTER ROLE {} RENAME TO {}",
-                            op.name.pg_quote(),
-                            new_name.pg_quote()
+                            op.name.quote(),
+                            new_name.quote()
                         );
 
                         warn!("renaming role '{}' to '{}'", op.name, new_name);
@@ -164,7 +162,7 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
             }
 
             if update_role {
-                let mut query: String = format!("ALTER ROLE {} ", name.pg_quote());
+                let mut query: String = format!("ALTER ROLE {} ", name.quote());
                 info_print!(" -> update");
 
                 query.push_str(&role.to_pg_options());
@@ -172,7 +170,7 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
             }
         } else {
             info!("role name: '{}'", &name);
-            let mut query: String = format!("CREATE ROLE {} ", name.pg_quote());
+            let mut query: String = format!("CREATE ROLE {} ", name.quote());
             info!("role create query: '{}'", &query);
             info_print!(" -> create");
 
@@ -181,7 +179,7 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
 
             let grant_query = format!(
                 "GRANT pg_read_all_data, pg_write_all_data TO {}",
-                name.pg_quote()
+                name.quote()
             );
             xact.execute(grant_query.as_str(), &[])?;
             info!("role grant query: '{}'", &grant_query);
@@ -217,7 +215,7 @@ pub fn handle_role_deletions(node: &ComputeNode, client: &mut Client) -> Result<
             // We do not check either role exists or not,
             // Postgres will take care of it for us
             if op.action == "delete_role" {
-                let query: String = format!("DROP ROLE IF EXISTS {}", &op.name.pg_quote());
+                let query: String = format!("DROP ROLE IF EXISTS {}", &op.name.quote());
 
                 warn!("deleting role '{}'", &op.name);
                 xact.execute(query.as_str(), &[])?;
@@ -232,16 +230,17 @@ pub fn handle_role_deletions(node: &ComputeNode, client: &mut Client) -> Result<
 fn reassign_owned_objects(node: &ComputeNode, role_name: &PgIdent) -> Result<()> {
     for db in &node.spec.cluster.databases {
         if db.owner != *role_name {
-            let mut conf = Config::from_str(node.connstr.as_str())?;
-            conf.dbname(&db.name);
+            let mut connstr = node.connstr.clone();
+            // database name is always the last and the only component of the path
+            connstr.set_path(&db.name);
 
-            let mut client = conf.connect(NoTls)?;
+            let mut client = Client::connect(connstr.as_str(), NoTls)?;
 
             // This will reassign all dependent objects to the db owner
             let reassign_query = format!(
                 "REASSIGN OWNED BY {} TO {}",
-                role_name.pg_quote(),
-                db.owner.pg_quote()
+                role_name.quote(),
+                db.owner.quote()
             );
             info!(
                 "reassigning objects owned by '{}' in db '{}' to '{}'",
@@ -250,7 +249,7 @@ fn reassign_owned_objects(node: &ComputeNode, role_name: &PgIdent) -> Result<()>
             client.simple_query(&reassign_query)?;
 
             // This now will only drop privileges of the role
-            let drop_query = format!("DROP OWNED BY {}", role_name.pg_quote());
+            let drop_query = format!("DROP OWNED BY {}", role_name.quote());
             client.simple_query(&drop_query)?;
         }
     }
@@ -280,7 +279,7 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
                 // We do not check either DB exists or not,
                 // Postgres will take care of it for us
                 "delete_db" => {
-                    let query: String = format!("DROP DATABASE IF EXISTS {}", &op.name.pg_quote());
+                    let query: String = format!("DROP DATABASE IF EXISTS {}", &op.name.quote());
 
                     warn!("deleting database '{}'", &op.name);
                     client.execute(query.as_str(), &[])?;
@@ -292,8 +291,8 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
                     if existing_dbs.iter().any(|r| r.name == op.name) {
                         let query: String = format!(
                             "ALTER DATABASE {} RENAME TO {}",
-                            op.name.pg_quote(),
-                            new_name.pg_quote()
+                            op.name.quote(),
+                            new_name.quote()
                         );
 
                         warn!("renaming database '{}' to '{}'", op.name, new_name);
@@ -321,7 +320,7 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
             // XXX: db owner name is returned as quoted string from Postgres,
             // when quoting is needed.
             let new_owner = if r.owner.starts_with('"') {
-                db.owner.pg_quote()
+                db.owner.quote()
             } else {
                 db.owner.clone()
             };
@@ -329,15 +328,15 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
             if new_owner != r.owner {
                 let query: String = format!(
                     "ALTER DATABASE {} OWNER TO {}",
-                    name.pg_quote(),
-                    db.owner.pg_quote()
+                    name.quote(),
+                    db.owner.quote()
                 );
                 info_print!(" -> update");
 
                 client.execute(query.as_str(), &[])?;
             }
         } else {
-            let mut query: String = format!("CREATE DATABASE {} ", name.pg_quote());
+            let mut query: String = format!("CREATE DATABASE {} ", name.quote());
             info_print!(" -> create");
 
             query.push_str(&db.to_pg_options());
@@ -367,7 +366,7 @@ pub fn handle_grants(node: &ComputeNode, client: &mut Client) -> Result<()> {
         .cluster
         .roles
         .iter()
-        .map(|r| r.name.pg_quote())
+        .map(|r| r.name.quote())
         .collect::<Vec<_>>();
 
     for db in &spec.cluster.databases {
@@ -375,7 +374,7 @@ pub fn handle_grants(node: &ComputeNode, client: &mut Client) -> Result<()> {
 
         let query: String = format!(
             "GRANT CREATE ON DATABASE {} TO {}",
-            dbname.pg_quote(),
+            dbname.quote(),
             roles.join(", ")
         );
         info!("grant query {}", &query);
@@ -386,11 +385,12 @@ pub fn handle_grants(node: &ComputeNode, client: &mut Client) -> Result<()> {
     // Do some per-database access adjustments. We'd better do this at db creation time,
     // but CREATE DATABASE isn't transactional. So we cannot create db + do some grants
     // atomically.
+    let mut db_connstr = node.connstr.clone();
     for db in &node.spec.cluster.databases {
-        let mut conf = Config::from_str(node.connstr.as_str())?;
-        conf.dbname(&db.name);
+        // database name is always the last and the only component of the path
+        db_connstr.set_path(&db.name);
 
-        let mut db_client = conf.connect(NoTls)?;
+        let mut db_client = Client::connect(db_connstr.as_str(), NoTls)?;
 
         // This will only change ownership on the schema itself, not the objects
         // inside it. Without it owner of the `public` schema will be `cloud_admin`
@@ -419,36 +419,9 @@ pub fn handle_grants(node: &ComputeNode, client: &mut Client) -> Result<()> {
                     END IF;\n\
                 END\n\
             $$;",
-            db.owner.pg_quote()
+            db.owner.quote()
         );
         db_client.simple_query(&alter_query)?;
-
-        // Explicitly grant CREATE ON SCHEMA PUBLIC to the web_access user.
-        // This is needed because since postgres 15 this privilege is removed by default.
-        let grant_query = "DO $$\n\
-                BEGIN\n\
-                    IF EXISTS(\n\
-                        SELECT nspname\n\
-                        FROM pg_catalog.pg_namespace\n\
-                        WHERE nspname = 'public'\n\
-                    ) AND\n\
-                    current_setting('server_version_num')::int/10000 >= 15\n\
-                    THEN\n\
-                        IF EXISTS(\n\
-                            SELECT rolname\n\
-                            FROM pg_catalog.pg_roles\n\
-                            WHERE rolname = 'web_access'\n\
-                        )\n\
-                        THEN\n\
-                            GRANT CREATE ON SCHEMA public TO web_access;\n\
-                        END IF;\n\
-                    END IF;\n\
-                END\n\
-            $$;"
-        .to_string();
-
-        info!("grant query for db {} : {}", &db.name, &grant_query);
-        db_client.simple_query(&grant_query)?;
     }
 
     Ok(())

compute_tools/tests/pg_helpers_tests.rs

@@ -33,9 +33,9 @@ mod pg_helpers_tests {
     }
 
     #[test]
-    fn ident_pg_quote() {
+    fn quote_ident() {
         let ident: PgIdent = PgIdent::from("\"name\";\\n select 1;");
 
-        assert_eq!(ident.pg_quote(), "\"\"\"name\"\";\\n select 1;\"");
+        assert_eq!(ident.quote(), "\"\"\"name\"\";\\n select 1;\"");
     }
 }

control_plane/Cargo.toml

@@ -4,25 +4,22 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-anyhow = "1.0"
-clap = "4.0"
-comfy-table = "6.1"
+clap = "3.0"
+comfy-table = "5.0.1"
 git-version = "0.3.5"
-nix = "0.25"
-once_cell = "1.13.0"
-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev = "d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-regex = "1"
-reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "rustls-tls"] }
-serde = { version = "1.0", features = ["derive"] }
-serde_with = "2.0"
 tar = "0.4.38"
-thiserror = "1"
+postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+serde = { version = "1.0", features = ["derive"] }
+serde_with = "1.12.0"
 toml = "0.5"
-url = "2.2.2"
+once_cell = "1.13.0"
+regex = "1"
+anyhow = "1.0"
+thiserror = "1"
+nix = "0.23"
+reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "rustls-tls"] }
 
-# Note: Do not directly depend on pageserver or safekeeper; use pageserver_api or safekeeper_api
-# instead, so that recompile times are better.
-pageserver_api = { path = "../libs/pageserver_api" }
-safekeeper_api = { path = "../libs/safekeeper_api" }
+pageserver = { path = "../pageserver" }
+safekeeper = { path = "../safekeeper" }
 utils = { path = "../libs/utils" }
 workspace_hack = { version = "0.1", path = "../workspace_hack" }

control_plane/src/background_process.rs

@@ -1,264 +0,0 @@
-//! Spawns and kills background processes that are needed by Neon CLI.
-//! Applies common set-up such as log and pid files (if needed) to every process.
-//!
-//! Neon CLI does not run in background, so it needs to store the information about
-//! spawned processes, which it does in this module.
-//! We do that by storing the pid of the process in the "${process_name}.pid" file.
-//! The pid file can be created by the process itself
-//! (Neon storage binaries do that and also ensure that a lock is taken onto that file)
-//! or we create such file after starting the process
-//! (non-Neon binaries don't necessarily follow our pidfile conventions).
-//! The pid stored in the file is later used to stop the service.
-//!
-//! See [`lock_file`] module for more info.
-
-use std::ffi::OsStr;
-use std::io::Write;
-use std::path::Path;
-use std::process::{Child, Command};
-use std::time::Duration;
-use std::{fs, io, thread};
-
-use anyhow::{anyhow, bail, Context, Result};
-use nix::errno::Errno;
-use nix::sys::signal::{kill, Signal};
-use nix::unistd::Pid;
-
-use utils::lock_file;
-
-const RETRIES: u32 = 15;
-const RETRY_TIMEOUT_MILLIS: u64 = 500;
-
-/// Argument to `start_process`, to indicate whether it should create pidfile or if the process creates
-/// it itself.
-pub enum InitialPidFile<'t> {
-    /// Create a pidfile, to allow future CLI invocations to manipulate the process.
-    Create(&'t Path),
-    /// The process will create the pidfile itself, need to wait for that event.
-    Expect(&'t Path),
-}
-
-/// Start a background child process using the parameters given.
-pub fn start_process<F, S: AsRef<OsStr>>(
-    process_name: &str,
-    datadir: &Path,
-    command: &Path,
-    args: &[S],
-    initial_pid_file: InitialPidFile,
-    process_status_check: F,
-) -> anyhow::Result<Child>
-where
-    F: Fn() -> anyhow::Result<bool>,
-{
-    let log_path = datadir.join(format!("{process_name}.log"));
-    let process_log_file = fs::OpenOptions::new()
-        .create(true)
-        .write(true)
-        .append(true)
-        .open(&log_path)
-        .with_context(|| {
-            format!("Could not open {process_name} log file {log_path:?} for writing")
-        })?;
-    let same_file_for_stderr = process_log_file.try_clone().with_context(|| {
-        format!("Could not reuse {process_name} log file {log_path:?} for writing stderr")
-    })?;
-
-    let mut command = Command::new(command);
-    let background_command = command
-        .stdout(process_log_file)
-        .stderr(same_file_for_stderr)
-        .args(args);
-    let filled_cmd = fill_aws_secrets_vars(fill_rust_env_vars(background_command));
-
-    let mut spawned_process = filled_cmd.spawn().with_context(|| {
-        format!("Could not spawn {process_name}, see console output and log files for details.")
-    })?;
-    let pid = spawned_process.id();
-    let pid = Pid::from_raw(
-        i32::try_from(pid)
-            .with_context(|| format!("Subprocess {process_name} has invalid pid {pid}"))?,
-    );
-
-    let pid_file_to_check = match initial_pid_file {
-        InitialPidFile::Create(target_pid_file_path) => {
-            match lock_file::create_lock_file(target_pid_file_path, pid.to_string()) {
-                lock_file::LockCreationResult::Created { .. } => {
-                    // We use "lock" file here only to create the pid file. The lock on the pidfile will be dropped as soon
-                    // as this CLI invocation exits, so it's a bit useless, but doesn't any harm either.
-                }
-                lock_file::LockCreationResult::AlreadyLocked { .. } => {
-                    anyhow::bail!("Cannot write pid file for {process_name} at path {target_pid_file_path:?}: file is already locked by another process")
-                }
-                lock_file::LockCreationResult::CreationFailed(e) => {
-                    return Err(e.context(format!(
-                    "Failed to create pid file for {process_name} at path {target_pid_file_path:?}"
-                )))
-                }
-            }
-            None
-        }
-        InitialPidFile::Expect(pid_file_path) => Some(pid_file_path),
-    };
-
-    for retries in 0..RETRIES {
-        match process_started(pid, pid_file_to_check, &process_status_check) {
-            Ok(true) => {
-                println!("\n{process_name} started, pid: {pid}");
-                return Ok(spawned_process);
-            }
-            Ok(false) => {
-                if retries < 5 {
-                    print!(".");
-                    io::stdout().flush().unwrap();
-                } else {
-                    if retries == 5 {
-                        println!() // put a line break after dots for second message
-                    }
-                    println!("{process_name} has not started yet, retrying ({retries})...");
-                }
-                thread::sleep(Duration::from_millis(RETRY_TIMEOUT_MILLIS));
-            }
-            Err(e) => {
-                println!("{process_name} failed to start: {e:#}");
-                if let Err(e) = spawned_process.kill() {
-                    println!("Could not stop {process_name} subprocess: {e:#}")
-                };
-                return Err(e);
-            }
-        }
-    }
-    anyhow::bail!("{process_name} could not start in {RETRIES} attempts");
-}
-
-/// Stops the process, using the pid file given. Returns Ok also if the process is already not running.
-pub fn stop_process(immediate: bool, process_name: &str, pid_file: &Path) -> anyhow::Result<()> {
-    if !pid_file.exists() {
-        println!("{process_name} is already stopped: no pid file {pid_file:?} is present");
-        return Ok(());
-    }
-    let pid = read_pidfile(pid_file)?;
-
-    let sig = if immediate {
-        print!("Stopping {process_name} with pid {pid} immediately..");
-        Signal::SIGQUIT
-    } else {
-        print!("Stopping {process_name} with pid {pid} gracefully..");
-        Signal::SIGTERM
-    };
-    io::stdout().flush().unwrap();
-    match kill(pid, sig) {
-        Ok(()) => (),
-        Err(Errno::ESRCH) => {
-            println!(
-                "{process_name} with pid {pid} does not exist, but a pid file {pid_file:?} was found"
-            );
-            return Ok(());
-        }
-        Err(e) => anyhow::bail!("Failed to send signal to {process_name} with pid {pid}: {e}"),
-    }
-
-    // Wait until process is gone
-    for _ in 0..RETRIES {
-        match process_has_stopped(pid) {
-            Ok(true) => {
-                println!("\n{process_name} stopped");
-                if let Err(e) = fs::remove_file(pid_file) {
-                    if e.kind() != io::ErrorKind::NotFound {
-                        eprintln!("Failed to remove pid file {pid_file:?} after stopping the process: {e:#}");
-                    }
-                }
-                return Ok(());
-            }
-            Ok(false) => {
-                print!(".");
-                io::stdout().flush().unwrap();
-                thread::sleep(Duration::from_secs(1))
-            }
-            Err(e) => {
-                println!("{process_name} with pid {pid} failed to stop: {e:#}");
-                return Err(e);
-            }
-        }
-    }
-
-    anyhow::bail!("{process_name} with pid {pid} failed to stop in {RETRIES} attempts");
-}
-
-fn fill_rust_env_vars(cmd: &mut Command) -> &mut Command {
-    let mut filled_cmd = cmd.env_clear().env("RUST_BACKTRACE", "1");
-
-    let var = "LLVM_PROFILE_FILE";
-    if let Some(val) = std::env::var_os(var) {
-        filled_cmd = filled_cmd.env(var, val);
-    }
-
-    const RUST_LOG_KEY: &str = "RUST_LOG";
-    if let Ok(rust_log_value) = std::env::var(RUST_LOG_KEY) {
-        filled_cmd.env(RUST_LOG_KEY, rust_log_value)
-    } else {
-        filled_cmd
-    }
-}
-
-fn fill_aws_secrets_vars(mut cmd: &mut Command) -> &mut Command {
-    for env_key in [
-        "AWS_ACCESS_KEY_ID",
-        "AWS_SECRET_ACCESS_KEY",
-        "AWS_SESSION_TOKEN",
-    ] {
-        if let Ok(value) = std::env::var(env_key) {
-            cmd = cmd.env(env_key, value);
-        }
-    }
-    cmd
-}
-
-fn process_started<F>(
-    pid: Pid,
-    pid_file_to_check: Option<&Path>,
-    status_check: &F,
-) -> anyhow::Result<bool>
-where
-    F: Fn() -> anyhow::Result<bool>,
-{
-    match status_check() {
-        Ok(true) => match pid_file_to_check {
-            Some(pid_file_path) => {
-                if pid_file_path.exists() {
-                    let pid_in_file = read_pidfile(pid_file_path)?;
-                    Ok(pid_in_file == pid)
-                } else {
-                    Ok(false)
-                }
-            }
-            None => Ok(true),
-        },
-        Ok(false) => Ok(false),
-        Err(e) => anyhow::bail!("process failed to start: {e}"),
-    }
-}
-
-/// Read a PID file
-///
-/// We expect a file that contains a single integer.
-fn read_pidfile(pidfile: &Path) -> Result<Pid> {
-    let pid_str = fs::read_to_string(pidfile)
-        .with_context(|| format!("failed to read pidfile {pidfile:?}"))?;
-    let pid: i32 = pid_str
-        .parse()
-        .map_err(|_| anyhow!("failed to parse pidfile {pidfile:?}"))?;
-    if pid < 1 {
-        bail!("pidfile {pidfile:?} contained bad value '{pid}'");
-    }
-    Ok(Pid::from_raw(pid))
-}
-
-fn process_has_stopped(pid: Pid) -> anyhow::Result<bool> {
-    match kill(pid, None) {
-        // Process exists, keep waiting
-        Ok(_) => Ok(false),
-        // Process not found, we're done
-        Err(Errno::ESRCH) => Ok(true),
-        Err(err) => anyhow::bail!("Failed to send signal to process with pid {pid}: {err}"),
-    }
-}

control_plane/src/bin/neon_local.rs

@@ -6,18 +6,18 @@
 //! rely on `neon_local` to set up the environment for each test.
 //!
 use anyhow::{anyhow, bail, Context, Result};
-use clap::{value_parser, Arg, ArgAction, ArgMatches, Command};
+use clap::{App, AppSettings, Arg, ArgMatches};
 use control_plane::compute::ComputeControlPlane;
 use control_plane::local_env::{EtcdBroker, LocalEnv};
-use control_plane::pageserver::PageServerNode;
 use control_plane::safekeeper::SafekeeperNode;
+use control_plane::storage::PageServerNode;
 use control_plane::{etcd, local_env};
-use pageserver_api::models::TimelineInfo;
-use pageserver_api::{
+use pageserver::config::defaults::{
     DEFAULT_HTTP_LISTEN_ADDR as DEFAULT_PAGESERVER_HTTP_ADDR,
     DEFAULT_PG_LISTEN_ADDR as DEFAULT_PAGESERVER_PG_ADDR,
 };
-use safekeeper_api::{
+use pageserver::http::models::TimelineInfo;
+use safekeeper::defaults::{
     DEFAULT_HTTP_LISTEN_PORT as DEFAULT_SAFEKEEPER_HTTP_PORT,
     DEFAULT_PG_LISTEN_PORT as DEFAULT_SAFEKEEPER_PG_PORT,
 };
@@ -85,7 +85,212 @@ struct TimelineTreeEl {
 //   * Providing CLI api to the pageserver
 //   * TODO: export/import to/from usual postgres
 fn main() -> Result<()> {
-    let matches = cli().get_matches();
+    let branch_name_arg = Arg::new("branch-name")
+        .long("branch-name")
+        .takes_value(true)
+        .help("Name of the branch to be created or used as an alias for other services")
+        .required(false);
+
+    let pg_node_arg = Arg::new("node").help("Postgres node name").required(false);
+
+    let safekeeper_id_arg = Arg::new("id").help("safekeeper id").required(false);
+
+    let tenant_id_arg = Arg::new("tenant-id")
+        .long("tenant-id")
+        .help("Tenant id. Represented as a hexadecimal string 32 symbols length")
+        .takes_value(true)
+        .required(false);
+
+    let timeline_id_arg = Arg::new("timeline-id")
+        .long("timeline-id")
+        .help("Timeline id. Represented as a hexadecimal string 32 symbols length")
+        .takes_value(true)
+        .required(false);
+
+    let pg_version_arg = Arg::new("pg-version")
+        .long("pg-version")
+        .help("Postgres version to use for the initial tenant")
+        .required(false)
+        .takes_value(true)
+        .default_value(DEFAULT_PG_VERSION);
+
+    let port_arg = Arg::new("port")
+        .long("port")
+        .required(false)
+        .value_name("port");
+
+    let stop_mode_arg = Arg::new("stop-mode")
+        .short('m')
+        .takes_value(true)
+        .possible_values(&["fast", "immediate"])
+        .help("If 'immediate', don't flush repository data at shutdown")
+        .required(false)
+        .value_name("stop-mode");
+
+    let pageserver_config_args = Arg::new("pageserver-config-override")
+        .long("pageserver-config-override")
+        .takes_value(true)
+        .number_of_values(1)
+        .multiple_occurrences(true)
+        .help("Additional pageserver's configuration options or overrides, refer to pageserver's 'config-override' CLI parameter docs for more")
+        .required(false);
+
+    let lsn_arg = Arg::new("lsn")
+        .long("lsn")
+        .help("Specify Lsn on the timeline to start from. By default, end of the timeline would be used.")
+        .takes_value(true)
+        .required(false);
+
+    let matches = App::new("Neon CLI")
+        .setting(AppSettings::ArgRequiredElseHelp)
+        .version(GIT_VERSION)
+        .subcommand(
+            App::new("init")
+                .about("Initialize a new Neon repository")
+                .arg(pageserver_config_args.clone())
+                .arg(timeline_id_arg.clone().help("Use a specific timeline id when creating a tenant and its initial timeline"))
+                .arg(
+                    Arg::new("config")
+                        .long("config")
+                        .required(false)
+                        .value_name("config"),
+                )
+                .arg(pg_version_arg.clone())
+        )
+        .subcommand(
+            App::new("timeline")
+            .about("Manage timelines")
+            .subcommand(App::new("list")
+                .about("List all timelines, available to this pageserver")
+                .arg(tenant_id_arg.clone()))
+            .subcommand(App::new("branch")
+                .about("Create a new timeline, using another timeline as a base, copying its data")
+                .arg(tenant_id_arg.clone())
+                .arg(branch_name_arg.clone())
+                .arg(Arg::new("ancestor-branch-name").long("ancestor-branch-name").takes_value(true)
+                    .help("Use last Lsn of another timeline (and its data) as base when creating the new timeline. The timeline gets resolved by its branch name.").required(false))
+                .arg(Arg::new("ancestor-start-lsn").long("ancestor-start-lsn").takes_value(true)
+                    .help("When using another timeline as base, use a specific Lsn in it instead of the latest one").required(false)))
+            .subcommand(App::new("create")
+                .about("Create a new blank timeline")
+                .arg(tenant_id_arg.clone())
+                .arg(branch_name_arg.clone())
+                .arg(pg_version_arg.clone())
+            )
+            .subcommand(App::new("import")
+                .about("Import timeline from basebackup directory")
+                .arg(tenant_id_arg.clone())
+                .arg(timeline_id_arg.clone())
+                .arg(Arg::new("node-name").long("node-name").takes_value(true)
+                    .help("Name to assign to the imported timeline"))
+                .arg(Arg::new("base-tarfile").long("base-tarfile").takes_value(true)
+                    .help("Basebackup tarfile to import"))
+                .arg(Arg::new("base-lsn").long("base-lsn").takes_value(true)
+                    .help("Lsn the basebackup starts at"))
+                .arg(Arg::new("wal-tarfile").long("wal-tarfile").takes_value(true)
+                    .help("Wal to add after base"))
+                .arg(Arg::new("end-lsn").long("end-lsn").takes_value(true)
+                    .help("Lsn the basebackup ends at"))
+                .arg(pg_version_arg.clone())
+            )
+        ).subcommand(
+            App::new("tenant")
+            .setting(AppSettings::ArgRequiredElseHelp)
+            .about("Manage tenants")
+            .subcommand(App::new("list"))
+            .subcommand(App::new("create")
+                .arg(tenant_id_arg.clone())
+                .arg(timeline_id_arg.clone().help("Use a specific timeline id when creating a tenant and its initial timeline"))
+                .arg(Arg::new("config").short('c').takes_value(true).multiple_occurrences(true).required(false))
+                .arg(pg_version_arg.clone())
+                )
+            .subcommand(App::new("config")
+                .arg(tenant_id_arg.clone())
+                .arg(Arg::new("config").short('c').takes_value(true).multiple_occurrences(true).required(false))
+                )
+        )
+        .subcommand(
+            App::new("pageserver")
+                .setting(AppSettings::ArgRequiredElseHelp)
+                .about("Manage pageserver")
+                .subcommand(App::new("status"))
+                .subcommand(App::new("start").about("Start local pageserver").arg(pageserver_config_args.clone()))
+                .subcommand(App::new("stop").about("Stop local pageserver")
+                            .arg(stop_mode_arg.clone()))
+                .subcommand(App::new("restart").about("Restart local pageserver").arg(pageserver_config_args.clone()))
+        )
+        .subcommand(
+            App::new("safekeeper")
+                .setting(AppSettings::ArgRequiredElseHelp)
+                .about("Manage safekeepers")
+                .subcommand(App::new("start")
+                            .about("Start local safekeeper")
+                            .arg(safekeeper_id_arg.clone())
+                )
+                .subcommand(App::new("stop")
+                            .about("Stop local safekeeper")
+                            .arg(safekeeper_id_arg.clone())
+                            .arg(stop_mode_arg.clone())
+                )
+                .subcommand(App::new("restart")
+                            .about("Restart local safekeeper")
+                            .arg(safekeeper_id_arg.clone())
+                            .arg(stop_mode_arg.clone())
+                )
+        )
+        .subcommand(
+            App::new("pg")
+                .setting(AppSettings::ArgRequiredElseHelp)
+                .about("Manage postgres instances")
+                .subcommand(App::new("list").arg(tenant_id_arg.clone()))
+                .subcommand(App::new("create")
+                    .about("Create a postgres compute node")
+                    .arg(pg_node_arg.clone())
+                    .arg(branch_name_arg.clone())
+                    .arg(tenant_id_arg.clone())
+                    .arg(lsn_arg.clone())
+                    .arg(port_arg.clone())
+                    .arg(
+                        Arg::new("config-only")
+                            .help("Don't do basebackup, create compute node with only config files")
+                            .long("config-only")
+                            .required(false))
+                    .arg(pg_version_arg.clone())
+                )
+                .subcommand(App::new("start")
+                    .about("Start a postgres compute node.\n This command actually creates new node from scratch, but preserves existing config files")
+                    .arg(pg_node_arg.clone())
+                    .arg(tenant_id_arg.clone())
+                    .arg(branch_name_arg.clone())
+                    .arg(timeline_id_arg.clone())
+                    .arg(lsn_arg.clone())
+                    .arg(port_arg.clone())
+                    .arg(pg_version_arg.clone())
+                )
+                .subcommand(
+                    App::new("stop")
+                    .arg(pg_node_arg.clone())
+                    .arg(tenant_id_arg.clone())
+                    .arg(
+                        Arg::new("destroy")
+                            .help("Also delete data directory (now optional, should be default in future)")
+                            .long("destroy")
+                            .required(false)
+                    )
+                    )
+
+        )
+        .subcommand(
+            App::new("start")
+                .about("Start page server and safekeepers")
+                .arg(pageserver_config_args)
+        )
+        .subcommand(
+            App::new("stop")
+                .about("Stop page server and safekeepers")
+                .arg(stop_mode_arg.clone())
+        )
+        .get_matches();
 
     let (sub_name, sub_args) = match matches.subcommand() {
         Some(subcommand_data) => subcommand_data,
@@ -153,7 +358,9 @@ fn print_timelines_tree(
 
     // Memorize all direct children of each timeline.
     for timeline in timelines.iter() {
-        if let Some(ancestor_timeline_id) = timeline.ancestor_timeline_id {
+        if let Some(ancestor_timeline_id) =
+            timeline.local.as_ref().and_then(|l| l.ancestor_timeline_id)
+        {
             timelines_hash
                 .get_mut(&ancestor_timeline_id)
                 .context("missing timeline info in the HashMap")?
@@ -164,7 +371,13 @@ fn print_timelines_tree(
 
     for timeline in timelines_hash.values() {
         // Start with root local timelines (no ancestors) first.
-        if timeline.info.ancestor_timeline_id.is_none() {
+        if timeline
+            .info
+            .local
+            .as_ref()
+            .and_then(|l| l.ancestor_timeline_id)
+            .is_none()
+        {
             print_timeline(0, &Vec::from([true]), timeline, &timelines_hash)?;
         }
     }
@@ -181,8 +394,17 @@ fn print_timeline(
     timeline: &TimelineTreeEl,
     timelines: &HashMap<TimelineId, TimelineTreeEl>,
 ) -> Result<()> {
+    let local_remote = match (timeline.info.local.as_ref(), timeline.info.remote.as_ref()) {
+        (None, None) => unreachable!("in this case no info for a timeline is found"),
+        (None, Some(_)) => "(R)",
+        (Some(_), None) => "(L)",
+        (Some(_), Some(_)) => "(L+R)",
+    };
+    // Draw main padding
+    print!("{} ", local_remote);
+
     if nesting_level > 0 {
-        let ancestor_lsn = match timeline.info.ancestor_lsn {
+        let ancestor_lsn = match timeline.info.local.as_ref().and_then(|i| i.ancestor_lsn) {
             Some(lsn) => lsn.to_string(),
             None => "Unknown Lsn".to_string(),
         };
@@ -270,16 +492,16 @@ fn get_tenant_id(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow::R
 
 fn parse_tenant_id(sub_match: &ArgMatches) -> anyhow::Result<Option<TenantId>> {
     sub_match
-        .get_one::<String>("tenant-id")
-        .map(|tenant_id| TenantId::from_str(tenant_id))
+        .value_of("tenant-id")
+        .map(TenantId::from_str)
         .transpose()
         .context("Failed to parse tenant id from the argument string")
 }
 
 fn parse_timeline_id(sub_match: &ArgMatches) -> anyhow::Result<Option<TimelineId>> {
     sub_match
-        .get_one::<String>("timeline-id")
-        .map(|timeline_id| TimelineId::from_str(timeline_id))
+        .value_of("timeline-id")
+        .map(TimelineId::from_str)
         .transpose()
         .context("Failed to parse timeline id from the argument string")
 }
@@ -288,22 +510,19 @@ fn handle_init(init_match: &ArgMatches) -> anyhow::Result<LocalEnv> {
     let initial_timeline_id_arg = parse_timeline_id(init_match)?;
 
     // Create config file
-    let toml_file: String = if let Some(config_path) = init_match.get_one::<PathBuf>("config") {
+    let toml_file: String = if let Some(config_path) = init_match.value_of("config") {
         // load and parse the file
-        std::fs::read_to_string(config_path).with_context(|| {
-            format!(
-                "Could not read configuration file '{}'",
-                config_path.display()
-            )
-        })?
+        std::fs::read_to_string(std::path::Path::new(config_path))
+            .with_context(|| format!("Could not read configuration file '{config_path}'"))?
     } else {
         // Built-in default config
         default_conf(&EtcdBroker::locate_etcd()?)
     };
 
     let pg_version = init_match
-        .get_one::<u32>("pg-version")
-        .copied()
+        .value_of("pg-version")
+        .unwrap()
+        .parse::<u32>()
         .context("Failed to parse postgres version from the argument string")?;
 
     let mut env =
@@ -339,10 +558,9 @@ fn handle_init(init_match: &ArgMatches) -> anyhow::Result<LocalEnv> {
 
 fn pageserver_config_overrides(init_match: &ArgMatches) -> Vec<&str> {
     init_match
-        .get_many::<String>("pageserver-config-override")
+        .values_of("pageserver-config-override")
         .into_iter()
         .flatten()
-        .map(|s| s.as_str())
         .collect()
 }
 
@@ -357,7 +575,7 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an
         Some(("create", create_match)) => {
             let initial_tenant_id = parse_tenant_id(create_match)?;
             let tenant_conf: HashMap<_, _> = create_match
-                .get_many::<String>("config")
+                .values_of("config")
                 .map(|vals| vals.flat_map(|c| c.split_once(':')).collect())
                 .unwrap_or_default();
             let new_tenant_id = pageserver.tenant_create(initial_tenant_id, tenant_conf)?;
@@ -366,8 +584,9 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an
             // Create an initial timeline for the new tenant
             let new_timeline_id = parse_timeline_id(create_match)?;
             let pg_version = create_match
-                .get_one::<u32>("pg-version")
-                .copied()
+                .value_of("pg-version")
+                .unwrap()
+                .parse::<u32>()
                 .context("Failed to parse postgres version from the argument string")?;
 
             let timeline_info = pageserver.timeline_create(
@@ -378,7 +597,10 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an
                 Some(pg_version),
             )?;
             let new_timeline_id = timeline_info.timeline_id;
-            let last_record_lsn = timeline_info.last_record_lsn;
+            let last_record_lsn = timeline_info
+                .local
+                .context(format!("Failed to get last record LSN: no local timeline info for timeline {new_timeline_id}"))?
+                .last_record_lsn;
 
             env.register_branch_mapping(
                 DEFAULT_BRANCH_NAME.to_string(),
@@ -393,7 +615,7 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an
         Some(("config", create_match)) => {
             let tenant_id = get_tenant_id(create_match, env)?;
             let tenant_conf: HashMap<_, _> = create_match
-                .get_many::<String>("config")
+                .values_of("config")
                 .map(|vals| vals.flat_map(|c| c.split_once(':')).collect())
                 .unwrap_or_default();
 
@@ -420,19 +642,23 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) -
         Some(("create", create_match)) => {
             let tenant_id = get_tenant_id(create_match, env)?;
             let new_branch_name = create_match
-                .get_one::<String>("branch-name")
+                .value_of("branch-name")
                 .ok_or_else(|| anyhow!("No branch name provided"))?;
 
             let pg_version = create_match
-                .get_one::<u32>("pg-version")
-                .copied()
+                .value_of("pg-version")
+                .unwrap()
+                .parse::<u32>()
                 .context("Failed to parse postgres version from the argument string")?;
 
             let timeline_info =
                 pageserver.timeline_create(tenant_id, None, None, None, Some(pg_version))?;
             let new_timeline_id = timeline_info.timeline_id;
 
-            let last_record_lsn = timeline_info.last_record_lsn;
+            let last_record_lsn = timeline_info
+                .local
+                .expect("no local timeline info")
+                .last_record_lsn;
             env.register_branch_mapping(new_branch_name.to_string(), tenant_id, new_timeline_id)?;
 
             println!(
@@ -444,32 +670,35 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) -
             let tenant_id = get_tenant_id(import_match, env)?;
             let timeline_id = parse_timeline_id(import_match)?.expect("No timeline id provided");
             let name = import_match
-                .get_one::<String>("node-name")
+                .value_of("node-name")
                 .ok_or_else(|| anyhow!("No node name provided"))?;
 
             // Parse base inputs
             let base_tarfile = import_match
-                .get_one::<PathBuf>("base-tarfile")
-                .ok_or_else(|| anyhow!("No base-tarfile provided"))?
-                .to_owned();
+                .value_of("base-tarfile")
+                .map(|s| PathBuf::from_str(s).unwrap())
+                .ok_or_else(|| anyhow!("No base-tarfile provided"))?;
             let base_lsn = Lsn::from_str(
                 import_match
-                    .get_one::<String>("base-lsn")
+                    .value_of("base-lsn")
                     .ok_or_else(|| anyhow!("No base-lsn provided"))?,
             )?;
             let base = (base_lsn, base_tarfile);
 
             // Parse pg_wal inputs
-            let wal_tarfile = import_match.get_one::<PathBuf>("wal-tarfile").cloned();
+            let wal_tarfile = import_match
+                .value_of("wal-tarfile")
+                .map(|s| PathBuf::from_str(s).unwrap());
             let end_lsn = import_match
-                .get_one::<String>("end-lsn")
+                .value_of("end-lsn")
                 .map(|s| Lsn::from_str(s).unwrap());
             // TODO validate both or none are provided
             let pg_wal = end_lsn.zip(wal_tarfile);
 
             let pg_version = import_match
-                .get_one::<u32>("pg-version")
-                .copied()
+                .value_of("pg-version")
+                .unwrap()
+                .parse::<u32>()
                 .context("Failed to parse postgres version from the argument string")?;
 
             let mut cplane = ComputeControlPlane::load(env.clone())?;
@@ -484,11 +713,10 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) -
         Some(("branch", branch_match)) => {
             let tenant_id = get_tenant_id(branch_match, env)?;
             let new_branch_name = branch_match
-                .get_one::<String>("branch-name")
+                .value_of("branch-name")
                 .ok_or_else(|| anyhow!("No branch name provided"))?;
             let ancestor_branch_name = branch_match
-                .get_one::<String>("ancestor-branch-name")
-                .map(|s| s.as_str())
+                .value_of("ancestor-branch-name")
                 .unwrap_or(DEFAULT_BRANCH_NAME);
             let ancestor_timeline_id = env
                 .get_branch_timeline_id(ancestor_branch_name, tenant_id)
@@ -497,8 +725,8 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) -
                 })?;
 
             let start_lsn = branch_match
-                .get_one::<String>("ancestor-start-lsn")
-                .map(|lsn_str| Lsn::from_str(lsn_str))
+                .value_of("ancestor-start-lsn")
+                .map(Lsn::from_str)
                 .transpose()
                 .context("Failed to parse ancestor start Lsn from the request")?;
             let timeline_info = pageserver.timeline_create(
@@ -510,7 +738,10 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) -
             )?;
             let new_timeline_id = timeline_info.timeline_id;
 
-            let last_record_lsn = timeline_info.last_record_lsn;
+            let last_record_lsn = timeline_info
+                .local
+                .expect("no local timeline info")
+                .last_record_lsn;
 
             env.register_branch_mapping(new_branch_name.to_string(), tenant_id, new_timeline_id)?;
 
@@ -570,7 +801,7 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> {
                         // Use the LSN at the end of the timeline.
                         timeline_infos
                             .get(&node.timeline_id)
-                            .map(|bi| bi.last_record_lsn.to_string())
+                            .and_then(|bi| bi.local.as_ref().map(|l| l.last_record_lsn.to_string()))
                             .unwrap_or_else(|| "?".to_string())
                     }
                     Some(lsn) => {
@@ -599,39 +830,45 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> {
         }
         "create" => {
             let branch_name = sub_args
-                .get_one::<String>("branch-name")
-                .map(|s| s.as_str())
+                .value_of("branch-name")
                 .unwrap_or(DEFAULT_BRANCH_NAME);
             let node_name = sub_args
-                .get_one::<String>("node")
-                .map(|node_name| node_name.to_string())
-                .unwrap_or_else(|| format!("{branch_name}_node"));
+                .value_of("node")
+                .map(ToString::to_string)
+                .unwrap_or_else(|| format!("{}_node", branch_name));
 
             let lsn = sub_args
-                .get_one::<String>("lsn")
-                .map(|lsn_str| Lsn::from_str(lsn_str))
+                .value_of("lsn")
+                .map(Lsn::from_str)
                 .transpose()
                 .context("Failed to parse Lsn from the request")?;
             let timeline_id = env
                 .get_branch_timeline_id(branch_name, tenant_id)
-                .ok_or_else(|| anyhow!("Found no timeline id for branch name '{branch_name}'"))?;
+                .ok_or_else(|| anyhow!("Found no timeline id for branch name '{}'", branch_name))?;
 
-            let port: Option<u16> = sub_args.get_one::<u16>("port").copied();
+            let port: Option<u16> = match sub_args.value_of("port") {
+                Some(p) => Some(p.parse()?),
+                None => None,
+            };
 
             let pg_version = sub_args
-                .get_one::<u32>("pg-version")
-                .copied()
+                .value_of("pg-version")
+                .unwrap()
+                .parse::<u32>()
                 .context("Failed to parse postgres version from the argument string")?;
 
             cplane.new_node(tenant_id, &node_name, timeline_id, lsn, port, pg_version)?;
         }
         "start" => {
-            let port: Option<u16> = sub_args.get_one::<u16>("port").copied();
+            let port: Option<u16> = match sub_args.value_of("port") {
+                Some(p) => Some(p.parse()?),
+                None => None,
+            };
             let node_name = sub_args
-                .get_one::<String>("node")
+                .value_of("node")
                 .ok_or_else(|| anyhow!("No node name was provided to start"))?;
 
-            let node = cplane.nodes.get(&(tenant_id, node_name.to_string()));
+            let node = cplane.nodes.get(&(tenant_id, node_name.to_owned()));
 
             let auth_token = if matches!(env.pageserver.auth_type, AuthType::NeonJWT) {
                 let claims = Claims::new(Some(tenant_id), Scope::Tenant);
@@ -642,33 +879,36 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> {
             };
 
             if let Some(node) = node {
-                println!("Starting existing postgres {node_name}...");
+                println!("Starting existing postgres {}...", node_name);
                 node.start(&auth_token)?;
             } else {
                 let branch_name = sub_args
-                    .get_one::<String>("branch-name")
-                    .map(|s| s.as_str())
+                    .value_of("branch-name")
                     .unwrap_or(DEFAULT_BRANCH_NAME);
                 let timeline_id = env
                     .get_branch_timeline_id(branch_name, tenant_id)
                     .ok_or_else(|| {
-                        anyhow!("Found no timeline id for branch name '{branch_name}'")
+                        anyhow!("Found no timeline id for branch name '{}'", branch_name)
                     })?;
                 let lsn = sub_args
-                    .get_one::<String>("lsn")
-                    .map(|lsn_str| Lsn::from_str(lsn_str))
+                    .value_of("lsn")
+                    .map(Lsn::from_str)
                     .transpose()
                     .context("Failed to parse Lsn from the request")?;
                 let pg_version = sub_args
-                    .get_one::<u32>("pg-version")
-                    .copied()
-                    .context("Failed to `pg-version` from the argument string")?;
+                    .value_of("pg-version")
+                    .unwrap()
+                    .parse::<u32>()
+                    .context("Failed to parse postgres version from the argument string")?;
                 // when used with custom port this results in non obvious behaviour
                 // port is remembered from first start command, i e
                 // start --port X
                 // stop
                 // start <-- will also use port X even without explicit port argument
-                println!("Starting new postgres (v{pg_version}) {node_name} on timeline {timeline_id} ...");
+                println!(
+                    "Starting new postgres (v{}) {} on timeline {} ...",
+                    pg_version, node_name, timeline_id
+                );
 
                 let node =
                     cplane.new_node(tenant_id, node_name, timeline_id, lsn, port, pg_version)?;
@@ -677,18 +917,18 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> {
         }
         "stop" => {
             let node_name = sub_args
-                .get_one::<String>("node")
+                .value_of("node")
                 .ok_or_else(|| anyhow!("No node name was provided to stop"))?;
-            let destroy = sub_args.get_flag("destroy");
+            let destroy = sub_args.is_present("destroy");
 
             let node = cplane
                 .nodes
-                .get(&(tenant_id, node_name.to_string()))
-                .with_context(|| format!("postgres {node_name} is not found"))?;
+                .get(&(tenant_id, node_name.to_owned()))
+                .with_context(|| format!("postgres {} is not found", node_name))?;
             node.stop(destroy)?;
         }
 
-        _ => bail!("Unexpected pg subcommand '{sub_name}'"),
+        _ => bail!("Unexpected pg subcommand '{}'", sub_name),
     }
 
     Ok(())
@@ -706,10 +946,7 @@ fn handle_pageserver(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul
         }
 
         Some(("stop", stop_match)) => {
-            let immediate = stop_match
-                .get_one::<String>("stop-mode")
-                .map(|s| s.as_str())
-                == Some("immediate");
+            let immediate = stop_match.value_of("stop-mode") == Some("immediate");
 
             if let Err(e) = pageserver.stop(immediate) {
                 eprintln!("pageserver stop failed: {}", e);
@@ -759,7 +996,7 @@ fn handle_safekeeper(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul
     };
 
     // All the commands take an optional safekeeper name argument
-    let sk_id = if let Some(id_str) = sub_args.get_one::<String>("id") {
+    let sk_id = if let Some(id_str) = sub_args.value_of("id") {
         NodeId(id_str.parse().context("while parsing safekeeper id")?)
     } else {
         DEFAULT_SAFEKEEPER_ID
@@ -775,8 +1012,7 @@ fn handle_safekeeper(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul
         }
 
         "stop" => {
-            let immediate =
-                sub_args.get_one::<String>("stop-mode").map(|s| s.as_str()) == Some("immediate");
+            let immediate = sub_args.value_of("stop-mode") == Some("immediate");
 
             if let Err(e) = safekeeper.stop(immediate) {
                 eprintln!("safekeeper stop failed: {}", e);
@@ -785,8 +1021,7 @@ fn handle_safekeeper(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul
         }
 
         "restart" => {
-            let immediate =
-                sub_args.get_one::<String>("stop-mode").map(|s| s.as_str()) == Some("immediate");
+            let immediate = sub_args.value_of("stop-mode") == Some("immediate");
 
             if let Err(e) = safekeeper.stop(immediate) {
                 eprintln!("safekeeper stop failed: {}", e);
@@ -830,8 +1065,7 @@ fn handle_start_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow
 }
 
 fn handle_stop_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> {
-    let immediate =
-        sub_match.get_one::<String>("stop-mode").map(|s| s.as_str()) == Some("immediate");
+    let immediate = sub_match.value_of("stop-mode") == Some("immediate");
 
     let pageserver = PageServerNode::from_env(env);
 
@@ -864,219 +1098,3 @@ fn try_stop_etcd_process(env: &local_env::LocalEnv) {
         eprintln!("etcd stop failed: {e}");
     }
 }
-
-fn cli() -> Command {
-    let branch_name_arg = Arg::new("branch-name")
-        .long("branch-name")
-        .help("Name of the branch to be created or used as an alias for other services")
-        .required(false);
-
-    let pg_node_arg = Arg::new("node").help("Postgres node name").required(false);
-
-    let safekeeper_id_arg = Arg::new("id").help("safekeeper id").required(false);
-
-    let tenant_id_arg = Arg::new("tenant-id")
-        .long("tenant-id")
-        .help("Tenant id. Represented as a hexadecimal string 32 symbols length")
-        .required(false);
-
-    let timeline_id_arg = Arg::new("timeline-id")
-        .long("timeline-id")
-        .help("Timeline id. Represented as a hexadecimal string 32 symbols length")
-        .required(false);
-
-    let pg_version_arg = Arg::new("pg-version")
-        .long("pg-version")
-        .help("Postgres version to use for the initial tenant")
-        .required(false)
-        .value_parser(value_parser!(u32))
-        .default_value(DEFAULT_PG_VERSION);
-
-    let port_arg = Arg::new("port")
-        .long("port")
-        .required(false)
-        .value_parser(value_parser!(u16))
-        .value_name("port");
-
-    let stop_mode_arg = Arg::new("stop-mode")
-        .short('m')
-        .value_parser(["fast", "immediate"])
-        .help("If 'immediate', don't flush repository data at shutdown")
-        .required(false)
-        .value_name("stop-mode");
-
-    let pageserver_config_args = Arg::new("pageserver-config-override")
-        .long("pageserver-config-override")
-        .num_args(1)
-        .action(ArgAction::Append)
-        .help("Additional pageserver's configuration options or overrides, refer to pageserver's 'config-override' CLI parameter docs for more")
-        .required(false);
-
-    let lsn_arg = Arg::new("lsn")
-        .long("lsn")
-        .help("Specify Lsn on the timeline to start from. By default, end of the timeline would be used.")
-        .required(false);
-
-    Command::new("Neon CLI")
-        .arg_required_else_help(true)
-        .version(GIT_VERSION)
-        .subcommand(
-            Command::new("init")
-                .about("Initialize a new Neon repository")
-                .arg(pageserver_config_args.clone())
-                .arg(timeline_id_arg.clone().help("Use a specific timeline id when creating a tenant and its initial timeline"))
-                .arg(
-                    Arg::new("config")
-                        .long("config")
-                        .required(false)
-                        .value_parser(value_parser!(PathBuf))
-                        .value_name("config"),
-                )
-                .arg(pg_version_arg.clone())
-        )
-        .subcommand(
-            Command::new("timeline")
-            .about("Manage timelines")
-            .subcommand(Command::new("list")
-                .about("List all timelines, available to this pageserver")
-                .arg(tenant_id_arg.clone()))
-            .subcommand(Command::new("branch")
-                .about("Create a new timeline, using another timeline as a base, copying its data")
-                .arg(tenant_id_arg.clone())
-                .arg(branch_name_arg.clone())
-                .arg(Arg::new("ancestor-branch-name").long("ancestor-branch-name")
-                    .help("Use last Lsn of another timeline (and its data) as base when creating the new timeline. The timeline gets resolved by its branch name.").required(false))
-                .arg(Arg::new("ancestor-start-lsn").long("ancestor-start-lsn")
-                    .help("When using another timeline as base, use a specific Lsn in it instead of the latest one").required(false)))
-            .subcommand(Command::new("create")
-                .about("Create a new blank timeline")
-                .arg(tenant_id_arg.clone())
-                .arg(branch_name_arg.clone())
-                .arg(pg_version_arg.clone())
-            )
-            .subcommand(Command::new("import")
-                .about("Import timeline from basebackup directory")
-                .arg(tenant_id_arg.clone())
-                .arg(timeline_id_arg.clone())
-                .arg(Arg::new("node-name").long("node-name")
-                    .help("Name to assign to the imported timeline"))
-                .arg(Arg::new("base-tarfile")
-                    .long("base-tarfile")
-                    .value_parser(value_parser!(PathBuf))
-                    .help("Basebackup tarfile to import")
-                )
-                .arg(Arg::new("base-lsn").long("base-lsn")
-                    .help("Lsn the basebackup starts at"))
-                .arg(Arg::new("wal-tarfile")
-                    .long("wal-tarfile")
-                    .value_parser(value_parser!(PathBuf))
-                    .help("Wal to add after base")
-                )
-                .arg(Arg::new("end-lsn").long("end-lsn")
-                    .help("Lsn the basebackup ends at"))
-                .arg(pg_version_arg.clone())
-            )
-        ).subcommand(
-            Command::new("tenant")
-            .arg_required_else_help(true)
-            .about("Manage tenants")
-            .subcommand(Command::new("list"))
-            .subcommand(Command::new("create")
-                .arg(tenant_id_arg.clone())
-                .arg(timeline_id_arg.clone().help("Use a specific timeline id when creating a tenant and its initial timeline"))
-                .arg(Arg::new("config").short('c').num_args(1).action(ArgAction::Append).required(false))
-                .arg(pg_version_arg.clone())
-                )
-            .subcommand(Command::new("config")
-                .arg(tenant_id_arg.clone())
-                .arg(Arg::new("config").short('c').num_args(1).action(ArgAction::Append).required(false))
-                )
-        )
-        .subcommand(
-            Command::new("pageserver")
-                .arg_required_else_help(true)
-                .about("Manage pageserver")
-                .subcommand(Command::new("status"))
-                .subcommand(Command::new("start").about("Start local pageserver").arg(pageserver_config_args.clone()))
-                .subcommand(Command::new("stop").about("Stop local pageserver")
-                            .arg(stop_mode_arg.clone()))
-                .subcommand(Command::new("restart").about("Restart local pageserver").arg(pageserver_config_args.clone()))
-        )
-        .subcommand(
-            Command::new("safekeeper")
-                .arg_required_else_help(true)
-                .about("Manage safekeepers")
-                .subcommand(Command::new("start")
-                            .about("Start local safekeeper")
-                            .arg(safekeeper_id_arg.clone())
-                )
-                .subcommand(Command::new("stop")
-                            .about("Stop local safekeeper")
-                            .arg(safekeeper_id_arg.clone())
-                            .arg(stop_mode_arg.clone())
-                )
-                .subcommand(Command::new("restart")
-                            .about("Restart local safekeeper")
-                            .arg(safekeeper_id_arg)
-                            .arg(stop_mode_arg.clone())
-                )
-        )
-        .subcommand(
-            Command::new("pg")
-                .arg_required_else_help(true)
-                .about("Manage postgres instances")
-                .subcommand(Command::new("list").arg(tenant_id_arg.clone()))
-                .subcommand(Command::new("create")
-                    .about("Create a postgres compute node")
-                    .arg(pg_node_arg.clone())
-                    .arg(branch_name_arg.clone())
-                    .arg(tenant_id_arg.clone())
-                    .arg(lsn_arg.clone())
-                    .arg(port_arg.clone())
-                    .arg(
-                        Arg::new("config-only")
-                            .help("Don't do basebackup, create compute node with only config files")
-                            .long("config-only")
-                            .required(false))
-                    .arg(pg_version_arg.clone())
-                )
-                .subcommand(Command::new("start")
-                    .about("Start a postgres compute node.\n This command actually creates new node from scratch, but preserves existing config files")
-                    .arg(pg_node_arg.clone())
-                    .arg(tenant_id_arg.clone())
-                    .arg(branch_name_arg)
-                    .arg(timeline_id_arg)
-                    .arg(lsn_arg)
-                    .arg(port_arg)
-                    .arg(pg_version_arg)
-                )
-                .subcommand(
-                    Command::new("stop")
-                    .arg(pg_node_arg)
-                    .arg(tenant_id_arg)
-                    .arg(
-                        Arg::new("destroy")
-                            .help("Also delete data directory (now optional, should be default in future)")
-                            .long("destroy")
-                            .action(ArgAction::SetTrue)
-                            .required(false)
-                        )
-                )
-
-        )
-        .subcommand(
-            Command::new("start")
-                .about("Start page server and safekeepers")
-                .arg(pageserver_config_args)
-        )
-        .subcommand(
-            Command::new("stop")
-                .about("Stop page server and safekeepers")
-                .arg(stop_mode_arg)
-        )
-}
-
-#[test]
-fn verify_cli() {
-    cli().debug_assert();
-}

control_plane/src/compute.rs

@@ -12,14 +12,15 @@ use std::time::Duration;
 
 use anyhow::{Context, Result};
 use utils::{
+    connstring::connection_host_port,
     id::{TenantId, TimelineId},
     lsn::Lsn,
     postgres_backend::AuthType,
 };
 
 use crate::local_env::{LocalEnv, DEFAULT_PG_VERSION};
-use crate::pageserver::PageServerNode;
 use crate::postgresql_conf::PostgresConf;
+use crate::storage::PageServerNode;
 
 //
 // ComputeControlPlane
@@ -182,18 +183,18 @@ impl PostgresNode {
     }
 
     fn sync_safekeepers(&self, auth_token: &Option<String>, pg_version: u32) -> Result<Lsn> {
-        let pg_path = self.env.pg_bin_dir(pg_version)?.join("postgres");
+        let pg_path = self.env.pg_bin_dir(pg_version).join("postgres");
         let mut cmd = Command::new(&pg_path);
 
         cmd.arg("--sync-safekeepers")
             .env_clear()
             .env(
                 "LD_LIBRARY_PATH",
-                self.env.pg_lib_dir(pg_version)?.to_str().unwrap(),
+                self.env.pg_lib_dir(pg_version).to_str().unwrap(),
             )
             .env(
                 "DYLD_LIBRARY_PATH",
-                self.env.pg_lib_dir(pg_version)?.to_str().unwrap(),
+                self.env.pg_lib_dir(pg_version).to_str().unwrap(),
             )
             .env("PGDATA", self.pgdata().to_str().unwrap())
             .stdout(Stdio::piped())
@@ -281,6 +282,8 @@ impl PostgresNode {
     fn setup_pg_conf(&self, auth_type: AuthType) -> Result<()> {
         let mut conf = PostgresConf::new();
         conf.append("max_wal_senders", "10");
+        // wal_log_hints is mandatory when running against pageserver (see gh issue#192)
+        // TODO: is it possible to check wal_log_hints at pageserver side via XLOG_PARAMETER_CHANGE?
         conf.append("wal_log_hints", "off");
         conf.append("max_replication_slots", "10");
         conf.append("hot_standby", "on");
@@ -299,8 +302,7 @@ impl PostgresNode {
 
         // Configure the node to fetch pages from pageserver
         let pageserver_connstr = {
-            let config = &self.pageserver.pg_connection_config;
-            let (host, port) = (config.host(), config.port());
+            let (host, port) = connection_host_port(&self.pageserver.pg_connection_config);
 
             // Set up authentication
             //
@@ -420,7 +422,7 @@ impl PostgresNode {
     }
 
     fn pg_ctl(&self, args: &[&str], auth_token: &Option<String>) -> Result<()> {
-        let pg_ctl_path = self.env.pg_bin_dir(self.pg_version)?.join("pg_ctl");
+        let pg_ctl_path = self.env.pg_bin_dir(self.pg_version).join("pg_ctl");
         let mut cmd = Command::new(pg_ctl_path);
         cmd.args(
             [
@@ -438,11 +440,11 @@ impl PostgresNode {
         .env_clear()
         .env(
             "LD_LIBRARY_PATH",
-            self.env.pg_lib_dir(self.pg_version)?.to_str().unwrap(),
+            self.env.pg_lib_dir(self.pg_version).to_str().unwrap(),
         )
         .env(
             "DYLD_LIBRARY_PATH",
-            self.env.pg_lib_dir(self.pg_version)?.to_str().unwrap(),
+            self.env.pg_lib_dir(self.pg_version).to_str().unwrap(),
         );
         if let Some(token) = auth_token {
             cmd.env("ZENITH_AUTH_TOKEN", token);

control_plane/src/connection.rs

@@ -1,57 +0,0 @@
-use url::Url;
-
-#[derive(Debug)]
-pub struct PgConnectionConfig {
-    url: Url,
-}
-
-impl PgConnectionConfig {
-    pub fn host(&self) -> &str {
-        self.url.host_str().expect("BUG: no host")
-    }
-
-    pub fn port(&self) -> u16 {
-        self.url.port().expect("BUG: no port")
-    }
-
-    /// Return a `<host>:<port>` string.
-    pub fn raw_address(&self) -> String {
-        format!("{}:{}", self.host(), self.port())
-    }
-
-    /// Connect using postgres protocol with TLS disabled.
-    pub fn connect_no_tls(&self) -> Result<postgres::Client, postgres::Error> {
-        postgres::Client::connect(self.url.as_str(), postgres::NoTls)
-    }
-}
-
-impl std::str::FromStr for PgConnectionConfig {
-    type Err = anyhow::Error;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        let mut url: Url = s.parse()?;
-
-        match url.scheme() {
-            "postgres" | "postgresql" => {}
-            other => anyhow::bail!("invalid scheme: {other}"),
-        }
-
-        // It's not a valid connection url if host is unavailable.
-        if url.host().is_none() {
-            anyhow::bail!(url::ParseError::EmptyHost);
-        }
-
-        // E.g. `postgres:bar`.
-        if url.cannot_be_a_base() {
-            anyhow::bail!("URL cannot be a base");
-        }
-
-        // Set the default PG port if it's missing.
-        if url.port().is_none() {
-            url.set_port(Some(5432))
-                .expect("BUG: couldn't set the default port");
-        }
-
-        Ok(Self { url })
-    }
-}

control_plane/src/etcd.rs

@@ -1,75 +1,95 @@
-use std::{fs, path::PathBuf};
+use std::{
+    fs,
+    path::PathBuf,
+    process::{Command, Stdio},
+};
 
 use anyhow::Context;
+use nix::{
+    sys::signal::{kill, Signal},
+    unistd::Pid,
+};
 
-use crate::{background_process, local_env};
+use crate::{local_env, read_pidfile};
 
 pub fn start_etcd_process(env: &local_env::LocalEnv) -> anyhow::Result<()> {
     let etcd_broker = &env.etcd_broker;
     println!(
-        "Starting etcd broker using {:?}",
-        etcd_broker.etcd_binary_path
+        "Starting etcd broker using {}",
+        etcd_broker.etcd_binary_path.display()
     );
 
     let etcd_data_dir = env.base_data_dir.join("etcd");
-    fs::create_dir_all(&etcd_data_dir)
-        .with_context(|| format!("Failed to create etcd data dir {etcd_data_dir:?}"))?;
+    fs::create_dir_all(&etcd_data_dir).with_context(|| {
+        format!(
+            "Failed to create etcd data dir: {}",
+            etcd_data_dir.display()
+        )
+    })?;
 
+    let etcd_stdout_file =
+        fs::File::create(etcd_data_dir.join("etcd.stdout.log")).with_context(|| {
+            format!(
+                "Failed to create etcd stout file in directory {}",
+                etcd_data_dir.display()
+            )
+        })?;
+    let etcd_stderr_file =
+        fs::File::create(etcd_data_dir.join("etcd.stderr.log")).with_context(|| {
+            format!(
+                "Failed to create etcd stderr file in directory {}",
+                etcd_data_dir.display()
+            )
+        })?;
     let client_urls = etcd_broker.comma_separated_endpoints();
-    let args = [
-        format!("--data-dir={}", etcd_data_dir.display()),
-        format!("--listen-client-urls={client_urls}"),
-        format!("--advertise-client-urls={client_urls}"),
-        // Set --quota-backend-bytes to keep the etcd virtual memory
-        // size smaller. Our test etcd clusters are very small.
-        // See https://github.com/etcd-io/etcd/issues/7910
-        "--quota-backend-bytes=100000000".to_string(),
-        // etcd doesn't compact (vacuum) with default settings,
-        // enable it to prevent space exhaustion.
-        "--auto-compaction-mode=revision".to_string(),
-        "--auto-compaction-retention=1".to_string(),
-    ];
 
-    let pid_file_path = etcd_pid_file_path(env);
+    let etcd_process = Command::new(&etcd_broker.etcd_binary_path)
+        .args(&[
+            format!("--data-dir={}", etcd_data_dir.display()),
+            format!("--listen-client-urls={client_urls}"),
+            format!("--advertise-client-urls={client_urls}"),
+            // Set --quota-backend-bytes to keep the etcd virtual memory
+            // size smaller. Our test etcd clusters are very small.
+            // See https://github.com/etcd-io/etcd/issues/7910
+            "--quota-backend-bytes=100000000".to_string(),
+        ])
+        .stdout(Stdio::from(etcd_stdout_file))
+        .stderr(Stdio::from(etcd_stderr_file))
+        .spawn()
+        .context("Failed to spawn etcd subprocess")?;
+    let pid = etcd_process.id();
 
-    let client = reqwest::blocking::Client::new();
-
-    background_process::start_process(
-        "etcd",
-        &etcd_data_dir,
-        &etcd_broker.etcd_binary_path,
-        &args,
-        background_process::InitialPidFile::Create(&pid_file_path),
-        || {
-            for broker_endpoint in &etcd_broker.broker_endpoints {
-                let request = broker_endpoint
-                    .join("health")
-                    .with_context(|| {
-                        format!(
-                            "Failed to append /health path to broker endopint {}",
-                            broker_endpoint
-                        )
-                    })
-                    .and_then(|url| {
-                        client.get(&url.to_string()).build().with_context(|| {
-                            format!("Failed to construct request to etcd endpoint {url}")
-                        })
-                    })?;
-                if client.execute(request).is_ok() {
-                    return Ok(true);
-                }
-            }
-
-            Ok(false)
-        },
-    )
-    .context("Failed to spawn etcd subprocess")?;
+    let etcd_pid_file_path = etcd_pid_file_path(env);
+    fs::write(&etcd_pid_file_path, pid.to_string()).with_context(|| {
+        format!(
+            "Failed to create etcd pid file at {}",
+            etcd_pid_file_path.display()
+        )
+    })?;
 
     Ok(())
 }
 
 pub fn stop_etcd_process(env: &local_env::LocalEnv) -> anyhow::Result<()> {
-    background_process::stop_process(true, "etcd", &etcd_pid_file_path(env))
+    let etcd_path = &env.etcd_broker.etcd_binary_path;
+    println!("Stopping etcd broker at {}", etcd_path.display());
+
+    let etcd_pid_file_path = etcd_pid_file_path(env);
+    let pid = Pid::from_raw(read_pidfile(&etcd_pid_file_path).with_context(|| {
+        format!(
+            "Failed to read etcd pid file at {}",
+            etcd_pid_file_path.display()
+        )
+    })?);
+
+    kill(pid, Signal::SIGTERM).with_context(|| {
+        format!(
+            "Failed to stop etcd with pid {pid} at {}",
+            etcd_pid_file_path.display()
+        )
+    })?;
+
+    Ok(())
 }
 
 fn etcd_pid_file_path(env: &local_env::LocalEnv) -> PathBuf {

control_plane/src/lib.rs

@@ -6,12 +6,59 @@
 // Intended to be used in integration tests and in CLI tools for
 // local installations.
 //
+use anyhow::{anyhow, bail, Context, Result};
+use std::fs;
+use std::path::Path;
+use std::process::Command;
 
-mod background_process;
 pub mod compute;
-pub mod connection;
 pub mod etcd;
 pub mod local_env;
-pub mod pageserver;
 pub mod postgresql_conf;
 pub mod safekeeper;
+pub mod storage;
+
+/// Read a PID file
+///
+/// We expect a file that contains a single integer.
+/// We return an i32 for compatibility with libc and nix.
+pub fn read_pidfile(pidfile: &Path) -> Result<i32> {
+    let pid_str = fs::read_to_string(pidfile)
+        .with_context(|| format!("failed to read pidfile {:?}", pidfile))?;
+    let pid: i32 = pid_str
+        .parse()
+        .map_err(|_| anyhow!("failed to parse pidfile {:?}", pidfile))?;
+    if pid < 1 {
+        bail!("pidfile {:?} contained bad value '{}'", pidfile, pid);
+    }
+    Ok(pid)
+}
+
+fn fill_rust_env_vars(cmd: &mut Command) -> &mut Command {
+    let cmd = cmd.env_clear().env("RUST_BACKTRACE", "1");
+
+    let var = "LLVM_PROFILE_FILE";
+    if let Some(val) = std::env::var_os(var) {
+        cmd.env(var, val);
+    }
+
+    const RUST_LOG_KEY: &str = "RUST_LOG";
+    if let Ok(rust_log_value) = std::env::var(RUST_LOG_KEY) {
+        cmd.env(RUST_LOG_KEY, rust_log_value)
+    } else {
+        cmd
+    }
+}
+
+fn fill_aws_secrets_vars(mut cmd: &mut Command) -> &mut Command {
+    for env_key in [
+        "AWS_ACCESS_KEY_ID",
+        "AWS_SECRET_ACCESS_KEY",
+        "AWS_SESSION_TOKEN",
+    ] {
+        if let Ok(value) = std::env::var(env_key) {
+            cmd = cmd.env(env_key, value);
+        }
+    }
+    cmd
+}

control_plane/src/local_env.rs

@@ -201,37 +201,37 @@ impl LocalEnv {
         self.pg_distrib_dir.clone()
     }
 
-    pub fn pg_distrib_dir(&self, pg_version: u32) -> anyhow::Result<PathBuf> {
+    pub fn pg_distrib_dir(&self, pg_version: u32) -> PathBuf {
         let path = self.pg_distrib_dir.clone();
 
         match pg_version {
-            14 => Ok(path.join(format!("v{pg_version}"))),
-            15 => Ok(path.join(format!("v{pg_version}"))),
-            _ => bail!("Unsupported postgres version: {}", pg_version),
+            14 => path.join(format!("v{pg_version}")),
+            15 => path.join(format!("v{pg_version}")),
+            _ => panic!("Unsupported postgres version: {}", pg_version),
         }
     }
 
-    pub fn pg_bin_dir(&self, pg_version: u32) -> anyhow::Result<PathBuf> {
+    pub fn pg_bin_dir(&self, pg_version: u32) -> PathBuf {
         match pg_version {
-            14 => Ok(self.pg_distrib_dir(pg_version)?.join("bin")),
-            15 => Ok(self.pg_distrib_dir(pg_version)?.join("bin")),
-            _ => bail!("Unsupported postgres version: {}", pg_version),
+            14 => self.pg_distrib_dir(pg_version).join("bin"),
+            15 => self.pg_distrib_dir(pg_version).join("bin"),
+            _ => panic!("Unsupported postgres version: {}", pg_version),
         }
     }
-    pub fn pg_lib_dir(&self, pg_version: u32) -> anyhow::Result<PathBuf> {
+    pub fn pg_lib_dir(&self, pg_version: u32) -> PathBuf {
         match pg_version {
-            14 => Ok(self.pg_distrib_dir(pg_version)?.join("lib")),
-            15 => Ok(self.pg_distrib_dir(pg_version)?.join("lib")),
-            _ => bail!("Unsupported postgres version: {}", pg_version),
+            14 => self.pg_distrib_dir(pg_version).join("lib"),
+            15 => self.pg_distrib_dir(pg_version).join("lib"),
+            _ => panic!("Unsupported postgres version: {}", pg_version),
         }
     }
 
-    pub fn pageserver_bin(&self) -> PathBuf {
-        self.neon_distrib_dir.join("pageserver")
+    pub fn pageserver_bin(&self) -> anyhow::Result<PathBuf> {
+        Ok(self.neon_distrib_dir.join("pageserver"))
     }
 
-    pub fn safekeeper_bin(&self) -> PathBuf {
-        self.neon_distrib_dir.join("safekeeper")
+    pub fn safekeeper_bin(&self) -> anyhow::Result<PathBuf> {
+        Ok(self.neon_distrib_dir.join("safekeeper"))
     }
 
     pub fn pg_data_dirs_path(&self) -> PathBuf {
@@ -422,10 +422,10 @@ impl LocalEnv {
             "directory '{}' already exists. Perhaps already initialized?",
             base_path.display()
         );
-        if !self.pg_bin_dir(pg_version)?.join("postgres").exists() {
+        if !self.pg_bin_dir(pg_version).join("postgres").exists() {
             bail!(
                 "Can't find postgres binary at {}",
-                self.pg_bin_dir(pg_version)?.display()
+                self.pg_bin_dir(pg_version).display()
             );
         }
         for binary in ["pageserver", "safekeeper"] {

control_plane/src/safekeeper.rs

@@ -1,22 +1,29 @@
 use std::io::Write;
 use std::path::PathBuf;
-use std::process::Child;
+use std::process::Command;
 use std::sync::Arc;
-use std::{io, result};
+use std::time::Duration;
+use std::{io, result, thread};
 
-use anyhow::Context;
+use anyhow::bail;
+use nix::errno::Errno;
+use nix::sys::signal::{kill, Signal};
+use nix::unistd::Pid;
+use postgres::Config;
 use reqwest::blocking::{Client, RequestBuilder, Response};
 use reqwest::{IntoUrl, Method};
+use safekeeper::http::models::TimelineCreateRequest;
 use thiserror::Error;
-use utils::{http::error::HttpErrorBody, id::NodeId};
-
-use crate::connection::PgConnectionConfig;
-use crate::pageserver::PageServerNode;
-use crate::{
-    background_process,
-    local_env::{LocalEnv, SafekeeperConf},
+use utils::{
+    connstring::connection_address,
+    http::error::HttpErrorBody,
+    id::{NodeId, TenantId, TimelineId},
 };
 
+use crate::local_env::{LocalEnv, SafekeeperConf};
+use crate::storage::PageServerNode;
+use crate::{fill_aws_secrets_vars, fill_rust_env_vars, read_pidfile};
+
 #[derive(Error, Debug)]
 pub enum SafekeeperHttpError {
     #[error("Reqwest error: {0}")]
@@ -61,7 +68,7 @@ pub struct SafekeeperNode {
 
     pub conf: SafekeeperConf,
 
-    pub pg_connection_config: PgConnectionConfig,
+    pub pg_connection_config: Config,
     pub env: LocalEnv,
     pub http_client: Client,
     pub http_base_url: String,
@@ -85,15 +92,15 @@ impl SafekeeperNode {
     }
 
     /// Construct libpq connection string for connecting to this safekeeper.
-    fn safekeeper_connection_config(port: u16) -> PgConnectionConfig {
+    fn safekeeper_connection_config(port: u16) -> Config {
         // TODO safekeeper authentication not implemented yet
-        format!("postgresql://no_user@127.0.0.1:{port}/no_db")
+        format!("postgresql://no_user@127.0.0.1:{}/no_db", port)
             .parse()
             .unwrap()
     }
 
     pub fn datadir_path_by_id(env: &LocalEnv, sk_id: NodeId) -> PathBuf {
-        env.safekeeper_data_dir(&format!("sk{sk_id}"))
+        env.safekeeper_data_dir(format!("sk{}", sk_id).as_ref())
     }
 
     pub fn datadir_path(&self) -> PathBuf {
@@ -104,78 +111,92 @@ impl SafekeeperNode {
         self.datadir_path().join("safekeeper.pid")
     }
 
-    pub fn start(&self) -> anyhow::Result<Child> {
+    pub fn start(&self) -> anyhow::Result<()> {
         print!(
             "Starting safekeeper at '{}' in '{}'",
-            self.pg_connection_config.raw_address(),
+            connection_address(&self.pg_connection_config),
             self.datadir_path().display()
         );
         io::stdout().flush().unwrap();
 
         let listen_pg = format!("127.0.0.1:{}", self.conf.pg_port);
         let listen_http = format!("127.0.0.1:{}", self.conf.http_port);
-        let id = self.id;
-        let datadir = self.datadir_path();
 
-        let id_string = id.to_string();
-        let mut args = vec![
-            "-D",
-            datadir.to_str().with_context(|| {
-                format!("Datadir path {datadir:?} cannot be represented as a unicode string")
-            })?,
-            "--id",
-            &id_string,
-            "--listen-pg",
-            &listen_pg,
-            "--listen-http",
-            &listen_http,
-        ];
+        let mut cmd = Command::new(self.env.safekeeper_bin()?);
+        fill_rust_env_vars(
+            cmd.args(&["-D", self.datadir_path().to_str().unwrap()])
+                .args(&["--id", self.id.to_string().as_ref()])
+                .args(&["--listen-pg", &listen_pg])
+                .args(&["--listen-http", &listen_http])
+                .args(&["--recall", "1 second"])
+                .arg("--daemonize"),
+        );
         if !self.conf.sync {
-            args.push("--no-sync");
+            cmd.arg("--no-sync");
         }
 
         let comma_separated_endpoints = self.env.etcd_broker.comma_separated_endpoints();
         if !comma_separated_endpoints.is_empty() {
-            args.extend(["--broker-endpoints", &comma_separated_endpoints]);
+            cmd.args(&["--broker-endpoints", &comma_separated_endpoints]);
         }
         if let Some(prefix) = self.env.etcd_broker.broker_etcd_prefix.as_deref() {
-            args.extend(["--broker-etcd-prefix", prefix]);
+            cmd.args(&["--broker-etcd-prefix", prefix]);
         }
-
-        let mut backup_threads = String::new();
         if let Some(threads) = self.conf.backup_threads {
-            backup_threads = threads.to_string();
-            args.extend(["--backup-threads", &backup_threads]);
-        } else {
-            drop(backup_threads);
+            cmd.args(&["--backup-threads", threads.to_string().as_ref()]);
         }
-
         if let Some(ref remote_storage) = self.conf.remote_storage {
-            args.extend(["--remote-storage", remote_storage]);
+            cmd.args(&["--remote-storage", remote_storage]);
         }
-
-        let key_path = self.env.base_data_dir.join("auth_public_key.pem");
         if self.conf.auth_enabled {
-            args.extend([
-                "--auth-validation-public-key-path",
-                key_path.to_str().with_context(|| {
-                    format!("Key path {key_path:?} cannot be represented as a unicode string")
-                })?,
-            ]);
+            cmd.arg("--auth-validation-public-key-path");
+            // PathBuf is better be passed as is, not via `String`.
+            cmd.arg(self.env.base_data_dir.join("auth_public_key.pem"));
         }
 
-        background_process::start_process(
-            &format!("safekeeper {id}"),
-            &datadir,
-            &self.env.safekeeper_bin(),
-            &args,
-            background_process::InitialPidFile::Expect(&self.pid_file()),
-            || match self.check_status() {
-                Ok(()) => Ok(true),
-                Err(SafekeeperHttpError::Transport(_)) => Ok(false),
-                Err(e) => Err(anyhow::anyhow!("Failed to check node status: {e}")),
-            },
-        )
+        fill_aws_secrets_vars(&mut cmd);
+
+        if !cmd.status()?.success() {
+            bail!(
+                "Safekeeper failed to start. See '{}' for details.",
+                self.datadir_path().join("safekeeper.log").display()
+            );
+        }
+
+        // It takes a while for the safekeeper to start up. Wait until it is
+        // open for business.
+        const RETRIES: i8 = 15;
+        for retries in 1..RETRIES {
+            match self.check_status() {
+                Ok(_) => {
+                    println!("\nSafekeeper started");
+                    return Ok(());
+                }
+                Err(err) => {
+                    match err {
+                        SafekeeperHttpError::Transport(err) => {
+                            if err.is_connect() && retries < 5 {
+                                print!(".");
+                                io::stdout().flush().unwrap();
+                            } else {
+                                if retries == 5 {
+                                    println!() // put a line break after dots for second message
+                                }
+                                println!(
+                                    "Safekeeper not responding yet, err {} retrying ({})...",
+                                    err, retries
+                                );
+                            }
+                        }
+                        SafekeeperHttpError::Response(msg) => {
+                            bail!("safekeeper failed to start: {} ", msg)
+                        }
+                    }
+                    thread::sleep(Duration::from_secs(1));
+                }
+            }
+        }
+        bail!("safekeeper failed to start in {} seconds", RETRIES);
     }
 
     ///
@@ -187,11 +208,63 @@ impl SafekeeperNode {
     /// If the server is not running, returns success
     ///
     pub fn stop(&self, immediate: bool) -> anyhow::Result<()> {
-        background_process::stop_process(
-            immediate,
-            &format!("safekeeper {}", self.id),
-            &self.pid_file(),
-        )
+        let pid_file = self.pid_file();
+        if !pid_file.exists() {
+            println!("Safekeeper {} is already stopped", self.id);
+            return Ok(());
+        }
+        let pid = read_pidfile(&pid_file)?;
+        let pid = Pid::from_raw(pid);
+
+        let sig = if immediate {
+            print!("Stopping safekeeper {} immediately..", self.id);
+            Signal::SIGQUIT
+        } else {
+            print!("Stopping safekeeper {} gracefully..", self.id);
+            Signal::SIGTERM
+        };
+        io::stdout().flush().unwrap();
+        match kill(pid, sig) {
+            Ok(_) => (),
+            Err(Errno::ESRCH) => {
+                println!(
+                    "Safekeeper with pid {} does not exist, but a PID file was found",
+                    pid
+                );
+                return Ok(());
+            }
+            Err(err) => bail!(
+                "Failed to send signal to safekeeper with pid {}: {}",
+                pid,
+                err.desc()
+            ),
+        }
+
+        // Wait until process is gone
+        for i in 0..600 {
+            let signal = None; // Send no signal, just get the error code
+            match kill(pid, signal) {
+                Ok(_) => (), // Process exists, keep waiting
+                Err(Errno::ESRCH) => {
+                    // Process not found, we're done
+                    println!("done!");
+                    return Ok(());
+                }
+                Err(err) => bail!(
+                    "Failed to send signal to pageserver with pid {}: {}",
+                    pid,
+                    err.desc()
+                ),
+            };
+
+            if i % 10 == 0 {
+                print!(".");
+                io::stdout().flush().unwrap();
+            }
+            thread::sleep(Duration::from_millis(100));
+        }
+
+        bail!("Failed to stop safekeeper with pid {}", pid);
     }
 
     fn http_request<U: IntoUrl>(&self, method: Method, url: U) -> RequestBuilder {
@@ -208,4 +281,24 @@ impl SafekeeperNode {
             .error_from_body()?;
         Ok(())
     }
+
+    pub fn timeline_create(
+        &self,
+        tenant_id: TenantId,
+        timeline_id: TimelineId,
+        peer_ids: Vec<NodeId>,
+    ) -> Result<()> {
+        Ok(self
+            .http_request(
+                Method::POST,
+                format!("{}/tenant/{}/timeline", self.http_base_url, tenant_id),
+            )
+            .json(&TimelineCreateRequest {
+                timeline_id,
+                peer_ids,
+            })
+            .send()?
+            .error_from_body()?
+            .json()?)
+    }
 }

control_plane/src/storage.rs

@@ -1,27 +1,33 @@
 use std::collections::HashMap;
-use std::fs::{self, File};
+use std::fs::File;
 use std::io::{BufReader, Write};
 use std::num::NonZeroU64;
 use std::path::{Path, PathBuf};
-use std::process::Child;
-use std::{io, result};
+use std::process::Command;
+use std::time::Duration;
+use std::{io, result, thread};
 
-use crate::connection::PgConnectionConfig;
 use anyhow::{bail, Context};
-use pageserver_api::models::{
+use nix::errno::Errno;
+use nix::sys::signal::{kill, Signal};
+use nix::unistd::Pid;
+use pageserver::http::models::{
     TenantConfigRequest, TenantCreateRequest, TenantInfo, TimelineCreateRequest, TimelineInfo,
 };
+use postgres::{Config, NoTls};
 use reqwest::blocking::{Client, RequestBuilder, Response};
 use reqwest::{IntoUrl, Method};
 use thiserror::Error;
 use utils::{
+    connstring::connection_address,
     http::error::HttpErrorBody,
     id::{TenantId, TimelineId},
     lsn::Lsn,
     postgres_backend::AuthType,
 };
 
-use crate::{background_process, local_env::LocalEnv};
+use crate::local_env::LocalEnv;
+use crate::{fill_aws_secrets_vars, fill_rust_env_vars, read_pidfile};
 
 #[derive(Error, Debug)]
 pub enum PageserverHttpError {
@@ -69,7 +75,7 @@ impl ResponseErrorMessageExt for Response {
 //
 #[derive(Debug)]
 pub struct PageServerNode {
-    pub pg_connection_config: PgConnectionConfig,
+    pub pg_connection_config: Config,
     pub env: LocalEnv,
     pub http_client: Client,
     pub http_base_url: String,
@@ -95,7 +101,7 @@ impl PageServerNode {
     }
 
     /// Construct libpq connection string for connecting to the pageserver.
-    fn pageserver_connection_config(password: &str, listen_addr: &str) -> PgConnectionConfig {
+    fn pageserver_connection_config(password: &str, listen_addr: &str) -> Config {
         format!("postgresql://no_user:{password}@{listen_addr}/no_db")
             .parse()
             .unwrap()
@@ -155,15 +161,7 @@ impl PageServerNode {
             init_config_overrides.push("auth_validation_public_key_path='auth_public_key.pem'");
         }
 
-        let mut pageserver_process = self
-            .start_node(&init_config_overrides, &self.env.base_data_dir, true)
-            .with_context(|| {
-                format!(
-                    "Failed to start a process for pageserver {}",
-                    self.env.pageserver.id,
-                )
-            })?;
-
+        self.start_node(&init_config_overrides, &self.env.base_data_dir, true)?;
         let init_result = self
             .try_init_timeline(create_tenant, initial_timeline_id, pg_version)
             .context("Failed to create initial tenant and timeline for pageserver");
@@ -173,29 +171,7 @@ impl PageServerNode {
             }
             Err(e) => eprintln!("{e:#}"),
         }
-        match pageserver_process.kill() {
-            Err(e) => {
-                eprintln!(
-                    "Failed to stop pageserver {} process with pid {}: {e:#}",
-                    self.env.pageserver.id,
-                    pageserver_process.id(),
-                )
-            }
-            Ok(()) => {
-                println!(
-                    "Stopped pageserver {} process with pid {}",
-                    self.env.pageserver.id,
-                    pageserver_process.id(),
-                );
-                // cleanup after pageserver startup, since we do not call regular `stop_process` during init
-                let pid_file = self.pid_file();
-                if let Err(e) = fs::remove_file(&pid_file) {
-                    if e.kind() != io::ErrorKind::NotFound {
-                        eprintln!("Failed to remove pid file {pid_file:?} after stopping the process: {e:#}");
-                    }
-                }
-            }
-        }
+        self.stop(false)?;
         init_result
     }
 
@@ -220,14 +196,11 @@ impl PageServerNode {
         self.env.pageserver_data_dir()
     }
 
-    /// The pid file is created by the pageserver process, with its pid stored inside.
-    /// Other pageservers cannot lock the same file and overwrite it for as long as the current
-    /// pageserver runs. (Unless someone removes the file manually; never do that!)
-    fn pid_file(&self) -> PathBuf {
+    pub fn pid_file(&self) -> PathBuf {
         self.repo_path().join("pageserver.pid")
     }
 
-    pub fn start(&self, config_overrides: &[&str]) -> anyhow::Result<Child> {
+    pub fn start(&self, config_overrides: &[&str]) -> anyhow::Result<()> {
         self.start_node(config_overrides, &self.repo_path(), false)
     }
 
@@ -236,10 +209,10 @@ impl PageServerNode {
         config_overrides: &[&str],
         datadir: &Path,
         update_config: bool,
-    ) -> anyhow::Result<Child> {
+    ) -> anyhow::Result<()> {
         println!(
             "Starting pageserver at '{}' in '{}'",
-            self.pg_connection_config.raw_address(),
+            connection_address(&self.pg_connection_config),
             datadir.display()
         );
         io::stdout().flush()?;
@@ -247,7 +220,10 @@ impl PageServerNode {
         let mut args = vec![
             "-D",
             datadir.to_str().with_context(|| {
-                format!("Datadir path {datadir:?} cannot be represented as a unicode string")
+                format!(
+                    "Datadir path '{}' cannot be represented as a unicode string",
+                    datadir.display()
+                )
             })?,
         ];
 
@@ -259,18 +235,48 @@ impl PageServerNode {
             args.extend(["-c", config_override]);
         }
 
-        background_process::start_process(
-            "pageserver",
-            datadir,
-            &self.env.pageserver_bin(),
-            &args,
-            background_process::InitialPidFile::Expect(&self.pid_file()),
-            || match self.check_status() {
-                Ok(()) => Ok(true),
-                Err(PageserverHttpError::Transport(_)) => Ok(false),
-                Err(e) => Err(anyhow::anyhow!("Failed to check node status: {e}")),
-            },
-        )
+        let mut cmd = Command::new(self.env.pageserver_bin()?);
+        let mut filled_cmd = fill_rust_env_vars(cmd.args(&args).arg("--daemonize"));
+        filled_cmd = fill_aws_secrets_vars(filled_cmd);
+
+        if !filled_cmd.status()?.success() {
+            bail!(
+                "Pageserver failed to start. See console output and '{}' for details.",
+                datadir.join("pageserver.log").display()
+            );
+        }
+
+        // It takes a while for the page server to start up. Wait until it is
+        // open for business.
+        const RETRIES: i8 = 15;
+        for retries in 1..RETRIES {
+            match self.check_status() {
+                Ok(()) => {
+                    println!("\nPageserver started");
+                    return Ok(());
+                }
+                Err(err) => {
+                    match err {
+                        PageserverHttpError::Transport(err) => {
+                            if err.is_connect() && retries < 5 {
+                                print!(".");
+                                io::stdout().flush().unwrap();
+                            } else {
+                                if retries == 5 {
+                                    println!() // put a line break after dots for second message
+                                }
+                                println!("Pageserver not responding yet, err {err} retrying ({retries})...");
+                            }
+                        }
+                        PageserverHttpError::Response(msg) => {
+                            bail!("pageserver failed to start: {msg} ")
+                        }
+                    }
+                    thread::sleep(Duration::from_secs(1));
+                }
+            }
+        }
+        bail!("pageserver failed to start in {RETRIES} seconds");
     }
 
     ///
@@ -282,18 +288,69 @@ impl PageServerNode {
     /// If the server is not running, returns success
     ///
     pub fn stop(&self, immediate: bool) -> anyhow::Result<()> {
-        background_process::stop_process(immediate, "pageserver", &self.pid_file())
+        let pid_file = self.pid_file();
+        if !pid_file.exists() {
+            println!("Pageserver is already stopped");
+            return Ok(());
+        }
+        let pid = Pid::from_raw(read_pidfile(&pid_file)?);
+
+        let sig = if immediate {
+            print!("Stopping pageserver immediately..");
+            Signal::SIGQUIT
+        } else {
+            print!("Stopping pageserver gracefully..");
+            Signal::SIGTERM
+        };
+        io::stdout().flush().unwrap();
+        match kill(pid, sig) {
+            Ok(_) => (),
+            Err(Errno::ESRCH) => {
+                println!("Pageserver with pid {pid} does not exist, but a PID file was found");
+                return Ok(());
+            }
+            Err(err) => bail!(
+                "Failed to send signal to pageserver with pid {pid}: {}",
+                err.desc()
+            ),
+        }
+
+        // Wait until process is gone
+        for i in 0..600 {
+            let signal = None; // Send no signal, just get the error code
+            match kill(pid, signal) {
+                Ok(_) => (), // Process exists, keep waiting
+                Err(Errno::ESRCH) => {
+                    // Process not found, we're done
+                    println!("done!");
+                    return Ok(());
+                }
+                Err(err) => bail!(
+                    "Failed to send signal to pageserver with pid {}: {}",
+                    pid,
+                    err.desc()
+                ),
+            };
+
+            if i % 10 == 0 {
+                print!(".");
+                io::stdout().flush().unwrap();
+            }
+            thread::sleep(Duration::from_millis(100));
+        }
+
+        bail!("Failed to stop pageserver with pid {pid}");
     }
 
     pub fn page_server_psql(&self, sql: &str) -> Vec<postgres::SimpleQueryMessage> {
-        let mut client = self.pg_connection_config.connect_no_tls().unwrap();
+        let mut client = self.pg_connection_config.connect(NoTls).unwrap();
 
         println!("Pageserver query: '{sql}'");
         client.simple_query(sql).unwrap()
     }
 
     pub fn page_server_psql_client(&self) -> result::Result<postgres::Client, postgres::Error> {
-        self.pg_connection_config.connect_no_tls()
+        self.pg_connection_config.connect(NoTls)
     }
 
     fn http_request<U: IntoUrl>(&self, method: Method, url: U) -> RequestBuilder {
@@ -362,11 +419,6 @@ impl PageServerNode {
                 .map(|x| x.parse::<NonZeroU64>())
                 .transpose()
                 .context("Failed to parse 'max_lsn_wal_lag' as non zero integer")?,
-            trace_read_requests: settings
-                .remove("trace_read_requests")
-                .map(|x| x.parse::<bool>())
-                .transpose()
-                .context("Failed to parse 'trace_read_requests' as bool")?,
         };
         if !settings.is_empty() {
             bail!("Unrecognized tenant settings: {settings:?}")
@@ -429,11 +481,6 @@ impl PageServerNode {
                     .map(|x| x.parse::<NonZeroU64>())
                     .transpose()
                     .context("Failed to parse 'max_lsn_wal_lag' as non zero integer")?,
-                trace_read_requests: settings
-                    .get("trace_read_requests")
-                    .map(|x| x.parse::<bool>())
-                    .transpose()
-                    .context("Failed to parse 'trace_read_requests' as bool")?,
             })
             .send()?
             .error_from_body()?;
@@ -502,7 +549,7 @@ impl PageServerNode {
         pg_wal: Option<(Lsn, PathBuf)>,
         pg_version: u32,
     ) -> anyhow::Result<()> {
-        let mut client = self.pg_connection_config.connect_no_tls().unwrap();
+        let mut client = self.pg_connection_config.connect(NoTls).unwrap();
 
         // Init base reader
         let (start_lsn, base_tarfile_path) = base;

docker-compose/compute_wrapper/Dockerfile

@@ -1,13 +0,0 @@
-ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com
-ARG COMPUTE_IMAGE=compute-node-v14
-ARG TAG=latest
-
-FROM $REPOSITORY/${COMPUTE_IMAGE}:$TAG
-
-USER root
-RUN apt-get update &&       \
-    apt-get install -y curl \
-                       jq   \
-                       netcat
-
-USER postgres

docker-compose/compute_wrapper/shell/compute.sh

@@ -1,48 +0,0 @@
-#!/bin/bash
-set -eux
-
-PG_VERSION=${PG_VERSION:-14}
-
-SPEC_FILE_ORG=/var/db/postgres/specs/spec.json
-SPEC_FILE=/tmp/spec.json
-
-echo "Waiting pageserver become ready."
-while ! nc -z pageserver 6400; do
-     sleep 1;
-done
-echo "Page server is ready."
-
-echo "Create a tenant and timeline"
-PARAMS=(
-     -sb 
-     -X POST
-     -H "Content-Type: application/json"
-     -d "{}"
-     http://pageserver:9898/v1/tenant/
-)
-tenant_id=$(curl "${PARAMS[@]}" | sed 's/"//g')
-
-PARAMS=(
-     -sb 
-     -X POST
-     -H "Content-Type: application/json"
-     -d "{\"tenant_id\":\"${tenant_id}\", \"pg_version\": ${PG_VERSION}}"
-     "http://pageserver:9898/v1/tenant/${tenant_id}/timeline/"
-)
-result=$(curl "${PARAMS[@]}")
-echo $result | jq .
-
-echo "Overwrite tenant id and timeline id in spec file"
-tenant_id=$(echo ${result} | jq -r .tenant_id)
-timeline_id=$(echo ${result} | jq -r .timeline_id)
-
-sed "s/TENANT_ID/${tenant_id}/" ${SPEC_FILE_ORG} > ${SPEC_FILE}
-sed -i "s/TIMELINE_ID/${timeline_id}/" ${SPEC_FILE}
-
-cat ${SPEC_FILE}
-
-echo "Start compute node"
-/usr/local/bin/compute_ctl --pgdata /var/db/postgres/compute \
-     -C "postgresql://cloud_admin@localhost:55433/postgres"  \
-     -b /usr/local/bin/postgres                              \
-     -S ${SPEC_FILE}

docker-compose/compute_wrapper/var/db/postgres/specs/spec.json

@@ -1,141 +0,0 @@
-{
-    "format_version": 1.0,
-
-    "timestamp": "2022-10-12T18:00:00.000Z",
-    "operation_uuid": "0f657b36-4b0f-4a2d-9c2e-1dcd615e7d8c",
-
-    "cluster": {
-        "cluster_id": "docker_compose",
-        "name": "docker_compose_test",
-        "state": "restarted",
-        "roles": [
-            {
-                "name": "cloud_admin",
-                "encrypted_password": "b093c0d3b281ba6da1eacc608620abd8",
-                "options": null
-            }
-        ],
-        "databases": [
-        ],
-        "settings": [
-            {
-                "name": "fsync",
-                "value": "off",
-                "vartype": "bool"
-            },
-            {
-                "name": "wal_level",
-                "value": "replica",
-                "vartype": "enum"
-            },
-            {
-                "name": "hot_standby",
-                "value": "on",
-                "vartype": "bool"
-            },
-            {
-                "name": "wal_log_hints",
-                "value": "on",
-                "vartype": "bool"
-            },
-            {
-                "name": "log_connections",
-                "value": "on",
-                "vartype": "bool"
-            },
-            {
-                "name": "port",
-                "value": "55433",
-                "vartype": "integer"
-            },
-            {
-                "name": "shared_buffers",
-                "value": "1MB",
-                "vartype": "string"
-            },
-            {
-                "name": "max_connections",
-                "value": "100",
-                "vartype": "integer"
-            },
-            {
-                "name": "listen_addresses",
-                "value": "0.0.0.0",
-                "vartype": "string"
-            },
-            {
-                "name": "max_wal_senders",
-                "value": "10",
-                "vartype": "integer"
-            },
-            {
-                "name": "max_replication_slots",
-                "value": "10",
-                "vartype": "integer"
-            },
-            {
-                "name": "wal_sender_timeout",
-                "value": "5s",
-                "vartype": "string"
-            },
-            {
-                "name": "wal_keep_size",
-                "value": "0",
-                "vartype": "integer"
-            },
-            {
-                "name": "password_encryption",
-                "value": "md5",
-                "vartype": "enum"
-            },
-            {
-                "name": "restart_after_crash",
-                "value": "off",
-                "vartype": "bool"
-            },
-            {
-                "name": "synchronous_standby_names",
-                "value": "walproposer",
-                "vartype": "string"
-            },
-            {
-                "name": "shared_preload_libraries",
-                "value": "neon",
-                "vartype": "string"
-            },
-            {
-                "name": "neon.safekeepers",
-                "value": "safekeeper1:5454,safekeeper2:5454,safekeeper3:5454",
-                "vartype": "string"
-            },
-            {
-                "name": "neon.timeline_id",
-                "value": "TIMELINE_ID",
-                "vartype": "string"
-            },
-            {
-                "name": "neon.tenant_id",
-                "value": "TENANT_ID",
-                "vartype": "string"
-            },
-            {
-                "name": "neon.pageserver_connstring",
-                "value": "host=pageserver port=6400",
-                "vartype": "string"
-            },
-            {
-                "name": "max_replication_write_lag",
-                "value": "500MB",
-                "vartype": "string"
-            },
-            {
-                "name": "max_replication_flush_lag",
-                "value": "10GB",
-                "vartype": "string"
-            }
-        ]
-    },
-
-    "delta_operations": [
-    ]
-}

docker-compose/docker-compose.yml

@@ -1,209 +0,0 @@
-version: '3'
-
-services:
-  etcd:
-    restart: always
-    image: quay.io/coreos/etcd:v3.5.4
-    ports:
-      - 2379:2379
-      - 2380:2380
-    environment:
-      # This signifficantly speeds up etcd and we anyway don't data persistency there.
-      ETCD_UNSAFE_NO_FSYNC: "1"
-    command:
-      - "etcd"
-      - "--auto-compaction-mode=revision"
-      - "--auto-compaction-retention=1"
-      - "--name=etcd-cluster"
-      - "--initial-cluster-state=new"
-      - "--initial-cluster-token=etcd-cluster-1"
-      - "--initial-cluster=etcd-cluster=http://etcd:2380"
-      - "--initial-advertise-peer-urls=http://etcd:2380"
-      - "--advertise-client-urls=http://etcd:2379"
-      - "--listen-client-urls=http://0.0.0.0:2379"
-      - "--listen-peer-urls=http://0.0.0.0:2380"
-      - "--quota-backend-bytes=134217728" # 128 MB
-
-  minio:
-    restart: always
-    image: quay.io/minio/minio:RELEASE.2022-10-20T00-55-09Z
-    ports:
-      - 9000:9000
-      - 9001:9001
-    environment:
-      - MINIO_ROOT_USER=minio
-      - MINIO_ROOT_PASSWORD=password
-    command: server /data --address :9000 --console-address ":9001"
-
-  minio_create_buckets:
-    image: minio/mc
-    environment:
-      - MINIO_ROOT_USER=minio
-      - MINIO_ROOT_PASSWORD=password
-    entrypoint:
-      - "/bin/sh"
-      - "-c"
-    command:
-      - "until (/usr/bin/mc alias set minio http://minio:9000 $$MINIO_ROOT_USER $$MINIO_ROOT_PASSWORD) do
-             echo 'Waiting to start minio...' && sleep 1;
-         done;
-         /usr/bin/mc mb minio/neon --region=eu-north-1;
-         exit 0;"
-    depends_on:
-      - minio
-
-  pageserver:
-    restart: always
-    image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
-    environment:
-      - BROKER_ENDPOINT='http://etcd:2379'
-      - AWS_ACCESS_KEY_ID=minio
-      - AWS_SECRET_ACCESS_KEY=password
-      #- RUST_BACKTRACE=1
-    ports:
-       #- 6400:6400  # pg protocol handler
-       - 9898:9898 # http endpoints
-    entrypoint:
-      - "/bin/sh"
-      - "-c"
-    command:
-      - "/usr/local/bin/pageserver -D /data/.neon/
-                                   -c \"broker_endpoints=[$$BROKER_ENDPOINT]\"
-                                   -c \"listen_pg_addr='0.0.0.0:6400'\"
-                                   -c \"listen_http_addr='0.0.0.0:9898'\"
-                                   -c \"remote_storage={endpoint='http://minio:9000',
-                                                        bucket_name='neon',
-                                                        bucket_region='eu-north-1',
-                                                        prefix_in_bucket='/pageserver/'}\""
-    depends_on:
-      - etcd
-      - minio_create_buckets
-
-  safekeeper1:
-    restart: always
-    image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
-    environment:
-      - SAFEKEEPER_ADVERTISE_URL=safekeeper1:5454
-      - SAFEKEEPER_ID=1
-      - BROKER_ENDPOINT=http://etcd:2379
-      - AWS_ACCESS_KEY_ID=minio
-      - AWS_SECRET_ACCESS_KEY=password
-      #- RUST_BACKTRACE=1
-    ports:
-      #- 5454:5454 # pg protocol handler
-      - 7676:7676 # http endpoints
-    entrypoint:
-      - "/bin/sh"
-      - "-c"
-    command:
-      - "safekeeper --listen-pg=$$SAFEKEEPER_ADVERTISE_URL
-                    --listen-http='0.0.0.0:7676'
-                    --id=$$SAFEKEEPER_ID
-                    --broker-endpoints=$$BROKER_ENDPOINT
-                    -D /data
-                    --remote-storage=\"{endpoint='http://minio:9000',
-                                        bucket_name='neon',
-                                        bucket_region='eu-north-1',
-                                        prefix_in_bucket='/safekeeper/'}\""
-    depends_on:
-      - etcd
-      - minio_create_buckets
-
-  safekeeper2:
-    restart: always
-    image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
-    environment:
-      - SAFEKEEPER_ADVERTISE_URL=safekeeper2:5454
-      - SAFEKEEPER_ID=2
-      - BROKER_ENDPOINT=http://etcd:2379
-      - AWS_ACCESS_KEY_ID=minio
-      - AWS_SECRET_ACCESS_KEY=password
-      #- RUST_BACKTRACE=1
-    ports:
-      #- 5454:5454 # pg protocol handler
-      - 7677:7676 # http endpoints
-    entrypoint:
-      - "/bin/sh"
-      - "-c"
-    command:
-      - "safekeeper --listen-pg=$$SAFEKEEPER_ADVERTISE_URL
-                    --listen-http='0.0.0.0:7676'
-                    --id=$$SAFEKEEPER_ID
-                    --broker-endpoints=$$BROKER_ENDPOINT
-                    -D /data
-                    --remote-storage=\"{endpoint='http://minio:9000',
-                                        bucket_name='neon',
-                                        bucket_region='eu-north-1',
-                                        prefix_in_bucket='/safekeeper/'}\""
-    depends_on:
-      - etcd
-      - minio_create_buckets
-
-  safekeeper3:
-    restart: always
-    image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
-    environment:
-      - SAFEKEEPER_ADVERTISE_URL=safekeeper3:5454
-      - SAFEKEEPER_ID=3
-      - BROKER_ENDPOINT=http://etcd:2379
-      - AWS_ACCESS_KEY_ID=minio
-      - AWS_SECRET_ACCESS_KEY=password
-      #- RUST_BACKTRACE=1
-    ports:
-      #- 5454:5454 # pg protocol handler
-      - 7678:7676 # http endpoints
-    entrypoint:
-      - "/bin/sh"
-      - "-c"
-    command:
-      - "safekeeper --listen-pg=$$SAFEKEEPER_ADVERTISE_URL
-                    --listen-http='0.0.0.0:7676'
-                    --id=$$SAFEKEEPER_ID
-                    --broker-endpoints=$$BROKER_ENDPOINT
-                    -D /data
-                    --remote-storage=\"{endpoint='http://minio:9000',
-                                        bucket_name='neon',
-                                        bucket_region='eu-north-1',
-                                        prefix_in_bucket='/safekeeper/'}\""
-    depends_on:
-      - etcd
-      - minio_create_buckets
-
-  compute:
-    restart: always
-    build:
-      context: ./compute_wrapper/
-      args:
-        - COMPUTE_IMAGE=compute-node-v${PG_VERSION:-14}
-        - TAG=${TAG:-latest}
-        - http_proxy=$http_proxy
-        - https_proxy=$https_proxy
-    environment:
-      - PG_VERSION=${PG_VERSION:-14}
-      #- RUST_BACKTRACE=1
-    # Mount the test files directly, for faster editing cycle.
-    volumes:
-      - ./compute_wrapper/var/db/postgres/specs/:/var/db/postgres/specs/
-      - ./compute_wrapper/shell/:/shell/
-    ports:
-      - 55433:55433 # pg protocol handler
-      - 3080:3080 # http endpoints
-    entrypoint:
-      - "/shell/compute.sh"
-    depends_on:
-      - safekeeper1
-      - safekeeper2
-      - safekeeper3
-      - pageserver
-
-  compute_is_ready:
-    image: postgres:latest
-    entrypoint:
-      - "/bin/bash"
-      - "-c"
-    command:
-      - "until pg_isready -h compute -p 55433 ; do
-            echo 'Waiting to start compute...' && sleep 1;
-         done"
-    depends_on:
-      - compute

docker-compose/docker_compose_test.sh

@@ -1,60 +0,0 @@
-#!/bin/bash
-
-# A basic test to ensure Docker images are built correctly.
-# Build a wrapper around the compute, start all services and runs a simple SQL query.
-# Repeats the process for all currenly supported Postgres versions.
-
-# Implicitly accepts `REPOSITORY` and `TAG` env vars that are passed into the compose file
-# Their defaults point at DockerHub `neondatabase/neon:latest` image.`,
-# to verify custom image builds (e.g pre-published ones).
-
-# XXX: Current does not work on M1 macs due to x86_64 Docker images compiled only, and no seccomp support in M1 Docker emulation layer.
-
-set -eux -o pipefail
-
-SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
-COMPOSE_FILE=$SCRIPT_DIR/docker-compose.yml
-
-COMPUTE_CONTAINER_NAME=docker-compose-compute-1
-SQL="CREATE TABLE t(key int primary key, value text); insert into t values(1,1); select * from t;"
-PSQL_OPTION="-h localhost -U cloud_admin -p 55433 -c '$SQL' postgres"
-
-cleanup() {
-    echo "show container information"
-    docker ps
-    docker compose -f $COMPOSE_FILE logs
-    echo "stop containers..."
-    docker compose -f $COMPOSE_FILE down
-}
-
-echo "clean up containers if exists"
-cleanup
-
-for pg_version in 14 15; do
-    echo "start containers (pg_version=$pg_version)."
-    PG_VERSION=$pg_version docker compose -f $COMPOSE_FILE up --build -d
-
-    echo "wait until the compute is ready. timeout after 60s. "
-    cnt=0
-    while sleep 1; do
-        # check timeout
-        cnt=`expr $cnt + 1`
-        if [ $cnt -gt 60 ]; then
-            echo "timeout before the compute is ready."
-            cleanup
-            exit 1
-        fi
-
-        # check if the compute is ready
-        set +o pipefail
-        result=`docker compose -f $COMPOSE_FILE logs "compute_is_ready" | grep "accepting connections" | wc -l`
-        set -o pipefail
-        if [ $result -eq 1 ]; then
-            echo "OK. The compute is ready to connect."
-            echo "execute simple queries."
-            docker exec $COMPUTE_CONTAINER_NAME /bin/bash -c "psql $PSQL_OPTION"
-            cleanup
-            break
-        fi
-    done
-done

docs/SUMMARY.md

@@ -37,7 +37,7 @@
 
 - [Source view](./sourcetree.md)
   - [docker.md](./docker.md) — Docker images and building pipeline.
-  - [Error handling and logging](./error-handling.md)
+  - [Error handling and logging]()
   - [Testing]()
     - [Unit testing]()
     - [Integration testing]()
@@ -80,6 +80,4 @@
 - [015-storage-messaging](rfcs/015-storage-messaging.md)
 - [016-connection-routing](rfcs/016-connection-routing.md)
 - [017-timeline-data-management](rfcs/017-timeline-data-management.md)
-- [018-storage-messaging-2](rfcs/018-storage-messaging-2.md)
-- [019-tenant-timeline-lifecycles](rfcs/019-tenant-timeline-lifecycles.md)
 - [cluster-size-limits](rfcs/cluster-size-limits.md)

docs/docker.md

@@ -18,67 +18,3 @@ We build all images after a successful `release` tests run and push automaticall
 1. `neondatabase/compute-tools` and `neondatabase/compute-node`
 
 2. `neondatabase/neon`
-
-## Docker Compose example
-
-You can see a [docker compose](https://docs.docker.com/compose/) example to create a neon cluster in [/docker-compose/docker-compose.yml](/docker-compose/docker-compose.yml). It creates the following conatainers.
-
-- etcd x 1
-- pageserver x 1
-- safekeeper x 3
-- compute x 1
-- MinIO x 1        # This is Amazon S3 compatible object storage
-
-### How to use
-
-1. create containers
-
-You can specify version of neon cluster using following environment values.
-- PG_VERSION: postgres version for compute (default is 14)
-- TAG: the tag version of [docker image](https://registry.hub.docker.com/r/neondatabase/neon/tags) (default is latest), which is tagged in [CI test](/.github/workflows/build_and_test.yml)
-```
-$ cd docker-compose/docker-compose.yml
-$ docker-compose down   # remove the conainers if exists
-$ PG_VERSION=15 TAG=2221 docker-compose up --build -d  # You can specify the postgres and image version
-Creating network "dockercompose_default" with the default driver
-Creating dockercompose_etcd3_1 ...
-(...omit...)
-```
-
-2. connect compute node
-```
-$ echo "localhost:55433:postgres:cloud_admin:cloud_admin" >> ~/.pgpass
-$ psql -h localhost -p 55433 -U cloud_admin
-postgres=# CREATE TABLE t(key int primary key, value text);
-CREATE TABLE
-postgres=# insert into t values(1,1);
-INSERT 0 1
-postgres=# select * from t;
- key | value
------+-------
-   1 | 1
-(1 row)
-```
-
-3. If you want to see the log, you can use `docker-compose logs` command.
-```
-# check the container name you want to see
-$ docker ps
-CONTAINER ID   IMAGE                                              COMMAND                  CREATED         STATUS         PORTS                                                                                                                                  NAMES
-d6968a5ae912   dockercompose_compute                              "/shell/compute.sh"      5 minutes ago   Up 5 minutes   0.0.0.0:3080->3080/tcp, 0.0.0.0:55433->55433/tcp                                                                                       dockercompose_compute_1
-(...omit...)
-
-$ docker logs -f dockercompose_compute_1
-2022-10-21 06:15:48.757 GMT [56] LOG:  connection authorized: user=cloud_admin database=postgres application_name=psql
-2022-10-21 06:17:00.307 GMT [56] LOG:  [NEON_SMGR] libpagestore: connected to 'host=pageserver port=6400'
-(...omit...)
-```
-
-4. If you want to see durable data in MinIO which is s3 compatible storage
-
-Access http://localhost:9001 and sign in.
-
-- Username: `minio`
-- Password: `password`
-
-You can see durable pages and WAL data in `neon` bucket.

docs/error-handling.md

@@ -1,198 +0,0 @@
-# Error handling and logging
-
-## Logging errors
-
-The principle is that errors are logged when they are handled. If you
-just propagate an error to the caller in a function, you don't need to
-log it; the caller will. But if you consume an error in a function,
-you *must* log it (if it needs to be logged at all).
-
-For example:
-
-```rust
-fn read_motd_file() -> std::io::Result<String> {
-    let mut f = File::open("/etc/motd")?;
-    let mut result = String::new();
-    f.read_to_string(&mut result)?;
-    result
-}
-```
-
-Opening or reading the file could fail, but there is no need to log
-the error here. The function merely propagates the error to the
-caller, and it is up to the caller to log the error or propagate it
-further, if the failure is not expected. But if, for example, it is
-normal that the "/etc/motd" file doesn't exist, the caller can choose
-to silently ignore the error, or log it as an INFO or DEBUG level
-message:
-
-```rust
-fn get_message_of_the_day() -> String {
-    // Get the motd from /etc/motd, or return the default proverb
-    match read_motd_file() {
-        Ok(motd) => motd,
-        Err(err)  => {
-            // It's normal that /etc/motd doesn't exist, but if we fail to
-            // read it for some other reason, that's unexpected. The message
-            // of the day isn't very important though, so we just WARN and
-            // continue with the default in any case.
-            if err.kind() != std::io::ErrorKind::NotFound {
-                 tracing::warn!("could not read \"/etc/motd\": {err:?}");
-            }
-            "An old error is always more popular than a new truth. - German proverb"
-        }
-    }
-}
-```
-
-## Error types
-
-We use the `anyhow` crate widely. It contains many convenient macros
-like `bail!` and `ensure!` to construct and return errors, and to
-propagate many kinds of low-level errors, wrapped in `anyhow::Error`.
-
-A downside of `anyhow::Error` is that the caller cannot distinguish
-between different error cases. Most errors are propagated all the way
-to the mgmt API handler function, or the main loop that handles a
-connection with the compute node, and they are all handled the same
-way: the error is logged and returned to the client as an HTTP or
-libpq error.
-
-But in some cases, we need to distinguish between errors and handle
-them differently. For example, attaching a tenant to the pageserver
-could fail either because the tenant has already been attached, or
-because we could not load its metadata from cloud storage. The first
-case is more or less expected. The console sends the Attach request to
-the pageserver, and the pageserver completes the operation, but the
-network connection might be lost before the console receives the
-response. The console will retry the operation in that case, but the
-tenant has already been attached. It is important that the pagserver
-responds with the HTTP 403 Already Exists error in that case, rather
-than a generic HTTP 500 Internal Server Error.
-
-If you need to distinguish between different kinds of errors, create a
-new `Error` type. The `thiserror` crate is useful for that. But in
-most cases `anyhow::Error` is good enough.
-
-## Panics
-
-Depending on where a panic happens, it can cause the whole pageserver
-or safekeeper to restart, or just a single tenant. In either case,
-that is pretty bad and causes an outage. Avoid panics. Never use
-`unwrap()` or other calls that might panic, to verify inputs from the
-network or from disk.
-
-It is acceptable to use functions that might panic, like `unwrap()`, if
-it is obvious that it cannot panic. For example, if you have just
-checked that a variable is not None, it is OK to call `unwrap()` on it,
-but it is still preferable to use `expect("reason")` instead to explain
-why the function cannot fail.
-
-`assert!` and `panic!` are reserved for checking clear invariants and
-very obvious "can't happen" cases. When in doubt, use anyhow `ensure!`
-or `bail!` instead.
-
-## Error levels
-
-`tracing::Level` doesn't provide very clear guidelines on what the
-different levels mean, or when to use which level. Here is how we use
-them:
-
-### Error
-
-Examples:
-- could not open file "foobar"
-- invalid tenant id
-
-Errors are not expected to happen during normal operation. Incorrect
-inputs from client can cause ERRORs. For example, if a client tries to
-call a mgmt API that doesn't exist, or if a compute node sends passes
-an LSN that has already been garbage collected away.
-
-These should *not* happen during normal operations. "Normal
-operations" is not a very precise concept. But for example, disk
-errors are not expected to happen when the system is working, so those
-count as Errors. However, if a TCP connection to a compute node is
-lost, that is not considered an Error, because it doesn't affect the
-pageserver's or safekeeper's operation in any way, and happens fairly
-frequently when compute nodes are shut down, or are killed abruptly
-because of errors in the compute.
-
-**Errors are monitored, and always need human investigation to determine
-the cause.**
-
-Whether something should be logged at ERROR, WARNING or INFO level can
-depend on the callers and clients. For example, it might be unexpected
-and a sign of a serious issue if the console calls the
-"timeline_detail" mgmt API for a timeline that doesn't exist. ERROR
-would be appropriate in that case. But if the console routinely calls
-the API after deleting a timeline, to check if the deletion has
-completed, then it would be totally normal and an INFO or DEBUG level
-message would be more appropriate. If a message is logged as an ERROR,
-but it in fact happens frequently in production and never requires any
-action, it should probably be demoted to an INFO level message.
-
-### Warn
-
-Examples:
-- could not remove temporary file "foobar.temp"
-- unrecognized file "foobar" in timeline directory
-
-Warnings are similar to Errors, in that they should not happen
-when the system is operating normally. The difference between Error and
-Warning is that an Error means that the operation failed, whereas Warning
-means that something unexpected happened, but the operation continued anyway.
-For example, if deleting a file fails because the file already didn't exist,
-it should be logged as Warning.
-
-> **Note:** The python regression tests, under `test_regress`, check the
-> pageserver log after each test for any ERROR and WARN lines. If there are
-> any ERRORs or WARNs that have not been explicitly listed in the test as
-> allowed, the test is marked a failed. This is to catch unexpected errors
-> e.g. in background operations, that don't cause immediate misbehaviour in
-> the tested functionality.
-
-### Info
-
-Info level is used to log useful information when the system is
-operating normally. Info level is appropriate e.g. for logging state
-changes, background operations, and network connections.
-
-Examples:
-- "system is shutting down"
-- "tenant was created"
-- "retrying S3 upload"
-
-### Debug & Trace
-
-Debug and Trace level messages are not printed to the log in our normal
-production configuration, but could be enabled for a specific server or
-tenant, to aid debugging. (Although we don't actually have that
-capability as of this writing).
-
-## Context
-
-We use logging "spans" to hold context information about the current
-operation. Almost every operation happens on a particular tenant and
-timeline, so we enter a span with the "tenant_id" and "timeline_id"
-very early when processing an incoming API request, for example. All
-background operations should also run in a span containing at least
-those two fields, and any other parameters or information that might
-be useful when debugging an error that might happen when performing
-the operation.
-
-TODO: Spans are not captured in the Error when it is created, but when
-the error is logged. It would be more useful to capture them at Error
-creation. We should consider using `tracing_error::SpanTrace` to do
-that.
-
-## Error message style
-
-PostgreSQL has a style guide for writing error messages:
-
-https://www.postgresql.org/docs/current/error-style-guide.html
-
-Follow that guide when writing error messages in the PostgreSQL
-extension. We don't follow it strictly in the pageserver and
-safekeeper, but the advice in the PostgreSQL style guide is generally
-good, and you can't go wrong by following it.

docs/rfcs/018-storage-messaging-2.md

@@ -1,163 +0,0 @@
-# Storage messaging
-
-Safekeepers need to communicate to each other to
-* Trim WAL on safekeepers;
-* Decide on which SK should push WAL to the S3;
-* Decide on when to shut down SK<->pageserver connection;
-* Understand state of each other to perform peer recovery;
-
-Pageservers need to communicate to safekeepers to decide which SK should provide
-WAL to the pageserver.
-
-This is an iteration on [015-storage-messaging](https://github.com/neondatabase/neon/blob/main/docs/rfcs/015-storage-messaging.md) describing current situation,
-potential performance issue and ways to address it.
-
-## Background
-
-What we have currently is very close to etcd variant described in
-015-storage-messaging. Basically, we have single `SkTimelineInfo` message
-periodically sent by all safekeepers to etcd for each timeline.
-* Safekeepers subscribe to it to learn status of peers (currently they subscribe to
-  'everything', but they can and should fetch data only for timelines they hold).
-* Pageserver subscribes to it (separate watch per timeline) to learn safekeepers
-  positions; based on that, it decides from which safekeepers to pull WAL.
-
-Also, safekeepers use etcd elections API to make sure only single safekeeper
-offloads WAL.
-
-It works, and callmemaybe is gone. However, this has a performance
-hazard. Currently deployed etcd can do about 6k puts per second (using its own
-`benchmark` tool); on my 6 core laptop, while running on tmpfs, this gets to
-35k. Making benchmark closer to our usage [etcd watch bench](https://github.com/arssher/etcd-client/blob/watch-bench/examples/watch_bench.rs),
-I get ~10k received messages per second with various number of publisher-subscribers
-(laptop, tmpfs). Diving this by 12 (3 sks generate msg, 1 ps + 3 sk consume them) we
-get about 800 active timelines, if message is sent each second. Not extremely
-low, but quite reachable.
-
-A lot of idle watches seem to be ok though -- which is good, as pageserver
-subscribes to all its timelines regardless of their activity.
-
-Also, running etcd with fsyncs disabled is messy -- data dir must be wiped on
-each restart or there is a risk of corruption errors.
-
-The reason is etcd making much more than what we need; it is a fault tolerant
-store with strong consistency, but I claim all we need here is just simplest pub
-sub with best effort delivery, because
-* We already have centralized source of truth for long running data, like which
-  tlis are on which nodes  -- the console.
-* Momentary data (safekeeper/pageserver progress) doesn't make sense to persist.
-  Instead of putting each change to broker, expecting it to reliably deliver it
-  is better to just have constant flow of data for active timelines: 1) they
-  serve as natural heartbeats -- if node can't send, we shouldn't pull WAL from
-  it 2) it is simpler -- no need to track delivery to/from the broker.
-  Moreover, latency here is important: the faster we obtain fresh data, the
-  faster we can switch to proper safekeeper after failure.
-* As for WAL offloading leader election, it is trivial to achieve through these
-  heartbeats -- just take suitable node through deterministic rule (min node
-  id).  Once network is stable, this is a converging process (well, except
-  complicated failure topology, but even then making it converge is not
-  hard). Such elections bear some risk of several offloaders running
-  concurrently for a short period of time, but that's harmless.
-
-  Generally, if one needs strong consistency, electing leader per se is not
-  enough; it must be accompanied with number (logical clock ts), checked at
-  every action to track causality. s3 doesn't provide CAS, so it can't
-  differentiate old/new leader, this must be solved differently.
-
-  We could use etcd CAS (its most powerful/useful primitive actually) to issue
-  these leader numbers (and e.g. prefix files in s3), but currently I don't see
-  need for that.
-
-
-Obviously best effort pub sub is much more simpler and performant; the one proposed is
-
-## gRPC broker
-
-I took tonic and [prototyped](https://github.com/neondatabase/neon/blob/asher/neon-broker/broker/src/broker.rs) the replacement of functionality we currently use
-with grpc streams and tokio mpsc channels. The implementation description is at the file header.
-
-It is just 500 lines of code and core functionality is complete. 1-1 pub sub
-gives about 120k received messages per second; having multiple subscribers in
-different connecitons quickly scales to 1 million received messages per second.
-I had concerns about many concurrent streams in singe connection, but 2^20
-subscribers still work (though eat memory, with 10 publishers 20GB are consumed;
-in this implementation each publisher holds full copy of all subscribers). There
-is `bench.rs` nearby which I used for testing.
-
-`SkTimelineInfo` is wired here, but another message can be added (e.g. if
-pageservers want to communicate with each other) with templating.
-
-### Fault tolerance
-
-Since such broker is stateless, we can run it under k8s. Or add proxying to
-other members, with best-effort this is simple.
-
-### Security implications
-
-Communication happens in a private network that is not exposed to users;
-additionaly we can add auth to the broker.
-
-## Alternative: get existing pub-sub
-
-We could take some existing pub sub solution, e.g. RabbitMQ, Redis. But in this
-case IMV simplicity of our own outweights external dependency costs (RabbitMQ is
-much more complicated and needs VM; Redis Rust client maintenance is not
-ideal...). Also note that projects like CockroachDB and TiDB are based on gRPC
-as well.
-
-## Alternative: direct communication
-
-Apart from being transport, broker solves one more task: discovery, i.e. letting
-safekeepers and pageservers find each other. We can let safekeepers know, for
-each timeline, both other safekeepers for this timeline and pageservers serving
-it. In this case direct communication is possible:
- - each safekeeper pushes to each other safekeeper status of timelines residing
-   on both of them, letting remove WAL, decide who offloads, decide on peer
-   recovery;
- - each safekeeper pushes to each pageserver status of timelines residing on
-   both of them, letting pageserver choose from which sk to pull WAL;
-
-It was mostly described in [014-safekeeper-gossip](https://github.com/neondatabase/neon/blob/main/docs/rfcs/014-safekeepers-gossip.md), but I want to recap on that.
-
-The main pro is less one dependency: less moving parts, easier to run Neon
-locally/manually, less places to monitor. Fault tolerance for broker disappears,
-no kuber or something. To me this is a big thing.
-
-Also (though not a big thing) idle watches for inactive timelines disappear:
-naturally safekeepers learn about compute connection first and start pushing
-status to pageserver(s), notifying it should pull.
-
-Importantly, I think that eventually knowing and persisting peers and
-pageservers on safekeepers is inevitable:
-- Knowing peer safekeepers for the timeline is required for correct
-  automatic membership change -- new member set must be hardened on old
-  majority before proceeding. It is required to get rid of sync-safekeepers
-  as well (peer recovery up to flush_lsn).
-- Knowing pageservers where the timeline is attached is needed to
-  1. Understand when to shut down activity on the timeline, i.e. push data to
-     the broker. We can have a lot of timelines sleeping quietly which
-	 shouldn't occupy resources.
-  2. Preserve WAL for these (currently we offload to s3 and take it from there,
-     but serving locally is better, and we get one less condition on which WAL
-     can be removed from s3).
-
-I suppose this membership data should be passed to safekeepers directly from the
-console because
-1. Console is the original source of this data, conceptually this is the
-   simplest way (rather than passing it through compute or something).
-2. We already have similar code for deleting timeline on safekeepers
-   (and attaching/detaching timeline on pageserver), this is a typical
-   action -- queue operation against storage node and execute it until it
-   completes (or timeline is dropped).
-
-Cons of direct communication are
-- It is more complicated: each safekeeper should maintain set of peers it talks
-  to, and set of timelines for each such peer -- they ought to be multiplexed
-  into single connection.
-- Totally, we have O(n^2) connections instead of O(n) with broker schema
-  (still O(n) on each node). However, these are relatively stable, async and
-  thus not very expensive, I don't think this is a big problem. Up to 10k
-  storage nodes I doubt connection overhead would be noticeable.
-
-I'd use gRPC for direct communication, and in this sense gRPC based broker is a
-step towards it.

docs/rfcs/019-tenant-timeline-lifecycles.md

@@ -1,91 +0,0 @@
-# Managing Tenant and Timeline lifecycles
-
-## Summary
-
-The pageserver has a Tenant object in memory for each tenant it manages, and a
-Timeline for each timeline. There are a lot of tasks that operate on the tenants
-and timelines with references to those objects. We have some mechanisms to track
-which tasks are operating on each Tenant and Timeline, and to request them to
-shutdown when a tenant or timeline is deleted, but it does not cover all uses,
-and as a result we have many race conditions around tenant/timeline shutdown.
-
-## Motivation
-
-We have a bunch of race conditions that can produce weird errors and can be hard
-to track down.
-
-## Non Goals
-
-This RFC only covers the problem of ensuring that a task/thread isn't operating
-on a Tenant or Timeline. It does not cover what states, aside from Active and
-non-Active, each Tenant and Timeline should have, or when exactly the transitions
-should happen.
-
-## Impacted components (e.g. pageserver, safekeeper, console, etc)
-
-Pageserver. Although I wonder if the safekeeper should have a similar mechanism.
-
-## Current situation
-
-Most pageserver tasks of are managed by task_mgr.rs:
-
-- LibpqEndpointListener
-- HttpEndPointListener
-- WalReceiverManager and -Connection
-- GarbageCollector and Compaction
-- InitialLogicalSizeCalculation
-
-In addition to those tasks, the walreceiver performs some direct tokio::spawn
-calls to spawn tasks that are not registered with 'task_mgr'. And all of these
-tasks can spawn extra operations with tokio spawn_blocking.
-
-Whenever a tenant or timeline is removed from the system, by pageserver
-shutdown, delete_timeline or tenant-detach operation, we rely on the task
-registry in 'task_mgr.rs' to wait until there are no tasks operating on the
-tenant or timeline, before its Tenant/Timeline object is removed. That relies on
-each task to register itself with the tenant/timeline ID in
-'task_mgr.rs'. However, there are many gaps in that. For example,
-GarbageCollection and Compaction tasks are registered with the tenant, but when
-they proceed to operate on a particular timeline of the tenant, they don't
-register with timeline ID. Because of that, the timeline can be deleted while GC
-or compaction is running on it, causing failures in the GC or compaction (see
-https://github.com/neondatabase/neon/issues/2442).
-
-Another problem is that the task registry only works for tokio Tasks. There is
-no way to register a piece of code that runs inside spawn_blocking(), for
-example.
-
-## Proposed implementation
-
-This "voluntary" registration of tasks is fragile. Let's use Rust language features
-to enforce that a tenant/timeline cannot be removed from the system when there is
-still some code operating on it.
-
-Let's introduce new Guard objects for Tenant and Timeline, and do all actions through
-the Guard object. Something like:
-
-TenantActiveGuard: Guard object over Arc<Tenant>. When you acquire the guard,
-the code checks that the tenant is in Active state. If it's not, you get an
-error. You can change the state of the tenant to Stopping while there are
-ActiveTenantGuard objects still on it, to prevent new ActiveTenantGuards from
-being acquired, but the Tenant cannot be removed until all the guards are gone.
-
-TenantMaintenanceGuard: Like ActiveTenantGuard, but can be held even when the
-tenant is not in Active state. Used for operations like attach/detach. Perhaps
-allow only one such guard on a Tenant at a time.
-
-Similarly for Timelines. We don't currentl have a "state" on Timeline, but I think
-we need at least two states: Active and Stopping. The Stopping state is used at
-deletion, to prevent new TimelineActiveGuards from appearing, while you wait for
-existing TimelineActiveGuards to die out.
-
-The shutdown-signaling, using shutdown_watcher() and is_shutdown_requested(),
-probably also needs changes to deal with the new Guards. The rule is that if you
-have a TenantActiveGuard, and the tenant's state changes from Active to
-Stopping, the is_shutdown_requested() function should return true, and
-shutdown_watcher() future should return.
-
-This signaling doesn't neessarily need to cover all cases. For example, if you
-have a block of code in spawn_blocking(), it might be acceptable if
-is_shutdown_requested() doesn't return true even though the tenant is in
-Stopping state, as long as the code finishes reasonably fast.

docs/rfcs/020-pageserver-s3-coordination.md

@@ -1,246 +0,0 @@
-# Coordinating access of multiple pageservers to the same s3 data
-
-## Motivation
-
-There are some blind spots around coordinating access of multiple pageservers
-to the same s3 data. Currently this is applicable only to tenant relocation
-case, but in the future we'll need to solve similar problems for
-replica/standby pageservers.
-
-## Impacted components (e.g. pageserver, safekeeper, console, etc)
-
-Pageserver
-
-## The problem
-
-### Relocation
-
-During relocation both pageservers can write to s3. This should be ok for all
-data except the `index_part.json`. For index part it causes problems during
-compaction/gc because they remove files from index/s3.
-
-Imagine this case:
-
-```mermaid
-sequenceDiagram
-    autonumber
-    participant PS1
-    participant S3
-    participant PS2
-
-    PS1->>S3: Uploads L1, L2 <br/> Index contains L1 L2
-    PS2->>S3: Attach called, sees L1, L2
-    PS1->>S3: Compaction comes <br/> Removes L1, adds L3
-    note over S3: Index now L2, L3
-    PS2->>S3: Uploads new layer L4 <br/> (added to previous view of the index)
-    note over S3: Index now L1, L2, L4
-```
-
-At this point it is not possible to restore from index, it contains L2 which
-is no longer available in s3 and doesnt contain L3 added by compaction by the
-first pageserver. So if any of the pageservers restart initial sync will fail
-(or in on-demand world it will fail a bit later during page request from
-missing layer)
-
-### Standby pageserver
-
-Another related case is standby pageserver. In this case second pageserver can
-be used as a replica to scale reads and serve as a failover target in case
-first one fails.
-
-In this mode second pageserver needs to have the same picture of s3 files to
-be able to load layers on-demand. To accomplish that second pageserver
-cannot run gc/compaction jobs. Instead it needs to receive updates for index
-contents. (There is no need to run walreceiver on the second pageserver then).
-
-## Observations
-
-- If both pageservers ingest wal then their layer set diverges, because layer
-  file generation is not deterministic
-- If one of the pageservers does not ingest wal (and just picks up layer
-  updates) then it lags behind and cannot really answer queries in the same
-  pace as the primary one
-- Can compaction help make layers deterministic? E g we do not upload level
-  zero layers and construction of higher levels should be deterministic.
-  This way we can guarantee that layer creation by timeout wont mess things up.
-  This way one pageserver uploads data and second one can just ingest it.
-  But we still need some form of election
-
-## Solutions
-
-### Manual orchestration
-
-One possible solution for relocation case is to orchestrate background jobs
-from outside. The oracle who runs migration can turn off background jobs on
-PS1 before migration and then run migration -> enable them on PS2. The problem
-comes if migration fails. In this case in order to resume background jobs
-oracle needs to guarantee that PS2 doesnt run background jobs and if it doesnt
-respond then PS1 is stuck unable to run compaction/gc. This cannot be solved
-without human ensuring that no upload from PS2 can happen. In order to be able
-to resolve this automatically CAS is required on S3 side so pageserver can
-avoid overwriting index part if it is no longer the leading one
-
-Note that flag that disables background jobs needs to be persistent, because
-otherwise pageserver restart will clean it
-
-### Avoid index_part.json
-
-Index part consists of two parts, list of layers and metadata. List of layers
-can be easily obtained by `ListObjects` S3 API method. But what to do with
-metadata? Create metadata instance for each checkpoint and add some counter
-to the file name?
-
-Back to potentially long s3 ls.
-
-### Coordination based approach
-
-Do it like safekeepers chose leader for WAL upload. Ping each other and decide
-based on some heuristics e g smallest node id. During relocation PS1 sends
-"resign" ping message so others can start election without waiting for a timeout.
-
-This still leaves metadata question open and non deterministic layers are a
-problem as well
-
-### Avoid metadata file
-
-One way to eliminate metadata file is to store it in layer files under some
-special key. This may resonate with intention to keep all relation sizes in
-some special segment to avoid initial download during size calculation.
-Maybe with that we can even store pre calculated value.
-
-As a downside each checkpoint gets 512 bytes larger.
-
-If we entirely avoid metadata file this opens up many approaches
-
-* * *
-
-During discussion it seems that we converged on the approach consisting of:
-
-- index files stored per pageserver in the same timeline directory. With that
-  index file name starts to look like: `<pageserver_node_id>_index_part.json`.
-  In such set up there are no concurrent overwrites of index file by different
-  pageservers.
-- For replica pageservers the solution would be for primary to broadcast index
-  changes to any followers with an ability to check index files in s3 and
-  restore the full state. To properly merge changes with index files we can use
-  a counter that is persisted in an index file, is incremented on every change
-  to it and passed along with broadcasted change. This way we can determine
-  whether we need to apply change to the index state or not.
-- Responsibility for running background jobs is assigned externally. Pageserver
-  keeps locally persistent flag for each tenant that indicates whether this
-  pageserver is considered as primary one or not. TODO what happends if we
-  crash and cannot start for some extended period of time? Control plane can
-  assign ownership to some other pageserver. Pageserver needs some way to check
-  if its still the blessed one. Maybe by explicit request to control plane on
-  start.
-
-Requirement for deterministic layer generation was considered overly strict
-because of two reasons:
-
-- It can limit possible optimizations e g when pageserver wants to reshuffle
-  some data locally and doesnt want to coordinate this
-- The deterministic algorithm itself can change so during deployments for some
-  time there will be two different version running at the same time which can
-  cause non determinism
-
-### External elections
-
-The above case with lost state in this schema with externally managed
-leadership is represented like this:
-
-Note that here we keep objects list in the index file.
-
-```mermaid
-sequenceDiagram
-    autonumber
-    participant PS1
-    participant CP as Control Plane
-    participant S3
-    participant PS2
-
-    note over PS1,PS2: PS1 starts up and still a leader
-    PS1->>CP: Am I still the leader for Tenant X?
-    activate CP
-    CP->>PS1: Yes
-    deactivate CP
-    PS1->>S3: Fetch PS1 index.
-    note over PS1: Continue operations, start backround jobs
-    note over PS1,PS2: PS1 starts up and still and is not a leader anymore
-    PS1->>CP: Am I still the leader for Tenant X?
-    CP->>PS1: No
-    PS1->>PS2: Subscribe to index changes
-    PS1->>S3: Fetch PS1 and PS2 indexes
-    note over PS1: Combine index file to include layers <br> from both indexes to be able <br> to see newer files from leader (PS2)
-    note over PS1: Continue operations, do not start background jobs
-```
-
-### Internal elections
-
-To manage leadership internally we can use broker to exchange pings so nodes
-can decide on the leader roles. In case multiple pageservers are active leader
-is the one with lowest node id.
-
-Operations with internally managed elections:
-
-```mermaid
-sequenceDiagram
-    autonumber
-    participant PS1
-    participant S3
-
-    note over PS1: Starts up
-    note over PS1: Subscribes to changes, waits for two ping <br> timeouts to see if there is a leader
-    PS1->>S3: Fetch indexes from s3
-    alt there is a leader
-        note over PS1: do not start background jobs, <br> continue applying index updates
-    else there is no leader
-        note over PS1: start background jobs, <br> broadcast index changes
-    end
-
-    note over PS1,S3: Then the picture is similar to external elections <br> the difference is that follower can become a leader <br> if there are no pings after some timeout new leader gets elected
-```
-
-### Eviction
-
-When two pageservers operate on a tenant for extended period of time follower
-doesnt perform write operations in s3. When layer is evicted follower relies
-on updates from primary to get info about layers it needs to cover range for
-evicted layer.
-
-Note that it wont match evicted layer exactly, so layers will overlap and
-lookup code needs to correctly handle that.
-
-### Relocation flow
-
-Actions become:
-
-- Attach tenant to new pageserver
-- New pageserver becomes follower since previous one is still leading
-- New pageserver starts replicating from safekeepers but does not upload layers
-- Detach is called on the old one
-- New pageserver becomes leader after it realizes that old one disappeared
-
-### Index File
-
-Using `s3 ls` on startup simplifies things, but we still need metadata, so we
-need to fetch index files anyway. If they contain list of files we can combine
-them and avoid costly `s3 ls`
-
-### Remaining issues
-
-- More than one remote consistent lsn for safekeepers to know
-
-Anything else?
-
-### Proposed solution
-
-To recap. On meeting we converged on approach with external elections but I
-think it will be overall harder to manage and will introduce a dependency on
-control plane for pageserver. Using separate index files for each pageserver
-consisting of log of operations and a metadata snapshot should be enough.
-
-### What we need to get there?
-
-- Change index file structure to contain log of changes instead of just the
-  file list
-- Implement pinging/elections for pageservers

docs/sourcetree.md

@@ -52,10 +52,6 @@ PostgreSQL extension that implements storage manager API and network communicati
 
 PostgreSQL extension that contains functions needed for testing and debugging.
 
-`/pgxn/neon_walredo`:
-
-Library to run Postgres as a "WAL redo process" in the pageserver.
-
 `/safekeeper`:
 
 The neon WAL service that receives WAL from a primary compute nodes and streams it to the pageserver.

libs/etcd_broker/Cargo.toml

@@ -8,7 +8,7 @@
  regex = "1.4.5"
  serde = { version = "1.0", features = ["derive"] }
  serde_json = "1"
- serde_with = "2.0"
+ serde_with = "1.12.0"
  once_cell = "1.13.0"
 
  utils = { path = "../utils" }

libs/etcd_broker/src/subscription_value.rs

@@ -29,9 +29,6 @@ pub struct SkTimelineInfo {
     #[serde_as(as = "Option<DisplayFromStr>")]
     #[serde(default)]
     pub peer_horizon_lsn: Option<Lsn>,
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    #[serde(default)]
-    pub local_start_lsn: Option<Lsn>,
     /// A connection string to use for WAL receiving.
     #[serde(default)]
     pub safekeeper_connstr: Option<String>,

libs/metrics/src/lib.rs

@@ -3,7 +3,7 @@
 //! Otherwise, we might not see all metrics registered via
 //! a default registry.
 use once_cell::sync::Lazy;
-use prometheus::core::{AtomicU64, Collector, GenericGauge, GenericGaugeVec};
+use prometheus::core::{AtomicU64, GenericGauge, GenericGaugeVec};
 pub use prometheus::opts;
 pub use prometheus::register;
 pub use prometheus::{core, default_registry, proto};
@@ -17,7 +17,6 @@ pub use prometheus::{register_int_counter_vec, IntCounterVec};
 pub use prometheus::{register_int_gauge, IntGauge};
 pub use prometheus::{register_int_gauge_vec, IntGaugeVec};
 pub use prometheus::{Encoder, TextEncoder};
-use prometheus::{Registry, Result};
 
 mod wrappers;
 pub use wrappers::{CountedReader, CountedWriter};
@@ -33,27 +32,13 @@ macro_rules! register_uint_gauge_vec {
     }};
 }
 
-/// Special internal registry, to collect metrics independently from the default registry.
-/// Was introduced to fix deadlock with lazy registration of metrics in the default registry.
-static INTERNAL_REGISTRY: Lazy<Registry> = Lazy::new(Registry::new);
-
-/// Register a collector in the internal registry. MUST be called before the first call to `gather()`.
-/// Otherwise, we can have a deadlock in the `gather()` call, trying to register a new collector
-/// while holding the lock.
-pub fn register_internal(c: Box<dyn Collector>) -> Result<()> {
-    INTERNAL_REGISTRY.register(c)
-}
-
 /// Gathers all Prometheus metrics and records the I/O stats just before that.
 ///
 /// Metrics gathering is a relatively simple and standalone operation, so
 /// it might be fine to do it this way to keep things simple.
 pub fn gather() -> Vec<prometheus::proto::MetricFamily> {
     update_rusage_metrics();
-    let mut mfs = prometheus::gather();
-    let mut internal_mfs = INTERNAL_REGISTRY.gather();
-    mfs.append(&mut internal_mfs);
-    mfs
+    prometheus::gather()
 }
 
 static DISK_IO_BYTES: Lazy<IntGaugeVec> = Lazy::new(|| {
@@ -77,16 +62,6 @@ pub const DISK_WRITE_SECONDS_BUCKETS: &[f64] = &[
     0.000_050, 0.000_100, 0.000_500, 0.001, 0.003, 0.005, 0.01, 0.05, 0.1, 0.3, 0.5,
 ];
 
-pub fn set_build_info_metric(revision: &str) {
-    let metric = register_int_gauge_vec!(
-        "libmetrics_build_info",
-        "Build/version information",
-        &["revision"]
-    )
-    .expect("Failed to register build info metric");
-    metric.with_label_values(&[revision]).set(1);
-}
-
 // Records I/O stats in a "cross-platform" way.
 // Compiles both on macOS and Linux, but current macOS implementation always returns 0 as values for I/O stats.
 // An alternative is to read procfs (`/proc/[pid]/io`) which does not work under macOS at all, hence abandoned.

libs/pageserver_api/Cargo.toml

@@ -1,16 +0,0 @@
-[package]
-name = "pageserver_api"
-version = "0.1.0"
-edition = "2021"
-
-[dependencies]
-serde = { version = "1.0", features = ["derive"] }
-serde_with = "2.0"
-const_format = "0.2.21"
-anyhow = { version = "1.0", features = ["backtrace"] }
-bytes = "1.0.1"
-byteorder = "1.4.3"
-
-utils = { path = "../utils" }
-postgres_ffi = { path = "../postgres_ffi" }
-workspace_hack = { version = "0.1", path = "../../workspace_hack" }

libs/pageserver_api/src/lib.rs

@@ -1,10 +0,0 @@
-use const_format::formatcp;
-
-/// Public API types
-pub mod models;
-pub mod reltag;
-
-pub const DEFAULT_PG_LISTEN_PORT: u16 = 64000;
-pub const DEFAULT_PG_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_PG_LISTEN_PORT}");
-pub const DEFAULT_HTTP_LISTEN_PORT: u16 = 9898;
-pub const DEFAULT_HTTP_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_HTTP_LISTEN_PORT}");

libs/pageserver_api/src/models.rs

@@ -1,488 +0,0 @@
-use std::num::NonZeroU64;
-
-use byteorder::{BigEndian, ReadBytesExt};
-use serde::{Deserialize, Serialize};
-use serde_with::{serde_as, DisplayFromStr};
-use utils::{
-    id::{NodeId, TenantId, TimelineId},
-    lsn::Lsn,
-};
-
-use crate::reltag::RelTag;
-use anyhow::bail;
-use bytes::{BufMut, Bytes, BytesMut};
-
-/// A state of a tenant in pageserver's memory.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
-pub enum TenantState {
-    /// Tenant is fully operational, its background jobs might be running or not.
-    Active { background_jobs_running: bool },
-    /// A tenant is recognized by pageserver, but not yet ready to operate:
-    /// e.g. not present locally and being downloaded or being read into memory from the file system.
-    Paused,
-    /// A tenant is recognized by the pageserver, but no longer used for any operations, as failed to get activated.
-    Broken,
-}
-
-/// A state of a timeline in pageserver's memory.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
-pub enum TimelineState {
-    /// Timeline is fully operational, its background jobs are running.
-    Active,
-    /// A timeline is recognized by pageserver, but not yet ready to operate.
-    /// The status indicates, that the timeline could eventually go back to Active automatically:
-    /// for example, if the owning tenant goes back to Active again.
-    Suspended,
-    /// A timeline is recognized by pageserver, but not yet ready to operate and not allowed to
-    /// automatically become Active after certain events: only a management call can change this status.
-    Paused,
-    /// A timeline is recognized by the pageserver, but no longer used for any operations, as failed to get activated.
-    Broken,
-}
-
-#[serde_as]
-#[derive(Serialize, Deserialize)]
-pub struct TimelineCreateRequest {
-    #[serde(default)]
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub new_timeline_id: Option<TimelineId>,
-    #[serde(default)]
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub ancestor_timeline_id: Option<TimelineId>,
-    #[serde(default)]
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub ancestor_start_lsn: Option<Lsn>,
-    pub pg_version: Option<u32>,
-}
-
-#[serde_as]
-#[derive(Serialize, Deserialize, Default)]
-pub struct TenantCreateRequest {
-    #[serde(default)]
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub new_tenant_id: Option<TenantId>,
-    pub checkpoint_distance: Option<u64>,
-    pub checkpoint_timeout: Option<String>,
-    pub compaction_target_size: Option<u64>,
-    pub compaction_period: Option<String>,
-    pub compaction_threshold: Option<usize>,
-    pub gc_horizon: Option<u64>,
-    pub gc_period: Option<String>,
-    pub image_creation_threshold: Option<usize>,
-    pub pitr_interval: Option<String>,
-    pub walreceiver_connect_timeout: Option<String>,
-    pub lagging_wal_timeout: Option<String>,
-    pub max_lsn_wal_lag: Option<NonZeroU64>,
-    pub trace_read_requests: Option<bool>,
-}
-
-#[serde_as]
-#[derive(Serialize, Deserialize)]
-#[serde(transparent)]
-pub struct TenantCreateResponse(#[serde_as(as = "DisplayFromStr")] pub TenantId);
-
-#[derive(Serialize)]
-pub struct StatusResponse {
-    pub id: NodeId,
-}
-
-impl TenantCreateRequest {
-    pub fn new(new_tenant_id: Option<TenantId>) -> TenantCreateRequest {
-        TenantCreateRequest {
-            new_tenant_id,
-            ..Default::default()
-        }
-    }
-}
-
-#[serde_as]
-#[derive(Serialize, Deserialize)]
-pub struct TenantConfigRequest {
-    pub tenant_id: TenantId,
-    #[serde(default)]
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub checkpoint_distance: Option<u64>,
-    pub checkpoint_timeout: Option<String>,
-    pub compaction_target_size: Option<u64>,
-    pub compaction_period: Option<String>,
-    pub compaction_threshold: Option<usize>,
-    pub gc_horizon: Option<u64>,
-    pub gc_period: Option<String>,
-    pub image_creation_threshold: Option<usize>,
-    pub pitr_interval: Option<String>,
-    pub walreceiver_connect_timeout: Option<String>,
-    pub lagging_wal_timeout: Option<String>,
-    pub max_lsn_wal_lag: Option<NonZeroU64>,
-    pub trace_read_requests: Option<bool>,
-}
-
-impl TenantConfigRequest {
-    pub fn new(tenant_id: TenantId) -> TenantConfigRequest {
-        TenantConfigRequest {
-            tenant_id,
-            checkpoint_distance: None,
-            checkpoint_timeout: None,
-            compaction_target_size: None,
-            compaction_period: None,
-            compaction_threshold: None,
-            gc_horizon: None,
-            gc_period: None,
-            image_creation_threshold: None,
-            pitr_interval: None,
-            walreceiver_connect_timeout: None,
-            lagging_wal_timeout: None,
-            max_lsn_wal_lag: None,
-            trace_read_requests: None,
-        }
-    }
-}
-
-#[serde_as]
-#[derive(Serialize, Deserialize, Clone)]
-pub struct TenantInfo {
-    #[serde_as(as = "DisplayFromStr")]
-    pub id: TenantId,
-    pub state: TenantState,
-    pub current_physical_size: Option<u64>, // physical size is only included in `tenant_status` endpoint
-    pub has_in_progress_downloads: Option<bool>,
-}
-
-/// This represents the output of the "timeline_detail" and "timeline_list" API calls.
-#[serde_as]
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct TimelineInfo {
-    #[serde_as(as = "DisplayFromStr")]
-    pub tenant_id: TenantId,
-    #[serde_as(as = "DisplayFromStr")]
-    pub timeline_id: TimelineId,
-
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub ancestor_timeline_id: Option<TimelineId>,
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub ancestor_lsn: Option<Lsn>,
-    #[serde_as(as = "DisplayFromStr")]
-    pub last_record_lsn: Lsn,
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub prev_record_lsn: Option<Lsn>,
-    #[serde_as(as = "DisplayFromStr")]
-    pub latest_gc_cutoff_lsn: Lsn,
-    #[serde_as(as = "DisplayFromStr")]
-    pub disk_consistent_lsn: Lsn,
-    pub current_logical_size: Option<u64>, // is None when timeline is Unloaded
-    pub current_physical_size: Option<u64>, // is None when timeline is Unloaded
-    pub current_logical_size_non_incremental: Option<u64>,
-    pub current_physical_size_non_incremental: Option<u64>,
-
-    pub wal_source_connstr: Option<String>,
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub last_received_msg_lsn: Option<Lsn>,
-    /// the timestamp (in microseconds) of the last received message
-    pub last_received_msg_ts: Option<u128>,
-    pub pg_version: u32,
-
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub remote_consistent_lsn: Option<Lsn>,
-    pub awaits_download: bool,
-
-    pub state: TimelineState,
-
-    // Some of the above fields are duplicated in 'local' and 'remote', for backwards-
-    // compatility with older clients.
-    pub local: LocalTimelineInfo,
-    pub remote: RemoteTimelineInfo,
-}
-
-#[serde_as]
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct LocalTimelineInfo {
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub ancestor_timeline_id: Option<TimelineId>,
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub ancestor_lsn: Option<Lsn>,
-    pub current_logical_size: Option<u64>, // is None when timeline is Unloaded
-    pub current_physical_size: Option<u64>, // is None when timeline is Unloaded
-}
-
-#[serde_as]
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct RemoteTimelineInfo {
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub remote_consistent_lsn: Option<Lsn>,
-}
-
-pub type ConfigureFailpointsRequest = Vec<FailpointConfig>;
-
-/// Information for configuring a single fail point
-#[derive(Debug, Serialize, Deserialize)]
-pub struct FailpointConfig {
-    /// Name of the fail point
-    pub name: String,
-    /// List of actions to take, using the format described in `fail::cfg`
-    ///
-    /// We also support `actions = "exit"` to cause the fail point to immediately exit.
-    pub actions: String,
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct TimelineGcRequest {
-    pub gc_horizon: Option<u64>,
-}
-
-// Wrapped in libpq CopyData
-#[derive(PartialEq, Eq)]
-pub enum PagestreamFeMessage {
-    Exists(PagestreamExistsRequest),
-    Nblocks(PagestreamNblocksRequest),
-    GetPage(PagestreamGetPageRequest),
-    DbSize(PagestreamDbSizeRequest),
-}
-
-// Wrapped in libpq CopyData
-pub enum PagestreamBeMessage {
-    Exists(PagestreamExistsResponse),
-    Nblocks(PagestreamNblocksResponse),
-    GetPage(PagestreamGetPageResponse),
-    Error(PagestreamErrorResponse),
-    DbSize(PagestreamDbSizeResponse),
-}
-
-#[derive(Debug, PartialEq, Eq)]
-pub struct PagestreamExistsRequest {
-    pub latest: bool,
-    pub lsn: Lsn,
-    pub rel: RelTag,
-}
-
-#[derive(Debug, PartialEq, Eq)]
-pub struct PagestreamNblocksRequest {
-    pub latest: bool,
-    pub lsn: Lsn,
-    pub rel: RelTag,
-}
-
-#[derive(Debug, PartialEq, Eq)]
-pub struct PagestreamGetPageRequest {
-    pub latest: bool,
-    pub lsn: Lsn,
-    pub rel: RelTag,
-    pub blkno: u32,
-}
-
-#[derive(Debug, PartialEq, Eq)]
-pub struct PagestreamDbSizeRequest {
-    pub latest: bool,
-    pub lsn: Lsn,
-    pub dbnode: u32,
-}
-
-#[derive(Debug)]
-pub struct PagestreamExistsResponse {
-    pub exists: bool,
-}
-
-#[derive(Debug)]
-pub struct PagestreamNblocksResponse {
-    pub n_blocks: u32,
-}
-
-#[derive(Debug)]
-pub struct PagestreamGetPageResponse {
-    pub page: Bytes,
-}
-
-#[derive(Debug)]
-pub struct PagestreamErrorResponse {
-    pub message: String,
-}
-
-#[derive(Debug)]
-pub struct PagestreamDbSizeResponse {
-    pub db_size: i64,
-}
-
-impl PagestreamFeMessage {
-    pub fn serialize(&self) -> Bytes {
-        let mut bytes = BytesMut::new();
-
-        match self {
-            Self::Exists(req) => {
-                bytes.put_u8(0);
-                bytes.put_u8(if req.latest { 1 } else { 0 });
-                bytes.put_u64(req.lsn.0);
-                bytes.put_u32(req.rel.spcnode);
-                bytes.put_u32(req.rel.dbnode);
-                bytes.put_u32(req.rel.relnode);
-                bytes.put_u8(req.rel.forknum);
-            }
-
-            Self::Nblocks(req) => {
-                bytes.put_u8(1);
-                bytes.put_u8(if req.latest { 1 } else { 0 });
-                bytes.put_u64(req.lsn.0);
-                bytes.put_u32(req.rel.spcnode);
-                bytes.put_u32(req.rel.dbnode);
-                bytes.put_u32(req.rel.relnode);
-                bytes.put_u8(req.rel.forknum);
-            }
-
-            Self::GetPage(req) => {
-                bytes.put_u8(2);
-                bytes.put_u8(if req.latest { 1 } else { 0 });
-                bytes.put_u64(req.lsn.0);
-                bytes.put_u32(req.rel.spcnode);
-                bytes.put_u32(req.rel.dbnode);
-                bytes.put_u32(req.rel.relnode);
-                bytes.put_u8(req.rel.forknum);
-                bytes.put_u32(req.blkno);
-            }
-
-            Self::DbSize(req) => {
-                bytes.put_u8(3);
-                bytes.put_u8(if req.latest { 1 } else { 0 });
-                bytes.put_u64(req.lsn.0);
-                bytes.put_u32(req.dbnode);
-            }
-        }
-
-        bytes.into()
-    }
-
-    pub fn parse<R: std::io::Read>(body: &mut R) -> anyhow::Result<PagestreamFeMessage> {
-        // TODO these gets can fail
-
-        // these correspond to the NeonMessageTag enum in pagestore_client.h
-        //
-        // TODO: consider using protobuf or serde bincode for less error prone
-        // serialization.
-        let msg_tag = body.read_u8()?;
-        match msg_tag {
-            0 => Ok(PagestreamFeMessage::Exists(PagestreamExistsRequest {
-                latest: body.read_u8()? != 0,
-                lsn: Lsn::from(body.read_u64::<BigEndian>()?),
-                rel: RelTag {
-                    spcnode: body.read_u32::<BigEndian>()?,
-                    dbnode: body.read_u32::<BigEndian>()?,
-                    relnode: body.read_u32::<BigEndian>()?,
-                    forknum: body.read_u8()?,
-                },
-            })),
-            1 => Ok(PagestreamFeMessage::Nblocks(PagestreamNblocksRequest {
-                latest: body.read_u8()? != 0,
-                lsn: Lsn::from(body.read_u64::<BigEndian>()?),
-                rel: RelTag {
-                    spcnode: body.read_u32::<BigEndian>()?,
-                    dbnode: body.read_u32::<BigEndian>()?,
-                    relnode: body.read_u32::<BigEndian>()?,
-                    forknum: body.read_u8()?,
-                },
-            })),
-            2 => Ok(PagestreamFeMessage::GetPage(PagestreamGetPageRequest {
-                latest: body.read_u8()? != 0,
-                lsn: Lsn::from(body.read_u64::<BigEndian>()?),
-                rel: RelTag {
-                    spcnode: body.read_u32::<BigEndian>()?,
-                    dbnode: body.read_u32::<BigEndian>()?,
-                    relnode: body.read_u32::<BigEndian>()?,
-                    forknum: body.read_u8()?,
-                },
-                blkno: body.read_u32::<BigEndian>()?,
-            })),
-            3 => Ok(PagestreamFeMessage::DbSize(PagestreamDbSizeRequest {
-                latest: body.read_u8()? != 0,
-                lsn: Lsn::from(body.read_u64::<BigEndian>()?),
-                dbnode: body.read_u32::<BigEndian>()?,
-            })),
-            _ => bail!("unknown smgr message tag: {:?}", msg_tag),
-        }
-    }
-}
-
-impl PagestreamBeMessage {
-    pub fn serialize(&self) -> Bytes {
-        let mut bytes = BytesMut::new();
-
-        match self {
-            Self::Exists(resp) => {
-                bytes.put_u8(100); /* tag from pagestore_client.h */
-                bytes.put_u8(resp.exists as u8);
-            }
-
-            Self::Nblocks(resp) => {
-                bytes.put_u8(101); /* tag from pagestore_client.h */
-                bytes.put_u32(resp.n_blocks);
-            }
-
-            Self::GetPage(resp) => {
-                bytes.put_u8(102); /* tag from pagestore_client.h */
-                bytes.put(&resp.page[..]);
-            }
-
-            Self::Error(resp) => {
-                bytes.put_u8(103); /* tag from pagestore_client.h */
-                bytes.put(resp.message.as_bytes());
-                bytes.put_u8(0); // null terminator
-            }
-            Self::DbSize(resp) => {
-                bytes.put_u8(104); /* tag from pagestore_client.h */
-                bytes.put_i64(resp.db_size);
-            }
-        }
-
-        bytes.into()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use bytes::Buf;
-
-    use super::*;
-
-    #[test]
-    fn test_pagestream() {
-        // Test serialization/deserialization of PagestreamFeMessage
-        let messages = vec![
-            PagestreamFeMessage::Exists(PagestreamExistsRequest {
-                latest: true,
-                lsn: Lsn(4),
-                rel: RelTag {
-                    forknum: 1,
-                    spcnode: 2,
-                    dbnode: 3,
-                    relnode: 4,
-                },
-            }),
-            PagestreamFeMessage::Nblocks(PagestreamNblocksRequest {
-                latest: false,
-                lsn: Lsn(4),
-                rel: RelTag {
-                    forknum: 1,
-                    spcnode: 2,
-                    dbnode: 3,
-                    relnode: 4,
-                },
-            }),
-            PagestreamFeMessage::GetPage(PagestreamGetPageRequest {
-                latest: true,
-                lsn: Lsn(4),
-                rel: RelTag {
-                    forknum: 1,
-                    spcnode: 2,
-                    dbnode: 3,
-                    relnode: 4,
-                },
-                blkno: 7,
-            }),
-            PagestreamFeMessage::DbSize(PagestreamDbSizeRequest {
-                latest: true,
-                lsn: Lsn(4),
-                dbnode: 7,
-            }),
-        ];
-        for msg in messages {
-            let bytes = msg.serialize();
-            let reconstructed = PagestreamFeMessage::parse(&mut bytes.reader()).unwrap();
-            assert!(msg == reconstructed);
-        }
-    }
-}

libs/postgres_ffi/Cargo.toml

@@ -13,7 +13,7 @@ crc32c = "0.6.0"
 hex = "0.4.3"
 once_cell = "1.13.0"
 log = "0.4.14"
-memoffset = "0.7"
+memoffset = "0.6.2"
 thiserror = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 utils = { path = "../utils" }
@@ -26,4 +26,4 @@ wal_craft = { path = "wal_craft" }
 
 [build-dependencies]
 anyhow = "1.0"
-bindgen = "0.61"
+bindgen = "0.60.1"

libs/postgres_ffi/wal_craft/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 
 [dependencies]
 anyhow = "1.0"
-clap = "4.0"
+clap = "3.0"
 env_logger = "0.9"
 log = "0.4"
 once_cell = "1.13.0"

libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs

@@ -1,19 +1,68 @@
 use anyhow::*;
-use clap::{value_parser, Arg, ArgMatches, Command};
-use std::{path::PathBuf, str::FromStr};
+use clap::{App, Arg, ArgMatches};
+use std::str::FromStr;
 use wal_craft::*;
 
 fn main() -> Result<()> {
     env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("wal_craft=info"))
         .init();
-    let arg_matches = cli().get_matches();
+    let type_arg = &Arg::new("type")
+        .takes_value(true)
+        .help("Type of WAL to craft")
+        .possible_values([
+            Simple::NAME,
+            LastWalRecordXlogSwitch::NAME,
+            LastWalRecordXlogSwitchEndsOnPageBoundary::NAME,
+            WalRecordCrossingSegmentFollowedBySmallOne::NAME,
+            LastWalRecordCrossingSegment::NAME,
+        ])
+        .required(true);
+    let arg_matches = App::new("Postgres WAL crafter")
+        .about("Crafts Postgres databases with specific WAL properties")
+        .subcommand(
+            App::new("print-postgres-config")
+                .about("Print the configuration required for PostgreSQL server before running this script")
+        )
+        .subcommand(
+            App::new("with-initdb")
+                .about("Craft WAL in a new data directory first initialized with initdb")
+                .arg(type_arg)
+                .arg(
+                    Arg::new("datadir")
+                        .takes_value(true)
+                        .help("Data directory for the Postgres server")
+                        .required(true)
+                )
+                .arg(
+                    Arg::new("pg-distrib-dir")
+                        .long("pg-distrib-dir")
+                        .takes_value(true)
+                        .help("Directory with Postgres distributions (bin and lib directories, e.g. pg_install containing subpath `v14/bin/postgresql`)")
+                        .default_value("/usr/local")
+                )
+                .arg(
+                    Arg::new("pg-version")
+                    .long("pg-version")
+                    .help("Postgres version to use for the initial tenant")
+                    .required(true)
+                    .takes_value(true)
+                )
+        )
+        .subcommand(
+            App::new("in-existing")
+                .about("Craft WAL at an existing recently created Postgres database. Note that server may append new WAL entries on shutdown.")
+                .arg(type_arg)
+                .arg(
+                    Arg::new("connection")
+                        .takes_value(true)
+                        .help("Connection string to the Postgres database to populate")
+                        .required(true)
+                )
+        )
+        .get_matches();
 
     let wal_craft = |arg_matches: &ArgMatches, client| {
-        let (intermediate_lsns, end_of_wal_lsn) = match arg_matches
-            .get_one::<String>("type")
-            .map(|s| s.as_str())
-            .context("'type' is required")?
-        {
+        let (intermediate_lsns, end_of_wal_lsn) = match arg_matches.value_of("type").unwrap() {
             Simple::NAME => Simple::craft(client)?,
             LastWalRecordXlogSwitch::NAME => LastWalRecordXlogSwitch::craft(client)?,
             LastWalRecordXlogSwitchEndsOnPageBoundary::NAME => {
@@ -23,12 +72,12 @@ fn main() -> Result<()> {
                 WalRecordCrossingSegmentFollowedBySmallOne::craft(client)?
             }
             LastWalRecordCrossingSegment::NAME => LastWalRecordCrossingSegment::craft(client)?,
-            a => panic!("Unknown --type argument: {a}"),
+            a => panic!("Unknown --type argument: {}", a),
         };
         for lsn in intermediate_lsns {
-            println!("intermediate_lsn = {lsn}");
+            println!("intermediate_lsn = {}", lsn);
         }
-        println!("end_of_wal = {end_of_wal_lsn}");
+        println!("end_of_wal = {}", end_of_wal_lsn);
         Ok(())
     };
 
@@ -36,24 +85,20 @@ fn main() -> Result<()> {
         None => panic!("No subcommand provided"),
         Some(("print-postgres-config", _)) => {
             for cfg in REQUIRED_POSTGRES_CONFIG.iter() {
-                println!("{cfg}");
+                println!("{}", cfg);
             }
             Ok(())
         }
 
         Some(("with-initdb", arg_matches)) => {
             let cfg = Conf {
-                pg_version: *arg_matches
-                    .get_one::<u32>("pg-version")
-                    .context("'pg-version' is required")?,
-                pg_distrib_dir: arg_matches
-                    .get_one::<PathBuf>("pg-distrib-dir")
-                    .context("'pg-distrib-dir' is required")?
-                    .to_owned(),
-                datadir: arg_matches
-                    .get_one::<PathBuf>("datadir")
-                    .context("'datadir' is required")?
-                    .to_owned(),
+                pg_version: arg_matches
+                    .value_of("pg-version")
+                    .unwrap()
+                    .parse::<u32>()
+                    .context("Failed to parse postgres version from the argument string")?,
+                pg_distrib_dir: arg_matches.value_of("pg-distrib-dir").unwrap().into(),
+                datadir: arg_matches.value_of("datadir").unwrap().into(),
             };
             cfg.initdb()?;
             let srv = cfg.start_server()?;
@@ -63,77 +108,9 @@ fn main() -> Result<()> {
         }
         Some(("in-existing", arg_matches)) => wal_craft(
             arg_matches,
-            &mut postgres::Config::from_str(
-                arg_matches
-                    .get_one::<String>("connection")
-                    .context("'connection' is required")?,
-            )
-            .context(
-                "'connection' argument value could not be parsed as a postgres connection string",
-            )?
-            .connect(postgres::NoTls)?,
+            &mut postgres::Config::from_str(arg_matches.value_of("connection").unwrap())?
+                .connect(postgres::NoTls)?,
         ),
         Some(_) => panic!("Unknown subcommand"),
     }
 }
-
-fn cli() -> Command {
-    let type_arg = &Arg::new("type")
-        .help("Type of WAL to craft")
-        .value_parser([
-            Simple::NAME,
-            LastWalRecordXlogSwitch::NAME,
-            LastWalRecordXlogSwitchEndsOnPageBoundary::NAME,
-            WalRecordCrossingSegmentFollowedBySmallOne::NAME,
-            LastWalRecordCrossingSegment::NAME,
-        ])
-        .required(true);
-
-    Command::new("Postgres WAL crafter")
-        .about("Crafts Postgres databases with specific WAL properties")
-        .subcommand(
-            Command::new("print-postgres-config")
-                .about("Print the configuration required for PostgreSQL server before running this script")
-        )
-        .subcommand(
-            Command::new("with-initdb")
-                .about("Craft WAL in a new data directory first initialized with initdb")
-                .arg(type_arg)
-                .arg(
-                    Arg::new("datadir")
-                        .help("Data directory for the Postgres server")
-                        .value_parser(value_parser!(PathBuf))
-                        .required(true)
-                )
-                .arg(
-                    Arg::new("pg-distrib-dir")
-                        .long("pg-distrib-dir")
-                        .value_parser(value_parser!(PathBuf))
-                        .help("Directory with Postgres distributions (bin and lib directories, e.g. pg_install containing subpath `v14/bin/postgresql`)")
-                        .default_value("/usr/local")
-                )
-                .arg(
-                    Arg::new("pg-version")
-                    .long("pg-version")
-                    .help("Postgres version to use for the initial tenant")
-                    .value_parser(value_parser!(u32))
-                    .required(true)
-
-                )
-        )
-        .subcommand(
-            Command::new("in-existing")
-                .about("Craft WAL at an existing recently created Postgres database. Note that server may append new WAL entries on shutdown.")
-                .arg(type_arg)
-                .arg(
-                    Arg::new("connection")
-                        .help("Connection string to the Postgres database to populate")
-                        .required(true)
-                )
-        )
-}
-
-#[test]
-fn verify_cli() {
-    cli().debug_assert();
-}

libs/postgres_ffi/wal_craft/src/lib.rs

@@ -37,22 +37,22 @@ pub static REQUIRED_POSTGRES_CONFIG: Lazy<Vec<&'static str>> = Lazy::new(|| {
 });
 
 impl Conf {
-    pub fn pg_distrib_dir(&self) -> anyhow::Result<PathBuf> {
+    pub fn pg_distrib_dir(&self) -> PathBuf {
         let path = self.pg_distrib_dir.clone();
 
         match self.pg_version {
-            14 => Ok(path.join(format!("v{}", self.pg_version))),
-            15 => Ok(path.join(format!("v{}", self.pg_version))),
-            _ => bail!("Unsupported postgres version: {}", self.pg_version),
+            14 => path.join(format!("v{}", self.pg_version)),
+            15 => path.join(format!("v{}", self.pg_version)),
+            _ => panic!("Unsupported postgres version: {}", self.pg_version),
         }
     }
 
-    fn pg_bin_dir(&self) -> anyhow::Result<PathBuf> {
-        Ok(self.pg_distrib_dir()?.join("bin"))
+    fn pg_bin_dir(&self) -> PathBuf {
+        self.pg_distrib_dir().join("bin")
     }
 
-    fn pg_lib_dir(&self) -> anyhow::Result<PathBuf> {
-        Ok(self.pg_distrib_dir()?.join("lib"))
+    fn pg_lib_dir(&self) -> PathBuf {
+        self.pg_distrib_dir().join("lib")
     }
 
     pub fn wal_dir(&self) -> PathBuf {
@@ -60,12 +60,12 @@ impl Conf {
     }
 
     fn new_pg_command(&self, command: impl AsRef<Path>) -> Result<Command> {
-        let path = self.pg_bin_dir()?.join(command);
+        let path = self.pg_bin_dir().join(command);
         ensure!(path.exists(), "Command {:?} does not exist", path);
         let mut cmd = Command::new(path);
         cmd.env_clear()
-            .env("LD_LIBRARY_PATH", self.pg_lib_dir()?)
-            .env("DYLD_LIBRARY_PATH", self.pg_lib_dir()?);
+            .env("LD_LIBRARY_PATH", self.pg_lib_dir())
+            .env("DYLD_LIBRARY_PATH", self.pg_lib_dir());
         Ok(cmd)
     }
 

libs/pq_proto/Cargo.toml

@@ -1,16 +0,0 @@
-[package]
-name = "pq_proto"
-version = "0.1.0"
-edition = "2021"
-
-[dependencies]
-anyhow = "1.0"
-bytes = "1.0.1"
-pin-project-lite = "0.2.7"
-postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-rand = "0.8.3"
-serde = { version = "1.0", features = ["derive"] }
-tokio = { version = "1.17", features = ["macros"] }
-tracing = "0.1"
-
-workspace_hack = { version = "0.1", path = "../../workspace_hack" }

libs/remote_storage/Cargo.toml

@@ -15,7 +15,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
 tokio = { version = "1.17", features = ["sync", "macros", "fs", "io-util"] }
 tokio-util = { version = "0.7", features = ["io"] }
-toml_edit = { version = "0.14", features = ["easy"] }
+toml_edit = { version = "0.13", features = ["easy"] }
 tracing = "0.1.27"
 
 workspace_hack = { version = "0.1", path = "../../workspace_hack" }

libs/remote_storage/src/local_fs.rs

@@ -16,7 +16,7 @@ use tokio::{
     io::{self, AsyncReadExt, AsyncSeekExt, AsyncWriteExt},
 };
 use tracing::*;
-use utils::crashsafe::path_with_suffix_extension;
+use utils::crashsafe_dir::path_with_suffix_extension;
 
 use crate::{Download, DownloadError, RemoteObjectId};
 

libs/safekeeper_api/Cargo.toml

@@ -1,12 +0,0 @@
-[package]
-name = "safekeeper_api"
-version = "0.1.0"
-edition = "2021"
-
-[dependencies]
-serde = { version = "1.0", features = ["derive"] }
-serde_with = "2.0"
-const_format = "0.2.21"
-
-utils = { path = "../utils" }
-workspace_hack = { version = "0.1", path = "../../workspace_hack" }

libs/safekeeper_api/src/lib.rs

@@ -1,10 +0,0 @@
-use const_format::formatcp;
-
-/// Public API types
-pub mod models;
-
-pub const DEFAULT_PG_LISTEN_PORT: u16 = 5454;
-pub const DEFAULT_PG_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_PG_LISTEN_PORT}");
-
-pub const DEFAULT_HTTP_LISTEN_PORT: u16 = 7676;
-pub const DEFAULT_HTTP_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_HTTP_LISTEN_PORT}");

libs/safekeeper_api/src/models.rs

@@ -1,24 +0,0 @@
-use serde::{Deserialize, Serialize};
-use serde_with::{serde_as, DisplayFromStr};
-
-use utils::{
-    id::{NodeId, TenantId, TimelineId},
-    lsn::Lsn,
-};
-
-#[serde_as]
-#[derive(Serialize, Deserialize)]
-pub struct TimelineCreateRequest {
-    #[serde_as(as = "DisplayFromStr")]
-    pub tenant_id: TenantId,
-    #[serde_as(as = "DisplayFromStr")]
-    pub timeline_id: TimelineId,
-    pub peer_ids: Option<Vec<NodeId>>,
-    pub pg_version: u32,
-    pub system_id: Option<u64>,
-    pub wal_seg_size: Option<u32>,
-    #[serde_as(as = "DisplayFromStr")]
-    pub commit_lsn: Lsn,
-    // If not passed, it is assigned to the beginning of commit_lsn segment.
-    pub local_start_lsn: Option<Lsn>,
-}

libs/tenant_size_model/.gitignore

@@ -1,3 +0,0 @@
-*.dot
-*.png
-*.svg

libs/tenant_size_model/Cargo.toml

@@ -1,8 +0,0 @@
-[package]
-name = "tenant_size_model"
-version = "0.1.0"
-edition = "2021"
-publish = false
-
-[dependencies]
-workspace_hack = { version = "0.1", path = "../../workspace_hack" }

libs/tenant_size_model/Makefile

@@ -1,13 +0,0 @@
-all: 1.svg 2.svg 3.svg 4.svg 1.png 2.png 3.png 4.png
-
-../../target/debug/tenant_size_model: Cargo.toml src/main.rs src/lib.rs
-	cargo build --bin tenant_size_model
-
-%.svg: %.dot
-	dot -Tsvg $< > $@
-
-%.png: %.dot
-	dot -Tpng $< > $@
-
-%.dot: ../../target/debug/tenant_size_model
-	../../target/debug/tenant_size_model $* > $@

libs/tenant_size_model/README.md

@@ -1,7 +0,0 @@
-# Logical size + WAL pricing
-
-This is a simulator to calculate the tenant size in different scenarios,
-using the "Logical size + WAL" method. Makefile produces diagrams used in a
-private presentation:
-
-https://docs.google.com/presentation/d/1OapE4k11xmcwMh7I7YvNWGC63yCRLh6udO9bXZ-fZmo/edit?usp=sharing

libs/tenant_size_model/src/lib.rs

@@ -1,349 +0,0 @@
-use std::borrow::Cow;
-use std::collections::HashMap;
-
-/// Pricing model or history size builder.
-///
-/// Maintains knowledge of the branches and their modifications. Generic over the branch name key
-/// type.
-pub struct Storage<K: 'static> {
-    segments: Vec<Segment>,
-
-    /// Mapping from the branch name to the index of a segment describing it's latest state.
-    branches: HashMap<K, usize>,
-}
-
-/// Snapshot of a branch.
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub struct Segment {
-    /// Previous segment index into ['Storage::segments`], if any.
-    parent: Option<usize>,
-
-    /// Description of how did we get to this state.
-    ///
-    /// Mainly used in the original scenarios 1..=4 with insert, delete and update. Not used when
-    /// modifying a branch directly.
-    pub op: Cow<'static, str>,
-
-    /// LSN before this state
-    start_lsn: u64,
-
-    /// LSN at this state
-    pub end_lsn: u64,
-
-    /// Logical size before this state
-    start_size: u64,
-
-    /// Logical size at this state
-    pub end_size: u64,
-
-    /// Indices to [`Storage::segments`]
-    ///
-    /// FIXME: this could be an Option<usize>
-    children_after: Vec<usize>,
-
-    /// Determined by `retention_period` given to [`Storage::calculate`]
-    pub needed: bool,
-}
-
-//
-//
-//
-//
-//                 *-g--*---D--->
-//                /
-//               /
-//              /                 *---b----*-B--->
-//             /                 /
-//            /                 /
-//      -----*--e---*-----f----* C
-//           E                  \
-//                               \
-//                                *--a---*---A-->
-//
-// If A and B need to be retained, is it cheaper to store
-// snapshot at C+a+b, or snapshots at A and B ?
-//
-// If D also needs to be retained, which is cheaper:
-//
-// 1. E+g+e+f+a+b
-// 2. D+C+a+b
-// 3. D+A+B
-
-/// [`Segment`] which has had it's size calculated.
-pub struct SegmentSize {
-    pub seg_id: usize,
-
-    pub method: SegmentMethod,
-
-    this_size: u64,
-
-    pub children: Vec<SegmentSize>,
-}
-
-impl SegmentSize {
-    fn total(&self) -> u64 {
-        self.this_size + self.children.iter().fold(0, |acc, x| acc + x.total())
-    }
-
-    pub fn total_children(&self) -> u64 {
-        if self.method == SnapshotAfter {
-            self.this_size + self.children.iter().fold(0, |acc, x| acc + x.total())
-        } else {
-            self.children.iter().fold(0, |acc, x| acc + x.total())
-        }
-    }
-}
-
-/// Different methods to retain history from a particular state
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub enum SegmentMethod {
-    SnapshotAfter,
-    Wal,
-    WalNeeded,
-    Skipped,
-}
-
-use SegmentMethod::*;
-
-impl<K: std::hash::Hash + Eq + 'static> Storage<K> {
-    /// Creates a new storage with the given default branch name.
-    pub fn new(initial_branch: K) -> Storage<K> {
-        let init_segment = Segment {
-            op: "".into(),
-            needed: false,
-            parent: None,
-            start_lsn: 0,
-            end_lsn: 0,
-            start_size: 0,
-            end_size: 0,
-            children_after: Vec::new(),
-        };
-
-        Storage {
-            segments: vec![init_segment],
-            branches: HashMap::from([(initial_branch, 0)]),
-        }
-    }
-
-    /// Advances the branch with the named operation, by the relative LSN and logical size bytes.
-    pub fn modify_branch<Q: ?Sized>(
-        &mut self,
-        branch: &Q,
-        op: Cow<'static, str>,
-        lsn_bytes: u64,
-        size_bytes: i64,
-    ) where
-        K: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq,
-    {
-        let lastseg_id = *self.branches.get(branch).unwrap();
-        let newseg_id = self.segments.len();
-        let lastseg = &mut self.segments[lastseg_id];
-
-        let newseg = Segment {
-            op,
-            parent: Some(lastseg_id),
-            start_lsn: lastseg.end_lsn,
-            end_lsn: lastseg.end_lsn + lsn_bytes,
-            start_size: lastseg.end_size,
-            end_size: (lastseg.end_size as i64 + size_bytes) as u64,
-            children_after: Vec::new(),
-            needed: false,
-        };
-        lastseg.children_after.push(newseg_id);
-
-        self.segments.push(newseg);
-        *self.branches.get_mut(branch).expect("read already") = newseg_id;
-    }
-
-    pub fn insert<Q: ?Sized>(&mut self, branch: &Q, bytes: u64)
-    where
-        K: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq,
-    {
-        self.modify_branch(branch, "insert".into(), bytes, bytes as i64);
-    }
-
-    pub fn update<Q: ?Sized>(&mut self, branch: &Q, bytes: u64)
-    where
-        K: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq,
-    {
-        self.modify_branch(branch, "update".into(), bytes, 0i64);
-    }
-
-    pub fn delete<Q: ?Sized>(&mut self, branch: &Q, bytes: u64)
-    where
-        K: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq,
-    {
-        self.modify_branch(branch, "delete".into(), bytes, -(bytes as i64));
-    }
-
-    /// Panics if the parent branch cannot be found.
-    pub fn branch<Q: ?Sized>(&mut self, parent: &Q, name: K)
-    where
-        K: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq,
-    {
-        // Find the right segment
-        let branchseg_id = *self
-            .branches
-            .get(parent)
-            .expect("should had found the parent by key");
-        let _branchseg = &mut self.segments[branchseg_id];
-
-        // Create branch name for it
-        self.branches.insert(name, branchseg_id);
-    }
-
-    pub fn calculate(&mut self, retention_period: u64) -> SegmentSize {
-        // Phase 1: Mark all the segments that need to be retained
-        for (_branch, &last_seg_id) in self.branches.iter() {
-            let last_seg = &self.segments[last_seg_id];
-            let cutoff_lsn = last_seg.start_lsn.saturating_sub(retention_period);
-            let mut seg_id = last_seg_id;
-            loop {
-                let seg = &mut self.segments[seg_id];
-                if seg.end_lsn < cutoff_lsn {
-                    break;
-                }
-                seg.needed = true;
-                if let Some(prev_seg_id) = seg.parent {
-                    seg_id = prev_seg_id;
-                } else {
-                    break;
-                }
-            }
-        }
-
-        // Phase 2: For each oldest segment in a chain that needs to be retained,
-        // calculate if we should store snapshot or WAL
-        self.size_from_snapshot_later(0)
-    }
-
-    fn size_from_wal(&self, seg_id: usize) -> SegmentSize {
-        let seg = &self.segments[seg_id];
-
-        let this_size = seg.end_lsn - seg.start_lsn;
-
-        let mut children = Vec::new();
-
-        // try both ways
-        for &child_id in seg.children_after.iter() {
-            // try each child both ways
-            let child = &self.segments[child_id];
-            let p1 = self.size_from_wal(child_id);
-
-            let p = if !child.needed {
-                let p2 = self.size_from_snapshot_later(child_id);
-                if p1.total() < p2.total() {
-                    p1
-                } else {
-                    p2
-                }
-            } else {
-                p1
-            };
-            children.push(p);
-        }
-        SegmentSize {
-            seg_id,
-            method: if seg.needed { WalNeeded } else { Wal },
-            this_size,
-            children,
-        }
-    }
-
-    fn size_from_snapshot_later(&self, seg_id: usize) -> SegmentSize {
-        // If this is needed, then it's time to do the snapshot and continue
-        // with wal method.
-        let seg = &self.segments[seg_id];
-        //eprintln!("snap: seg{}: {} needed: {}", seg_id, seg.children_after.len(), seg.needed);
-        if seg.needed {
-            let mut children = Vec::new();
-
-            for &child_id in seg.children_after.iter() {
-                // try each child both ways
-                let child = &self.segments[child_id];
-                let p1 = self.size_from_wal(child_id);
-
-                let p = if !child.needed {
-                    let p2 = self.size_from_snapshot_later(child_id);
-                    if p1.total() < p2.total() {
-                        p1
-                    } else {
-                        p2
-                    }
-                } else {
-                    p1
-                };
-                children.push(p);
-            }
-            SegmentSize {
-                seg_id,
-                method: WalNeeded,
-                this_size: seg.start_size,
-                children,
-            }
-        } else {
-            // If any of the direct children are "needed", need to be able to reconstruct here
-            let mut children_needed = false;
-            for &child in seg.children_after.iter() {
-                let seg = &self.segments[child];
-                if seg.needed {
-                    children_needed = true;
-                    break;
-                }
-            }
-
-            let method1 = if !children_needed {
-                let mut children = Vec::new();
-                for child in seg.children_after.iter() {
-                    children.push(self.size_from_snapshot_later(*child));
-                }
-                Some(SegmentSize {
-                    seg_id,
-                    method: Skipped,
-                    this_size: 0,
-                    children,
-                })
-            } else {
-                None
-            };
-
-            // If this a junction, consider snapshotting here
-            let method2 = if children_needed || seg.children_after.len() >= 2 {
-                let mut children = Vec::new();
-                for child in seg.children_after.iter() {
-                    children.push(self.size_from_wal(*child));
-                }
-                Some(SegmentSize {
-                    seg_id,
-                    method: SnapshotAfter,
-                    this_size: seg.end_size,
-                    children,
-                })
-            } else {
-                None
-            };
-
-            match (method1, method2) {
-                (None, None) => panic!(),
-                (Some(method), None) => method,
-                (None, Some(method)) => method,
-                (Some(method1), Some(method2)) => {
-                    if method1.total() < method2.total() {
-                        method1
-                    } else {
-                        method2
-                    }
-                }
-            }
-        }
-    }
-
-    pub fn into_segments(self) -> Vec<Segment> {
-        self.segments
-    }
-}

libs/tenant_size_model/src/main.rs

@@ -1,268 +0,0 @@
-//! Tenant size model testing ground.
-//!
-//! Has a number of scenarios and a `main` for invoking these by number, calculating the history
-//! size, outputs graphviz graph. Makefile in directory shows how to use graphviz to turn scenarios
-//! into pngs.
-
-use tenant_size_model::{Segment, SegmentSize, Storage};
-
-// Main branch only. Some updates on it.
-fn scenario_1() -> (Vec<Segment>, SegmentSize) {
-    // Create main branch
-    let mut storage = Storage::new("main");
-
-    // Bulk load 5 GB of data to it
-    storage.insert("main", 5_000);
-
-    // Stream of updates
-    for _ in 0..5 {
-        storage.update("main", 1_000);
-    }
-
-    let size = storage.calculate(1000);
-
-    (storage.into_segments(), size)
-}
-
-// Main branch only. Some updates on it.
-fn scenario_2() -> (Vec<Segment>, SegmentSize) {
-    // Create main branch
-    let mut storage = Storage::new("main");
-
-    // Bulk load 5 GB of data to it
-    storage.insert("main", 5_000);
-
-    // Stream of updates
-    for _ in 0..5 {
-        storage.update("main", 1_000);
-    }
-
-    // Branch
-    storage.branch("main", "child");
-    storage.update("child", 1_000);
-
-    // More updates on parent
-    storage.update("main", 1_000);
-
-    let size = storage.calculate(1000);
-
-    (storage.into_segments(), size)
-}
-
-// Like 2, but more updates on main
-fn scenario_3() -> (Vec<Segment>, SegmentSize) {
-    // Create main branch
-    let mut storage = Storage::new("main");
-
-    // Bulk load 5 GB of data to it
-    storage.insert("main", 5_000);
-
-    // Stream of updates
-    for _ in 0..5 {
-        storage.update("main", 1_000);
-    }
-
-    // Branch
-    storage.branch("main", "child");
-    storage.update("child", 1_000);
-
-    // More updates on parent
-    for _ in 0..5 {
-        storage.update("main", 1_000);
-    }
-
-    let size = storage.calculate(1000);
-
-    (storage.into_segments(), size)
-}
-
-// Diverged branches
-fn scenario_4() -> (Vec<Segment>, SegmentSize) {
-    // Create main branch
-    let mut storage = Storage::new("main");
-
-    // Bulk load 5 GB of data to it
-    storage.insert("main", 5_000);
-
-    // Stream of updates
-    for _ in 0..5 {
-        storage.update("main", 1_000);
-    }
-
-    // Branch
-    storage.branch("main", "child");
-    storage.update("child", 1_000);
-
-    // More updates on parent
-    for _ in 0..8 {
-        storage.update("main", 1_000);
-    }
-
-    let size = storage.calculate(1000);
-
-    (storage.into_segments(), size)
-}
-
-fn scenario_5() -> (Vec<Segment>, SegmentSize) {
-    let mut storage = Storage::new("a");
-    storage.insert("a", 5000);
-    storage.branch("a", "b");
-    storage.update("b", 4000);
-    storage.update("a", 2000);
-    storage.branch("a", "c");
-    storage.insert("c", 4000);
-    storage.insert("a", 2000);
-
-    let size = storage.calculate(5000);
-
-    (storage.into_segments(), size)
-}
-
-fn scenario_6() -> (Vec<Segment>, SegmentSize) {
-    use std::borrow::Cow;
-
-    const NO_OP: Cow<'static, str> = Cow::Borrowed("");
-
-    let branches = [
-        Some(0x7ff1edab8182025f15ae33482edb590a_u128),
-        Some(0xb1719e044db05401a05a2ed588a3ad3f),
-        Some(0xb68d6691c895ad0a70809470020929ef),
-    ];
-
-    // compared to other scenarios, this one uses bytes instead of kB
-
-    let mut storage = Storage::new(None);
-
-    storage.branch(&None, branches[0]); // at 0
-    storage.modify_branch(&branches[0], NO_OP, 108951064, 43696128); // at 108951064
-    storage.branch(&branches[0], branches[1]); // at 108951064
-    storage.modify_branch(&branches[1], NO_OP, 15560408, -1851392); // at 124511472
-    storage.modify_branch(&branches[0], NO_OP, 174464360, -1531904); // at 283415424
-    storage.branch(&branches[0], branches[2]); // at 283415424
-    storage.modify_branch(&branches[2], NO_OP, 15906192, 8192); // at 299321616
-    storage.modify_branch(&branches[0], NO_OP, 18909976, 32768); // at 302325400
-
-    let size = storage.calculate(100_000);
-
-    (storage.into_segments(), size)
-}
-
-fn main() {
-    let args: Vec<String> = std::env::args().collect();
-
-    let scenario = if args.len() < 2 { "1" } else { &args[1] };
-
-    let (segments, size) = match scenario {
-        "1" => scenario_1(),
-        "2" => scenario_2(),
-        "3" => scenario_3(),
-        "4" => scenario_4(),
-        "5" => scenario_5(),
-        "6" => scenario_6(),
-        other => {
-            eprintln!("invalid scenario {}", other);
-            std::process::exit(1);
-        }
-    };
-
-    graphviz_tree(&segments, &size);
-}
-
-fn graphviz_recurse(segments: &[Segment], node: &SegmentSize) {
-    use tenant_size_model::SegmentMethod::*;
-
-    let seg_id = node.seg_id;
-    let seg = segments.get(seg_id).unwrap();
-    let lsn = seg.end_lsn;
-    let size = seg.end_size;
-    let method = node.method;
-
-    println!("  {{");
-    println!("    node [width=0.1 height=0.1 shape=oval]");
-
-    let tenant_size = node.total_children();
-
-    let penwidth = if seg.needed { 6 } else { 3 };
-    let x = match method {
-        SnapshotAfter =>
-            format!("label=\"lsn: {lsn}\\nsize: {size}\\ntenant_size: {tenant_size}\" style=filled penwidth={penwidth}"),
-        Wal =>
-            format!("label=\"lsn: {lsn}\\nsize: {size}\\ntenant_size: {tenant_size}\" color=\"black\" penwidth={penwidth}"),
-        WalNeeded =>
-            format!("label=\"lsn: {lsn}\\nsize: {size}\\ntenant_size: {tenant_size}\" color=\"black\" penwidth={penwidth}"),
-        Skipped =>
-            format!("label=\"lsn: {lsn}\\nsize: {size}\\ntenant_size: {tenant_size}\" color=\"gray\" penwidth={penwidth}"),
-    };
-
-    println!("    \"seg{seg_id}\" [{x}]");
-    println!("  }}");
-
-    // Recurse. Much of the data is actually on the edge
-    for child in node.children.iter() {
-        let child_id = child.seg_id;
-        graphviz_recurse(segments, child);
-
-        let edge_color = match child.method {
-            SnapshotAfter => "gray",
-            Wal => "black",
-            WalNeeded => "black",
-            Skipped => "gray",
-        };
-
-        println!("  {{");
-        println!("    edge [] ");
-        print!("    \"seg{seg_id}\" -> \"seg{child_id}\" [");
-        print!("color={edge_color}");
-        if child.method == WalNeeded {
-            print!(" penwidth=6");
-        }
-        if child.method == Wal {
-            print!(" penwidth=3");
-        }
-
-        let next = segments.get(child_id).unwrap();
-
-        if next.op.is_empty() {
-            print!(
-                " label=\"{} / {}\"",
-                next.end_lsn - seg.end_lsn,
-                (next.end_size as i128 - seg.end_size as i128)
-            );
-        } else {
-            print!(" label=\"{}: {}\"", next.op, next.end_lsn - seg.end_lsn);
-        }
-        println!("]");
-        println!("  }}");
-    }
-}
-
-fn graphviz_tree(segments: &[Segment], tree: &SegmentSize) {
-    println!("digraph G {{");
-    println!("  fontname=\"Helvetica,Arial,sans-serif\"");
-    println!("  node [fontname=\"Helvetica,Arial,sans-serif\"]");
-    println!("  edge [fontname=\"Helvetica,Arial,sans-serif\"]");
-    println!("  graph [center=1 rankdir=LR]");
-    println!("  edge [dir=none]");
-
-    graphviz_recurse(segments, tree);
-
-    println!("}}");
-}
-
-#[test]
-fn scenarios_return_same_size() {
-    type ScenarioFn = fn() -> (Vec<Segment>, SegmentSize);
-    let truths: &[(u32, ScenarioFn, _)] = &[
-        (line!(), scenario_1, 8000),
-        (line!(), scenario_2, 9000),
-        (line!(), scenario_3, 13000),
-        (line!(), scenario_4, 16000),
-        (line!(), scenario_5, 17000),
-        (line!(), scenario_6, 333_792_000),
-    ];
-
-    for (line, scenario, expected) in truths {
-        let (_, size) = scenario();
-        assert_eq!(*expected, size.total_children(), "scenario on line {line}");
-    }
-}

libs/utils/Cargo.toml

@@ -9,6 +9,9 @@ anyhow = "1.0"
 bincode = "1.3"
 bytes = "1.0.1"
 hyper = { version = "0.14.7", features = ["full"] }
+pin-project-lite = "0.2.7"
+postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
 routerify = "3"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
@@ -16,8 +19,8 @@ thiserror = "1.0"
 tokio = { version = "1.17", features = ["macros"]}
 tokio-rustls = "0.23"
 tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
-nix = "0.25"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+nix = "0.23.0"
 signal-hook = "0.3.10"
 rand = "0.8.3"
 jsonwebtoken = "8"
@@ -25,13 +28,11 @@ hex = { version = "0.4.3", features = ["serde"] }
 rustls = "0.20.2"
 rustls-split = "0.3.0"
 git-version = "0.3.5"
-serde_with = "2.0"
+serde_with = "1.12.0"
 once_cell = "1.13.0"
-strum = "0.24"
-strum_macros = "0.24"
+
 
 metrics = { path = "../metrics" }
-pq_proto = { path = "../pq_proto" }
 workspace_hack = { version = "0.1", path = "../../workspace_hack" }
 
 [dev-dependencies]
@@ -39,7 +40,7 @@ byteorder = "1.4.3"
 bytes = "1.0.1"
 hex-literal = "0.3"
 tempfile = "3.2"
-criterion = "0.4"
+criterion = "0.3"
 rustls-pemfile = "1"
 
 [[bench]]

libs/utils/src/connstring.rs

@@ -0,0 +1,52 @@
+use postgres::Config;
+
+pub fn connection_host_port(config: &Config) -> (String, u16) {
+    assert_eq!(
+        config.get_hosts().len(),
+        1,
+        "only one pair of host and port is supported in connection string"
+    );
+    assert_eq!(
+        config.get_ports().len(),
+        1,
+        "only one pair of host and port is supported in connection string"
+    );
+    let host = match &config.get_hosts()[0] {
+        postgres::config::Host::Tcp(host) => host.as_ref(),
+        postgres::config::Host::Unix(host) => host.to_str().unwrap(),
+    };
+    (host.to_owned(), config.get_ports()[0])
+}
+
+pub fn connection_address(config: &Config) -> String {
+    let (host, port) = connection_host_port(config);
+    format!("{}:{}", host, port)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_connection_host_port() {
+        let config: Config = "postgresql://no_user@localhost:64000/no_db"
+            .parse()
+            .unwrap();
+        assert_eq!(
+            connection_host_port(&config),
+            ("localhost".to_owned(), 64000)
+        );
+    }
+
+    #[test]
+    #[should_panic(expected = "only one pair of host and port is supported in connection string")]
+    fn test_connection_host_port_multiple_ports() {
+        let config: Config = "postgresql://no_user@localhost:64000,localhost:64001/no_db"
+            .parse()
+            .unwrap();
+        assert_eq!(
+            connection_host_port(&config),
+            ("localhost".to_owned(), 64000)
+        );
+    }
+}

libs/utils/src/crashsafe_dir.rs

@@ -12,8 +12,16 @@ pub fn create_dir(path: impl AsRef<Path>) -> io::Result<()> {
     let path = path.as_ref();
 
     fs::create_dir(path)?;
-    fsync_file_and_parent(path)?;
-    Ok(())
+    File::open(path)?.sync_all()?;
+
+    if let Some(parent) = path.parent() {
+        File::open(parent)?.sync_all()
+    } else {
+        Err(io::Error::new(
+            io::ErrorKind::InvalidInput,
+            "can't find parent",
+        ))
+    }
 }
 
 /// Similar to [`std::fs::create_dir_all`], except we fsync all
@@ -57,12 +65,12 @@ pub fn create_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
 
     // Fsync the created directories from child to parent.
     for &path in dirs_to_create.iter() {
-        fsync(path)?;
+        File::open(path)?.sync_all()?;
     }
 
     // If we created any new directories, fsync the parent.
     if !dirs_to_create.is_empty() {
-        fsync(path)?;
+        File::open(path)?.sync_all()?;
     }
 
     Ok(())
@@ -84,33 +92,6 @@ pub fn path_with_suffix_extension(original_path: impl AsRef<Path>, suffix: &str)
         .with_extension(new_extension.as_ref())
 }
 
-pub fn fsync_file_and_parent(file_path: &Path) -> io::Result<()> {
-    let parent = file_path.parent().ok_or_else(|| {
-        io::Error::new(
-            io::ErrorKind::Other,
-            format!("File {file_path:?} has no parent"),
-        )
-    })?;
-
-    fsync(file_path)?;
-    fsync(parent)?;
-    Ok(())
-}
-
-pub fn fsync(path: &Path) -> io::Result<()> {
-    File::open(path)
-        .map_err(|e| io::Error::new(e.kind(), format!("Failed to open the file {path:?}: {e}")))
-        .and_then(|file| {
-            file.sync_all().map_err(|e| {
-                io::Error::new(
-                    e.kind(),
-                    format!("Failed to sync file {path:?} data and metadata: {e}"),
-                )
-            })
-        })
-        .map_err(|e| io::Error::new(e.kind(), format!("Failed to fsync file {path:?}: {e}")))
-}
-
 #[cfg(test)]
 mod tests {
     use tempfile::tempdir;

libs/utils/src/http/endpoint.rs

@@ -9,7 +9,6 @@ use once_cell::sync::Lazy;
 use routerify::ext::RequestExt;
 use routerify::RequestInfo;
 use routerify::{Middleware, Router, RouterBuilder, RouterService};
-use tokio::task::JoinError;
 use tracing::info;
 
 use std::future::Future;
@@ -36,13 +35,7 @@ async fn prometheus_metrics_handler(_req: Request<Body>) -> Result<Response<Body
     let mut buffer = vec![];
     let encoder = TextEncoder::new();
 
-    let metrics = tokio::task::spawn_blocking(move || {
-        // Currently we take a lot of mutexes while collecting metrics, so it's
-        // better to spawn a blocking task to avoid blocking the event loop.
-        metrics::gather()
-    })
-    .await
-    .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))?;
+    let metrics = metrics::gather();
     encoder.encode(&metrics, &mut buffer).unwrap();
 
     let response = Response::builder()

libs/utils/src/id.rs

@@ -75,12 +75,6 @@ impl From<[u8; 16]> for Id {
     }
 }
 
-impl From<Id> for u128 {
-    fn from(id: Id) -> Self {
-        u128::from_le_bytes(id.0)
-    }
-}
-
 impl fmt::Display for Id {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_str(&self.hex_encode())
@@ -142,12 +136,6 @@ macro_rules! id_newtype {
             }
         }
 
-        impl From<$t> for u128 {
-            fn from(id: $t) -> Self {
-                u128::from(id.0)
-            }
-        }
-
         impl fmt::Display for $t {
             fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
                 self.0.fmt(f)
@@ -204,17 +192,6 @@ pub struct TenantId(Id);
 
 id_newtype!(TenantId);
 
-/// Neon Connection Id identifies long-lived connections (for example a pagestream
-/// connection with the page_service). Is used for better logging and tracing
-///
-/// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look
-/// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`.
-/// See [`Id`] for alternative ways to serialize it.
-#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)]
-pub struct ConnectionId(Id);
-
-id_newtype!(ConnectionId);
-
 // A pair uniquely identifying Neon instance.
 #[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct TenantTimelineId {

libs/utils/src/lib.rs

@@ -1,6 +1,8 @@
 //! `utils` is intended to be a place to put code that is shared
 //! between other crates in this repository.
 
+#![allow(clippy::manual_range_contains)]
+
 /// `Lsn` type implements common tasks on Log Sequence Numbers
 pub mod lsn;
 /// SeqWait allows waiting for a future sequence number to arrive
@@ -15,9 +17,13 @@ pub mod vec_map;
 pub mod bin_ser;
 pub mod postgres_backend;
 pub mod postgres_backend_async;
+pub mod pq_proto;
 
-// helper functions for creating and fsyncing
-pub mod crashsafe;
+// dealing with connstring parsing and handy access to it's parts
+pub mod connstring;
+
+// helper functions for creating and fsyncing directories/trees
+pub mod crashsafe_dir;
 
 // common authentication routines
 pub mod auth;
@@ -33,12 +39,13 @@ pub mod sock_split;
 // common log initialisation routine
 pub mod logging;
 
-pub mod lock_file;
-
 // Misc
 pub mod accum;
 pub mod shutdown;
 
+// Tools for calling certain async methods in sync contexts
+pub mod sync;
+
 // Utility for binding TcpListeners with proper socket options.
 pub mod tcp_listener;