Add query cancel tests

.circleci/ansible/ansible.cfg

@@ -1,10 +0,0 @@
-[defaults]
-
-localhost_warning = False
-host_key_checking = False
-timeout = 30
-
-[ssh_connection]
-ssh_args   = -F ./ansible.ssh.cfg
-scp_if_ssh = True
-pipelining = True

.circleci/ansible/ansible.ssh.cfg

@@ -1,11 +0,0 @@
-Host tele.zenith.tech
-    User admin
-    Port 3023
-    StrictHostKeyChecking no
-    UserKnownHostsFile /dev/null
-
-Host * !tele.zenith.tech
-    User admin
-    StrictHostKeyChecking no
-    UserKnownHostsFile /dev/null
-    ProxyJump tele.zenith.tech

.circleci/ansible/deploy.yaml

@@ -1,174 +0,0 @@
-- name: Upload Zenith binaries
-  hosts: pageservers:safekeepers
-  gather_facts: False
-  remote_user: admin
-  vars:
-    force_deploy: false
-
-  tasks:
-
-    - name: get latest version of Zenith binaries
-      ignore_errors: true
-      register: current_version_file
-      set_fact:
-        current_version: "{{ lookup('file', '.zenith_current_version') | trim }}"
-      tags:
-      - pageserver
-      - safekeeper
-
-    - name: set zero value for current_version
-      when: current_version_file is failed
-      set_fact:
-        current_version: "0"
-      tags:
-      - pageserver
-      - safekeeper
-
-    - name: get deployed version from content of remote file
-      ignore_errors: true
-      ansible.builtin.slurp:
-        src: /usr/local/.zenith_current_version
-      register: remote_version_file
-      tags:
-      - pageserver
-      - safekeeper
-
-    - name: decode remote file content
-      when: remote_version_file is succeeded
-      set_fact:
-        remote_version: "{{ remote_version_file['content'] | b64decode | trim }}"
-      tags:
-      - pageserver
-      - safekeeper
-
-    - name: set zero value for remote_version
-      when: remote_version_file is failed
-      set_fact:
-        remote_version: "0"
-      tags:
-      - pageserver
-      - safekeeper
-
-    - name: inform about versions
-      debug: msg="Version to deploy - {{ current_version }}, version on storage node - {{ remote_version }}"
-      tags:
-      - pageserver
-      - safekeeper
-
-
-    - name: upload and extract Zenith binaries to /usr/local
-      when: current_version > remote_version or force_deploy
-      ansible.builtin.unarchive:
-        owner: root
-        group: root
-        src: zenith_install.tar.gz
-        dest: /usr/local
-      become: true
-      tags:
-      - pageserver
-      - safekeeper
-      - binaries
-      - putbinaries
-
-- name: Deploy pageserver
-  hosts: pageservers
-  gather_facts: False
-  remote_user: admin
-  vars:
-    force_deploy: false
-
-  tasks:
-    - name: init pageserver
-      when: current_version > remote_version or force_deploy
-      shell:
-        cmd: sudo -u pageserver /usr/local/bin/pageserver -c "pg_distrib_dir='/usr/local'" --init -D /storage/pageserver/data
-      args:
-        creates: "/storage/pageserver/data/tenants"
-      environment:
-        ZENITH_REPO_DIR: "/storage/pageserver/data"
-        LD_LIBRARY_PATH: "/usr/local/lib"
-      become: true
-      tags:
-      - pageserver
-
-    - name: upload systemd service definition
-      when: current_version > remote_version or force_deploy
-      ansible.builtin.template:
-        src: systemd/pageserver.service
-        dest: /etc/systemd/system/pageserver.service
-        owner: root
-        group: root
-        mode: '0644'
-      become: true
-      tags:
-      - pageserver
-
-    - name: start systemd service
-      when: current_version > remote_version or force_deploy
-      ansible.builtin.systemd:
-        daemon_reload: yes
-        name: pageserver
-        enabled: yes
-        state: restarted
-      become: true
-      tags:
-      - pageserver
-
-    - name: post version to console
-      when: (current_version > remote_version or force_deploy) and console_mgmt_base_url is defined
-      shell:
-        cmd: |
-          INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
-          curl -sfS -d '{"version": {{ current_version }} }' -X POST {{ console_mgmt_base_url }}/api/v1/pageservers/$INSTANCE_ID
-      tags:
-      - pageserver
-
-- name: Deploy safekeeper
-  hosts: safekeepers
-  gather_facts: False
-  remote_user: admin
-  vars:
-    force_deploy: false
-
-  tasks:
-
-    # in the future safekeepers should discover pageservers byself
-    # but currently use first pageserver that was discovered
-    - name: set first pageserver var for safekeepers
-      when: current_version > remote_version or force_deploy
-      set_fact:
-        first_pageserver: "{{ hostvars[groups['pageservers'][0]]['inventory_hostname'] }}"
-      tags:
-      - safekeeper
-
-    - name: upload systemd service definition
-      when: current_version > remote_version or force_deploy
-      ansible.builtin.template:
-        src: systemd/safekeeper.service
-        dest: /etc/systemd/system/safekeeper.service
-        owner: root
-        group: root
-        mode: '0644'
-      become: true
-      tags:
-      - safekeeper
-
-    - name: start systemd service
-      when: current_version > remote_version or force_deploy
-      ansible.builtin.systemd:
-        daemon_reload: yes
-        name: safekeeper
-        enabled: yes
-        state: restarted
-      become: true
-      tags:
-      - safekeeper
-
-    - name: post version to console
-      when: (current_version > remote_version or force_deploy) and console_mgmt_base_url is defined
-      shell:
-        cmd: |
-          INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
-          curl -sfS -d '{"version": {{ current_version }} }' -X POST {{ console_mgmt_base_url }}/api/v1/safekeepers/$INSTANCE_ID
-      tags:
-      - safekeeper

.circleci/ansible/get_binaries.sh

@@ -1,52 +0,0 @@
-#!/bin/bash
-
-set -e
-
-RELEASE=${RELEASE:-false}
-
-# look at docker hub for latest tag fo zenith docker image
-if [ "${RELEASE}" = "true" ]; then
-    echo "search latest relase tag"
-    VERSION=$(curl -s https://registry.hub.docker.com/v1/repositories/zenithdb/zenith/tags |jq -r -S '.[].name' | grep release | sed 's/release-//g' | tail -1)
-    if [ -z "${VERSION}" ]; then
-        echo "no any docker tags found, exiting..."
-        exit 1
-    else
-        TAG="release-${VERSION}"
-    fi
-else
-    echo "search latest dev tag"
-    VERSION=$(curl -s https://registry.hub.docker.com/v1/repositories/zenithdb/zenith/tags |jq -r -S '.[].name' | grep -v release | tail -1)
-    if [ -z "${VERSION}" ]; then
-        echo "no any docker tags found, exiting..."
-        exit 1
-    else
-        TAG="${VERSION}"
-    fi
-fi
-
-echo "found ${VERSION}"
-
-# do initial cleanup
-rm -rf zenith_install postgres_install.tar.gz zenith_install.tar.gz .zenith_current_version
-mkdir zenith_install
-
-# retrive binaries from docker image
-echo "getting binaries from docker image"
-docker pull --quiet zenithdb/zenith:${TAG}
-ID=$(docker create zenithdb/zenith:${TAG})
-docker cp ${ID}:/data/postgres_install.tar.gz .
-tar -xzf postgres_install.tar.gz -C zenith_install
-docker cp ${ID}:/usr/local/bin/pageserver zenith_install/bin/
-docker cp ${ID}:/usr/local/bin/safekeeper zenith_install/bin/
-docker cp ${ID}:/usr/local/bin/proxy zenith_install/bin/
-docker cp ${ID}:/usr/local/bin/postgres zenith_install/bin/
-docker rm -vf ${ID}
-
-# store version to file (for ansible playbooks) and create binaries tarball
-echo ${VERSION} > zenith_install/.zenith_current_version
-echo ${VERSION} > .zenith_current_version
-tar -czf zenith_install.tar.gz -C zenith_install .
-
-# do final cleaup
-rm -rf zenith_install postgres_install.tar.gz

.circleci/ansible/production.hosts

@@ -1,7 +0,0 @@
-[pageservers]
-zenith-1-ps-1
-
-[safekeepers]
-zenith-1-sk-1
-zenith-1-sk-2
-zenith-1-sk-3

.circleci/ansible/staging.hosts

@@ -1,7 +0,0 @@
-[pageservers]
-zenith-us-stage-ps-1
-
-[safekeepers]
-zenith-us-stage-sk-1
-zenith-us-stage-sk-2
-zenith-us-stage-sk-3

.circleci/ansible/systemd/pageserver.service

@@ -1,18 +0,0 @@
-[Unit]
-Description=Zenith pageserver
-After=network.target auditd.service
-
-[Service]
-Type=simple
-User=pageserver
-Environment=RUST_BACKTRACE=1 ZENITH_REPO_DIR=/storage/pageserver LD_LIBRARY_PATH=/usr/local/lib
-ExecStart=/usr/local/bin/pageserver -c "pg_distrib_dir='/usr/local'" -c "listen_pg_addr='0.0.0.0:6400'" -c "listen_http_addr='0.0.0.0:9898'" -D /storage/pageserver/data
-ExecReload=/bin/kill -HUP $MAINPID
-KillMode=mixed
-KillSignal=SIGINT
-Restart=on-failure
-TimeoutSec=10
-LimitNOFILE=30000000
-
-[Install]
-WantedBy=multi-user.target

.circleci/ansible/systemd/safekeeper.service

@@ -1,18 +0,0 @@
-[Unit]
-Description=Zenith safekeeper
-After=network.target auditd.service
-
-[Service]
-Type=simple
-User=safekeeper
-Environment=RUST_BACKTRACE=1 ZENITH_REPO_DIR=/storage/safekeeper/data LD_LIBRARY_PATH=/usr/local/lib
-ExecStart=/usr/local/bin/safekeeper -l {{ inventory_hostname }}.local:6500 --listen-http {{ inventory_hostname }}.local:7676 -p {{ first_pageserver }}:6400 -D /storage/safekeeper/data
-ExecReload=/bin/kill -HUP $MAINPID
-KillMode=mixed
-KillSignal=SIGINT
-Restart=on-failure
-TimeoutSec=10
-LimitNOFILE=30000000
-
-[Install]
-WantedBy=multi-user.target

.circleci/config.yml

@@ -440,14 +440,8 @@ jobs:
           command: |
             echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
             DOCKER_TAG=$(git log --oneline|wc -l)
-            docker build \
-              --pull \
-              --build-arg GIT_VERSION=${CIRCLE_SHA1} \
-              --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \
-              --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \
-              --tag zenithdb/zenith:${DOCKER_TAG} --tag zenithdb/zenith:latest .
-            docker push zenithdb/zenith:${DOCKER_TAG}
-            docker push zenithdb/zenith:latest
+            docker build --build-arg GIT_VERSION=$CIRCLE_SHA1 -t zenithdb/zenith:latest . && docker push zenithdb/zenith:latest
+            docker tag zenithdb/zenith:latest zenithdb/zenith:${DOCKER_TAG} && docker push zenithdb/zenith:${DOCKER_TAG}
 
   # Build zenithdb/compute-node:latest image and push it to Docker hub
   docker-image-compute:
@@ -474,63 +468,8 @@ jobs:
           command: |
             echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
             DOCKER_TAG=$(git log --oneline|wc -l)
-            docker build --tag zenithdb/compute-node:${DOCKER_TAG} --tag zenithdb/compute-node:latest vendor/postgres
-            docker push zenithdb/compute-node:${DOCKER_TAG}
-            docker push zenithdb/compute-node:latest
-
-  # Build production zenithdb/zenith:release image and push it to Docker hub
-  docker-image-release:
-    docker:
-      - image: cimg/base:2021.04
-    steps:
-      - checkout
-      - setup_remote_docker:
-          docker_layer_caching: true
-      - run:
-          name: Init postgres submodule
-          command: git submodule update --init --depth 1
-      - run:
-          name: Build and push Docker image
-          command: |
-            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
-            DOCKER_TAG="release-$(git log --oneline|wc -l)"
-            docker build \
-              --pull \
-              --build-arg GIT_VERSION=${CIRCLE_SHA1} \
-              --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \
-              --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \
-              --tag zenithdb/zenith:${DOCKER_TAG} --tag zenithdb/zenith:release .
-            docker push zenithdb/zenith:${DOCKER_TAG}
-            docker push zenithdb/zenith:release
-
-  # Build production zenithdb/compute-node:release image and push it to Docker hub
-  docker-image-compute-release:
-    docker:
-      - image: cimg/base:2021.04
-    steps:
-      - checkout
-      - setup_remote_docker:
-          docker_layer_caching: true
-      # Build zenithdb/compute-tools:release image and push it to Docker hub
-      # TODO: this should probably also use versioned tag, not just :latest.
-      # XXX: but should it? We build and use it only locally now.
-      - run:
-          name: Build and push compute-tools Docker image
-          command: |
-            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
-            docker build -t zenithdb/compute-tools:release -f Dockerfile.compute-tools .
-            docker push zenithdb/compute-tools:release
-      - run:
-          name: Init postgres submodule
-          command: git submodule update --init --depth 1
-      - run:
-          name: Build and push compute-node Docker image
-          command: |
-            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
-            DOCKER_TAG="release-$(git log --oneline|wc -l)"
-            docker build --tag zenithdb/compute-node:${DOCKER_TAG} --tag zenithdb/compute-node:release vendor/postgres
-            docker push zenithdb/compute-node:${DOCKER_TAG}
-            docker push zenithdb/compute-node:release
+            docker build -t zenithdb/compute-node:latest vendor/postgres && docker push zenithdb/compute-node:latest
+            docker tag zenithdb/compute-node:latest zenithdb/compute-node:${DOCKER_TAG} && docker push zenithdb/compute-node:${DOCKER_TAG}
 
   deploy-staging:
     docker:
@@ -538,25 +477,40 @@ jobs:
     steps:
       - checkout
       - setup_remote_docker
+      - run:
+          name: Get Zenith binaries
+          command: |
+            rm -rf zenith_install postgres_install.tar.gz zenith_install.tar.gz
+            mkdir zenith_install
+            DOCKER_TAG=$(git log --oneline|wc -l)
+            docker pull --quiet zenithdb/zenith:${DOCKER_TAG}
+            ID=$(docker create zenithdb/zenith:${DOCKER_TAG})
+            docker cp $ID:/data/postgres_install.tar.gz .
+            tar -xzf postgres_install.tar.gz -C zenith_install && rm postgres_install.tar.gz
+            docker cp $ID:/usr/local/bin/pageserver zenith_install/bin/
+            docker cp $ID:/usr/local/bin/safekeeper zenith_install/bin/
+            docker cp $ID:/usr/local/bin/proxy zenith_install/bin/
+            docker cp $ID:/usr/local/bin/postgres zenith_install/bin/
+            docker rm -v $ID
+            echo ${DOCKER_TAG} | tee zenith_install/.zenith_current_version
+            tar -czf zenith_install.tar.gz -C zenith_install .
+            ls -la zenith_install.tar.gz
       - run:
           name: Setup ansible
           command: |
             pip install --progress-bar off --user ansible boto3
+            ansible-galaxy collection install amazon.aws
       - run:
-          name: Redeploy
+          name: Apply re-deploy playbook
+          environment:
+            ANSIBLE_HOST_KEY_CHECKING: false
           command: |
-            cd "$(pwd)/.circleci/ansible"
-
-            ./get_binaries.sh
-
-            echo "${TELEPORT_SSH_KEY}"  | tr -d '\n'| base64 --decode >ssh-key
-            echo "${TELEPORT_SSH_CERT}" | tr -d '\n'| base64 --decode >ssh-key-cert.pub
-            chmod 0600 ssh-key
-            ssh-add ssh-key
-            rm -f ssh-key ssh-key-cert.pub
-
-            ansible-playbook deploy.yaml -i staging.hosts
-            rm -f zenith_install.tar.gz .zenith_current_version
+            echo "${STAGING_SSH_KEY}" | base64 --decode | ssh-add -
+            export AWS_REGION=${STAGING_AWS_REGION}
+            export AWS_ACCESS_KEY_ID=${STAGING_AWS_ACCESS_KEY_ID}
+            export AWS_SECRET_ACCESS_KEY=${STAGING_AWS_SECRET_ACCESS_KEY}
+            ansible-playbook .circleci/storage-redeploy.playbook.yml
+            rm -f zenith_install.tar.gz
 
   deploy-staging-proxy:
     docker:
@@ -579,57 +533,7 @@ jobs:
           name: Re-deploy proxy
           command: |
             DOCKER_TAG=$(git log --oneline|wc -l)
-            helm upgrade zenith-proxy zenithdb/zenith-proxy --install -f .circleci/helm-values/staging.proxy.yaml --set image.tag=${DOCKER_TAG} --wait
-
-
-  deploy-release:
-    docker:
-      - image: cimg/python:3.10
-    steps:
-      - checkout
-      - setup_remote_docker
-      - run:
-          name: Setup ansible
-          command: |
-            pip install --progress-bar off --user ansible boto3
-      - run:
-          name: Redeploy
-          command: |
-            cd "$(pwd)/.circleci/ansible"
-
-            RELEASE=true ./get_binaries.sh
-
-            echo "${TELEPORT_SSH_KEY}"  | tr -d '\n'| base64 --decode >ssh-key
-            echo "${TELEPORT_SSH_CERT}" | tr -d '\n'| base64 --decode >ssh-key-cert.pub
-            chmod 0600 ssh-key
-            ssh-add ssh-key
-            rm -f ssh-key ssh-key-cert.pub
-
-            ansible-playbook deploy.yaml -i production.hosts -e console_mgmt_base_url=http://console-release.local
-            rm -f zenith_install.tar.gz .zenith_current_version
-
-  deploy-release-proxy:
-    docker:
-      - image: cimg/base:2021.04
-    environment:
-      KUBECONFIG: .kubeconfig
-    steps:
-      - checkout
-      - run:
-          name: Store kubeconfig file
-          command: |
-            echo "${PRODUCTION_KUBECONFIG_DATA}" | base64 --decode > ${KUBECONFIG}
-            chmod 0600 ${KUBECONFIG}
-      - run:
-          name: Setup helm v3
-          command: |
-            curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
-            helm repo add zenithdb https://zenithdb.github.io/helm-charts
-      - run:
-          name: Re-deploy proxy
-          command: |
-            DOCKER_TAG="release-$(git log --oneline|wc -l)"
-            helm upgrade zenith-proxy zenithdb/zenith-proxy --install -f .circleci/helm-values/production.proxy.yaml --set image.tag=${DOCKER_TAG} --wait
+            helm upgrade zenith-proxy zenithdb/zenith-proxy --install -f .circleci/proxy.staging.yaml --set image.tag=${DOCKER_TAG} --wait
 
   # Trigger a new remote CI job
   remote-ci-trigger:
@@ -765,47 +669,6 @@ workflows:
                 - main
           requires:
             - docker-image
-
-      - docker-image-release:
-          # Context gives an ability to login
-          context: Docker Hub
-          # Build image only for commits to main
-          filters:
-            branches:
-              only:
-                - release
-          requires:
-            - pg_regress-tests-release
-            - other-tests-release
-      - docker-image-compute-release:
-          # Context gives an ability to login
-          context: Docker Hub
-          # Build image only for commits to main
-          filters:
-            branches:
-              only:
-                - release
-          requires:
-            - pg_regress-tests-release
-            - other-tests-release
-      - deploy-release:
-          # Context gives an ability to login
-          context: Docker Hub
-          # deploy only for commits to main
-          filters:
-            branches:
-              only:
-                - release
-          requires:
-            - docker-image-release
-      - deploy-release-proxy:
-          # deploy only for commits to main
-          filters:
-            branches:
-              only:
-                - release
-          requires:
-            - docker-image-release
       - remote-ci-trigger:
           # Context passes credentials for gh api
           context: CI_ACCESS_TOKEN

.circleci/helm-values/production.proxy.yaml

@@ -1,35 +0,0 @@
-# Helm chart values for zenith-proxy.
-# This is a YAML-formatted file.
-
-settings:
-  authEndpoint: "https://console.zenith.tech/authenticate_proxy_request/"
-  uri: "https://console.zenith.tech/psql_session/"
-
-# -- Additional labels for zenith-proxy pods
-podLabels:
-  zenith_service: proxy
-  zenith_env: production
-  zenith_region: us-west-2
-  zenith_region_slug: oregon
-
-service:
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internal
-    external-dns.alpha.kubernetes.io/hostname: proxy-release.local
-  type: LoadBalancer
-
-exposedService:
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: external
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
-    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
-    external-dns.alpha.kubernetes.io/hostname: start.zenith.tech
-
-metrics:
-  enabled: true
-  serviceMonitor:
-    enabled: true
-    selector:
-      release: kube-prometheus-stack

.circleci/storage-redeploy.playbook.yml

@@ -0,0 +1,138 @@
+- name: discover storage nodes
+  hosts: localhost
+  connection: local
+  gather_facts: False
+
+  tasks:
+
+    - name: discover safekeepers
+      no_log: true
+      ec2_instance_info:
+        filters:
+          "tag:zenith_env": "staging"
+          "tag:zenith_service": "safekeeper"
+      register: ec2_safekeepers
+
+    - name: discover pageservers
+      no_log: true
+      ec2_instance_info:
+        filters:
+          "tag:zenith_env": "staging"
+          "tag:zenith_service": "pageserver"
+      register: ec2_pageservers
+
+    - name: add safekeepers to host group
+      no_log: true
+      add_host:
+        name: safekeeper-{{ ansible_loop.index }}
+        ansible_host: "{{ item.public_ip_address }}"
+        groups:
+          - storage
+          - safekeepers
+      with_items: "{{ ec2_safekeepers.instances }}"
+      loop_control:
+        extended: yes
+
+    - name: add pageservers to host group
+      no_log: true
+      add_host:
+        name: pageserver-{{ ansible_loop.index }}
+        ansible_host: "{{ item.public_ip_address }}"
+        groups:
+          - storage
+          - pageservers
+      with_items: "{{ ec2_pageservers.instances }}"
+      loop_control:
+        extended: yes
+
+- name: Retrive versions
+  hosts: storage
+  gather_facts: False
+  remote_user: admin
+
+  tasks:
+
+    - name: Get current version of binaries
+      set_fact:
+        current_version: "{{lookup('file', '../zenith_install/.zenith_current_version') }}"
+
+    - name: Check that file with version exists on host
+      stat:
+        path: /usr/local/.zenith_current_version
+      register: version_file
+
+    - name: Try to get current version from the host
+      when: version_file.stat.exists
+      ansible.builtin.fetch:
+        src: /usr/local/.zenith_current_version
+        dest: .remote_version.{{ inventory_hostname }}
+        fail_on_missing: no
+        flat: yes
+
+    - name: Store remote version to variable
+      when: version_file.stat.exists
+      set_fact:
+        remote_version: "{{ lookup('file', '.remote_version.{{ inventory_hostname }}') }}"
+
+    - name: Store default value of remote version to variable in case when remote version file not found
+      when: not version_file.stat.exists
+      set_fact:
+        remote_version: "000"
+
+- name: Extract Zenith binaries
+  hosts: storage
+  gather_facts: False
+  remote_user: admin
+
+  tasks:
+
+    - name: Inform about version conflict
+      when: current_version <= remote_version
+      debug: msg="Current version {{ current_version }} LE than remote {{ remote_version }}"
+
+    - name: Extract Zenith binaries to /usr/local
+      when: current_version > remote_version
+      ansible.builtin.unarchive:
+        src: ../zenith_install.tar.gz
+        dest: /usr/local
+      become: true
+
+- name: Restart safekeepers
+  hosts: safekeepers
+  gather_facts: False
+  remote_user: admin
+
+  tasks:
+
+    - name: Inform about version conflict
+      when: current_version <= remote_version
+      debug: msg="Current version {{ current_version }} LE than remote {{ remote_version }}"
+
+    - name: Restart systemd service
+      when: current_version > remote_version
+      ansible.builtin.systemd:
+        daemon_reload: yes
+        name: safekeeper
+        enabled: yes
+        state: restarted
+      become: true
+
+- name: Restart pageservers
+  hosts: pageservers
+  gather_facts: False
+  remote_user: admin
+
+  tasks:
+
+    - name: Inform about version conflict
+      when: current_version <= remote_version
+      debug: msg="Current version {{ current_version }} LE than remote {{ remote_version }}"
+
+    - name: Restart systemd service
+      when: current_version > remote_version
+      ansible.builtin.systemd:
+        daemon_reload: yes
+        name: pageserver
+        enabled: yes
+        state: restarted
+      become: true

Cargo.lock

@@ -424,7 +424,6 @@ dependencies = [
  "thiserror",
  "toml",
  "url",
- "walkeeper",
  "workspace_hack",
  "zenith_utils",
 ]
@@ -1307,7 +1306,6 @@ dependencies = [
  "hex-literal",
  "humantime",
  "hyper",
- "itertools",
  "lazy_static",
  "log",
  "nix",

Dockerfile

@@ -1,62 +1,62 @@
-# Build Postgres
 #
-#FROM zimg/rust:1.56 AS pg-build
-FROM zenithdb/build:buster-20220309 AS pg-build
-WORKDIR /pg
-
-USER root
-
-COPY vendor/postgres vendor/postgres
-COPY Makefile Makefile
+# Docker image for console integration testing.
+#
 
+#
+# Build Postgres separately --- this layer will be rebuilt only if one of
+# mentioned paths will get any changes.
+#
+FROM zenithdb/build:buster AS pg-build
+WORKDIR /zenith
+COPY ./vendor/postgres vendor/postgres
+COPY ./Makefile Makefile
 ENV BUILD_TYPE release
-RUN set -e \
-    && make -j $(nproc) -s postgres \
-    && rm -rf tmp_install/build \
-    && tar -C tmp_install -czf /postgres_install.tar.gz .
+RUN make -j $(getconf _NPROCESSORS_ONLN) -s postgres
+RUN rm -rf postgres_install/build
 
+#
 # Build zenith binaries
 #
-#FROM zimg/rust:1.56 AS build
-FROM zenithdb/build:buster-20220309 AS build
-ARG GIT_VERSION=local
-
-ARG CACHEPOT_BUCKET=zenith-rust-cachepot
-ARG AWS_ACCESS_KEY_ID
-ARG AWS_SECRET_ACCESS_KEY
-#ENV RUSTC_WRAPPER cachepot
-ENV RUSTC_WRAPPER /usr/local/cargo/bin/cachepot
-
-COPY --from=pg-build /pg/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
-COPY . .
-
-RUN cargo build --release
-
-# Build final image
+# TODO: build cargo deps as separate layer. We used cargo-chef before but that was
+# net time waste in a lot of cases. Copying Cargo.lock with empty lib.rs should do the work.
 #
-FROM debian:bullseye-slim
+FROM zenithdb/build:buster AS build
+
+ARG GIT_VERSION
+RUN if [ -z "$GIT_VERSION" ]; then echo "GIT_VERSION is reqired, use build_arg to pass it"; exit 1; fi
+
+WORKDIR /zenith
+COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server
+
+COPY . .
+RUN GIT_VERSION=$GIT_VERSION cargo build --release
+
+#
+# Copy binaries to resulting image.
+#
+FROM debian:buster-slim
 WORKDIR /data
 
-RUN set -e \
-    && apt-get update \
-    && apt-get install -y \
-        libreadline-dev \
-        libseccomp-dev \
-        openssl \
-        ca-certificates \
-    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
-    && useradd -d /data zenith \
-    && chown -R zenith:zenith /data
-
-COPY --from=build --chown=zenith:zenith /home/circleci/project/target/release/pageserver /usr/local/bin
-COPY --from=build --chown=zenith:zenith /home/circleci/project/target/release/safekeeper /usr/local/bin
-COPY --from=build --chown=zenith:zenith /home/circleci/project/target/release/proxy      /usr/local/bin
-
-COPY --from=pg-build /pg/tmp_install/         /usr/local/
-COPY --from=pg-build /postgres_install.tar.gz /data/
+RUN apt-get update && apt-get -yq install libreadline-dev libseccomp-dev openssl ca-certificates && \
+    mkdir zenith_install
 
+COPY --from=build /zenith/target/release/pageserver /usr/local/bin
+COPY --from=build /zenith/target/release/safekeeper /usr/local/bin
+COPY --from=build /zenith/target/release/proxy /usr/local/bin
+COPY --from=pg-build /zenith/tmp_install postgres_install
 COPY docker-entrypoint.sh /docker-entrypoint.sh
 
+# Remove build artifacts (~ 500 MB)
+RUN rm -rf postgres_install/build && \
+    # 'Install' Postgres binaries locally
+    cp -r postgres_install/* /usr/local/ && \
+    # Prepare an archive of Postgres binaries (should be around 11 MB)
+    # and keep it inside container for an ease of deploy pipeline.
+    cd postgres_install && tar -czf /data/postgres_install.tar.gz . && cd .. && \
+    rm -rf postgres_install
+
+RUN useradd -d /data zenith && chown -R zenith:zenith /data
+
 VOLUME ["/data"]
 USER zenith
 EXPOSE 6400

Dockerfile.build

@@ -1,23 +1,16 @@
+#
+# Image with all the required dependencies to build https://github.com/zenithdb/zenith
+# and Postgres from https://github.com/zenithdb/postgres
+# Also includes some rust development and build tools.
+# NB: keep in sync with rust image version in .circle/config.yml
+#
 FROM rust:1.56.1-slim-buster
-WORKDIR /home/circleci/project
+WORKDIR /zenith
 
-RUN set -e \
-    && apt-get update \
-    && apt-get -yq install \
-        automake \
-        libtool \
-        build-essential \
-        bison \
-        flex \
-        libreadline-dev \
-        zlib1g-dev \
-        libxml2-dev \
-        libseccomp-dev \
-        pkg-config \
-        libssl-dev \
-        clang
+# Install postgres and zenith build dependencies
+# clang is for rocksdb
+RUN apt-get update && apt-get -yq install automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \
+                                          libseccomp-dev pkg-config libssl-dev clang
 
-RUN set -e \
-    && rustup component add clippy \
-    && cargo install cargo-audit \
-    && cargo install --git https://github.com/paritytech/cachepot
+# Install rust tools
+RUN rustup component add clippy && cargo install cargo-audit

compute_tools/src/pg_helpers.rs

@@ -171,7 +171,7 @@ impl PgQuote for PgIdent {
     /// always quotes provided string with `""` and escapes every `"`. Not idempotent,
     /// i.e. if string is already escaped it will be escaped again.
     fn quote(&self) -> String {
-        let result = format!("\"{}\"", self.replace('"', "\"\""));
+        let result = format!("\"{}\"", self.replace("\"", "\"\""));
         result
     }
 }

compute_tools/src/spec.rs

@@ -215,7 +215,7 @@ pub fn handle_databases(spec: &ClusterSpec, client: &mut Client) -> Result<()> {
         if let Some(r) = pg_db {
             // XXX: db owner name is returned as quoted string from Postgres,
             // when quoting is needed.
-            let new_owner = if r.owner.starts_with('"') {
+            let new_owner = if r.owner.starts_with('\"') {
                 db.owner.quote()
             } else {
                 db.owner.clone()

control_plane/Cargo.toml

@@ -17,6 +17,5 @@ url = "2.2.2"
 reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "rustls-tls"] }
 
 pageserver = { path = "../pageserver" }
-walkeeper = { path = "../walkeeper" }
 zenith_utils = { path = "../zenith_utils" }
 workspace_hack = { path = "../workspace_hack" }

control_plane/safekeepers.conf

@@ -5,16 +5,16 @@ listen_http_addr = '127.0.0.1:9898'
 auth_type = 'Trust'
 
 [[safekeepers]]
-id = 1
+name = 'sk1'
 pg_port = 5454
 http_port = 7676
 
 [[safekeepers]]
-id = 2
+name = 'sk2'
 pg_port = 5455
 http_port = 7677
 
 [[safekeepers]]
-id = 3
+name = 'sk3'
 pg_port = 5456
 http_port = 7678

control_plane/simple.conf

@@ -6,6 +6,6 @@ listen_http_addr = '127.0.0.1:9898'
 auth_type = 'Trust'
 
 [[safekeepers]]
-id = 1
+name = 'single'
 pg_port = 5454
 http_port = 7676

control_plane/src/local_env.rs

@@ -12,9 +12,7 @@ use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
 use zenith_utils::auth::{encode_from_key_file, Claims, Scope};
 use zenith_utils::postgres_backend::AuthType;
-use zenith_utils::zid::{HexZTenantId, ZNodeId, ZTenantId};
-
-use crate::safekeeper::SafekeeperNode;
+use zenith_utils::zid::{opt_display_serde, ZTenantId};
 
 //
 // This data structures represents zenith CLI config
@@ -47,8 +45,9 @@ pub struct LocalEnv {
 
     // Default tenant ID to use with the 'zenith' command line utility, when
     // --tenantid is not explicitly specified.
+    #[serde(with = "opt_display_serde")]
     #[serde(default)]
-    pub default_tenantid: Option<HexZTenantId>,
+    pub default_tenantid: Option<ZTenantId>,
 
     // used to issue tokens during e.g pg start
     #[serde(default)]
@@ -63,8 +62,6 @@ pub struct LocalEnv {
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(default)]
 pub struct PageServerConf {
-    // node id
-    pub id: ZNodeId,
     // Pageserver connection settings
     pub listen_pg_addr: String,
     pub listen_http_addr: String,
@@ -79,7 +76,6 @@ pub struct PageServerConf {
 impl Default for PageServerConf {
     fn default() -> Self {
         Self {
-            id: ZNodeId(0),
             listen_pg_addr: String::new(),
             listen_http_addr: String::new(),
             auth_type: AuthType::Trust,
@@ -91,7 +87,7 @@ impl Default for PageServerConf {
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(default)]
 pub struct SafekeeperConf {
-    pub id: ZNodeId,
+    pub name: String,
     pub pg_port: u16,
     pub http_port: u16,
     pub sync: bool,
@@ -100,7 +96,7 @@ pub struct SafekeeperConf {
 impl Default for SafekeeperConf {
     fn default() -> Self {
         Self {
-            id: ZNodeId(0),
+            name: String::new(),
             pg_port: 0,
             http_port: 0,
             sync: true,
@@ -140,8 +136,8 @@ impl LocalEnv {
         self.base_data_dir.clone()
     }
 
-    pub fn safekeeper_data_dir(&self, data_dir_name: &str) -> PathBuf {
-        self.base_data_dir.join("safekeepers").join(data_dir_name)
+    pub fn safekeeper_data_dir(&self, node_name: &str) -> PathBuf {
+        self.base_data_dir.join("safekeepers").join(node_name)
     }
 
     /// Create a LocalEnv from a config file.
@@ -184,7 +180,7 @@ impl LocalEnv {
 
         // If no initial tenant ID was given, generate it.
         if env.default_tenantid.is_none() {
-            env.default_tenantid = Some(HexZTenantId::from(ZTenantId::generate()));
+            env.default_tenantid = Some(ZTenantId::generate());
         }
 
         env.base_data_dir = base_path();
@@ -289,7 +285,7 @@ impl LocalEnv {
         fs::create_dir_all(self.pg_data_dirs_path())?;
 
         for safekeeper in &self.safekeepers {
-            fs::create_dir_all(SafekeeperNode::datadir_path_by_id(self, safekeeper.id))?;
+            fs::create_dir_all(self.safekeeper_data_dir(&safekeeper.name))?;
         }
 
         let mut conf_content = String::new();

control_plane/src/safekeeper.rs

@@ -14,9 +14,7 @@ use postgres::Config;
 use reqwest::blocking::{Client, RequestBuilder, Response};
 use reqwest::{IntoUrl, Method};
 use thiserror::Error;
-use walkeeper::http::models::TimelineCreateRequest;
 use zenith_utils::http::error::HttpErrorBody;
-use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId};
 
 use crate::local_env::{LocalEnv, SafekeeperConf};
 use crate::storage::PageServerNode;
@@ -63,7 +61,7 @@ impl ResponseErrorMessageExt for Response {
 //
 #[derive(Debug)]
 pub struct SafekeeperNode {
-    pub id: ZNodeId,
+    pub name: String,
 
     pub conf: SafekeeperConf,
 
@@ -79,10 +77,10 @@ impl SafekeeperNode {
     pub fn from_env(env: &LocalEnv, conf: &SafekeeperConf) -> SafekeeperNode {
         let pageserver = Arc::new(PageServerNode::from_env(env));
 
-        println!("initializing for sk {} for {}", conf.id, conf.http_port);
+        println!("initializing for {} for {}", conf.name, conf.http_port);
 
         SafekeeperNode {
-            id: conf.id,
+            name: conf.name.clone(),
             conf: conf.clone(),
             pg_connection_config: Self::safekeeper_connection_config(conf.pg_port),
             env: env.clone(),
@@ -100,12 +98,8 @@ impl SafekeeperNode {
             .unwrap()
     }
 
-    pub fn datadir_path_by_id(env: &LocalEnv, sk_id: ZNodeId) -> PathBuf {
-        env.safekeeper_data_dir(format!("sk{}", sk_id).as_ref())
-    }
-
     pub fn datadir_path(&self) -> PathBuf {
-        SafekeeperNode::datadir_path_by_id(&self.env, self.id)
+        self.env.safekeeper_data_dir(&self.name)
     }
 
     pub fn pid_file(&self) -> PathBuf {
@@ -126,7 +120,6 @@ impl SafekeeperNode {
         let mut cmd = Command::new(self.env.safekeeper_bin()?);
         fill_rust_env_vars(
             cmd.args(&["-D", self.datadir_path().to_str().unwrap()])
-                .args(&["--id", self.id.to_string().as_ref()])
                 .args(&["--listen-pg", &listen_pg])
                 .args(&["--listen-http", &listen_http])
                 .args(&["--recall", "1 second"])
@@ -190,7 +183,7 @@ impl SafekeeperNode {
     pub fn stop(&self, immediate: bool) -> anyhow::Result<()> {
         let pid_file = self.pid_file();
         if !pid_file.exists() {
-            println!("Safekeeper {} is already stopped", self.id);
+            println!("Safekeeper {} is already stopped", self.name);
             return Ok(());
         }
         let pid = read_pidfile(&pid_file)?;
@@ -262,25 +255,4 @@ impl SafekeeperNode {
             .error_from_body()?;
         Ok(())
     }
-
-    pub fn timeline_create(
-        &self,
-        tenant_id: ZTenantId,
-        timeline_id: ZTimelineId,
-        peer_ids: Vec<ZNodeId>,
-    ) -> Result<()> {
-        Ok(self
-            .http_request(
-                Method::POST,
-                format!("{}/{}", self.http_base_url, "timeline"),
-            )
-            .json(&TimelineCreateRequest {
-                tenant_id,
-                timeline_id,
-                peer_ids,
-            })
-            .send()?
-            .error_from_body()?
-            .json()?)
-    }
 }

control_plane/src/storage.rs

@@ -103,8 +103,6 @@ impl PageServerNode {
     ) -> anyhow::Result<()> {
         let mut cmd = Command::new(self.env.pageserver_bin()?);
 
-        let id = format!("id={}", self.env.pageserver.id);
-
         // FIXME: the paths should be shell-escaped to handle paths with spaces, quotas etc.
         let base_data_dir_param = self.env.base_data_dir.display().to_string();
         let pg_distrib_dir_param =
@@ -124,7 +122,6 @@ impl PageServerNode {
         args.extend(["-c", &authg_type_param]);
         args.extend(["-c", &listen_http_addr_param]);
         args.extend(["-c", &listen_pg_addr_param]);
-        args.extend(["-c", &id]);
 
         for config_override in config_overrides {
             args.extend(["-c", config_override]);

docker-entrypoint.sh

@@ -4,7 +4,7 @@ set -eux
 if [ "$1" = 'pageserver' ]; then
     if [ ! -d "/data/tenants" ]; then
         echo "Initializing pageserver data directory"
-        pageserver --init -D /data -c "pg_distrib_dir='/usr/local'" -c "id=10"
+        pageserver --init -D /data -c "pg_distrib_dir='/usr/local'"
     fi
     echo "Staring pageserver at 0.0.0.0:6400"
     pageserver -c "listen_pg_addr='0.0.0.0:6400'" -c "listen_http_addr='0.0.0.0:9898'" -D /data

docs/docker.md

@@ -7,14 +7,32 @@ Currently we build two main images:
 - [zenithdb/zenith](https://hub.docker.com/repository/docker/zenithdb/zenith) — image with pre-built `pageserver`, `safekeeper` and `proxy` binaries and all the required runtime dependencies. Built from [/Dockerfile](/Dockerfile).
 - [zenithdb/compute-node](https://hub.docker.com/repository/docker/zenithdb/compute-node) — compute node image with pre-built Postgres binaries from [zenithdb/postgres](https://github.com/zenithdb/postgres).
 
-And additional intermediate images:
+And two intermediate images used either to reduce build time or to deliver some additional binary tools from other repos:
 
+- [zenithdb/build](https://hub.docker.com/repository/docker/zenithdb/build) — image with all the dependencies required to build Zenith and compute node images. This image is based on `rust:slim-buster`, so it also has a proper `rust` environment. Built from [/Dockerfile.build](/Dockerfile.build).
 - [zenithdb/compute-tools](https://hub.docker.com/repository/docker/zenithdb/compute-tools) — compute node configuration management tools.
 
 ## Building pipeline
 
 1. Image `zenithdb/compute-tools` is re-built automatically.
 
-2. Image `zenithdb/compute-node` is built independently in the [zenithdb/postgres](https://github.com/zenithdb/postgres) repo.
+2. Image `zenithdb/build` is built manually. If you want to introduce any new compile time dependencies to Zenith or compute node you have to update this image as well, build it and push to Docker Hub.
 
-3. Image `zenithdb/zenith` is built in this repo after a successful `release` tests run and pushed to Docker Hub automatically.
+Build:
+```sh
+docker build -t zenithdb/build:buster -f Dockerfile.build .
+```
+
+Login:
+```sh
+docker login
+```
+
+Push to Docker Hub:
+```sh
+docker push zenithdb/build:buster
+```
+
+3. Image `zenithdb/compute-node` is built independently in the [zenithdb/postgres](https://github.com/zenithdb/postgres) repo.
+
+4. Image `zenithdb/zenith` is built in this repo after a successful `release` tests run and pushed to Docker Hub automatically.

pageserver/Cargo.toml

@@ -12,7 +12,6 @@ bytes = { version = "1.0.1", features = ['serde'] }
 byteorder = "1.4.3"
 futures = "0.3.13"
 hyper = "0.14"
-itertools = "0.10.3"
 lazy_static = "1.4.0"
 log = "0.4.14"
 clap = "3.0"

pageserver/src/basebackup.rs

@@ -22,7 +22,6 @@ use tar::{Builder, EntryType, Header};
 
 use crate::relish::*;
 use crate::repository::Timeline;
-use crate::DatadirTimelineImpl;
 use postgres_ffi::xlog_utils::*;
 use postgres_ffi::*;
 use zenith_utils::lsn::Lsn;
@@ -32,7 +31,7 @@ use zenith_utils::lsn::Lsn;
 /// used for constructing tarball.
 pub struct Basebackup<'a> {
     ar: Builder<&'a mut dyn Write>,
-    timeline: &'a Arc<DatadirTimelineImpl>,
+    timeline: &'a Arc<dyn Timeline>,
     pub lsn: Lsn,
     prev_record_lsn: Lsn,
 }
@@ -47,7 +46,7 @@ pub struct Basebackup<'a> {
 impl<'a> Basebackup<'a> {
     pub fn new(
         write: &'a mut dyn Write,
-        timeline: &'a Arc<DatadirTimelineImpl>,
+        timeline: &'a Arc<dyn Timeline>,
         req_lsn: Option<Lsn>,
     ) -> Result<Basebackup<'a>> {
         // Compute postgres doesn't have any previous WAL files, but the first
@@ -65,7 +64,7 @@ impl<'a> Basebackup<'a> {
         // prev_lsn to Lsn(0) if we cannot provide the correct value.
         let (backup_prev, backup_lsn) = if let Some(req_lsn) = req_lsn {
             // Backup was requested at a particular LSN. Wait for it to arrive.
-            timeline.tline.wait_lsn(req_lsn)?;
+            timeline.wait_lsn(req_lsn)?;
 
             // If the requested point is the end of the timeline, we can
             // provide prev_lsn. (get_last_record_rlsn() might return it as
@@ -116,24 +115,21 @@ impl<'a> Basebackup<'a> {
         }
 
         // Gather non-relational files from object storage pages.
-        for kind in [
-            SlruKind::Clog,
-            SlruKind::MultiXactOffsets,
-            SlruKind::MultiXactMembers,
-        ] {
-            for segno in self.timeline.list_slru_segments(kind, self.lsn)? {
-                self.add_slru_segment(kind, segno)?;
+        for obj in self.timeline.list_nonrels(self.lsn)? {
+            match obj {
+                RelishTag::Slru { slru, segno } => {
+                    self.add_slru_segment(slru, segno)?;
+                }
+                RelishTag::FileNodeMap { spcnode, dbnode } => {
+                    self.add_relmap_file(spcnode, dbnode)?;
+                }
+                RelishTag::TwoPhase { xid } => {
+                    self.add_twophase_file(xid)?;
+                }
+                _ => {}
             }
         }
 
-        // Create tablespace directories
-        for ((spcnode, dbnode), has_relmap_file) in self.timeline.list_dbdirs(self.lsn)? {
-            self.add_dbdir(spcnode, dbnode, has_relmap_file)?;
-        }
-        for xid in self.timeline.list_twophase_files(self.lsn)? {
-            self.add_twophase_file(xid)?;
-        }
-
         // Generate pg_control and bootstrap WAL segment.
         self.add_pgcontrol_file()?;
         self.ar.finish()?;
@@ -145,14 +141,27 @@ impl<'a> Basebackup<'a> {
     // Generate SLRU segment files from repository.
     //
     fn add_slru_segment(&mut self, slru: SlruKind, segno: u32) -> anyhow::Result<()> {
-        let nblocks = self.timeline.get_slru_segment_size(slru, segno, self.lsn)?;
+        let seg_size = self
+            .timeline
+            .get_relish_size(RelishTag::Slru { slru, segno }, self.lsn)?;
+
+        if seg_size == None {
+            trace!(
+                "SLRU segment {}/{:>04X} was truncated",
+                slru.to_str(),
+                segno
+            );
+            return Ok(());
+        }
+
+        let nblocks = seg_size.unwrap();
 
         let mut slru_buf: Vec<u8> =
             Vec::with_capacity(nblocks as usize * pg_constants::BLCKSZ as usize);
         for blknum in 0..nblocks {
-            let img = self
-                .timeline
-                .get_slru_page_at_lsn(slru, segno, blknum, self.lsn)?;
+            let img =
+                self.timeline
+                    .get_page_at_lsn(RelishTag::Slru { slru, segno }, blknum, self.lsn)?;
             assert!(img.len() == pg_constants::BLCKSZ as usize);
 
             slru_buf.extend_from_slice(&img);
@@ -167,26 +176,16 @@ impl<'a> Basebackup<'a> {
     }
 
     //
-    // Include database/tablespace directories.
+    // Extract pg_filenode.map files from repository
+    // Along with them also send PG_VERSION for each database.
     //
-    // Each directory contains a PG_VERSION file, and the default database
-    // directories also contain pg_filenode.map files.
-    //
-    fn add_dbdir(
-        &mut self,
-        spcnode: u32,
-        dbnode: u32,
-        has_relmap_file: bool,
-    ) -> anyhow::Result<()> {
-        let relmap_img = if has_relmap_file {
-            let img = self.timeline.get_relmap_file(spcnode, dbnode, self.lsn)?;
-            assert!(img.len() == 512);
-            Some(img)
-        } else {
-            None
-        };
-
-        if spcnode == pg_constants::GLOBALTABLESPACE_OID {
+    fn add_relmap_file(&mut self, spcnode: u32, dbnode: u32) -> anyhow::Result<()> {
+        let img = self.timeline.get_page_at_lsn(
+            RelishTag::FileNodeMap { spcnode, dbnode },
+            0,
+            self.lsn,
+        )?;
+        let path = if spcnode == pg_constants::GLOBALTABLESPACE_OID {
             let version_bytes = pg_constants::PG_MAJORVERSION.as_bytes();
             let header = new_tar_header("PG_VERSION", version_bytes.len() as u64)?;
             self.ar.append(&header, version_bytes)?;
@@ -194,32 +193,9 @@ impl<'a> Basebackup<'a> {
             let header = new_tar_header("global/PG_VERSION", version_bytes.len() as u64)?;
             self.ar.append(&header, version_bytes)?;
 
-            if let Some(img) = relmap_img {
-                // filenode map for global tablespace
-                let header = new_tar_header("global/pg_filenode.map", img.len() as u64)?;
-                self.ar.append(&header, &img[..])?;
-            } else {
-                warn!("global/pg_filenode.map is missing");
-            }
+            String::from("global/pg_filenode.map") // filenode map for global tablespace
         } else {
-            // User defined tablespaces are not supported. However, as
-            // a special case, if a tablespace/db directory is
-            // completely empty, we can leave it out altogether. This
-            // makes taking a base backup after the 'tablespace'
-            // regression test pass, because the test drops the
-            // created tablespaces after the tests.
-            //
-            // FIXME: this wouldn't be necessary, if we handled
-            // XLOG_TBLSPC_DROP records. But we probably should just
-            // throw an error on CREATE TABLESPACE in the first place.
-            if !has_relmap_file
-                && self
-                    .timeline
-                    .list_rels(spcnode, dbnode, self.lsn)?
-                    .is_empty()
-            {
-                return Ok(());
-            }
+            // User defined tablespaces are not supported
             assert!(spcnode == pg_constants::DEFAULTTABLESPACE_OID);
 
             // Append dir path for each database
@@ -227,17 +203,16 @@ impl<'a> Basebackup<'a> {
             let header = new_tar_header_dir(&path)?;
             self.ar.append(&header, &mut io::empty())?;
 
-            if let Some(img) = relmap_img {
-                let dst_path = format!("base/{}/PG_VERSION", dbnode);
-                let version_bytes = pg_constants::PG_MAJORVERSION.as_bytes();
-                let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
-                self.ar.append(&header, version_bytes)?;
+            let dst_path = format!("base/{}/PG_VERSION", dbnode);
+            let version_bytes = pg_constants::PG_MAJORVERSION.as_bytes();
+            let header = new_tar_header(&dst_path, version_bytes.len() as u64)?;
+            self.ar.append(&header, version_bytes)?;
 
-                let relmap_path = format!("base/{}/pg_filenode.map", dbnode);
-                let header = new_tar_header(&relmap_path, img.len() as u64)?;
-                self.ar.append(&header, &img[..])?;
-            }
+            format!("base/{}/pg_filenode.map", dbnode)
         };
+        assert!(img.len() == 512);
+        let header = new_tar_header(&path, img.len() as u64)?;
+        self.ar.append(&header, &img[..])?;
         Ok(())
     }
 
@@ -245,7 +220,9 @@ impl<'a> Basebackup<'a> {
     // Extract twophase state files
     //
     fn add_twophase_file(&mut self, xid: TransactionId) -> anyhow::Result<()> {
-        let img = self.timeline.get_twophase_file(xid, self.lsn)?;
+        let img = self
+            .timeline
+            .get_page_at_lsn(RelishTag::TwoPhase { xid }, 0, self.lsn)?;
 
         let mut buf = BytesMut::new();
         buf.extend_from_slice(&img[..]);
@@ -265,11 +242,11 @@ impl<'a> Basebackup<'a> {
     fn add_pgcontrol_file(&mut self) -> anyhow::Result<()> {
         let checkpoint_bytes = self
             .timeline
-            .get_checkpoint(self.lsn)
+            .get_page_at_lsn(RelishTag::Checkpoint, 0, self.lsn)
             .context("failed to get checkpoint bytes")?;
         let pg_control_bytes = self
             .timeline
-            .get_control_file(self.lsn)
+            .get_page_at_lsn(RelishTag::ControlFile, 0, self.lsn)
             .context("failed get control bytes")?;
         let mut pg_control = ControlFileData::decode(&pg_control_bytes)?;
         let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
@@ -290,7 +267,7 @@ impl<'a> Basebackup<'a> {
         // add zenith.signal file
         let mut zenith_signal = String::new();
         if self.prev_record_lsn == Lsn(0) {
-            if self.lsn == self.timeline.tline.get_ancestor_lsn() {
+            if self.lsn == self.timeline.get_ancestor_lsn() {
                 write!(zenith_signal, "PREV LSN: none")?;
             } else {
                 write!(zenith_signal, "PREV LSN: invalid")?;

pageserver/src/bin/pageserver.rs

@@ -61,7 +61,7 @@ fn main() -> Result<()> {
                 .number_of_values(1)
                 .multiple_occurrences(true)
                 .help("Additional configuration overrides of the ones from the toml config file (or new ones to add there).
-                Any option has to be a valid toml document, example: `-c=\"foo='hey'\"` `-c=\"foo={value=1}\"`"),
+                Any option has to be a valid toml document, example: `-c \"foo='hey'\"` `-c \"foo={value=1}\"`"),
         )
         .get_matches();
 
@@ -115,14 +115,7 @@ fn main() -> Result<()> {
                     option_line
                 )
             })?;
-
             for (key, item) in doc.iter() {
-                if key == "id" {
-                    anyhow::ensure!(
-                        init,
-                        "node id can only be set during pageserver init and cannot be overridden"
-                    );
-                }
                 toml.insert(key, item.clone());
             }
         }

pageserver/src/branches.rs

@@ -16,18 +16,16 @@ use std::{
 };
 use tracing::*;
 
+use zenith_utils::crashsafe_dir;
+use zenith_utils::logging;
 use zenith_utils::lsn::Lsn;
 use zenith_utils::zid::{ZTenantId, ZTimelineId};
-use zenith_utils::{crashsafe_dir, logging};
 
-use crate::config::PageServerConf;
-use crate::pgdatadir_mapping::DatadirTimeline;
-use crate::repository::{Repository, Timeline};
-use crate::tenant_mgr;
 use crate::walredo::WalRedoManager;
 use crate::CheckpointConfig;
-use crate::RepositoryImpl;
+use crate::{config::PageServerConf, repository::Repository};
 use crate::{import_datadir, LOG_FILE_NAME};
+use crate::{repository::RepositoryTimeline, tenant_mgr};
 
 #[derive(Serialize, Deserialize, Clone)]
 pub struct BranchInfo {
@@ -44,7 +42,7 @@ pub struct BranchInfo {
 impl BranchInfo {
     pub fn from_path<T: AsRef<Path>>(
         path: T,
-        tenantid: ZTenantId,
+        repo: &Arc<dyn Repository>,
         include_non_incremental_logical_size: bool,
     ) -> Result<Self> {
         let path = path.as_ref();
@@ -58,23 +56,18 @@ impl BranchInfo {
             })?
             .parse::<ZTimelineId>()?;
 
-        let timeline = match tenant_mgr::get_timeline_for_tenant(tenantid, timeline_id) {
-            Ok(timeline) => timeline,
-            Err(err) => {
-                // FIXME: this was:
-                // bail!("Timeline {} is remote, no branches to display", timeline_id)
-                //
-                // but we cannot distinguish that from other errors now. Have
-                // get_timeline_for_tenant() return a more specific error
-                return Err(err);
+        let timeline = match repo.get_timeline(timeline_id)? {
+            RepositoryTimeline::Local(local_entry) => local_entry,
+            RepositoryTimeline::Remote { .. } => {
+                bail!("Timeline {} is remote, no branches to display", timeline_id)
             }
         };
 
         // we use ancestor lsn zero if we don't have an ancestor, so turn this into an option based on timeline id
-        let (ancestor_id, ancestor_lsn) = match timeline.tline.get_ancestor_timeline_id() {
+        let (ancestor_id, ancestor_lsn) = match timeline.get_ancestor_timeline_id() {
             Some(ancestor_id) => (
                 Some(ancestor_id.to_string()),
-                Some(timeline.tline.get_ancestor_lsn().to_string()),
+                Some(timeline.get_ancestor_lsn().to_string()),
             ),
             None => (None, None),
         };
@@ -86,7 +79,6 @@ impl BranchInfo {
                 timeline.get_current_logical_size_non_incremental(timeline.get_last_record_lsn())
             })
             .transpose()?;
-        let current_logical_size = timeline.get_current_logical_size();
 
         Ok(BranchInfo {
             name,
@@ -94,7 +86,7 @@ impl BranchInfo {
             latest_valid_lsn: timeline.get_last_record_lsn(),
             ancestor_id,
             ancestor_lsn,
-            current_logical_size,
+            current_logical_size: timeline.get_current_logical_size(),
             current_logical_size_non_incremental,
         })
     }
@@ -138,7 +130,7 @@ pub fn create_repo(
     conf: &'static PageServerConf,
     tenantid: ZTenantId,
     wal_redo_manager: Arc<dyn WalRedoManager + Send + Sync>,
-) -> Result<Arc<RepositoryImpl>> {
+) -> Result<Arc<dyn Repository>> {
     let repo_dir = conf.tenant_path(&tenantid);
     if repo_dir.exists() {
         bail!("repo for {} already exists", tenantid)
@@ -160,19 +152,19 @@ pub fn create_repo(
 
     crashsafe_dir::create_dir(&timelinedir)?;
 
-    let repo = crate::layered_repository::LayeredRepository::new(
+    let repo = Arc::new(crate::layered_repository::LayeredRepository::new(
         conf,
         wal_redo_manager,
         tenantid,
         conf.remote_storage_config.is_some(),
-    );
+    ));
 
     // Load data into pageserver
     // TODO To implement zenith import we need to
     //      move data loading out of create_repo()
-    bootstrap_timeline(conf, tenantid, timeline_id, &repo)?;
+    bootstrap_timeline(conf, tenantid, timeline_id, repo.as_ref())?;
 
-    Ok(Arc::new(repo))
+    Ok(repo)
 }
 
 // Returns checkpoint LSN from controlfile
@@ -220,11 +212,11 @@ fn run_initdb(conf: &'static PageServerConf, initdbpath: &Path) -> Result<()> {
 // - run initdb to init temporary instance and get bootstrap data
 // - after initialization complete, remove the temp dir.
 //
-fn bootstrap_timeline<R: Repository>(
+fn bootstrap_timeline(
     conf: &'static PageServerConf,
     tenantid: ZTenantId,
     tli: ZTimelineId,
-    repo: &R,
+    repo: &dyn Repository,
 ) -> Result<()> {
     let _enter = info_span!("bootstrapping", timeline = %tli, tenant = %tenantid).entered();
 
@@ -241,16 +233,17 @@ fn bootstrap_timeline<R: Repository>(
     // Initdb lsn will be equal to last_record_lsn which will be set after import.
     // Because we know it upfront avoid having an option or dummy zero value by passing it to create_empty_timeline.
     let timeline = repo.create_empty_timeline(tli, lsn)?;
-
-    let mut page_tline: DatadirTimeline<R> = DatadirTimeline::new(timeline);
-
-    import_datadir::import_timeline_from_postgres_datadir(&pgdata_path, &mut page_tline, lsn)?;
-    page_tline.tline.checkpoint(CheckpointConfig::Forced)?;
+    import_datadir::import_timeline_from_postgres_datadir(
+        &pgdata_path,
+        timeline.writer().as_ref(),
+        lsn,
+    )?;
+    timeline.checkpoint(CheckpointConfig::Forced)?;
 
     println!(
         "created initial timeline {} timeline.lsn {}",
         tli,
-        page_tline.tline.get_last_record_lsn()
+        timeline.get_last_record_lsn()
     );
 
     let data = tli.to_string();
@@ -268,6 +261,8 @@ pub(crate) fn get_branches(
     tenantid: &ZTenantId,
     include_non_incremental_logical_size: bool,
 ) -> Result<Vec<BranchInfo>> {
+    let repo = tenant_mgr::get_repository_for_tenant(*tenantid)?;
+
     // Each branch has a corresponding record (text file) in the refs/branches
     // with timeline_id.
     let branches_dir = conf.branches_path(tenantid);
@@ -290,7 +285,7 @@ pub(crate) fn get_branches(
             })?;
             BranchInfo::from_path(
                 dir_entry.path(),
-                *tenantid,
+                &repo,
                 include_non_incremental_logical_size,
             )
         })

pageserver/src/config.rs

@@ -8,7 +8,7 @@ use anyhow::{bail, ensure, Context, Result};
 use toml_edit;
 use toml_edit::{Document, Item};
 use zenith_utils::postgres_backend::AuthType;
-use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId};
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
 use std::convert::TryInto;
 use std::env;
@@ -36,9 +36,6 @@ pub mod defaults {
     pub const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
     pub const DEFAULT_GC_PERIOD: &str = "100 s";
 
-    pub const DEFAULT_WAIT_LSN_TIMEOUT: &str = "60 s";
-    pub const DEFAULT_WAL_REDO_TIMEOUT: &str = "60 s";
-
     pub const DEFAULT_SUPERUSER: &str = "zenith_admin";
     pub const DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNC: usize = 100;
     pub const DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS: u32 = 10;
@@ -62,9 +59,6 @@ pub mod defaults {
 #gc_period = '{DEFAULT_GC_PERIOD}'
 #gc_horizon = {DEFAULT_GC_HORIZON}
 
-#wait_lsn_timeout = '{DEFAULT_WAIT_LSN_TIMEOUT}'
-#wal_redo_timeout = '{DEFAULT_WAL_REDO_TIMEOUT}'
-
 #max_file_descriptors = {DEFAULT_MAX_FILE_DESCRIPTORS}
 
 # initial superuser role name to use when creating a new tenant
@@ -78,10 +72,6 @@ pub mod defaults {
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct PageServerConf {
-    // Identifier of that particular pageserver so e g safekeepers
-    // can safely distinguish different pageservers
-    pub id: ZNodeId,
-
     /// Example (default): 127.0.0.1:64000
     pub listen_pg_addr: String,
     /// Example (default): 127.0.0.1:9898
@@ -95,12 +85,6 @@ pub struct PageServerConf {
 
     pub gc_horizon: u64,
     pub gc_period: Duration,
-
-    // Timeout when waiting for WAL receiver to catch up to an LSN given in a GetPage@LSN call.
-    pub wait_lsn_timeout: Duration,
-    // How long to wait for WAL redo to complete.
-    pub wal_redo_timeout: Duration,
-
     pub superuser: String,
 
     pub page_cache_size: usize,
@@ -122,206 +106,6 @@ pub struct PageServerConf {
     pub remote_storage_config: Option<RemoteStorageConfig>,
 }
 
-// use dedicated enum for builder to better indicate the intention
-// and avoid possible confusion with nested options
-pub enum BuilderValue<T> {
-    Set(T),
-    NotSet,
-}
-
-impl<T> BuilderValue<T> {
-    pub fn ok_or<E>(self, err: E) -> Result<T, E> {
-        match self {
-            Self::Set(v) => Ok(v),
-            Self::NotSet => Err(err),
-        }
-    }
-}
-
-// needed to simplify config construction
-struct PageServerConfigBuilder {
-    listen_pg_addr: BuilderValue<String>,
-
-    listen_http_addr: BuilderValue<String>,
-
-    checkpoint_distance: BuilderValue<u64>,
-    checkpoint_period: BuilderValue<Duration>,
-
-    gc_horizon: BuilderValue<u64>,
-    gc_period: BuilderValue<Duration>,
-
-    wait_lsn_timeout: BuilderValue<Duration>,
-    wal_redo_timeout: BuilderValue<Duration>,
-
-    superuser: BuilderValue<String>,
-
-    page_cache_size: BuilderValue<usize>,
-    max_file_descriptors: BuilderValue<usize>,
-
-    workdir: BuilderValue<PathBuf>,
-
-    pg_distrib_dir: BuilderValue<PathBuf>,
-
-    auth_type: BuilderValue<AuthType>,
-
-    //
-    auth_validation_public_key_path: BuilderValue<Option<PathBuf>>,
-    remote_storage_config: BuilderValue<Option<RemoteStorageConfig>>,
-
-    id: BuilderValue<ZNodeId>,
-}
-
-impl Default for PageServerConfigBuilder {
-    fn default() -> Self {
-        use self::BuilderValue::*;
-        use defaults::*;
-        Self {
-            listen_pg_addr: Set(DEFAULT_PG_LISTEN_ADDR.to_string()),
-            listen_http_addr: Set(DEFAULT_HTTP_LISTEN_ADDR.to_string()),
-            checkpoint_distance: Set(DEFAULT_CHECKPOINT_DISTANCE),
-            checkpoint_period: Set(humantime::parse_duration(DEFAULT_CHECKPOINT_PERIOD)
-                .expect("cannot parse default checkpoint period")),
-            gc_horizon: Set(DEFAULT_GC_HORIZON),
-            gc_period: Set(humantime::parse_duration(DEFAULT_GC_PERIOD)
-                .expect("cannot parse default gc period")),
-            wait_lsn_timeout: Set(humantime::parse_duration(DEFAULT_WAIT_LSN_TIMEOUT)
-                .expect("cannot parse default wait lsn timeout")),
-            wal_redo_timeout: Set(humantime::parse_duration(DEFAULT_WAL_REDO_TIMEOUT)
-                .expect("cannot parse default wal redo timeout")),
-            superuser: Set(DEFAULT_SUPERUSER.to_string()),
-            page_cache_size: Set(DEFAULT_PAGE_CACHE_SIZE),
-            max_file_descriptors: Set(DEFAULT_MAX_FILE_DESCRIPTORS),
-            workdir: Set(PathBuf::new()),
-            pg_distrib_dir: Set(env::current_dir()
-                .expect("cannot access current directory")
-                .join("tmp_install")),
-            auth_type: Set(AuthType::Trust),
-            auth_validation_public_key_path: Set(None),
-            remote_storage_config: Set(None),
-            id: NotSet,
-        }
-    }
-}
-
-impl PageServerConfigBuilder {
-    pub fn listen_pg_addr(&mut self, listen_pg_addr: String) {
-        self.listen_pg_addr = BuilderValue::Set(listen_pg_addr)
-    }
-
-    pub fn listen_http_addr(&mut self, listen_http_addr: String) {
-        self.listen_http_addr = BuilderValue::Set(listen_http_addr)
-    }
-
-    pub fn checkpoint_distance(&mut self, checkpoint_distance: u64) {
-        self.checkpoint_distance = BuilderValue::Set(checkpoint_distance)
-    }
-
-    pub fn checkpoint_period(&mut self, checkpoint_period: Duration) {
-        self.checkpoint_period = BuilderValue::Set(checkpoint_period)
-    }
-
-    pub fn gc_horizon(&mut self, gc_horizon: u64) {
-        self.gc_horizon = BuilderValue::Set(gc_horizon)
-    }
-
-    pub fn gc_period(&mut self, gc_period: Duration) {
-        self.gc_period = BuilderValue::Set(gc_period)
-    }
-
-    pub fn wait_lsn_timeout(&mut self, wait_lsn_timeout: Duration) {
-        self.wait_lsn_timeout = BuilderValue::Set(wait_lsn_timeout)
-    }
-
-    pub fn wal_redo_timeout(&mut self, wal_redo_timeout: Duration) {
-        self.wal_redo_timeout = BuilderValue::Set(wal_redo_timeout)
-    }
-
-    pub fn superuser(&mut self, superuser: String) {
-        self.superuser = BuilderValue::Set(superuser)
-    }
-
-    pub fn page_cache_size(&mut self, page_cache_size: usize) {
-        self.page_cache_size = BuilderValue::Set(page_cache_size)
-    }
-
-    pub fn max_file_descriptors(&mut self, max_file_descriptors: usize) {
-        self.max_file_descriptors = BuilderValue::Set(max_file_descriptors)
-    }
-
-    pub fn workdir(&mut self, workdir: PathBuf) {
-        self.workdir = BuilderValue::Set(workdir)
-    }
-
-    pub fn pg_distrib_dir(&mut self, pg_distrib_dir: PathBuf) {
-        self.pg_distrib_dir = BuilderValue::Set(pg_distrib_dir)
-    }
-
-    pub fn auth_type(&mut self, auth_type: AuthType) {
-        self.auth_type = BuilderValue::Set(auth_type)
-    }
-
-    pub fn auth_validation_public_key_path(
-        &mut self,
-        auth_validation_public_key_path: Option<PathBuf>,
-    ) {
-        self.auth_validation_public_key_path = BuilderValue::Set(auth_validation_public_key_path)
-    }
-
-    pub fn remote_storage_config(&mut self, remote_storage_config: Option<RemoteStorageConfig>) {
-        self.remote_storage_config = BuilderValue::Set(remote_storage_config)
-    }
-
-    pub fn id(&mut self, node_id: ZNodeId) {
-        self.id = BuilderValue::Set(node_id)
-    }
-
-    pub fn build(self) -> Result<PageServerConf> {
-        Ok(PageServerConf {
-            listen_pg_addr: self
-                .listen_pg_addr
-                .ok_or(anyhow::anyhow!("missing listen_pg_addr"))?,
-            listen_http_addr: self
-                .listen_http_addr
-                .ok_or(anyhow::anyhow!("missing listen_http_addr"))?,
-            checkpoint_distance: self
-                .checkpoint_distance
-                .ok_or(anyhow::anyhow!("missing checkpoint_distance"))?,
-            checkpoint_period: self
-                .checkpoint_period
-                .ok_or(anyhow::anyhow!("missing checkpoint_period"))?,
-            gc_horizon: self
-                .gc_horizon
-                .ok_or(anyhow::anyhow!("missing gc_horizon"))?,
-            gc_period: self.gc_period.ok_or(anyhow::anyhow!("missing gc_period"))?,
-            wait_lsn_timeout: self
-                .wait_lsn_timeout
-                .ok_or(anyhow::anyhow!("missing wait_lsn_timeout"))?,
-            wal_redo_timeout: self
-                .wal_redo_timeout
-                .ok_or(anyhow::anyhow!("missing wal_redo_timeout"))?,
-            superuser: self.superuser.ok_or(anyhow::anyhow!("missing superuser"))?,
-            page_cache_size: self
-                .page_cache_size
-                .ok_or(anyhow::anyhow!("missing page_cache_size"))?,
-            max_file_descriptors: self
-                .max_file_descriptors
-                .ok_or(anyhow::anyhow!("missing max_file_descriptors"))?,
-            workdir: self.workdir.ok_or(anyhow::anyhow!("missing workdir"))?,
-            pg_distrib_dir: self
-                .pg_distrib_dir
-                .ok_or(anyhow::anyhow!("missing pg_distrib_dir"))?,
-            auth_type: self.auth_type.ok_or(anyhow::anyhow!("missing auth_type"))?,
-            auth_validation_public_key_path: self
-                .auth_validation_public_key_path
-                .ok_or(anyhow::anyhow!("missing auth_validation_public_key_path"))?,
-            remote_storage_config: self
-                .remote_storage_config
-                .ok_or(anyhow::anyhow!("missing remote_storage_config"))?,
-            id: self.id.ok_or(anyhow::anyhow!("missing id"))?,
-        })
-    }
-}
-
 /// External backup storage configuration, enough for creating a client for that storage.
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct RemoteStorageConfig {
@@ -437,41 +221,57 @@ impl PageServerConf {
     ///
     /// This leaves any options not present in the file in the built-in defaults.
     pub fn parse_and_validate(toml: &Document, workdir: &Path) -> Result<Self> {
-        let mut builder = PageServerConfigBuilder::default();
-        builder.workdir(workdir.to_owned());
+        use defaults::*;
+
+        let mut conf = PageServerConf {
+            workdir: workdir.to_path_buf(),
+
+            listen_pg_addr: DEFAULT_PG_LISTEN_ADDR.to_string(),
+            listen_http_addr: DEFAULT_HTTP_LISTEN_ADDR.to_string(),
+            checkpoint_distance: DEFAULT_CHECKPOINT_DISTANCE,
+            checkpoint_period: humantime::parse_duration(DEFAULT_CHECKPOINT_PERIOD)?,
+            gc_horizon: DEFAULT_GC_HORIZON,
+            gc_period: humantime::parse_duration(DEFAULT_GC_PERIOD)?,
+            page_cache_size: DEFAULT_PAGE_CACHE_SIZE,
+            max_file_descriptors: DEFAULT_MAX_FILE_DESCRIPTORS,
+
+            pg_distrib_dir: PathBuf::new(),
+            auth_validation_public_key_path: None,
+            auth_type: AuthType::Trust,
+
+            remote_storage_config: None,
+
+            superuser: DEFAULT_SUPERUSER.to_string(),
+        };
 
         for (key, item) in toml.iter() {
             match key {
-                "listen_pg_addr" => builder.listen_pg_addr(parse_toml_string(key, item)?),
-                "listen_http_addr" => builder.listen_http_addr(parse_toml_string(key, item)?),
-                "checkpoint_distance" => builder.checkpoint_distance(parse_toml_u64(key, item)?),
-                "checkpoint_period" => builder.checkpoint_period(parse_toml_duration(key, item)?),
-                "gc_horizon" => builder.gc_horizon(parse_toml_u64(key, item)?),
-                "gc_period" => builder.gc_period(parse_toml_duration(key, item)?),
-                "wait_lsn_timeout" => builder.wait_lsn_timeout(parse_toml_duration(key, item)?),
-                "wal_redo_timeout" => builder.wal_redo_timeout(parse_toml_duration(key, item)?),
-                "initial_superuser_name" => builder.superuser(parse_toml_string(key, item)?),
-                "page_cache_size" => builder.page_cache_size(parse_toml_u64(key, item)? as usize),
+                "listen_pg_addr" => conf.listen_pg_addr = parse_toml_string(key, item)?,
+                "listen_http_addr" => conf.listen_http_addr = parse_toml_string(key, item)?,
+                "checkpoint_distance" => conf.checkpoint_distance = parse_toml_u64(key, item)?,
+                "checkpoint_period" => conf.checkpoint_period = parse_toml_duration(key, item)?,
+                "gc_horizon" => conf.gc_horizon = parse_toml_u64(key, item)?,
+                "gc_period" => conf.gc_period = parse_toml_duration(key, item)?,
+                "initial_superuser_name" => conf.superuser = parse_toml_string(key, item)?,
+                "page_cache_size" => conf.page_cache_size = parse_toml_u64(key, item)? as usize,
                 "max_file_descriptors" => {
-                    builder.max_file_descriptors(parse_toml_u64(key, item)? as usize)
+                    conf.max_file_descriptors = parse_toml_u64(key, item)? as usize
                 }
                 "pg_distrib_dir" => {
-                    builder.pg_distrib_dir(PathBuf::from(parse_toml_string(key, item)?))
+                    conf.pg_distrib_dir = PathBuf::from(parse_toml_string(key, item)?)
                 }
-                "auth_validation_public_key_path" => builder.auth_validation_public_key_path(Some(
-                    PathBuf::from(parse_toml_string(key, item)?),
-                )),
-                "auth_type" => builder.auth_type(parse_toml_auth_type(key, item)?),
+                "auth_validation_public_key_path" => {
+                    conf.auth_validation_public_key_path =
+                        Some(PathBuf::from(parse_toml_string(key, item)?))
+                }
+                "auth_type" => conf.auth_type = parse_toml_auth_type(key, item)?,
                 "remote_storage" => {
-                    builder.remote_storage_config(Some(Self::parse_remote_storage_config(item)?))
+                    conf.remote_storage_config = Some(Self::parse_remote_storage_config(item)?)
                 }
-                "id" => builder.id(ZNodeId(parse_toml_u64(key, item)?)),
                 _ => bail!("unrecognized pageserver option '{}'", key),
             }
         }
 
-        let mut conf = builder.build().context("invalid config")?;
-
         if conf.auth_type == AuthType::ZenithJWT {
             let auth_validation_public_key_path = conf
                 .auth_validation_public_key_path
@@ -485,6 +285,9 @@ impl PageServerConf {
             );
         }
 
+        if conf.pg_distrib_dir == PathBuf::new() {
+            conf.pg_distrib_dir = env::current_dir()?.join("tmp_install")
+        };
         if !conf.pg_distrib_dir.join("bin/postgres").exists() {
             bail!(
                 "Can't find postgres binary at {}",
@@ -579,13 +382,10 @@ impl PageServerConf {
     #[cfg(test)]
     pub fn dummy_conf(repo_dir: PathBuf) -> Self {
         PageServerConf {
-            id: ZNodeId(0),
             checkpoint_distance: defaults::DEFAULT_CHECKPOINT_DISTANCE,
             checkpoint_period: Duration::from_secs(10),
             gc_horizon: defaults::DEFAULT_GC_HORIZON,
             gc_period: Duration::from_secs(10),
-            wait_lsn_timeout: Duration::from_secs(60),
-            wal_redo_timeout: Duration::from_secs(60),
             page_cache_size: defaults::DEFAULT_PAGE_CACHE_SIZE,
             max_file_descriptors: defaults::DEFAULT_MAX_FILE_DESCRIPTORS,
             listen_pg_addr: defaults::DEFAULT_PG_LISTEN_ADDR.to_string(),
@@ -656,24 +456,20 @@ checkpoint_period = '111 s'
 gc_period = '222 s'
 gc_horizon = 222
 
-wait_lsn_timeout = '111 s'
-wal_redo_timeout = '111 s'
-
 page_cache_size = 444
 max_file_descriptors = 333
 
 # initial superuser role name to use when creating a new tenant
 initial_superuser_name = 'zzzz'
-id = 10
 
-"#;
+    "#;
 
     #[test]
     fn parse_defaults() -> anyhow::Result<()> {
         let tempdir = tempdir()?;
         let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?;
         // we have to create dummy pathes to overcome the validation errors
-        let config_string = format!("pg_distrib_dir='{}'\nid=10", pg_distrib_dir.display());
+        let config_string = format!("pg_distrib_dir='{}'", pg_distrib_dir.display());
         let toml = config_string.parse()?;
 
         let parsed_config =
@@ -684,15 +480,12 @@ id = 10
         assert_eq!(
             parsed_config,
             PageServerConf {
-                id: ZNodeId(10),
                 listen_pg_addr: defaults::DEFAULT_PG_LISTEN_ADDR.to_string(),
                 listen_http_addr: defaults::DEFAULT_HTTP_LISTEN_ADDR.to_string(),
                 checkpoint_distance: defaults::DEFAULT_CHECKPOINT_DISTANCE,
                 checkpoint_period: humantime::parse_duration(defaults::DEFAULT_CHECKPOINT_PERIOD)?,
                 gc_horizon: defaults::DEFAULT_GC_HORIZON,
                 gc_period: humantime::parse_duration(defaults::DEFAULT_GC_PERIOD)?,
-                wait_lsn_timeout: humantime::parse_duration(defaults::DEFAULT_WAIT_LSN_TIMEOUT)?,
-                wal_redo_timeout: humantime::parse_duration(defaults::DEFAULT_WAL_REDO_TIMEOUT)?,
                 superuser: defaults::DEFAULT_SUPERUSER.to_string(),
                 page_cache_size: defaults::DEFAULT_PAGE_CACHE_SIZE,
                 max_file_descriptors: defaults::DEFAULT_MAX_FILE_DESCRIPTORS,
@@ -728,15 +521,12 @@ id = 10
         assert_eq!(
             parsed_config,
             PageServerConf {
-                id: ZNodeId(10),
                 listen_pg_addr: "127.0.0.1:64000".to_string(),
                 listen_http_addr: "127.0.0.1:9898".to_string(),
                 checkpoint_distance: 111,
                 checkpoint_period: Duration::from_secs(111),
                 gc_horizon: 222,
                 gc_period: Duration::from_secs(222),
-                wait_lsn_timeout: Duration::from_secs(111),
-                wal_redo_timeout: Duration::from_secs(111),
                 superuser: "zzzz".to_string(),
                 page_cache_size: 444,
                 max_file_descriptors: 333,

pageserver/src/http/models.rs

@@ -1,7 +1,6 @@
 use serde::{Deserialize, Serialize};
 
 use crate::ZTenantId;
-use zenith_utils::zid::ZNodeId;
 
 #[derive(Serialize, Deserialize)]
 pub struct BranchCreateRequest {
@@ -16,8 +15,3 @@ pub struct TenantCreateRequest {
     #[serde(with = "hex")]
     pub tenant_id: ZTenantId,
 }
-
-#[derive(Serialize)]
-pub struct StatusResponse {
-    pub id: ZNodeId,
-}

pageserver/src/http/openapi_spec.yml

@@ -17,11 +17,6 @@ paths:
             application/json:
               schema:
                 type: object
-                required:
-                - id
-                properties:
-                  id:
-                    type: integer
   /v1/timeline/{tenant_id}:
     parameters:
       - name: tenant_id

pageserver/src/http/routes.rs

@@ -1,6 +1,7 @@
 use std::sync::Arc;
 
 use anyhow::{Context, Result};
+use hyper::header;
 use hyper::StatusCode;
 use hyper::{Body, Request, Response, Uri};
 use serde::Serialize;
@@ -19,16 +20,13 @@ use zenith_utils::http::{
 };
 use zenith_utils::http::{RequestExt, RouterBuilder};
 use zenith_utils::lsn::Lsn;
-use zenith_utils::zid::HexZTimelineId;
-use zenith_utils::zid::ZTimelineId;
+use zenith_utils::zid::{opt_display_serde, ZTimelineId};
 
 use super::models::BranchCreateRequest;
-use super::models::StatusResponse;
 use super::models::TenantCreateRequest;
 use crate::branches::BranchInfo;
 use crate::repository::RepositoryTimeline;
 use crate::repository::TimelineSyncState;
-use crate::repository::{Repository, Timeline};
 use crate::{branches, config::PageServerConf, tenant_mgr, ZTenantId};
 
 #[derive(Debug)]
@@ -66,12 +64,12 @@ fn get_config(request: &Request<Body>) -> &'static PageServerConf {
 }
 
 // healthcheck handler
-async fn status_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    let config = get_config(&request);
-    Ok(json_response(
-        StatusCode::OK,
-        StatusResponse { id: config.id },
-    )?)
+async fn status_handler(_: Request<Body>) -> Result<Response<Body>, ApiError> {
+    Ok(Response::builder()
+        .status(StatusCode::OK)
+        .header(header::CONTENT_TYPE, "application/json")
+        .body(Body::from("{}"))
+        .map_err(ApiError::from_err)?)
 }
 
 async fn branch_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -138,7 +136,8 @@ async fn branch_detail_handler(request: Request<Body>) -> Result<Response<Body>,
 
     let response_data = tokio::task::spawn_blocking(move || {
         let _enter = info_span!("branch_detail", tenant = %tenantid, branch=%branch_name).entered();
-        BranchInfo::from_path(path, tenantid, include_non_incremental_logical_size)
+        let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;
+        BranchInfo::from_path(path, &repo, include_non_incremental_logical_size)
     })
     .await
     .map_err(ApiError::from_err)??;
@@ -199,7 +198,8 @@ enum TimelineInfo {
         timeline_id: ZTimelineId,
         #[serde(with = "hex")]
         tenant_id: ZTenantId,
-        ancestor_timeline_id: Option<HexZTimelineId>,
+        #[serde(with = "opt_display_serde")]
+        ancestor_timeline_id: Option<ZTimelineId>,
         last_record_lsn: Lsn,
         prev_record_lsn: Lsn,
         disk_consistent_lsn: Lsn,
@@ -232,9 +232,7 @@ async fn timeline_detail_handler(request: Request<Body>) -> Result<Response<Body
             Some(timeline) => TimelineInfo::Local {
                 timeline_id,
                 tenant_id,
-                ancestor_timeline_id: timeline
-                    .get_ancestor_timeline_id()
-                    .map(HexZTimelineId::from),
+                ancestor_timeline_id: timeline.get_ancestor_timeline_id(),
                 disk_consistent_lsn: timeline.get_disk_consistent_lsn(),
                 last_record_lsn: timeline.get_last_record_lsn(),
                 prev_record_lsn: timeline.get_prev_record_lsn(),

pageserver/src/import_datadir.rs

@@ -11,15 +11,14 @@ use anyhow::{bail, ensure, Context, Result};
 use bytes::Bytes;
 use tracing::*;
 
-use crate::pgdatadir_mapping::*;
 use crate::relish::*;
-use crate::repository::Repository;
+use crate::repository::*;
 use crate::walingest::WalIngest;
 use postgres_ffi::relfile_utils::*;
 use postgres_ffi::waldecoder::*;
 use postgres_ffi::xlog_utils::*;
+use postgres_ffi::Oid;
 use postgres_ffi::{pg_constants, ControlFileData, DBState_DB_SHUTDOWNED};
-use postgres_ffi::{Oid, TransactionId};
 use zenith_utils::lsn::Lsn;
 
 ///
@@ -28,42 +27,42 @@ use zenith_utils::lsn::Lsn;
 /// This is currently only used to import a cluster freshly created by initdb.
 /// The code that deals with the checkpoint would not work right if the
 /// cluster was not shut down cleanly.
-pub fn import_timeline_from_postgres_datadir<R: Repository>(
+pub fn import_timeline_from_postgres_datadir(
     path: &Path,
-    tline: &mut DatadirTimeline<R>,
+    writer: &dyn TimelineWriter,
     lsn: Lsn,
 ) -> Result<()> {
     let mut pg_control: Option<ControlFileData> = None;
 
-    let mut writer = tline.begin_record(lsn);
-    writer.init_empty()?;
-
     // Scan 'global'
-    let mut relfiles: Vec<PathBuf> = Vec::new();
     for direntry in fs::read_dir(path.join("global"))? {
         let direntry = direntry?;
         match direntry.file_name().to_str() {
             None => continue,
 
             Some("pg_control") => {
-                pg_control = Some(import_control_file(&mut writer, &direntry.path())?);
-            }
-            Some("pg_filenode.map") => {
-                import_relmap_file(
-                    &mut writer,
-                    pg_constants::GLOBALTABLESPACE_OID,
-                    0,
-                    &direntry.path(),
-                )?;
+                pg_control = Some(import_control_file(writer, lsn, &direntry.path())?);
             }
+            Some("pg_filenode.map") => import_nonrel_file(
+                writer,
+                lsn,
+                RelishTag::FileNodeMap {
+                    spcnode: pg_constants::GLOBALTABLESPACE_OID,
+                    dbnode: 0,
+                },
+                &direntry.path(),
+            )?,
 
-            // Load any relation files into the page server (but only after the other files)
-            _ => relfiles.push(direntry.path()),
+            // Load any relation files into the page server
+            _ => import_relfile(
+                &direntry.path(),
+                writer,
+                lsn,
+                pg_constants::GLOBALTABLESPACE_OID,
+                0,
+            )?,
         }
     }
-    for relfile in relfiles {
-        import_relfile(&mut writer, &relfile, pg_constants::GLOBALTABLESPACE_OID, 0)?;
-    }
 
     // Scan 'base'. It contains database dirs, the database OID is the filename.
     // E.g. 'base/12345', where 12345 is the database OID.
@@ -77,56 +76,54 @@ pub fn import_timeline_from_postgres_datadir<R: Repository>(
 
         let dboid = direntry.file_name().to_str().unwrap().parse::<u32>()?;
 
-        let mut relfiles: Vec<PathBuf> = Vec::new();
         for direntry in fs::read_dir(direntry.path())? {
             let direntry = direntry?;
             match direntry.file_name().to_str() {
                 None => continue,
 
-                Some("PG_VERSION") => {
-                    //writer.put_dbdir_creation(pg_constants::DEFAULTTABLESPACE_OID, dboid)?;
-                }
-                Some("pg_filenode.map") => import_relmap_file(
-                    &mut writer,
-                    pg_constants::DEFAULTTABLESPACE_OID,
-                    dboid,
+                Some("PG_VERSION") => continue,
+                Some("pg_filenode.map") => import_nonrel_file(
+                    writer,
+                    lsn,
+                    RelishTag::FileNodeMap {
+                        spcnode: pg_constants::DEFAULTTABLESPACE_OID,
+                        dbnode: dboid,
+                    },
                     &direntry.path(),
                 )?,
 
                 // Load any relation files into the page server
-                _ => relfiles.push(direntry.path()),
+                _ => import_relfile(
+                    &direntry.path(),
+                    writer,
+                    lsn,
+                    pg_constants::DEFAULTTABLESPACE_OID,
+                    dboid,
+                )?,
             }
         }
-        for relfile in relfiles {
-            import_relfile(
-                &mut writer,
-                &relfile,
-                pg_constants::DEFAULTTABLESPACE_OID,
-                dboid,
-            )?;
-        }
     }
     for entry in fs::read_dir(path.join("pg_xact"))? {
         let entry = entry?;
-        import_slru_file(&mut writer, SlruKind::Clog, &entry.path())?;
+        import_slru_file(writer, lsn, SlruKind::Clog, &entry.path())?;
     }
     for entry in fs::read_dir(path.join("pg_multixact").join("members"))? {
         let entry = entry?;
-        import_slru_file(&mut writer, SlruKind::MultiXactMembers, &entry.path())?;
+        import_slru_file(writer, lsn, SlruKind::MultiXactMembers, &entry.path())?;
     }
     for entry in fs::read_dir(path.join("pg_multixact").join("offsets"))? {
         let entry = entry?;
-        import_slru_file(&mut writer, SlruKind::MultiXactOffsets, &entry.path())?;
+        import_slru_file(writer, lsn, SlruKind::MultiXactOffsets, &entry.path())?;
     }
     for entry in fs::read_dir(path.join("pg_twophase"))? {
         let entry = entry?;
         let xid = u32::from_str_radix(entry.path().to_str().unwrap(), 16)?;
-        import_twophase_file(&mut writer, xid, &entry.path())?;
+        import_nonrel_file(writer, lsn, RelishTag::TwoPhase { xid }, &entry.path())?;
     }
     // TODO: Scan pg_tblspc
 
     // We're done importing all the data files.
-    writer.finish()?;
+    writer.advance_last_record_lsn(lsn);
 
     // We expect the Postgres server to be shut down cleanly.
     let pg_control = pg_control.context("pg_control file not found")?;
@@ -144,7 +141,7 @@ pub fn import_timeline_from_postgres_datadir<R: Repository>(
     // *after* the checkpoint record. And crucially, it initializes the 'prev_lsn'.
     import_wal(
         &path.join("pg_wal"),
-        tline,
+        writer,
         Lsn(pg_control.checkPointCopy.redo),
         lsn,
     )?;
@@ -153,9 +150,10 @@ pub fn import_timeline_from_postgres_datadir<R: Repository>(
 }
 
 // subroutine of import_timeline_from_postgres_datadir(), to load one relation file.
-fn import_relfile<R: Repository>(
-    timeline: &mut DatadirTimelineWriter<R>,
+fn import_relfile(
     path: &Path,
+    timeline: &dyn TimelineWriter,
+    lsn: Lsn,
     spcoid: Oid,
     dboid: Oid,
 ) -> Result<()> {
@@ -172,35 +170,26 @@ fn import_relfile<R: Repository>(
     let mut file = File::open(path)?;
     let mut buf: [u8; 8192] = [0u8; 8192];
 
-    let len = file.metadata().unwrap().len();
-    ensure!(len % pg_constants::BLCKSZ as u64 == 0);
-    let nblocks = len / pg_constants::BLCKSZ as u64;
-
-    if segno != 0 {
-        todo!();
-    }
-
-    let rel = RelTag {
-        spcnode: spcoid,
-        dbnode: dboid,
-        relnode,
-        forknum,
-    };
-    timeline.put_rel_creation(rel, nblocks as u32)?;
-
     let mut blknum: u32 = segno * (1024 * 1024 * 1024 / pg_constants::BLCKSZ as u32);
     loop {
         let r = file.read_exact(&mut buf);
         match r {
             Ok(_) => {
-                timeline.put_rel_page_image(rel, blknum, Bytes::copy_from_slice(&buf))?;
+                let rel = RelTag {
+                    spcnode: spcoid,
+                    dbnode: dboid,
+                    relnode,
+                    forknum,
+                };
+                let tag = RelishTag::Relation(rel);
+                timeline.put_page_image(tag, blknum, lsn, Bytes::copy_from_slice(&buf))?;
             }
 
             // TODO: UnexpectedEof is expected
             Err(err) => match err.kind() {
                 std::io::ErrorKind::UnexpectedEof => {
                     // reached EOF. That's expected.
-                    ensure!(blknum == nblocks as u32, "unexpected EOF");
+                    // FIXME: maybe check that we read the full length of the file?
                     break;
                 }
                 _ => {
@@ -214,28 +203,16 @@ fn import_relfile<R: Repository>(
     Ok(())
 }
 
-/// Import a relmapper (pg_filenode.map) file into the repository
-fn import_relmap_file<R: Repository>(
-    timeline: &mut DatadirTimelineWriter<R>,
-    spcnode: Oid,
-    dbnode: Oid,
-    path: &Path,
-) -> Result<()> {
-    let mut file = File::open(path)?;
-    let mut buffer = Vec::new();
-    // read the whole file
-    file.read_to_end(&mut buffer)?;
-
-    trace!("importing relmap file {}", path.display());
-
-    timeline.put_relmap_file(spcnode, dbnode, Bytes::copy_from_slice(&buffer[..]))?;
-    Ok(())
-}
-
-/// Import a twophase state file (pg_twophase/<xid>) into the repository
-fn import_twophase_file<R: Repository>(
-    timeline: &mut DatadirTimelineWriter<R>,
-    xid: TransactionId,
+///
+/// Import a "non-blocky" file into the repository
+///
+/// This is used for small files like the control file, twophase files etc. that
+/// are just slurped into the repository as one blob.
+///
+fn import_nonrel_file(
+    timeline: &dyn TimelineWriter,
+    lsn: Lsn,
+    tag: RelishTag,
     path: &Path,
 ) -> Result<()> {
     let mut file = File::open(path)?;
@@ -245,7 +222,7 @@ fn import_twophase_file<R: Repository>(
 
     trace!("importing non-rel file {}", path.display());
 
-    timeline.put_twophase_file(xid, Bytes::copy_from_slice(&buffer[..]))?;
+    timeline.put_page_image(tag, 0, lsn, Bytes::copy_from_slice(&buffer[..]))?;
     Ok(())
 }
 
@@ -254,8 +231,9 @@ fn import_twophase_file<R: Repository>(
 ///
 /// The control file is imported as is, but we also extract the checkpoint record
 /// from it and store it separated.
-fn import_control_file<R: Repository>(
-    timeline: &mut DatadirTimelineWriter<R>,
+fn import_control_file(
+    timeline: &dyn TimelineWriter,
+    lsn: Lsn,
     path: &Path,
 ) -> Result<ControlFileData> {
     let mut file = File::open(path)?;
@@ -266,12 +244,17 @@ fn import_control_file<R: Repository>(
     trace!("importing control file {}", path.display());
 
     // Import it as ControlFile
-    timeline.put_control_file(Bytes::copy_from_slice(&buffer[..]))?;
+    timeline.put_page_image(
+        RelishTag::ControlFile,
+        0,
+        lsn,
+        Bytes::copy_from_slice(&buffer[..]),
+    )?;
 
     // Extract the checkpoint record and import it separately.
     let pg_control = ControlFileData::decode(&buffer)?;
     let checkpoint_bytes = pg_control.checkPointCopy.encode();
-    timeline.put_checkpoint(checkpoint_bytes)?;
+    timeline.put_page_image(RelishTag::Checkpoint, 0, lsn, checkpoint_bytes)?;
 
     Ok(pg_control)
 }
@@ -279,38 +262,37 @@ fn import_control_file<R: Repository>(
 ///
 /// Import an SLRU segment file
 ///
-fn import_slru_file<R: Repository>(
-    timeline: &mut DatadirTimelineWriter<R>,
+fn import_slru_file(
+    timeline: &dyn TimelineWriter,
+    lsn: Lsn,
     slru: SlruKind,
     path: &Path,
 ) -> Result<()> {
-    trace!("importing slru file {}", path.display());
-
+    // Does it look like an SLRU file?
     let mut file = File::open(path)?;
     let mut buf: [u8; 8192] = [0u8; 8192];
     let segno = u32::from_str_radix(path.file_name().unwrap().to_str().unwrap(), 16)?;
 
-    let len = file.metadata().unwrap().len();
-    ensure!(len % pg_constants::BLCKSZ as u64 == 0); // we assume SLRU block size is the same as BLCKSZ
-    let nblocks = len / pg_constants::BLCKSZ as u64;
-
-    ensure!(nblocks <= pg_constants::SLRU_PAGES_PER_SEGMENT as u64);
-
-    timeline.put_slru_segment_creation(slru, segno, nblocks as u32)?;
+    trace!("importing slru file {}", path.display());
 
     let mut rpageno = 0;
     loop {
         let r = file.read_exact(&mut buf);
         match r {
             Ok(_) => {
-                timeline.put_slru_page_image(slru, segno, rpageno, Bytes::copy_from_slice(&buf))?;
+                timeline.put_page_image(
+                    RelishTag::Slru { slru, segno },
+                    rpageno,
+                    lsn,
+                    Bytes::copy_from_slice(&buf),
+                )?;
             }
 
             // TODO: UnexpectedEof is expected
             Err(err) => match err.kind() {
                 std::io::ErrorKind::UnexpectedEof => {
                     // reached EOF. That's expected.
-                    ensure!(rpageno == nblocks as u32, "unexpected EOF");
+                    // FIXME: maybe check that we read the full length of the file?
                     break;
                 }
                 _ => {
@@ -319,6 +301,8 @@ fn import_slru_file<R: Repository>(
             },
         };
         rpageno += 1;
+
+        // TODO: Check that the file isn't unexpectedly large, not larger than SLRU_PAGES_PER_SEGMENT pages
     }
 
     Ok(())
@@ -326,9 +310,9 @@ fn import_slru_file<R: Repository>(
 
 /// Scan PostgreSQL WAL files in given directory and load all records between
 /// 'startpoint' and 'endpoint' into the repository.
-fn import_wal<R: Repository>(
+fn import_wal(
     walpath: &Path,
-    tline: &mut DatadirTimeline<R>,
+    writer: &dyn TimelineWriter,
     startpoint: Lsn,
     endpoint: Lsn,
 ) -> Result<()> {
@@ -338,7 +322,7 @@ fn import_wal<R: Repository>(
     let mut offset = startpoint.segment_offset(pg_constants::WAL_SEGMENT_SIZE);
     let mut last_lsn = startpoint;
 
-    let mut walingest = WalIngest::new(tline, startpoint)?;
+    let mut walingest = WalIngest::new(writer.deref(), startpoint)?;
 
     while last_lsn <= endpoint {
         // FIXME: assume postgresql tli 1 for now
@@ -371,7 +355,7 @@ fn import_wal<R: Repository>(
         let mut nrecords = 0;
         while last_lsn <= endpoint {
             if let Some((lsn, recdata)) = waldecoder.poll_decode()? {
-                walingest.ingest_record(tline, recdata, lsn)?;
+                walingest.ingest_record(writer, recdata, lsn)?;
                 last_lsn = lsn;
 
                 nrecords += 1;

pageserver/src/keyspace.rs

@@ -1,129 +0,0 @@
-use crate::repository::{key_range_size, singleton_range, Key};
-use postgres_ffi::pg_constants;
-use std::ops::Range;
-
-// Target file size, when creating image and delta layers
-pub const TARGET_FILE_SIZE_BYTES: u64 = 128 * 1024 * 1024; // 128 MB
-
-///
-/// Represents a set of Keys, in a compact form.
-///
-pub struct KeySpace {
-    // Contiguous ranges of keys that belong to the key space. In key order, and
-    // with no overlap.
-    ranges: Vec<Range<Key>>,
-}
-
-impl KeySpace {
-    ///
-    /// Partition a key space into roughly chunks of roughly 'target_size' bytes in
-    /// each patition.
-    ///
-    pub fn partition(&self, target_size: u64) -> KeyPartitioning {
-        // Assume that each value is 8k in size.
-        let target_nblocks = (target_size / pg_constants::BLCKSZ as u64) as usize;
-
-        let mut partitions = Vec::new();
-        let mut current_part = Vec::new();
-        let mut current_part_size: usize = 0;
-        for range in &self.ranges {
-            // If appending the next contiguous range in the keyspace to the current
-            // partition would cause it to be too large, start a new partition.
-            let this_size = key_range_size(range) as usize;
-            if current_part_size + this_size > target_nblocks && !current_part.is_empty() {
-                partitions.push(current_part);
-                current_part = Vec::new();
-                current_part_size = 0;
-            }
-
-            // If the next range is larger than 'target_size', split it into
-            // 'target_size' chunks.
-            let mut remain_size = this_size;
-            let mut start = range.start;
-            while remain_size > target_nblocks {
-                let next = start.add(target_nblocks as u32);
-                partitions.push(vec![start..next]);
-                start = next;
-                remain_size -= target_nblocks
-            }
-            current_part.push(start..range.end);
-            current_part_size += remain_size;
-        }
-
-        // add last partition that wasn't full yet.
-        if !current_part.is_empty() {
-            partitions.push(current_part);
-        }
-
-        KeyPartitioning { partitions }
-    }
-}
-
-///
-/// Represents a partitioning of the key space.
-///
-/// The only kind of partitioning we do is to partition the key space into
-/// partitions that are roughly equal in physical size (see KeySpace::partition).
-/// But this data structure could represent any partitioning.
-///
-#[derive(Clone, Debug, Default)]
-pub struct KeyPartitioning {
-    pub partitions: Vec<Vec<Range<Key>>>,
-}
-
-impl KeyPartitioning {
-    pub fn new() -> Self {
-        KeyPartitioning {
-            partitions: Vec::new(),
-        }
-    }
-}
-
-///
-/// A helper object, to collect a set of keys and key ranges into a KeySpace
-/// object. This takes care of merging adjacent keys and key ranges into
-/// contiguous ranges.
-///
-#[derive(Clone, Debug, Default)]
-pub struct KeySpaceAccum {
-    accum: Option<Range<Key>>,
-
-    ranges: Vec<Range<Key>>,
-}
-
-impl KeySpaceAccum {
-    pub fn new() -> Self {
-        Self {
-            accum: None,
-            ranges: Vec::new(),
-        }
-    }
-
-    pub fn add_key(&mut self, key: Key) {
-        self.add_range(singleton_range(key))
-    }
-
-    pub fn add_range(&mut self, range: Range<Key>) {
-        match self.accum.as_mut() {
-            Some(accum) => {
-                if range.start == accum.end {
-                    accum.end = range.end;
-                } else {
-                    assert!(range.start > accum.end);
-                    self.ranges.push(accum.clone());
-                    *accum = range;
-                }
-            }
-            None => self.accum = Some(range),
-        }
-    }
-
-    pub fn to_keyspace(mut self) -> KeySpace {
-        if let Some(accum) = self.accum.take() {
-            self.ranges.push(accum);
-        }
-        KeySpace {
-            ranges: self.ranges,
-        }
-    }
-}

pageserver/src/layered_repository/delta_layer.rs

@@ -1,5 +1,6 @@
+//!
 //! A DeltaLayer represents a collection of WAL records or page images in a range of
-//! LSNs, and in a range of Keys. It is stored on a file on disk.
+//! LSNs, for one segment. It is stored on a file on disk.
 //!
 //! Usually a delta layer only contains differences - in the form of WAL records against
 //! a base LSN. However, if a segment is newly created, by creating a new relation or
@@ -10,53 +11,56 @@
 //! can happen when you create a new branch in the middle of a delta layer, and the WAL
 //! records on the new branch are put in a new delta layer.
 //!
-//! When a delta file needs to be accessed, we slurp the 'index' metadata
+//! When a delta file needs to be accessed, we slurp the metadata and segsize chapters
 //! into memory, into the DeltaLayerInner struct. See load() and unload() functions.
-//! To access a particular value, we search `index` for the given key.
-//! The byte offset in the index can be used to find the value in
-//! VALUES_CHAPTER.
+//! To access a page/WAL record, we search `page_version_metas` for the block # and LSN.
+//! The byte ranges in the metadata can be used to find the page/WAL record in
+//! PAGE_VERSIONS_CHAPTER.
 //!
 //! On disk, the delta files are stored in timelines/<timelineid> directory.
 //! Currently, there are no subdirectories, and each delta file is named like this:
 //!
-//!    <key start>-<key end>__<start LSN>-<end LSN
+//!    <spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<start LSN>_<end LSN>
 //!
 //! For example:
 //!
-//!    000000067F000032BE0000400000000020B6-000000067F000032BE0000400000000030B6__000000578C6B29-0000000057A50051
+//!    1663_13990_2609_0_5_000000000169C348_000000000169C349
 //!
+//! If a relation is dropped, we add a '_DROPPED' to the end of the filename to indicate that.
+//! So the above example would become:
 //!
-//! A delta file is constructed using the 'bookfile' crate. Each file consists of three
-//! parts: the 'index', the values, and a short summary header. They are stored as
-//! separate chapters.
+//!    1663_13990_2609_0_5_000000000169C348_000000000169C349_DROPPED
+//!
+//! The end LSN indicates when it was dropped in that case, we don't store it in the
+//! file contents in any way.
+//!
+//! A detlta file is constructed using the 'bookfile' crate. Each file consists of two
+//! parts: the page versions and the segment sizes. They are stored as separate chapters.
 //!
 use crate::config::PageServerConf;
 use crate::layered_repository::filename::{DeltaFileName, PathOrConf};
 use crate::layered_repository::storage_layer::{
-    Layer, ValueReconstructResult, ValueReconstructState,
+    Layer, PageReconstructData, PageReconstructResult, PageVersion, SegmentBlk, SegmentTag,
+    RELISH_SEG_SIZE,
 };
-use crate::layered_repository::utils;
-use crate::repository::{Key, Value};
 use crate::virtual_file::VirtualFile;
 use crate::walrecord;
 use crate::{ZTenantId, ZTimelineId};
-use anyhow::{bail, Result};
+use anyhow::{bail, ensure, Result};
 use log::*;
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
 use zenith_utils::vec_map::VecMap;
 // avoid binding to Write (conflicts with std::io::Write)
 // while being able to use std::fmt::Write's methods
 use std::fmt::Write as _;
 use std::fs;
-use std::io::BufWriter;
-use std::io::Write;
-use std::ops::Range;
+use std::io::{BufWriter, Write};
+use std::ops::Bound::Included;
 use std::os::unix::fs::FileExt;
 use std::path::{Path, PathBuf};
-use std::sync::{RwLock, RwLockReadGuard};
+use std::sync::{Mutex, MutexGuard};
 
-use bookfile::{Book, BookWriter, ChapterWriter};
+use bookfile::{Book, BookWriter, BoundedReader, ChapterWriter};
 
 use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::Lsn;
@@ -64,23 +68,27 @@ use zenith_utils::lsn::Lsn;
 // Magic constant to identify a Zenith delta file
 pub const DELTA_FILE_MAGIC: u32 = 0x5A616E01;
 
-/// Mapping from (key, lsn) -> page/WAL record
-/// byte ranges in VALUES_CHAPTER
-static INDEX_CHAPTER: u64 = 1;
-
+/// Mapping from (block #, lsn) -> page/WAL record
+/// byte ranges in PAGE_VERSIONS_CHAPTER
+static PAGE_VERSION_METAS_CHAPTER: u64 = 1;
 /// Page/WAL bytes - cannot be interpreted
-/// without the page versions from the INDEX_CHAPTER
-static VALUES_CHAPTER: u64 = 2;
+/// without PAGE_VERSION_METAS_CHAPTER
+static PAGE_VERSIONS_CHAPTER: u64 = 2;
+static SEG_SIZES_CHAPTER: u64 = 3;
 
 /// Contains the [`Summary`] struct
-static SUMMARY_CHAPTER: u64 = 3;
+static SUMMARY_CHAPTER: u64 = 4;
 
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
 struct Summary {
     tenantid: ZTenantId,
     timelineid: ZTimelineId,
-    key_range: Range<Key>,
-    lsn_range: Range<Lsn>,
+    seg: SegmentTag,
+
+    start_lsn: Lsn,
+    end_lsn: Lsn,
+
+    dropped: bool,
 }
 
 impl From<&DeltaLayer> for Summary {
@@ -88,17 +96,33 @@ impl From<&DeltaLayer> for Summary {
         Self {
             tenantid: layer.tenantid,
             timelineid: layer.timelineid,
-            key_range: layer.key_range.clone(),
-            lsn_range: layer.lsn_range.clone(),
+            seg: layer.seg,
+
+            start_lsn: layer.start_lsn,
+            end_lsn: layer.end_lsn,
+
+            dropped: layer.dropped,
         }
     }
 }
 
+#[derive(Serialize, Deserialize)]
+struct BlobRange {
+    offset: u64,
+    size: usize,
+}
+
+fn read_blob<F: FileExt>(reader: &BoundedReader<&'_ F>, range: &BlobRange) -> Result<Vec<u8>> {
+    let mut buf = vec![0u8; range.size];
+    reader.read_exact_at(&mut buf, range.offset)?;
+    Ok(buf)
+}
+
 ///
 /// DeltaLayer is the in-memory data structure associated with an
 /// on-disk delta file.  We keep a DeltaLayer in memory for each
 /// file, in the LayerMap. If a layer is in "loaded" state, we have a
-/// copy of the index in memory, in 'inner'. Otherwise the struct is
+/// copy of the file in memory, in 'inner'. Otherwise the struct is
 /// just a placeholder for a file that exists on disk, and it needs to
 /// be loaded before using it in queries.
 ///
@@ -107,24 +131,47 @@ pub struct DeltaLayer {
 
     pub tenantid: ZTenantId,
     pub timelineid: ZTimelineId,
-    pub key_range: Range<Key>,
-    pub lsn_range: Range<Lsn>,
+    pub seg: SegmentTag,
 
-    inner: RwLock<DeltaLayerInner>,
+    //
+    // This entry contains all the changes from 'start_lsn' to 'end_lsn'. The
+    // start is inclusive, and end is exclusive.
+    //
+    pub start_lsn: Lsn,
+    pub end_lsn: Lsn,
+
+    dropped: bool,
+
+    inner: Mutex<DeltaLayerInner>,
 }
 
 pub struct DeltaLayerInner {
-    /// If false, the 'index' has not been loaded into memory yet.
+    /// If false, the 'page_version_metas' and 'seg_sizes' have not been
+    /// loaded into memory yet.
     loaded: bool,
 
-    ///
-    /// All versions of all pages in the layer are kept here.
-    /// Indexed by block number and LSN. The value is an offset into the
-    /// chapter where the page version is stored.
-    ///
-    index: HashMap<Key, VecMap<Lsn, u64>>,
-
     book: Option<Book<VirtualFile>>,
+
+    /// All versions of all pages in the file are are kept here.
+    /// Indexed by block number and LSN.
+    page_version_metas: VecMap<(SegmentBlk, Lsn), BlobRange>,
+
+    /// `seg_sizes` tracks the size of the segment at different points in time.
+    seg_sizes: VecMap<Lsn, SegmentBlk>,
+}
+
+impl DeltaLayerInner {
+    fn get_seg_size(&self, lsn: Lsn) -> Result<SegmentBlk> {
+        // Scan the VecMap backwards, starting from the given entry.
+        let slice = self
+            .seg_sizes
+            .slice_range((Included(&Lsn(0)), Included(&lsn)));
+        if let Some((_entry_lsn, entry)) = slice.last() {
+            Ok(*entry)
+        } else {
+            bail!("could not find seg size in delta layer")
+        }
+    }
 }
 
 impl Layer for DeltaLayer {
@@ -136,82 +183,133 @@ impl Layer for DeltaLayer {
         self.timelineid
     }
 
-    fn get_key_range(&self) -> Range<Key> {
-        self.key_range.clone()
+    fn get_seg_tag(&self) -> SegmentTag {
+        self.seg
     }
 
-    fn get_lsn_range(&self) -> Range<Lsn> {
-        self.lsn_range.clone()
+    fn is_dropped(&self) -> bool {
+        self.dropped
+    }
+
+    fn get_start_lsn(&self) -> Lsn {
+        self.start_lsn
+    }
+
+    fn get_end_lsn(&self) -> Lsn {
+        self.end_lsn
     }
 
     fn filename(&self) -> PathBuf {
         PathBuf::from(self.layer_name().to_string())
     }
 
-    fn get_value_reconstruct_data(
+    /// Look up given page in the cache.
+    fn get_page_reconstruct_data(
         &self,
-        key: Key,
-        lsn_range: Range<Lsn>,
-        reconstruct_state: &mut ValueReconstructState,
-    ) -> Result<ValueReconstructResult> {
+        blknum: SegmentBlk,
+        lsn: Lsn,
+        cached_img_lsn: Option<Lsn>,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<PageReconstructResult> {
         let mut need_image = true;
 
-        assert!(self.key_range.contains(&key));
+        assert!((0..RELISH_SEG_SIZE).contains(&blknum));
+
+        match &cached_img_lsn {
+            Some(cached_lsn) if &self.end_lsn <= cached_lsn => {
+                return Ok(PageReconstructResult::Cached)
+            }
+            _ => {}
+        }
 
         {
             // Open the file and lock the metadata in memory
             let inner = self.load()?;
-            let values_reader = inner
+            let page_version_reader = inner
                 .book
                 .as_ref()
                 .expect("should be loaded in load call above")
-                .chapter_reader(VALUES_CHAPTER)?;
+                .chapter_reader(PAGE_VERSIONS_CHAPTER)?;
 
-            // Scan the page versions backwards, starting from `lsn`.
-            if let Some(vec_map) = inner.index.get(&key) {
-                let slice = vec_map.slice_range(lsn_range);
-                for (entry_lsn, pos) in slice.iter().rev() {
-                    let val = Value::des(&utils::read_blob_from_chapter(&values_reader, *pos)?)?;
-                    match val {
-                        Value::Image(img) => {
-                            reconstruct_state.img = Some((*entry_lsn, img));
+            // Scan the metadata VecMap backwards, starting from the given entry.
+            let minkey = (blknum, Lsn(0));
+            let maxkey = (blknum, lsn);
+            let iter = inner
+                .page_version_metas
+                .slice_range((Included(&minkey), Included(&maxkey)))
+                .iter()
+                .rev();
+            for ((_blknum, pv_lsn), blob_range) in iter {
+                match &cached_img_lsn {
+                    Some(cached_lsn) if pv_lsn <= cached_lsn => {
+                        return Ok(PageReconstructResult::Cached)
+                    }
+                    _ => {}
+                }
+
+                let pv = PageVersion::des(&read_blob(&page_version_reader, blob_range)?)?;
+
+                match pv {
+                    PageVersion::Page(img) => {
+                        // Found a page image, return it
+                        reconstruct_data.page_img = Some(img);
+                        need_image = false;
+                        break;
+                    }
+                    PageVersion::Wal(rec) => {
+                        let will_init = rec.will_init();
+                        reconstruct_data.records.push((*pv_lsn, rec));
+                        if will_init {
+                            // This WAL record initializes the page, so no need to go further back
                             need_image = false;
                             break;
                         }
-                        Value::WalRecord(rec) => {
-                            let will_init = rec.will_init();
-                            reconstruct_state.records.push((*entry_lsn, rec));
-                            if will_init {
-                                // This WAL record initializes the page, so no need to go further back
-                                need_image = false;
-                                break;
-                            }
-                        }
                     }
                 }
             }
+
+            // If we didn't find any records for this, check if the request is beyond EOF
+            if need_image
+                && reconstruct_data.records.is_empty()
+                && self.seg.rel.is_blocky()
+                && blknum >= inner.get_seg_size(lsn)?
+            {
+                return Ok(PageReconstructResult::Missing(self.start_lsn));
+            }
+
             // release metadata lock and close the file
         }
 
         // If an older page image is needed to reconstruct the page, let the
         // caller know.
         if need_image {
-            Ok(ValueReconstructResult::Continue)
+            Ok(PageReconstructResult::Continue(Lsn(self.start_lsn.0 - 1)))
         } else {
-            Ok(ValueReconstructResult::Complete)
+            Ok(PageReconstructResult::Complete)
         }
     }
 
-    fn iter(&self) -> Box<dyn Iterator<Item = Result<(Key, Lsn, Value)>> + '_> {
-        let inner = self.load().unwrap();
+    /// Get size of the relation at given LSN
+    fn get_seg_size(&self, lsn: Lsn) -> Result<SegmentBlk> {
+        assert!(lsn >= self.start_lsn);
+        ensure!(
+            self.seg.rel.is_blocky(),
+            "get_seg_size() called on a non-blocky rel"
+        );
 
-        let mut pairs: Vec<(&Key, &VecMap<Lsn, u64>)> = inner.index.iter().collect();
-        pairs.sort_by_key(|x| x.0);
+        let inner = self.load()?;
+        inner.get_seg_size(lsn)
+    }
 
-        match DeltaValueIter::new(inner) {
-            Ok(iter) => Box::new(iter),
-            Err(err) => Box::new(std::iter::once(Err(err))),
+    /// Does this segment exist at given LSN?
+    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
+        // Is the requested LSN after the rel was dropped?
+        if self.dropped && lsn >= self.end_lsn {
+            return Ok(false);
         }
+
+        // Otherwise, it exists.
+        Ok(true)
     }
 
     ///
@@ -219,14 +317,14 @@ impl Layer for DeltaLayer {
     /// it will need to be loaded back.
     ///
     fn unload(&self) -> Result<()> {
-        if let Ok(mut inner) = self.inner.try_write() {
-            inner.index = HashMap::default();
-            inner.loaded = false;
+        let mut inner = self.inner.lock().unwrap();
+        inner.page_version_metas = VecMap::default();
+        inner.seg_sizes = VecMap::default();
+        inner.loaded = false;
 
-            // Note: we keep the Book open. Is that a good idea? The virtual file
-            // machinery has its own rules for closing the file descriptor if it's not
-            // needed, but the Book struct uses up some memory, too.
-        }
+        // Note: we keep the Book open. Is that a good idea? The virtual file
+        // machinery has its own rules for closing the file descriptor if it's not
+        // needed, but the Book struct uses up some memory, too.
 
         Ok(())
     }
@@ -248,52 +346,45 @@ impl Layer for DeltaLayer {
     /// debugging function to print out the contents of the layer
     fn dump(&self) -> Result<()> {
         println!(
-            "----- delta layer for ten {} tli {} keys {}-{} lsn {}-{} ----",
-            self.tenantid,
-            self.timelineid,
-            self.key_range.start,
-            self.key_range.end,
-            self.lsn_range.start,
-            self.lsn_range.end
+            "----- delta layer for ten {} tli {} seg {} {}-{} ----",
+            self.tenantid, self.timelineid, self.seg, self.start_lsn, self.end_lsn
         );
 
+        println!("--- seg sizes ---");
         let inner = self.load()?;
+        for (k, v) in inner.seg_sizes.as_slice() {
+            println!("  {}: {}", k, v);
+        }
+        println!("--- page versions ---");
 
         let path = self.path();
         let file = std::fs::File::open(&path)?;
         let book = Book::new(file)?;
-        let chapter = book.chapter_reader(VALUES_CHAPTER)?;
 
-        let mut values: Vec<(&Key, &VecMap<Lsn, u64>)> = inner.index.iter().collect();
-        values.sort_by_key(|k| k.0);
+        let chapter = book.chapter_reader(PAGE_VERSIONS_CHAPTER)?;
+        for ((blk, lsn), blob_range) in inner.page_version_metas.as_slice() {
+            let mut desc = String::new();
 
-        for (key, versions) in values {
-            for (lsn, off) in versions.as_slice() {
-                let mut desc = String::new();
+            let buf = read_blob(&chapter, blob_range)?;
+            let pv = PageVersion::des(&buf)?;
 
-                let buf = utils::read_blob_from_chapter(&chapter, *off)?;
-                let val = Value::des(&buf);
-
-                match val {
-                    Ok(Value::Image(img)) => {
-                        write!(&mut desc, " img {} bytes", img.len())?;
-                    }
-                    Ok(Value::WalRecord(rec)) => {
-                        let wal_desc = walrecord::describe_wal_record(&rec);
-                        write!(
-                            &mut desc,
-                            " rec {} bytes will_init: {} {}",
-                            buf.len(),
-                            rec.will_init(),
-                            wal_desc
-                        )?;
-                    }
-                    Err(err) => {
-                        write!(&mut desc, " DESERIALIZATION ERROR: {}", err)?;
-                    }
+            match pv {
+                PageVersion::Page(img) => {
+                    write!(&mut desc, " img {} bytes", img.len())?;
+                }
+                PageVersion::Wal(rec) => {
+                    let wal_desc = walrecord::describe_wal_record(&rec);
+                    write!(
+                        &mut desc,
+                        " rec {} bytes will_init: {} {}",
+                        blob_range.size,
+                        rec.will_init(),
+                        wal_desc
+                    )?;
                 }
-                println!("  key {} at {}: {}", key, lsn, desc);
             }
+
+            println!("  blk {} at {}: {}", blk, lsn, desc);
         }
 
         Ok(())
@@ -318,61 +409,61 @@ impl DeltaLayer {
     ///
     /// Load the contents of the file into memory
     ///
-    fn load(&self) -> Result<RwLockReadGuard<DeltaLayerInner>> {
-        loop {
-            // quick exit if already loaded
-            {
-                let inner = self.inner.read().unwrap();
+    fn load(&self) -> Result<MutexGuard<DeltaLayerInner>> {
+        // quick exit if already loaded
+        let mut inner = self.inner.lock().unwrap();
 
-                if inner.loaded {
-                    return Ok(inner);
-                }
-            }
-            // need to upgrade to write lock
-            let mut inner = self.inner.write().unwrap();
-
-            let path = self.path();
-
-            // Open the file if it's not open already.
-            if inner.book.is_none() {
-                let file = VirtualFile::open(&path)?;
-                inner.book = Some(Book::new(file)?);
-            }
-            let book = inner.book.as_ref().unwrap();
-
-            match &self.path_or_conf {
-                PathOrConf::Conf(_) => {
-                    let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
-                    let actual_summary = Summary::des(&chapter)?;
-
-                    let expected_summary = Summary::from(self);
-
-                    if actual_summary != expected_summary {
-                        bail!("in-file summary does not match expected summary. actual = {:?} expected = {:?}", actual_summary, expected_summary);
-                    }
-                }
-                PathOrConf::Path(path) => {
-                    let actual_filename = Path::new(path.file_name().unwrap());
-                    let expected_filename = self.filename();
-
-                    if actual_filename != expected_filename {
-                        println!(
-                            "warning: filename does not match what is expected from in-file summary"
-                        );
-                        println!("actual: {:?}", actual_filename);
-                        println!("expected: {:?}", expected_filename);
-                    }
-                }
-            }
-
-            let chapter = book.read_chapter(INDEX_CHAPTER)?;
-            let index = HashMap::des(&chapter)?;
-
-            debug!("loaded from {}", &path.display());
-
-            inner.index = index;
-            inner.loaded = true;
+        if inner.loaded {
+            return Ok(inner);
         }
+
+        let path = self.path();
+
+        // Open the file if it's not open already.
+        if inner.book.is_none() {
+            let file = VirtualFile::open(&path)?;
+            inner.book = Some(Book::new(file)?);
+        }
+        let book = inner.book.as_ref().unwrap();
+
+        match &self.path_or_conf {
+            PathOrConf::Conf(_) => {
+                let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
+                let actual_summary = Summary::des(&chapter)?;
+
+                let expected_summary = Summary::from(self);
+
+                if actual_summary != expected_summary {
+                    bail!("in-file summary does not match expected summary. actual = {:?} expected = {:?}", actual_summary, expected_summary);
+                }
+            }
+            PathOrConf::Path(path) => {
+                let actual_filename = Path::new(path.file_name().unwrap());
+                let expected_filename = self.filename();
+
+                if actual_filename != expected_filename {
+                    println!(
+                        "warning: filename does not match what is expected from in-file summary"
+                    );
+                    println!("actual: {:?}", actual_filename);
+                    println!("expected: {:?}", expected_filename);
+                }
+            }
+        }
+
+        let chapter = book.read_chapter(PAGE_VERSION_METAS_CHAPTER)?;
+        let page_version_metas = VecMap::des(&chapter)?;
+
+        let chapter = book.read_chapter(SEG_SIZES_CHAPTER)?;
+        let seg_sizes = VecMap::des(&chapter)?;
+
+        debug!("loaded from {}", &path.display());
+
+        inner.page_version_metas = page_version_metas;
+        inner.seg_sizes = seg_sizes;
+        inner.loaded = true;
+
+        Ok(inner)
     }
 
     /// Create a DeltaLayer struct representing an existing file on disk.
@@ -386,12 +477,15 @@ impl DeltaLayer {
             path_or_conf: PathOrConf::Conf(conf),
             timelineid,
             tenantid,
-            key_range: filename.key_range.clone(),
-            lsn_range: filename.lsn_range.clone(),
-            inner: RwLock::new(DeltaLayerInner {
+            seg: filename.seg,
+            start_lsn: filename.start_lsn,
+            end_lsn: filename.end_lsn,
+            dropped: filename.dropped,
+            inner: Mutex::new(DeltaLayerInner {
                 loaded: false,
                 book: None,
-                index: HashMap::default(),
+                page_version_metas: VecMap::default(),
+                seg_sizes: VecMap::default(),
             }),
         }
     }
@@ -401,7 +495,7 @@ impl DeltaLayer {
     /// This variant is only used for debugging purposes, by the 'dump_layerfile' binary.
     pub fn new_for_path<F>(path: &Path, book: &Book<F>) -> Result<Self>
     where
-        F: FileExt,
+        F: std::os::unix::prelude::FileExt,
     {
         let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
         let summary = Summary::des(&chapter)?;
@@ -410,20 +504,25 @@ impl DeltaLayer {
             path_or_conf: PathOrConf::Path(path.to_path_buf()),
             timelineid: summary.timelineid,
             tenantid: summary.tenantid,
-            key_range: summary.key_range,
-            lsn_range: summary.lsn_range,
-            inner: RwLock::new(DeltaLayerInner {
+            seg: summary.seg,
+            start_lsn: summary.start_lsn,
+            end_lsn: summary.end_lsn,
+            dropped: summary.dropped,
+            inner: Mutex::new(DeltaLayerInner {
                 loaded: false,
                 book: None,
-                index: HashMap::default(),
+                page_version_metas: VecMap::default(),
+                seg_sizes: VecMap::default(),
             }),
         })
     }
 
     fn layer_name(&self) -> DeltaFileName {
         DeltaFileName {
-            key_range: self.key_range.clone(),
-            lsn_range: self.lsn_range.clone(),
+            seg: self.seg,
+            start_lsn: self.start_lsn,
+            end_lsn: self.end_lsn,
+            dropped: self.dropped,
         }
     }
 
@@ -444,24 +543,24 @@ impl DeltaLayer {
 ///
 /// 1. Create the DeltaLayerWriter by calling DeltaLayerWriter::new(...)
 ///
-/// 2. Write the contents by calling `put_value` for every page
+/// 2. Write the contents by calling `put_page_version` for every page
 ///    version to store in the layer.
 ///
 /// 3. Call `finish`.
 ///
 pub struct DeltaLayerWriter {
     conf: &'static PageServerConf,
-    path: PathBuf,
     timelineid: ZTimelineId,
     tenantid: ZTenantId,
+    seg: SegmentTag,
+    start_lsn: Lsn,
+    end_lsn: Lsn,
+    dropped: bool,
 
-    key_start: Key,
-    lsn_range: Range<Lsn>,
+    page_version_writer: ChapterWriter<BufWriter<VirtualFile>>,
+    pv_offset: u64,
 
-    index: HashMap<Key, VecMap<Lsn, u64>>,
-
-    values_writer: ChapterWriter<BufWriter<VirtualFile>>,
-    end_offset: u64,
+    page_version_metas: VecMap<(SegmentBlk, Lsn), BlobRange>,
 }
 
 impl DeltaLayerWriter {
@@ -472,85 +571,94 @@ impl DeltaLayerWriter {
         conf: &'static PageServerConf,
         timelineid: ZTimelineId,
         tenantid: ZTenantId,
-        key_start: Key,
-        lsn_range: Range<Lsn>,
+        seg: SegmentTag,
+        start_lsn: Lsn,
+        end_lsn: Lsn,
+        dropped: bool,
     ) -> Result<DeltaLayerWriter> {
         // Create the file
         //
         // Note: This overwrites any existing file. There shouldn't be any.
         // FIXME: throw an error instead?
-
-        let path = conf.timeline_path(&timelineid, &tenantid).join(format!(
-            "{}-XXX__{:016X}-{:016X}.temp",
-            key_start,
-            u64::from(lsn_range.start),
-            u64::from(lsn_range.end)
-        ));
-        info!("temp deltalayer path {}", path.display());
+        let path = DeltaLayer::path_for(
+            &PathOrConf::Conf(conf),
+            timelineid,
+            tenantid,
+            &DeltaFileName {
+                seg,
+                start_lsn,
+                end_lsn,
+                dropped,
+            },
+        );
         let file = VirtualFile::create(&path)?;
         let buf_writer = BufWriter::new(file);
         let book = BookWriter::new(buf_writer, DELTA_FILE_MAGIC)?;
 
         // Open the page-versions chapter for writing. The calls to
-        // `put_value` will use this to write the contents.
-        let values_writer = book.new_chapter(VALUES_CHAPTER);
+        // `put_page_version` will use this to write the contents.
+        let page_version_writer = book.new_chapter(PAGE_VERSIONS_CHAPTER);
 
         Ok(DeltaLayerWriter {
             conf,
-            path,
             timelineid,
             tenantid,
-            key_start,
-            lsn_range,
-            index: HashMap::new(),
-            values_writer,
-            end_offset: 0,
+            seg,
+            start_lsn,
+            end_lsn,
+            dropped,
+            page_version_writer,
+            page_version_metas: VecMap::default(),
+            pv_offset: 0,
         })
     }
 
     ///
-    /// Append a key-value pair to the file.
+    /// Append a page version to the file.
     ///
-    /// The values must be appended in key, lsn order.
+    /// 'buf' is a serialized PageVersion.
+    /// The page versions must be appended in blknum, lsn order.
     ///
-    pub fn put_value(&mut self, key: Key, lsn: Lsn, val: Value) -> Result<()> {
-        //info!("DELTA: key {} at {} on {}", key, lsn, self.path.display());
-        assert!(self.lsn_range.start <= lsn);
+    pub fn put_page_version(&mut self, blknum: SegmentBlk, lsn: Lsn, buf: &[u8]) -> Result<()> {
         // Remember the offset and size metadata. The metadata is written
         // to a separate chapter, in `finish`.
-        let off = self.end_offset;
-        let len = utils::write_blob(&mut self.values_writer, &Value::ser(&val)?)?;
-        self.end_offset += len;
-        let vec_map = self.index.entry(key).or_default();
-        let old = vec_map.append_or_update_last(lsn, off).unwrap().0;
-        if old.is_some() {
-            // We already had an entry for this LSN. That's odd..
-            bail!(
-                "Value for {} at {} already exists in delta layer being built",
-                key,
-                lsn
-            );
-        }
+        let blob_range = BlobRange {
+            offset: self.pv_offset,
+            size: buf.len(),
+        };
+        self.page_version_metas
+            .append((blknum, lsn), blob_range)
+            .unwrap();
+
+        // write the page version
+        self.page_version_writer.write_all(buf)?;
+        self.pv_offset += buf.len() as u64;
 
         Ok(())
     }
 
-    pub fn size(&self) -> u64 {
-        self.end_offset
-    }
-
     ///
     /// Finish writing the delta layer.
     ///
     /// 'seg_sizes' is a list of size changes to store with the actual data.
     ///
-    pub fn finish(self, key_end: Key) -> Result<DeltaLayer> {
-        // Close the values chapter
-        let book = self.values_writer.close()?;
+    pub fn finish(self, seg_sizes: VecMap<Lsn, SegmentBlk>) -> Result<DeltaLayer> {
+        // Close the page-versions chapter
+        let book = self.page_version_writer.close()?;
 
-        // Write out the index
-        let mut chapter = book.new_chapter(INDEX_CHAPTER);
-        let buf = HashMap::ser(&self.index)?;
+        // Write out page versions metadata
+        let mut chapter = book.new_chapter(PAGE_VERSION_METAS_CHAPTER);
+        let buf = VecMap::ser(&self.page_version_metas)?;
+        chapter.write_all(&buf)?;
+        let book = chapter.close()?;
+
+        if self.seg.rel.is_blocky() {
+            assert!(!seg_sizes.is_empty());
+        }
+
+        // and seg_sizes to separate chapter
+        let mut chapter = book.new_chapter(SEG_SIZES_CHAPTER);
+        let buf = VecMap::ser(&seg_sizes)?;
         chapter.write_all(&buf)?;
         let book = chapter.close()?;
 
@@ -558,8 +666,12 @@ impl DeltaLayerWriter {
         let summary = Summary {
             tenantid: self.tenantid,
             timelineid: self.timelineid,
-            key_range: self.key_start..key_end,
-            lsn_range: self.lsn_range.clone(),
+            seg: self.seg,
+
+            start_lsn: self.start_lsn,
+            end_lsn: self.end_lsn,
+
+            dropped: self.dropped,
         };
         Summary::ser_into(&summary, &mut chapter)?;
         let book = chapter.close()?;
@@ -574,110 +686,20 @@ impl DeltaLayerWriter {
             path_or_conf: PathOrConf::Conf(self.conf),
             tenantid: self.tenantid,
             timelineid: self.timelineid,
-            key_range: self.key_start..key_end,
-            lsn_range: self.lsn_range.clone(),
-            inner: RwLock::new(DeltaLayerInner {
+            seg: self.seg,
+            start_lsn: self.start_lsn,
+            end_lsn: self.end_lsn,
+            dropped: self.dropped,
+            inner: Mutex::new(DeltaLayerInner {
                 loaded: false,
-                index: HashMap::new(),
                 book: None,
+                page_version_metas: VecMap::default(),
+                seg_sizes: VecMap::default(),
             }),
         };
 
-        // Rename the file to its final name
-        //
-        // Note: This overwrites any existing file. There shouldn't be any.
-        // FIXME: throw an error instead?
-        let final_path = DeltaLayer::path_for(
-            &PathOrConf::Conf(self.conf),
-            self.timelineid,
-            self.tenantid,
-            &DeltaFileName {
-                key_range: self.key_start..key_end,
-                lsn_range: self.lsn_range,
-            },
-        );
-        std::fs::rename(self.path, &final_path)?;
-
-        trace!("created delta layer {}", final_path.display());
+        trace!("created delta layer {}", &layer.path().display());
 
         Ok(layer)
     }
-
-    pub fn abort(self) {
-        match self.values_writer.close() {
-            Ok(book) => {
-                if let Err(err) = book.close() {
-                    error!("error while closing delta layer file: {}", err);
-                }
-            }
-            Err(err) => {
-                error!("error while closing chapter writer: {}", err);
-            }
-        }
-        if let Err(err) = std::fs::remove_file(self.path) {
-            error!("error removing unfinished delta layer file: {}", err);
-        }
-    }
-}
-
-///
-/// Iterator over all key-value pairse stored in a delta layer
-///
-/// FIXME: This creates a Vector to hold the offsets of all key value pairs.
-/// That takes up quite a lot of memory. Should do this in a more streaming
-/// fashion.
-///
-struct DeltaValueIter<'a> {
-    all_offsets: Vec<(Key, Lsn, u64)>,
-    next_idx: usize,
-
-    inner: RwLockReadGuard<'a, DeltaLayerInner>,
-}
-
-impl<'a> Iterator for DeltaValueIter<'a> {
-    type Item = Result<(Key, Lsn, Value)>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.next_res().transpose()
-    }
-}
-
-impl<'a> DeltaValueIter<'a> {
-    fn new(inner: RwLockReadGuard<'a, DeltaLayerInner>) -> Result<Self> {
-        let mut index: Vec<(&Key, &VecMap<Lsn, u64>)> = inner.index.iter().collect();
-        index.sort_by_key(|x| x.0);
-
-        let mut all_offsets: Vec<(Key, Lsn, u64)> = Vec::new();
-        for (key, vec_map) in index.iter() {
-            for (lsn, off) in vec_map.as_slice().iter() {
-                all_offsets.push((**key, *lsn, *off));
-            }
-        }
-
-        Ok(DeltaValueIter {
-            all_offsets,
-            inner,
-            next_idx: 0,
-        })
-    }
-
-    fn next_res(&mut self) -> Result<Option<(Key, Lsn, Value)>> {
-        if self.next_idx < self.all_offsets.len() {
-            let (key, lsn, off) = self.all_offsets[self.next_idx];
-
-            let values_reader = self
-                .inner
-                .book
-                .as_ref()
-                .expect("should be loaded in load call above")
-                .chapter_reader(VALUES_CHAPTER)?;
-
-            let val = Value::des(&utils::read_blob_from_chapter(&values_reader, off)?)?;
-
-            self.next_idx += 1;
-            Ok(Some((key, lsn, val)))
-        } else {
-            Ok(None)
-        }
-    }
 }

pageserver/src/layered_repository/filename.rs

@@ -2,52 +2,29 @@
 //! Helper functions for dealing with filenames of the image and delta layer files.
 //!
 use crate::config::PageServerConf;
-use crate::repository::Key;
-use std::cmp::Ordering;
+use crate::layered_repository::storage_layer::SegmentTag;
+use crate::relish::*;
 use std::fmt;
-use std::ops::Range;
 use std::path::PathBuf;
 
 use zenith_utils::lsn::Lsn;
 
 // Note: LayeredTimeline::load_layer_map() relies on this sort order
-#[derive(Debug, PartialEq, Eq, Clone)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
 pub struct DeltaFileName {
-    pub key_range: Range<Key>,
-    pub lsn_range: Range<Lsn>,
-}
-
-impl PartialOrd for DeltaFileName {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl Ord for DeltaFileName {
-    fn cmp(&self, other: &Self) -> Ordering {
-        let mut cmp;
-
-        cmp = self.key_range.start.cmp(&other.key_range.start);
-        if cmp != Ordering::Equal {
-            return cmp;
-        }
-        cmp = self.key_range.end.cmp(&other.key_range.end);
-        if cmp != Ordering::Equal {
-            return cmp;
-        }
-        cmp = self.lsn_range.start.cmp(&other.lsn_range.start);
-        if cmp != Ordering::Equal {
-            return cmp;
-        }
-        cmp = self.lsn_range.end.cmp(&other.lsn_range.end);
-
-        cmp
-    }
+    pub seg: SegmentTag,
+    pub start_lsn: Lsn,
+    pub end_lsn: Lsn,
+    pub dropped: bool,
 }
 
 /// Represents the filename of a DeltaLayer
 ///
-///    <key start>-<key end>__<LSN start>-<LSN end>
+///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>
+///
+/// or if it was dropped:
+///
+///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<start LSN>_<end LSN>_DROPPED
 ///
 impl DeltaFileName {
     ///
@@ -55,123 +32,234 @@ impl DeltaFileName {
     /// match the expected pattern.
     ///
     pub fn parse_str(fname: &str) -> Option<Self> {
-        let mut parts = fname.split("__");
-        let mut key_parts = parts.next()?.split('-');
-        let mut lsn_parts = parts.next()?.split('-');
-
-        let key_start_str = key_parts.next()?;
-        let key_end_str = key_parts.next()?;
-        let lsn_start_str = lsn_parts.next()?;
-        let lsn_end_str = lsn_parts.next()?;
-        if parts.next().is_some() || key_parts.next().is_some() || key_parts.next().is_some() {
+        let rel;
+        let mut parts;
+        if let Some(rest) = fname.strip_prefix("rel_") {
+            parts = rest.split('_');
+            rel = RelishTag::Relation(RelTag {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+                relnode: parts.next()?.parse::<u32>().ok()?,
+                forknum: parts.next()?.parse::<u8>().ok()?,
+            });
+        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
+            parts = rest.split('_');
+            rel = RelishTag::FileNodeMap {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
+            parts = rest.split('_');
+            rel = RelishTag::TwoPhase {
+                xid: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
+            parts = rest.split('_');
+            rel = RelishTag::Checkpoint;
+        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
+            parts = rest.split('_');
+            rel = RelishTag::ControlFile;
+        } else {
             return None;
         }
 
-        let key_start = Key::from_hex(key_start_str).ok()?;
-        let key_end = Key::from_hex(key_end_str).ok()?;
+        let segno = parts.next()?.parse::<u32>().ok()?;
 
-        let start_lsn = Lsn::from_hex(lsn_start_str).ok()?;
-        let end_lsn = Lsn::from_hex(lsn_end_str).ok()?;
+        let seg = SegmentTag { rel, segno };
 
-        if start_lsn >= end_lsn {
-            return None;
-            // or panic?
+        let start_lsn = Lsn::from_hex(parts.next()?).ok()?;
+        let end_lsn = Lsn::from_hex(parts.next()?).ok()?;
+
+        let mut dropped = false;
+        if let Some(suffix) = parts.next() {
+            if suffix == "DROPPED" {
+                dropped = true;
+            } else {
+                return None;
+            }
         }
-
-        if key_start >= key_end {
+        if parts.next().is_some() {
             return None;
-            // or panic?
         }
 
         Some(DeltaFileName {
-            key_range: key_start..key_end,
-            lsn_range: start_lsn..end_lsn,
+            seg,
+            start_lsn,
+            end_lsn,
+            dropped,
         })
     }
 }
 
 impl fmt::Display for DeltaFileName {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let basename = match self.seg.rel {
+            RelishTag::Relation(reltag) => format!(
+                "rel_{}_{}_{}_{}",
+                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
+            ),
+            RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno,
+            } => format!("pg_xact_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno,
+            } => format!("pg_multixact_members_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno,
+            } => format!("pg_multixact_offsets_{:04X}", segno),
+            RelishTag::FileNodeMap { spcnode, dbnode } => {
+                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
+            }
+            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
+            RelishTag::Checkpoint => "pg_control_checkpoint".to_string(),
+            RelishTag::ControlFile => "pg_control".to_string(),
+        };
+
         write!(
             f,
-            "{}-{}__{:016X}-{:016X}",
-            self.key_range.start,
-            self.key_range.end,
-            u64::from(self.lsn_range.start),
-            u64::from(self.lsn_range.end),
+            "{}_{}_{:016X}_{:016X}{}",
+            basename,
+            self.seg.segno,
+            u64::from(self.start_lsn),
+            u64::from(self.end_lsn),
+            if self.dropped { "_DROPPED" } else { "" }
         )
     }
 }
 
-#[derive(Debug, PartialEq, Eq, Clone)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
 pub struct ImageFileName {
-    pub key_range: Range<Key>,
+    pub seg: SegmentTag,
     pub lsn: Lsn,
 }
 
-impl PartialOrd for ImageFileName {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl Ord for ImageFileName {
-    fn cmp(&self, other: &Self) -> Ordering {
-        let mut cmp;
-
-        cmp = self.key_range.start.cmp(&other.key_range.start);
-        if cmp != Ordering::Equal {
-            return cmp;
-        }
-        cmp = self.key_range.end.cmp(&other.key_range.end);
-        if cmp != Ordering::Equal {
-            return cmp;
-        }
-        cmp = self.lsn.cmp(&other.lsn);
-
-        cmp
-    }
-}
-
 ///
 /// Represents the filename of an ImageLayer
 ///
-///    <key start>-<key end>__<LSN>
+///    <spcnode>_<dbnode>_<relnode>_<forknum>_<seg>_<LSN>
+///
 impl ImageFileName {
     ///
     /// Parse a string as an image file name. Returns None if the filename does not
     /// match the expected pattern.
     ///
     pub fn parse_str(fname: &str) -> Option<Self> {
-        let mut parts = fname.split("__");
-        let mut key_parts = parts.next()?.split('-');
-
-        let key_start_str = key_parts.next()?;
-        let key_end_str = key_parts.next()?;
-        let lsn_str = parts.next()?;
-        if parts.next().is_some() || key_parts.next().is_some() {
+        let rel;
+        let mut parts;
+        if let Some(rest) = fname.strip_prefix("rel_") {
+            parts = rest.split('_');
+            rel = RelishTag::Relation(RelTag {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+                relnode: parts.next()?.parse::<u32>().ok()?,
+                forknum: parts.next()?.parse::<u8>().ok()?,
+            });
+        } else if let Some(rest) = fname.strip_prefix("pg_xact_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") {
+            parts = rest.split('_');
+            rel = RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno: u32::from_str_radix(parts.next()?, 16).ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") {
+            parts = rest.split('_');
+            rel = RelishTag::FileNodeMap {
+                spcnode: parts.next()?.parse::<u32>().ok()?,
+                dbnode: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_twophase_") {
+            parts = rest.split('_');
+            rel = RelishTag::TwoPhase {
+                xid: parts.next()?.parse::<u32>().ok()?,
+            };
+        } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") {
+            parts = rest.split('_');
+            rel = RelishTag::Checkpoint;
+        } else if let Some(rest) = fname.strip_prefix("pg_control_") {
+            parts = rest.split('_');
+            rel = RelishTag::ControlFile;
+        } else {
             return None;
         }
 
-        let key_start = Key::from_hex(key_start_str).ok()?;
-        let key_end = Key::from_hex(key_end_str).ok()?;
+        let segno = parts.next()?.parse::<u32>().ok()?;
 
-        let lsn = Lsn::from_hex(lsn_str).ok()?;
+        let seg = SegmentTag { rel, segno };
 
-        Some(ImageFileName {
-            key_range: key_start..key_end,
-            lsn,
-        })
+        let lsn = Lsn::from_hex(parts.next()?).ok()?;
+
+        if parts.next().is_some() {
+            return None;
+        }
+
+        Some(ImageFileName { seg, lsn })
     }
 }
 
 impl fmt::Display for ImageFileName {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let basename = match self.seg.rel {
+            RelishTag::Relation(reltag) => format!(
+                "rel_{}_{}_{}_{}",
+                reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum
+            ),
+            RelishTag::Slru {
+                slru: SlruKind::Clog,
+                segno,
+            } => format!("pg_xact_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactMembers,
+                segno,
+            } => format!("pg_multixact_members_{:04X}", segno),
+            RelishTag::Slru {
+                slru: SlruKind::MultiXactOffsets,
+                segno,
+            } => format!("pg_multixact_offsets_{:04X}", segno),
+            RelishTag::FileNodeMap { spcnode, dbnode } => {
+                format!("pg_filenodemap_{}_{}", spcnode, dbnode)
+            }
+            RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid),
+            RelishTag::Checkpoint => "pg_control_checkpoint".to_string(),
+            RelishTag::ControlFile => "pg_control".to_string(),
+        };
+
         write!(
             f,
-            "{}-{}__{:016X}",
-            self.key_range.start,
-            self.key_range.end,
+            "{}_{}_{:016X}",
+            basename,
+            self.seg.segno,
             u64::from(self.lsn),
         )
     }

pageserver/src/layered_repository/global_layer_map.rs

@@ -0,0 +1,142 @@
+//!
+//! Global registry of open layers.
+//!
+//! Whenever a new in-memory layer is created to hold incoming WAL, it is registered
+//! in [`GLOBAL_LAYER_MAP`], so that we can keep track of the total number of
+//! in-memory layers in the system, and know when we need to evict some to release
+//! memory.
+//!
+//! Each layer is assigned a unique ID when it's registered in the global registry.
+//! The ID can be used to relocate the layer later, without having to hold locks.
+//!
+
+use std::sync::atomic::{AtomicU8, Ordering};
+use std::sync::{Arc, RwLock};
+
+use super::inmemory_layer::InMemoryLayer;
+
+use lazy_static::lazy_static;
+
+const MAX_USAGE_COUNT: u8 = 5;
+
+lazy_static! {
+    pub static ref GLOBAL_LAYER_MAP: RwLock<InMemoryLayers> =
+        RwLock::new(InMemoryLayers::default());
+}
+
+// TODO these types can probably be smaller
+#[derive(PartialEq, Eq, Clone, Copy)]
+pub struct LayerId {
+    index: usize,
+    tag: u64, // to avoid ABA problem
+}
+
+enum SlotData {
+    Occupied(Arc<InMemoryLayer>),
+    /// Vacant slots form a linked list, the value is the index
+    /// of the next vacant slot in the list.
+    Vacant(Option<usize>),
+}
+
+struct Slot {
+    tag: u64,
+    data: SlotData,
+    usage_count: AtomicU8, // for clock algorithm
+}
+
+#[derive(Default)]
+pub struct InMemoryLayers {
+    slots: Vec<Slot>,
+    num_occupied: usize,
+
+    // Head of free-slot list.
+    next_empty_slot_idx: Option<usize>,
+}
+
+impl InMemoryLayers {
+    pub fn insert(&mut self, layer: Arc<InMemoryLayer>) -> LayerId {
+        let slot_idx = match self.next_empty_slot_idx {
+            Some(slot_idx) => slot_idx,
+            None => {
+                let idx = self.slots.len();
+                self.slots.push(Slot {
+                    tag: 0,
+                    data: SlotData::Vacant(None),
+                    usage_count: AtomicU8::new(0),
+                });
+                idx
+            }
+        };
+        let slots_len = self.slots.len();
+
+        let slot = &mut self.slots[slot_idx];
+
+        match slot.data {
+            SlotData::Occupied(_) => {
+                panic!("an occupied slot was in the free list");
+            }
+            SlotData::Vacant(next_empty_slot_idx) => {
+                self.next_empty_slot_idx = next_empty_slot_idx;
+            }
+        }
+
+        slot.data = SlotData::Occupied(layer);
+        slot.usage_count.store(1, Ordering::Relaxed);
+
+        self.num_occupied += 1;
+        assert!(self.num_occupied <= slots_len);
+
+        LayerId {
+            index: slot_idx,
+            tag: slot.tag,
+        }
+    }
+
+    pub fn get(&self, layer_id: &LayerId) -> Option<Arc<InMemoryLayer>> {
+        let slot = self.slots.get(layer_id.index)?; // TODO should out of bounds indexes just panic?
+        if slot.tag != layer_id.tag {
+            return None;
+        }
+
+        if let SlotData::Occupied(layer) = &slot.data {
+            let _ = slot.usage_count.fetch_update(
+                Ordering::Relaxed,
+                Ordering::Relaxed,
+                |old_usage_count| {
+                    if old_usage_count < MAX_USAGE_COUNT {
+                        Some(old_usage_count + 1)
+                    } else {
+                        None
+                    }
+                },
+            );
+            Some(Arc::clone(layer))
+        } else {
+            None
+        }
+    }
+
+    // TODO this won't be a public API in the future
+    pub fn remove(&mut self, layer_id: &LayerId) {
+        let slot = &mut self.slots[layer_id.index];
+
+        if slot.tag != layer_id.tag {
+            return;
+        }
+
+        match &slot.data {
+            SlotData::Occupied(_layer) => {
+                // TODO evict the layer
+            }
+            SlotData::Vacant(_) => unimplemented!(),
+        }
+
+        slot.data = SlotData::Vacant(self.next_empty_slot_idx);
+        self.next_empty_slot_idx = Some(layer_id.index);
+
+        assert!(self.num_occupied > 0);
+        self.num_occupied -= 1;
+
+        slot.tag = slot.tag.wrapping_add(1);
+    }
+}

pageserver/src/layered_repository/image_layer.rs

@@ -1,40 +1,41 @@
-//! An ImageLayer represents an image or a snapshot of a key-range at
-//! one particular LSN. It contains an image of all key-value pairs
-//! in its key-range. Any key that falls into the image layer's range
-//! but does not exist in the layer, does not exist.
+//! An ImageLayer represents an image or a snapshot of a segment at one particular LSN.
+//! It is stored in a file on disk.
 //!
-//! An image layer is stored in a file on disk. The file is stored in
-//! timelines/<timelineid> directory.  Currently, there are no
-//! subdirectories, and each image layer file is named like this:
+//! On disk, the image files are stored in timelines/<timelineid> directory.
+//! Currently, there are no subdirectories, and each image layer file is named like this:
 //!
-//!    <key start>-<key end>__<LSN>
+//! Note that segno is
+//!    <spcnode>_<dbnode>_<relnode>_<forknum>_<segno>_<LSN>
 //!
 //! For example:
 //!
-//!    000000067F000032BE0000400000000070B6-000000067F000032BE0000400000000080B6__00000000346BC568
+//!    1663_13990_2609_0_5_000000000169C348
 //!
 //! An image file is constructed using the 'bookfile' crate.
 //!
 //! Only metadata is loaded into memory by the load function.
 //! When images are needed, they are read directly from disk.
 //!
+//! For blocky relishes, the images are stored in BLOCKY_IMAGES_CHAPTER.
+//! All the images are required to be BLOCK_SIZE, which allows for random access.
+//!
+//! For non-blocky relishes, the image can be found in NONBLOCKY_IMAGE_CHAPTER.
+//!
 use crate::config::PageServerConf;
 use crate::layered_repository::filename::{ImageFileName, PathOrConf};
 use crate::layered_repository::storage_layer::{
-    Layer, ValueReconstructResult, ValueReconstructState,
+    Layer, PageReconstructData, PageReconstructResult, SegmentBlk, SegmentTag,
 };
-use crate::layered_repository::utils;
-use crate::repository::{Key, Value};
+use crate::layered_repository::RELISH_SEG_SIZE;
 use crate::virtual_file::VirtualFile;
 use crate::{ZTenantId, ZTimelineId};
-use anyhow::{bail, Context, Result};
+use anyhow::{anyhow, bail, ensure, Context, Result};
 use bytes::Bytes;
 use log::*;
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
+use std::convert::TryInto;
 use std::fs;
 use std::io::{BufWriter, Write};
-use std::ops::Range;
 use std::path::{Path, PathBuf};
 use std::sync::{Mutex, MutexGuard};
 
@@ -43,16 +44,12 @@ use bookfile::{Book, BookWriter, ChapterWriter};
 use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::Lsn;
 
-// Magic constant to identify a Zenith image layer file
-// FIXME: bump all magics
+// Magic constant to identify a Zenith segment image file
 pub const IMAGE_FILE_MAGIC: u32 = 0x5A616E01 + 1;
 
-/// Mapping from (key, lsn) -> page/WAL record
-/// byte ranges in VALUES_CHAPTER
-static INDEX_CHAPTER: u64 = 1;
-
 /// Contains each block in block # order
-const VALUES_CHAPTER: u64 = 2;
+const BLOCKY_IMAGES_CHAPTER: u64 = 1;
+const NONBLOCKY_IMAGE_CHAPTER: u64 = 2;
 
 /// Contains the [`Summary`] struct
 const SUMMARY_CHAPTER: u64 = 3;
@@ -61,7 +58,7 @@ const SUMMARY_CHAPTER: u64 = 3;
 struct Summary {
     tenantid: ZTenantId,
     timelineid: ZTimelineId,
-    key_range: Range<Key>,
+    seg: SegmentTag,
 
     lsn: Lsn,
 }
@@ -71,17 +68,19 @@ impl From<&ImageLayer> for Summary {
         Self {
             tenantid: layer.tenantid,
             timelineid: layer.timelineid,
-            key_range: layer.key_range.clone(),
+            seg: layer.seg,
 
             lsn: layer.lsn,
         }
     }
 }
 
+const BLOCK_SIZE: usize = 8192;
+
 ///
 /// ImageLayer is the in-memory data structure associated with an on-disk image
 /// file.  We keep an ImageLayer in memory for each file, in the LayerMap. If a
-/// layer is in "loaded" state, we have a copy of the index in memory, in 'inner'.
+/// layer is in "loaded" state, we have a copy of the file in memory, in 'inner'.
 /// Otherwise the struct is just a placeholder for a file that exists on disk,
 /// and it needs to be loaded before using it in queries.
 ///
@@ -89,7 +88,7 @@ pub struct ImageLayer {
     path_or_conf: PathOrConf,
     pub tenantid: ZTenantId,
     pub timelineid: ZTimelineId,
-    pub key_range: Range<Key>,
+    pub seg: SegmentTag,
 
     // This entry contains an image of all pages as of this LSN
     pub lsn: Lsn,
@@ -97,15 +96,18 @@ pub struct ImageLayer {
     inner: Mutex<ImageLayerInner>,
 }
 
-pub struct ImageLayerInner {
-    /// If false, the 'index' has not been loaded into memory yet.
-    loaded: bool,
+#[derive(Clone)]
+enum ImageType {
+    Blocky { num_blocks: SegmentBlk },
+    NonBlocky,
+}
 
-    /// If None, the 'image_type' has not been loaded into memory yet. FIXME
+pub struct ImageLayerInner {
+    /// If None, the 'image_type' has not been loaded into memory yet.
     book: Option<Book<VirtualFile>>,
 
-    /// offset of each value
-    index: HashMap<Key, u64>,
+    /// Derived from filename and bookfile chapter metadata
+    image_type: ImageType,
 }
 
 impl Layer for ImageLayer {
@@ -121,61 +123,97 @@ impl Layer for ImageLayer {
         self.timelineid
     }
 
-    fn get_key_range(&self) -> Range<Key> {
-        self.key_range.clone()
+    fn get_seg_tag(&self) -> SegmentTag {
+        self.seg
     }
 
-    fn get_lsn_range(&self) -> Range<Lsn> {
+    fn is_dropped(&self) -> bool {
+        false
+    }
+
+    fn get_start_lsn(&self) -> Lsn {
+        self.lsn
+    }
+
+    fn get_end_lsn(&self) -> Lsn {
         // End-bound is exclusive
-        self.lsn..(self.lsn + 1)
+        self.lsn + 1
     }
 
     /// Look up given page in the file
-    fn get_value_reconstruct_data(
+    fn get_page_reconstruct_data(
         &self,
-        key: Key,
-        lsn_range: Range<Lsn>,
-        reconstruct_state: &mut ValueReconstructState,
-    ) -> Result<ValueReconstructResult> {
-        assert!(self.key_range.contains(&key));
-        assert!(lsn_range.end >= self.lsn);
+        blknum: SegmentBlk,
+        lsn: Lsn,
+        cached_img_lsn: Option<Lsn>,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<PageReconstructResult> {
+        assert!((0..RELISH_SEG_SIZE).contains(&blknum));
+        assert!(lsn >= self.lsn);
+
+        match cached_img_lsn {
+            Some(cached_lsn) if self.lsn <= cached_lsn => return Ok(PageReconstructResult::Cached),
+            _ => {}
+        }
 
         let inner = self.load()?;
 
-        if let Some(offset) = inner.index.get(&key) {
-            let chapter = inner
-                .book
-                .as_ref()
-                .unwrap()
-                .chapter_reader(VALUES_CHAPTER)?;
+        let buf = match &inner.image_type {
+            ImageType::Blocky { num_blocks } => {
+                // Check if the request is beyond EOF
+                if blknum >= *num_blocks {
+                    return Ok(PageReconstructResult::Missing(lsn));
+                }
 
-            let blob = utils::read_blob_from_chapter(&chapter, *offset).with_context(|| {
-                format!(
-                    "failed to read value from data file {} at offset {}",
-                    self.filename().display(),
-                    offset
-                )
-            })?;
-            let value = Bytes::from(blob);
+                let mut buf = vec![0u8; BLOCK_SIZE];
+                let offset = BLOCK_SIZE as u64 * blknum as u64;
 
-            reconstruct_state.img = Some((self.lsn, value));
-            Ok(ValueReconstructResult::Complete)
-        } else {
-            Ok(ValueReconstructResult::Missing)
+                let chapter = inner
+                    .book
+                    .as_ref()
+                    .unwrap()
+                    .chapter_reader(BLOCKY_IMAGES_CHAPTER)?;
+
+                chapter.read_exact_at(&mut buf, offset).with_context(|| {
+                    format!(
+                        "failed to read page from data file {} at offset {}",
+                        self.filename().display(),
+                        offset
+                    )
+                })?;
+
+                buf
+            }
+            ImageType::NonBlocky => {
+                ensure!(blknum == 0);
+                inner
+                    .book
+                    .as_ref()
+                    .unwrap()
+                    .read_chapter(NONBLOCKY_IMAGE_CHAPTER)?
+                    .into_vec()
+            }
+        };
+
+        reconstruct_data.page_img = Some(Bytes::from(buf));
+        Ok(PageReconstructResult::Complete)
+    }
+
+    /// Get size of the segment
+    fn get_seg_size(&self, _lsn: Lsn) -> Result<SegmentBlk> {
+        let inner = self.load()?;
+        match inner.image_type {
+            ImageType::Blocky { num_blocks } => Ok(num_blocks),
+            ImageType::NonBlocky => Err(anyhow!("get_seg_size called for non-blocky segment")),
         }
     }
 
-    fn iter(&self) -> Box<dyn Iterator<Item = Result<(Key, Lsn, Value)>>> {
-        todo!();
+    /// Does this segment exist at given LSN?
+    fn get_seg_exists(&self, _lsn: Lsn) -> Result<bool> {
+        Ok(true)
     }
 
     fn unload(&self) -> Result<()> {
-        // TODO: unload 'segs'. Or even better, don't hold it in memory but
-        // access it directly from the file (using the buffer cache)
-        let mut inner = self.inner.lock().unwrap();
-        inner.index = HashMap::default();
-        inner.loaded = false;
-
         Ok(())
     }
 
@@ -196,17 +234,22 @@ impl Layer for ImageLayer {
     /// debugging function to print out the contents of the layer
     fn dump(&self) -> Result<()> {
         println!(
-            "----- image layer for ten {} tli {} key {}-{} at {} ----",
-            self.tenantid, self.timelineid, self.key_range.start, self.key_range.end, self.lsn
+            "----- image layer for ten {} tli {} seg {} at {} ----",
+            self.tenantid, self.timelineid, self.seg, self.lsn
         );
 
         let inner = self.load()?;
 
-        let mut index_vec: Vec<(&Key, &u64)> = inner.index.iter().collect();
-        index_vec.sort_by_key(|x| x.1);
-
-        for (key, offset) in index_vec {
-            println!("key: {} offset {}", key, offset);
+        match inner.image_type {
+            ImageType::Blocky { num_blocks } => println!("({}) blocks ", num_blocks),
+            ImageType::NonBlocky => {
+                let chapter = inner
+                    .book
+                    .as_ref()
+                    .unwrap()
+                    .read_chapter(NONBLOCKY_IMAGE_CHAPTER)?;
+                println!("non-blocky ({} bytes)", chapter.len());
+            }
         }
 
         Ok(())
@@ -235,21 +278,19 @@ impl ImageLayer {
         // quick exit if already loaded
         let mut inner = self.inner.lock().unwrap();
 
-        if inner.loaded {
+        if inner.book.is_some() {
             return Ok(inner);
         }
 
         let path = self.path();
-
-        // Open the file if it's not open already.
-        if inner.book.is_none() {
-            let file = VirtualFile::open(&path)
-                .with_context(|| format!("Failed to open file '{}'", path.display()))?;
-            inner.book = Some(Book::new(file).with_context(|| {
-                format!("Failed to open file '{}' as a bookfile", path.display())
-            })?);
-        }
-        let book = inner.book.as_ref().unwrap();
+        let file = VirtualFile::open(&path)
+            .with_context(|| format!("Failed to open virtual file '{}'", path.display()))?;
+        let book = Book::new(file).with_context(|| {
+            format!(
+                "Failed to open virtual file '{}' as a bookfile",
+                path.display()
+            )
+        })?;
 
         match &self.path_or_conf {
             PathOrConf::Conf(_) => {
@@ -276,13 +317,23 @@ impl ImageLayer {
             }
         }
 
-        let chapter = book.read_chapter(INDEX_CHAPTER)?;
-        let index = HashMap::des(&chapter)?;
+        let image_type = if self.seg.rel.is_blocky() {
+            let chapter = book.chapter_reader(BLOCKY_IMAGES_CHAPTER)?;
+            let images_len = chapter.len();
+            ensure!(images_len % BLOCK_SIZE as u64 == 0);
+            let num_blocks: SegmentBlk = (images_len / BLOCK_SIZE as u64).try_into()?;
+            ImageType::Blocky { num_blocks }
+        } else {
+            let _chapter = book.chapter_reader(NONBLOCKY_IMAGE_CHAPTER)?;
+            ImageType::NonBlocky
+        };
 
-        info!("loaded from {}", &path.display());
+        debug!("loaded from {}", &path.display());
 
-        inner.index = index;
-        inner.loaded = true;
+        *inner = ImageLayerInner {
+            book: Some(book),
+            image_type,
+        };
 
         Ok(inner)
     }
@@ -298,12 +349,11 @@ impl ImageLayer {
             path_or_conf: PathOrConf::Conf(conf),
             timelineid,
             tenantid,
-            key_range: filename.key_range.clone(),
+            seg: filename.seg,
             lsn: filename.lsn,
             inner: Mutex::new(ImageLayerInner {
                 book: None,
-                index: HashMap::new(),
-                loaded: false,
+                image_type: ImageType::Blocky { num_blocks: 0 },
             }),
         }
     }
@@ -322,19 +372,18 @@ impl ImageLayer {
             path_or_conf: PathOrConf::Path(path.to_path_buf()),
             timelineid: summary.timelineid,
             tenantid: summary.tenantid,
-            key_range: summary.key_range,
+            seg: summary.seg,
             lsn: summary.lsn,
             inner: Mutex::new(ImageLayerInner {
                 book: None,
-                index: HashMap::new(),
-                loaded: false,
+                image_type: ImageType::Blocky { num_blocks: 0 },
             }),
         })
     }
 
     fn layer_name(&self) -> ImageFileName {
         ImageFileName {
-            key_range: self.key_range.clone(),
+            seg: self.seg,
             lsn: self.lsn,
         }
     }
@@ -363,18 +412,15 @@ impl ImageLayer {
 ///
 pub struct ImageLayerWriter {
     conf: &'static PageServerConf,
-    path: PathBuf,
     timelineid: ZTimelineId,
     tenantid: ZTenantId,
-    key_range: Range<Key>,
+    seg: SegmentTag,
     lsn: Lsn,
 
-    values_writer: Option<ChapterWriter<BufWriter<VirtualFile>>>,
-    end_offset: u64,
+    num_blocks: SegmentBlk,
 
-    index: HashMap<Key, u64>,
-
-    finished: bool,
+    page_image_writer: ChapterWriter<BufWriter<VirtualFile>>,
+    num_blocks_written: SegmentBlk,
 }
 
 impl ImageLayerWriter {
@@ -382,8 +428,9 @@ impl ImageLayerWriter {
         conf: &'static PageServerConf,
         timelineid: ZTimelineId,
         tenantid: ZTenantId,
-        key_range: &Range<Key>,
+        seg: SegmentTag,
         lsn: Lsn,
+        num_blocks: SegmentBlk,
     ) -> Result<ImageLayerWriter> {
         // Create the file
         //
@@ -393,74 +440,70 @@ impl ImageLayerWriter {
             &PathOrConf::Conf(conf),
             timelineid,
             tenantid,
-            &ImageFileName {
-                key_range: key_range.clone(),
-                lsn,
-            },
+            &ImageFileName { seg, lsn },
         );
-        info!("new image layer {}", path.display());
         let file = VirtualFile::create(&path)?;
         let buf_writer = BufWriter::new(file);
         let book = BookWriter::new(buf_writer, IMAGE_FILE_MAGIC)?;
 
         // Open the page-images chapter for writing. The calls to
-        // `put_image` will use this to write the contents.
-        let chapter = book.new_chapter(VALUES_CHAPTER);
+        // `put_page_image` will use this to write the contents.
+        let chapter = if seg.rel.is_blocky() {
+            book.new_chapter(BLOCKY_IMAGES_CHAPTER)
+        } else {
+            assert_eq!(num_blocks, 1);
+            book.new_chapter(NONBLOCKY_IMAGE_CHAPTER)
+        };
 
         let writer = ImageLayerWriter {
             conf,
-            path,
             timelineid,
             tenantid,
-            key_range: key_range.clone(),
+            seg,
             lsn,
-            values_writer: Some(chapter),
-            index: HashMap::new(),
-            end_offset: 0,
-            finished: false,
+            num_blocks,
+            page_image_writer: chapter,
+            num_blocks_written: 0,
         };
 
         Ok(writer)
     }
 
     ///
-    /// Write next value to the file.
+    /// Write next page image to the file.
     ///
     /// The page versions must be appended in blknum order.
     ///
-    pub fn put_image(&mut self, key: Key, img: &[u8]) -> Result<()> {
-        assert!(self.key_range.contains(&key));
-        let off = self.end_offset;
-
-        if let Some(writer) = &mut self.values_writer {
-            let len = utils::write_blob(writer, img)?;
-            self.end_offset += len;
-
-            let old = self.index.insert(key, off);
-            assert!(old.is_none());
-        } else {
-            panic!()
+    pub fn put_page_image(&mut self, block_bytes: &[u8]) -> Result<()> {
+        assert!(self.num_blocks_written < self.num_blocks);
+        if self.seg.rel.is_blocky() {
+            assert_eq!(block_bytes.len(), BLOCK_SIZE);
         }
-
+        self.page_image_writer.write_all(block_bytes)?;
+        self.num_blocks_written += 1;
         Ok(())
     }
 
-    pub fn finish(&mut self) -> Result<ImageLayer> {
-        // Close the values chapter
-        let book = self.values_writer.take().unwrap().close()?;
+    pub fn finish(self) -> Result<ImageLayer> {
+        // Check that the `put_page_image' was called for every block.
+        assert!(self.num_blocks_written == self.num_blocks);
 
-        // Write out the index
-        let mut chapter = book.new_chapter(INDEX_CHAPTER);
-        let buf = HashMap::ser(&self.index)?;
-        chapter.write_all(&buf)?;
-        let book = chapter.close()?;
+        // Close the page-images chapter
+        let book = self.page_image_writer.close()?;
 
         // Write out the summary chapter
+        let image_type = if self.seg.rel.is_blocky() {
+            ImageType::Blocky {
+                num_blocks: self.num_blocks,
+            }
+        } else {
+            ImageType::NonBlocky
+        };
         let mut chapter = book.new_chapter(SUMMARY_CHAPTER);
         let summary = Summary {
             tenantid: self.tenantid,
             timelineid: self.timelineid,
-            key_range: self.key_range.clone(),
+            seg: self.seg,
             lsn: self.lsn,
         };
         Summary::ser_into(&summary, &mut chapter)?;
@@ -476,31 +519,15 @@ impl ImageLayerWriter {
             path_or_conf: PathOrConf::Conf(self.conf),
             timelineid: self.timelineid,
             tenantid: self.tenantid,
-            key_range: self.key_range.clone(),
+            seg: self.seg,
             lsn: self.lsn,
             inner: Mutex::new(ImageLayerInner {
                 book: None,
-                loaded: false,
-                index: HashMap::new(),
+                image_type,
             }),
         };
         trace!("created image layer {}", layer.path().display());
 
-        self.finished = true;
-
         Ok(layer)
     }
 }
-
-impl Drop for ImageLayerWriter {
-    fn drop(&mut self) {
-        if let Some(page_image_writer) = self.values_writer.take() {
-            if let Ok(book) = page_image_writer.close() {
-                let _ = book.close();
-            }
-        }
-        if !self.finished {
-            let _ = fs::remove_file(&self.path);
-        }
-    }
-}

pageserver/src/layered_repository/inmemory_layer.rs

@@ -1,32 +1,37 @@
-//! An in-memory layer stores recently received key-value pairs.
+//! An in-memory layer stores recently received PageVersions.
+//! The page versions are held in a BTreeMap. To avoid OOM errors, the map size is limited
+//! and layers can be spilled to disk into ephemeral files.
 //!
-//! The "in-memory" part of the name is a bit misleading: the actual page versions are
-//! held in an ephemeral file, not in memory. The metadata for each page version, i.e.
-//! its position in the file, is kept in memory, though.
+//! And there's another BTreeMap to track the size of the relation.
 //!
 use crate::config::PageServerConf;
 use crate::layered_repository::delta_layer::{DeltaLayer, DeltaLayerWriter};
 use crate::layered_repository::ephemeral_file::EphemeralFile;
+use crate::layered_repository::filename::DeltaFileName;
+use crate::layered_repository::image_layer::{ImageLayer, ImageLayerWriter};
 use crate::layered_repository::storage_layer::{
-    Layer, ValueReconstructResult, ValueReconstructState,
+    Layer, PageReconstructData, PageReconstructResult, PageVersion, SegmentBlk, SegmentTag,
+    RELISH_SEG_SIZE,
 };
-use crate::layered_repository::utils;
-use crate::repository::{Key, Value};
+use crate::layered_repository::LayeredTimeline;
+use crate::layered_repository::ZERO_PAGE;
+use crate::repository::ZenithWalRecord;
 use crate::{ZTenantId, ZTimelineId};
-use anyhow::Result;
+use anyhow::{ensure, Result};
+use bytes::Bytes;
 use log::*;
-use std::collections::HashMap;
-use std::ops::Range;
 use std::path::PathBuf;
-use std::sync::RwLock;
-use zenith_utils::bin_ser::BeSer;
+use std::sync::{Arc, RwLock};
 use zenith_utils::lsn::Lsn;
 use zenith_utils::vec_map::VecMap;
 
+use super::page_versions::PageVersions;
+
 pub struct InMemoryLayer {
     conf: &'static PageServerConf,
     tenantid: ZTenantId,
     timelineid: ZTimelineId,
+    seg: SegmentTag,
 
     ///
     /// This layer contains all the changes from 'start_lsn'. The
@@ -35,7 +40,7 @@ pub struct InMemoryLayer {
     start_lsn: Lsn,
 
     ///
-    /// LSN of the oldest value stored in this layer.
+    /// LSN of the oldest page version stored in this layer.
     ///
     /// This is different from 'start_lsn' in that we enforce that the 'start_lsn'
     /// of a layer always matches the 'end_lsn' of its predecessor, even if there
@@ -52,6 +57,9 @@ pub struct InMemoryLayer {
     /// The above fields never change. The parts that do change are in 'inner',
     /// and protected by mutex.
     inner: RwLock<InMemoryLayerInner>,
+
+    /// Predecessor layer might be needed?
+    incremental: bool,
 }
 
 pub struct InMemoryLayerInner {
@@ -59,25 +67,50 @@ pub struct InMemoryLayerInner {
     /// Writes are only allowed when this is None
     end_lsn: Option<Lsn>,
 
-    ///
-    /// All versions of all pages in the layer are kept here.  Indexed
-    /// by block number and LSN. The value is an offset into the
-    /// ephemeral file where the page version is stored.
-    ///
-    index: HashMap<Key, VecMap<Lsn, u64>>,
+    /// If this relation was dropped, remember when that happened.
+    /// The drop LSN is recorded in [`end_lsn`].
+    dropped: bool,
 
-    /// The values are stored in a serialized format in this file.
-    /// Each serialized Value is preceded by a 'u32' length field.
-    /// PerSeg::page_versions map stores offsets into this file.
-    file: EphemeralFile,
+    ///
+    /// All versions of all pages in the layer are are kept here.
+    /// Indexed by block number and LSN.
+    ///
+    page_versions: PageVersions,
 
-    end_offset: u64,
+    ///
+    /// `seg_sizes` tracks the size of the segment at different points in time.
+    ///
+    /// For a blocky rel, there is always one entry, at the layer's start_lsn,
+    /// so that determining the size never depends on the predecessor layer. For
+    /// a non-blocky rel, 'seg_sizes' is not used and is always empty.
+    ///
+    seg_sizes: VecMap<Lsn, SegmentBlk>,
+
+    ///
+    /// LSN of the newest page version stored in this layer.
+    ///
+    /// The difference between 'end_lsn' and 'latest_lsn' is the same as between
+    /// 'start_lsn' and 'oldest_lsn'. See comments in 'oldest_lsn'.
+    ///
+    latest_lsn: Lsn,
 }
 
 impl InMemoryLayerInner {
     fn assert_writeable(&self) {
         assert!(self.end_lsn.is_none());
     }
+
+    fn get_seg_size(&self, lsn: Lsn) -> SegmentBlk {
+        // Scan the BTreeMap backwards, starting from the given entry.
+        let slice = self.seg_sizes.slice_range(..=lsn);
+
+        // We make sure there is always at least one entry
+        if let Some((_entry_lsn, entry)) = slice.last() {
+            *entry
+        } else {
+            panic!("could not find seg size in in-memory layer");
+        }
+    }
 }
 
 impl Layer for InMemoryLayer {
@@ -87,12 +120,22 @@ impl Layer for InMemoryLayer {
     fn filename(&self) -> PathBuf {
         let inner = self.inner.read().unwrap();
 
-        let end_lsn = inner.end_lsn.unwrap_or(Lsn(u64::MAX));
+        let end_lsn;
+        if let Some(drop_lsn) = inner.end_lsn {
+            end_lsn = drop_lsn;
+        } else {
+            end_lsn = Lsn(u64::MAX);
+        }
 
-        PathBuf::from(format!(
-            "inmem-{:016X}-{:016X}",
-            self.start_lsn.0, end_lsn.0
-        ))
+        let delta_filename = DeltaFileName {
+            seg: self.seg,
+            start_lsn: self.start_lsn,
+            end_lsn,
+            dropped: inner.dropped,
+        }
+        .to_string();
+
+        PathBuf::from(format!("inmem-{}", delta_filename))
     }
 
     fn get_tenant_id(&self) -> ZTenantId {
@@ -103,54 +146,68 @@ impl Layer for InMemoryLayer {
         self.timelineid
     }
 
-    fn get_key_range(&self) -> Range<Key> {
-        Key::MIN..Key::MAX
+    fn get_seg_tag(&self) -> SegmentTag {
+        self.seg
     }
 
-    fn get_lsn_range(&self) -> Range<Lsn> {
+    fn get_start_lsn(&self) -> Lsn {
+        self.start_lsn
+    }
+
+    fn get_end_lsn(&self) -> Lsn {
         let inner = self.inner.read().unwrap();
 
-        let end_lsn = if let Some(end_lsn) = inner.end_lsn {
+        if let Some(end_lsn) = inner.end_lsn {
             end_lsn
         } else {
             Lsn(u64::MAX)
-        };
-        self.start_lsn..end_lsn
+        }
     }
 
-    /// Look up given value in the layer.
-    fn get_value_reconstruct_data(
+    fn is_dropped(&self) -> bool {
+        let inner = self.inner.read().unwrap();
+        inner.dropped
+    }
+
+    /// Look up given page in the cache.
+    fn get_page_reconstruct_data(
         &self,
-        key: Key,
-        lsn_range: Range<Lsn>,
-        reconstruct_state: &mut ValueReconstructState,
-    ) -> Result<ValueReconstructResult> {
-        assert!(lsn_range.start <= self.start_lsn);
+        blknum: SegmentBlk,
+        lsn: Lsn,
+        cached_img_lsn: Option<Lsn>,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<PageReconstructResult> {
         let mut need_image = true;
 
-        let inner = self.inner.read().unwrap();
+        assert!((0..RELISH_SEG_SIZE).contains(&blknum));
 
-        // Scan the page versions backwards, starting from `lsn`.
-        if let Some(vec_map) = inner.index.get(&key) {
-            let slice = vec_map.slice_range(lsn_range);
-            for (entry_lsn, pos) in slice.iter().rev() {
-                match &reconstruct_state.img {
-                    Some((cached_lsn, _)) if entry_lsn <= cached_lsn => {
-                        return Ok(ValueReconstructResult::Complete)
+        {
+            let inner = self.inner.read().unwrap();
+
+            // Scan the page versions backwards, starting from `lsn`.
+            let iter = inner
+                .page_versions
+                .get_block_lsn_range(blknum, ..=lsn)
+                .iter()
+                .rev();
+            for (entry_lsn, pos) in iter {
+                match &cached_img_lsn {
+                    Some(cached_lsn) if entry_lsn <= cached_lsn => {
+                        return Ok(PageReconstructResult::Cached)
                     }
                     _ => {}
                 }
 
-                let value = Value::des(&utils::read_blob(&inner.file, *pos)?)?;
-                match value {
-                    Value::Image(img) => {
-                        reconstruct_state.img = Some((*entry_lsn, img));
-                        return Ok(ValueReconstructResult::Complete);
+                let pv = inner.page_versions.read_pv(*pos)?;
+                match pv {
+                    PageVersion::Page(img) => {
+                        reconstruct_data.page_img = Some(img);
+                        need_image = false;
+                        break;
                     }
-                    Value::WalRecord(rec) => {
-                        let will_init = rec.will_init();
-                        reconstruct_state.records.push((*entry_lsn, rec));
-                        if will_init {
+                    PageVersion::Wal(rec) => {
+                        reconstruct_data.records.push((*entry_lsn, rec.clone()));
+                        if rec.will_init() {
                             // This WAL record initializes the page, so no need to go further back
                             need_image = false;
                             break;
@@ -158,21 +215,66 @@ impl Layer for InMemoryLayer {
                     }
                 }
             }
+
+            // If we didn't find any records for this, check if the request is beyond EOF
+            if need_image
+                && reconstruct_data.records.is_empty()
+                && self.seg.rel.is_blocky()
+                && blknum >= self.get_seg_size(lsn)?
+            {
+                return Ok(PageReconstructResult::Missing(self.start_lsn));
+            }
+
+            // release lock on 'inner'
         }
 
-        // release lock on 'inner'
-
         // If an older page image is needed to reconstruct the page, let the
-        // caller know.
+        // caller know
         if need_image {
-            Ok(ValueReconstructResult::Continue)
+            if self.incremental {
+                Ok(PageReconstructResult::Continue(Lsn(self.start_lsn.0 - 1)))
+            } else {
+                Ok(PageReconstructResult::Missing(self.start_lsn))
+            }
         } else {
-            Ok(ValueReconstructResult::Complete)
+            Ok(PageReconstructResult::Complete)
         }
     }
 
-    fn iter(&self) -> Box<dyn Iterator<Item = Result<(Key, Lsn, Value)>>> {
-        todo!();
+    /// Get size of the relation at given LSN
+    fn get_seg_size(&self, lsn: Lsn) -> Result<SegmentBlk> {
+        assert!(lsn >= self.start_lsn);
+        ensure!(
+            self.seg.rel.is_blocky(),
+            "get_seg_size() called on a non-blocky rel"
+        );
+
+        let inner = self.inner.read().unwrap();
+        Ok(inner.get_seg_size(lsn))
+    }
+
+    /// Does this segment exist at given LSN?
+    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool> {
+        let inner = self.inner.read().unwrap();
+
+        // If the segment created after requested LSN,
+        // it doesn't exist in the layer. But we shouldn't
+        // have requested it in the first place.
+        assert!(lsn >= self.start_lsn);
+
+        // Is the requested LSN after the segment was dropped?
+        if inner.dropped {
+            if let Some(end_lsn) = inner.end_lsn {
+                if lsn >= end_lsn {
+                    return Ok(false);
+                }
+            } else {
+                panic!("dropped in-memory layer with no end LSN");
+            }
+        }
+
+        // Otherwise, it exists
+        Ok(true)
     }
 
     /// Cannot unload anything in an in-memory layer, since there's no backing
@@ -189,8 +291,7 @@ impl Layer for InMemoryLayer {
     }
 
     fn is_incremental(&self) -> bool {
-        // in-memory layer is always considered incremental.
-        true
+        self.incremental
     }
 
     fn is_in_memory(&self) -> bool {
@@ -208,39 +309,45 @@ impl Layer for InMemoryLayer {
             .unwrap_or_default();
 
         println!(
-            "----- in-memory layer for tli {} LSNs {}-{} ----",
-            self.timelineid,
-            self.start_lsn,
-            end_str,
-            //inner.dropped,
+            "----- in-memory layer for tli {} seg {} {}-{} {} ----",
+            self.timelineid, self.seg, self.start_lsn, end_str, inner.dropped,
         );
 
-        // FIXME
-        /*
-           for (blknum, versions) in page_versions {
-               for (lsn, off) in versions.as_slice() {
-                   let pv = inner.read_pv(*off);
-                   let pv_description = match pv {
-                       Ok(PageVersion::Page(_img)) => "page",
-                       Ok(PageVersion::Wal(_rec)) => "wal",
-                       Err(_err) => "INVALID",
-                   };
+        for (k, v) in inner.seg_sizes.as_slice() {
+            println!("seg_sizes {}: {}", k, v);
+        }
 
-                   println!("blk {} at {}: {}\n", blknum, lsn, pv_description);
-               }
-           }
-        */
+        for (blknum, lsn, pos) in inner.page_versions.ordered_page_version_iter(None) {
+            let pv = inner.page_versions.read_pv(pos)?;
+            let pv_description = match pv {
+                PageVersion::Page(_img) => "page",
+                PageVersion::Wal(_rec) => "wal",
+            };
+
+            println!("blk {} at {}: {}\n", blknum, lsn, pv_description);
+        }
 
         Ok(())
     }
 }
 
+/// A result of an inmemory layer data being written to disk.
+pub struct LayersOnDisk {
+    pub delta_layers: Vec<DeltaLayer>,
+    pub image_layers: Vec<ImageLayer>,
+}
+
 impl InMemoryLayer {
     /// Return the oldest page version that's stored in this layer
     pub fn get_oldest_lsn(&self) -> Lsn {
         self.oldest_lsn
     }
 
+    pub fn get_latest_lsn(&self) -> Lsn {
+        let inner = self.inner.read().unwrap();
+        inner.latest_lsn
+    }
+
     ///
     /// Create a new, empty, in-memory layer
     ///
@@ -248,88 +355,257 @@ impl InMemoryLayer {
         conf: &'static PageServerConf,
         timelineid: ZTimelineId,
         tenantid: ZTenantId,
+        seg: SegmentTag,
         start_lsn: Lsn,
         oldest_lsn: Lsn,
     ) -> Result<InMemoryLayer> {
         trace!(
-            "initializing new empty InMemoryLayer for writing on timeline {} at {}",
+            "initializing new empty InMemoryLayer for writing {} on timeline {} at {}",
+            seg,
             timelineid,
             start_lsn
         );
 
+        // The segment is initially empty, so initialize 'seg_sizes' with 0.
+        let mut seg_sizes = VecMap::default();
+        if seg.rel.is_blocky() {
+            seg_sizes.append(start_lsn, 0).unwrap();
+        }
+
         let file = EphemeralFile::create(conf, tenantid, timelineid)?;
 
         Ok(InMemoryLayer {
             conf,
             timelineid,
             tenantid,
+            seg,
             start_lsn,
             oldest_lsn,
+            incremental: false,
             inner: RwLock::new(InMemoryLayerInner {
                 end_lsn: None,
-                index: HashMap::new(),
-                file,
-                end_offset: 0,
+                dropped: false,
+                page_versions: PageVersions::new(file),
+                seg_sizes,
+                latest_lsn: oldest_lsn,
             }),
         })
     }
 
     // Write operations
 
+    /// Remember new page version, as a WAL record over previous version
+    pub fn put_wal_record(
+        &self,
+        lsn: Lsn,
+        blknum: SegmentBlk,
+        rec: ZenithWalRecord,
+    ) -> Result<u32> {
+        self.put_page_version(blknum, lsn, PageVersion::Wal(rec))
+    }
+
+    /// Remember new page version, as a full page image
+    pub fn put_page_image(&self, blknum: SegmentBlk, lsn: Lsn, img: Bytes) -> Result<u32> {
+        self.put_page_version(blknum, lsn, PageVersion::Page(img))
+    }
+
     /// Common subroutine of the public put_wal_record() and put_page_image() functions.
     /// Adds the page version to the in-memory tree
-    pub fn put_value(&self, key: Key, lsn: Lsn, val: Value) -> Result<()> {
-        trace!("put_value key {} at {}/{}", key, self.timelineid, lsn);
+    pub fn put_page_version(&self, blknum: SegmentBlk, lsn: Lsn, pv: PageVersion) -> Result<u32> {
+        assert!((0..RELISH_SEG_SIZE).contains(&blknum));
+
+        trace!(
+            "put_page_version blk {} of {} at {}/{}",
+            blknum,
+            self.seg.rel,
+            self.timelineid,
+            lsn
+        );
         let mut inner = self.inner.write().unwrap();
 
         inner.assert_writeable();
+        assert!(lsn >= inner.latest_lsn);
+        inner.latest_lsn = lsn;
 
-        let off = inner.end_offset;
-        let len = utils::write_blob(&mut inner.file, &Value::ser(&val)?)?;
-        inner.end_offset += len;
+        let old = inner.page_versions.append_or_update_last(blknum, lsn, pv)?;
 
-        let vec_map = inner.index.entry(key).or_default();
-        let old = vec_map.append_or_update_last(lsn, off).unwrap().0;
         if old.is_some() {
             // We already had an entry for this LSN. That's odd..
-            warn!("Key {} at {} already exists", key, lsn);
+            warn!(
+                "Page version of rel {} blk {} at {} already exists",
+                self.seg.rel, blknum, lsn
+            );
         }
 
-        Ok(())
+        // Also update the relation size, if this extended the relation.
+        if self.seg.rel.is_blocky() {
+            let newsize = blknum + 1;
+
+            // use inner get_seg_size, since calling self.get_seg_size will try to acquire the lock,
+            // which we've just acquired above
+            let oldsize = inner.get_seg_size(lsn);
+            if newsize > oldsize {
+                trace!(
+                    "enlarging segment {} from {} to {} blocks at {}",
+                    self.seg,
+                    oldsize,
+                    newsize,
+                    lsn
+                );
+
+                // If we are extending the relation by more than one page, initialize the "gap"
+                // with zeros
+                //
+                // XXX: What if the caller initializes the gap with subsequent call with same LSN?
+                // I don't think that can happen currently, but that is highly dependent on how
+                // PostgreSQL writes its WAL records and there's no guarantee of it. If it does
+                // happen, we would hit the "page version already exists" warning above on the
+                // subsequent call to initialize the gap page.
+                for gapblknum in oldsize..blknum {
+                    let zeropv = PageVersion::Page(ZERO_PAGE.clone());
+                    trace!(
+                        "filling gap blk {} with zeros for write of {}",
+                        gapblknum,
+                        blknum
+                    );
+                    let old = inner
+                        .page_versions
+                        .append_or_update_last(gapblknum, lsn, zeropv)?;
+                    // We already had an entry for this LSN. That's odd..
+
+                    if old.is_some() {
+                        warn!(
+                            "Page version of seg {} blk {} at {} already exists",
+                            self.seg, blknum, lsn
+                        );
+                    }
+                }
+
+                inner.seg_sizes.append_or_update_last(lsn, newsize).unwrap();
+                return Ok(newsize - oldsize);
+            }
+        }
+
+        Ok(0)
     }
 
-    pub fn put_tombstone(&self, _key_range: Range<Key>, _lsn: Lsn) -> Result<()> {
-        // TODO: Currently, we just leak the storage for any deleted keys
+    /// Remember that the relation was truncated at given LSN
+    pub fn put_truncation(&self, lsn: Lsn, new_size: SegmentBlk) {
+        assert!(
+            self.seg.rel.is_blocky(),
+            "put_truncation() called on a non-blocky rel"
+        );
 
-        Ok(())
+        let mut inner = self.inner.write().unwrap();
+        inner.assert_writeable();
+
+        // check that this we truncate to a smaller size than segment was before the truncation
+        let old_size = inner.get_seg_size(lsn);
+        assert!(new_size < old_size);
+
+        let (old, _delta_size) = inner
+            .seg_sizes
+            .append_or_update_last(lsn, new_size)
+            .unwrap();
+
+        if old.is_some() {
+            // We already had an entry for this LSN. That's odd..
+            warn!("Inserting truncation, but had an entry for the LSN already");
+        }
+    }
+
+    /// Remember that the segment was dropped at given LSN
+    pub fn drop_segment(&self, lsn: Lsn) {
+        let mut inner = self.inner.write().unwrap();
+
+        assert!(inner.end_lsn.is_none());
+        assert!(!inner.dropped);
+        inner.dropped = true;
+        assert!(self.start_lsn < lsn);
+        inner.end_lsn = Some(lsn);
+
+        trace!("dropped segment {} at {}", self.seg, lsn);
+    }
+
+    ///
+    /// Initialize a new InMemoryLayer for, by copying the state at the given
+    /// point in time from given existing layer.
+    ///
+    pub fn create_successor_layer(
+        conf: &'static PageServerConf,
+        src: Arc<dyn Layer>,
+        timelineid: ZTimelineId,
+        tenantid: ZTenantId,
+        start_lsn: Lsn,
+        oldest_lsn: Lsn,
+    ) -> Result<InMemoryLayer> {
+        let seg = src.get_seg_tag();
+
+        assert!(oldest_lsn.is_aligned());
+
+        trace!(
+            "initializing new InMemoryLayer for writing {} on timeline {} at {}",
+            seg,
+            timelineid,
+            start_lsn,
+        );
+
+        // Copy the segment size at the start LSN from the predecessor layer.
+        let mut seg_sizes = VecMap::default();
+        if seg.rel.is_blocky() {
+            let size = src.get_seg_size(start_lsn)?;
+            seg_sizes.append(start_lsn, size).unwrap();
+        }
+
+        let file = EphemeralFile::create(conf, tenantid, timelineid)?;
+
+        Ok(InMemoryLayer {
+            conf,
+            timelineid,
+            tenantid,
+            seg,
+            start_lsn,
+            oldest_lsn,
+            incremental: true,
+            inner: RwLock::new(InMemoryLayerInner {
+                end_lsn: None,
+                dropped: false,
+                page_versions: PageVersions::new(file),
+                seg_sizes,
+                latest_lsn: oldest_lsn,
+            }),
+        })
+    }
+
+    pub fn is_writeable(&self) -> bool {
+        let inner = self.inner.read().unwrap();
+        inner.end_lsn.is_none()
     }
 
     /// Make the layer non-writeable. Only call once.
     /// Records the end_lsn for non-dropped layers.
-    /// `end_lsn` is exclusive
+    /// `end_lsn` is inclusive
     pub fn freeze(&self, end_lsn: Lsn) {
         let mut inner = self.inner.write().unwrap();
 
-        assert!(self.start_lsn < end_lsn);
-        inner.end_lsn = Some(end_lsn);
+        if inner.end_lsn.is_some() {
+            assert!(inner.dropped);
+        } else {
+            assert!(!inner.dropped);
+            assert!(self.start_lsn < end_lsn + 1);
+            inner.end_lsn = Some(Lsn(end_lsn.0 + 1));
 
-        // FIXME
-        /*
-                for perseg in inner.segs.values() {
-                    if let Some((lsn, _)) = perseg.seg_sizes.as_slice().last() {
-                        assert!(lsn < &end_lsn, "{:?} {:?}", lsn, end_lsn);
-                    }
+            if let Some((lsn, _)) = inner.seg_sizes.as_slice().last() {
+                assert!(lsn <= &end_lsn, "{:?} {:?}", lsn, end_lsn);
+            }
 
-                    for (_blk, vec_map) in perseg.page_versions.iter() {
-                        for (lsn, _pos) in vec_map.as_slice() {
-                            assert!(*lsn < end_lsn);
-                        }
-                    }
-                }
-        */
+            for (_blk, lsn, _pv) in inner.page_versions.ordered_page_version_iter(None) {
+                assert!(lsn <= end_lsn);
+            }
+        }
     }
 
-    /// Write this frozen in-memory layer to disk.
+    /// Write the this frozen in-memory layer to disk.
     ///
     /// Returns new layers that replace this one.
     /// If not dropped and reconstruct_pages is true, returns a new image layer containing the page versions
@@ -337,7 +613,17 @@ impl InMemoryLayer {
     /// WAL records between start and end LSN. (The delta layer is not needed
     /// when a new relish is created with a single LSN, so that the start and
     /// end LSN are the same.)
-    pub fn write_to_disk(&self) -> Result<DeltaLayer> {
+    pub fn write_to_disk(
+        &self,
+        timeline: &LayeredTimeline,
+        reconstruct_pages: bool,
+    ) -> Result<LayersOnDisk> {
+        trace!(
+            "write_to_disk {} get_end_lsn is {}",
+            self.filename().display(),
+            self.get_end_lsn()
+        );
+
         // Grab the lock in read-mode. We hold it over the I/O, but because this
         // layer is not writeable anymore, no one should be trying to acquire the
         // write lock on it, so we shouldn't block anyone. There's one exception
@@ -349,30 +635,101 @@ impl InMemoryLayer {
         // rare though, so we just accept the potential latency hit for now.
         let inner = self.inner.read().unwrap();
 
-        let mut delta_layer_writer = DeltaLayerWriter::new(
-            self.conf,
-            self.timelineid,
-            self.tenantid,
-            Key::MIN,
-            self.start_lsn..inner.end_lsn.unwrap(),
-        )?;
+        // Since `end_lsn` is exclusive, subtract 1 to calculate the last LSN
+        // that is included.
+        let end_lsn_exclusive = inner.end_lsn.unwrap();
+        let end_lsn_inclusive = Lsn(end_lsn_exclusive.0 - 1);
 
-        let mut do_steps = || -> Result<()> {
-            for (key, vec_map) in inner.index.iter() {
-                // Write all page versions
-                for (lsn, pos) in vec_map.as_slice() {
-                    let val = Value::des(&utils::read_blob(&inner.file, *pos)?)?;
-                    delta_layer_writer.put_value(*key, *lsn, val)?;
-                }
-            }
-            Ok(())
-        };
-        if let Err(err) = do_steps() {
-            delta_layer_writer.abort();
-            return Err(err);
+        // Figure out if we should create a delta layer, image layer, or both.
+        let image_lsn: Option<Lsn>;
+        let delta_end_lsn: Option<Lsn>;
+        if self.is_dropped() || !reconstruct_pages {
+            // The segment was dropped. Create just a delta layer containing all the
+            // changes up to and including the drop.
+            delta_end_lsn = Some(end_lsn_exclusive);
+            image_lsn = None;
+        } else if self.start_lsn == end_lsn_inclusive {
+            // The layer contains exactly one LSN. It's enough to write an image
+            // layer at that LSN.
+            delta_end_lsn = None;
+            image_lsn = Some(end_lsn_inclusive);
+        } else {
+            // Create a delta layer with all the changes up to the end LSN,
+            // and an image layer at the end LSN.
+            //
+            // Note that we the delta layer does *not* include the page versions
+            // at the end LSN. They are included in the image layer, and there's
+            // no need to store them twice.
+            delta_end_lsn = Some(end_lsn_inclusive);
+            image_lsn = Some(end_lsn_inclusive);
         }
 
-        let delta_layer = delta_layer_writer.finish(Key::MAX)?;
-        Ok(delta_layer)
+        let mut delta_layers = Vec::new();
+        let mut image_layers = Vec::new();
+
+        if let Some(delta_end_lsn) = delta_end_lsn {
+            let mut delta_layer_writer = DeltaLayerWriter::new(
+                self.conf,
+                self.timelineid,
+                self.tenantid,
+                self.seg,
+                self.start_lsn,
+                delta_end_lsn,
+                self.is_dropped(),
+            )?;
+
+            // Write all page versions
+            let mut buf: Vec<u8> = Vec::new();
+
+            let page_versions_iter = inner
+                .page_versions
+                .ordered_page_version_iter(Some(delta_end_lsn));
+            for (blknum, lsn, pos) in page_versions_iter {
+                let len = inner.page_versions.read_pv_bytes(pos, &mut buf)?;
+                delta_layer_writer.put_page_version(blknum, lsn, &buf[..len])?;
+            }
+
+            // Create seg_sizes
+            let seg_sizes = if delta_end_lsn == end_lsn_exclusive {
+                inner.seg_sizes.clone()
+            } else {
+                inner.seg_sizes.split_at(&end_lsn_exclusive).0
+            };
+
+            let delta_layer = delta_layer_writer.finish(seg_sizes)?;
+            delta_layers.push(delta_layer);
+        }
+
+        drop(inner);
+
+        // Write a new base image layer at the cutoff point
+        if let Some(image_lsn) = image_lsn {
+            let size = if self.seg.rel.is_blocky() {
+                self.get_seg_size(image_lsn)?
+            } else {
+                1
+            };
+            let mut image_layer_writer = ImageLayerWriter::new(
+                self.conf,
+                self.timelineid,
+                self.tenantid,
+                self.seg,
+                image_lsn,
+                size,
+            )?;
+
+            for blknum in 0..size {
+                let img = timeline.materialize_page(self.seg, blknum, image_lsn, &*self)?;
+
+                image_layer_writer.put_page_image(&img)?;
+            }
+            let image_layer = image_layer_writer.finish()?;
+            image_layers.push(image_layer);
+        }
+
+        Ok(LayersOnDisk {
+            delta_layers,
+            image_layers,
+        })
     }
 }

pageserver/src/layered_repository/interval_tree.rs

@@ -0,0 +1,468 @@
+///
+/// IntervalTree is data structure for holding intervals. It is generic
+/// to make unit testing possible, but the only real user of it is the layer map,
+///
+/// It's inspired by the "segment tree" or a "statistic tree" as described in
+/// https://en.wikipedia.org/wiki/Segment_tree. However, we use a B-tree to hold
+/// the points instead of a binary tree. This is called an "interval tree" instead
+/// of "segment tree" because the term "segment" is already using Zenith to mean
+/// something else. To add to the confusion, there is another data structure known
+/// as "interval tree" out there (see https://en.wikipedia.org/wiki/Interval_tree),
+/// for storing intervals, but this isn't that.
+///
+/// The basic idea is to have a B-tree of "interesting Points". At each Point,
+/// there is a list of intervals that contain the point. The Points are formed
+/// from the start bounds of each interval; there is a Point for each distinct
+/// start bound.
+///
+/// Operations:
+///
+/// To find intervals that contain a given point, you search the b-tree to find
+/// the nearest Point <= search key. Then you just return the list of intervals.
+///
+/// To insert an interval, find the Point with start key equal to the inserted item.
+/// If the Point doesn't exist yet, create it, by copying all the items from the
+/// previous Point that cover the new Point. Then walk right, inserting the new
+/// interval to all the Points that are contained by the new interval (including the
+/// newly created Point).
+///
+/// To remove an interval, you scan the tree for all the Points that are contained by
+/// the removed interval, and remove it from the list in each Point.
+///
+/// Requirements and assumptions:
+///
+/// - Can store overlapping items
+/// - But there are not many overlapping items
+/// - The interval bounds don't change after it is added to the tree
+/// - Intervals are uniquely identified by pointer equality. You must not be insert the
+///   same interval object twice, and `remove` uses pointer equality to remove the right
+///   interval. It is OK to have two intervals with the same bounds, however.
+///
+use std::collections::BTreeMap;
+use std::fmt::Debug;
+use std::ops::Range;
+use std::sync::Arc;
+
+pub struct IntervalTree<I: ?Sized>
+where
+    I: IntervalItem,
+{
+    points: BTreeMap<I::Key, Point<I>>,
+}
+
+struct Point<I: ?Sized> {
+    /// All intervals that contain this point, in no particular order.
+    ///
+    /// We assume that there aren't a lot of overlappingg intervals, so that this vector
+    /// never grows very large. If that assumption doesn't hold, we could keep this ordered
+    /// by the end bound, to speed up `search`. But as long as there are only a few elements,
+    /// a linear search is OK.
+    elements: Vec<Arc<I>>,
+}
+
+/// Abstraction for an interval that can be stored in the tree
+///
+/// The start bound is inclusive and the end bound is exclusive. End must be greater
+/// than start.
+pub trait IntervalItem {
+    type Key: Ord + Copy + Debug + Sized;
+
+    fn start_key(&self) -> Self::Key;
+    fn end_key(&self) -> Self::Key;
+
+    fn bounds(&self) -> Range<Self::Key> {
+        self.start_key()..self.end_key()
+    }
+}
+
+impl<I: ?Sized> IntervalTree<I>
+where
+    I: IntervalItem,
+{
+    /// Return an element that contains 'key', or precedes it.
+    ///
+    /// If there are multiple candidates, returns the one with the highest 'end' key.
+    pub fn search(&self, key: I::Key) -> Option<Arc<I>> {
+        // Find the greatest point that precedes or is equal to the search key. If there is
+        // none, returns None.
+        let (_, p) = self.points.range(..=key).next_back()?;
+
+        // Find the element with the highest end key at this point
+        let highest_item = p
+            .elements
+            .iter()
+            .reduce(|a, b| {
+                // starting with Rust 1.53, could use `std::cmp::min_by_key` here
+                if a.end_key() > b.end_key() {
+                    a
+                } else {
+                    b
+                }
+            })
+            .unwrap();
+        Some(Arc::clone(highest_item))
+    }
+
+    /// Iterate over all items with start bound >= 'key'
+    pub fn iter_newer(&self, key: I::Key) -> IntervalIter<I> {
+        IntervalIter {
+            point_iter: self.points.range(key..),
+            elem_iter: None,
+        }
+    }
+
+    /// Iterate over all items
+    pub fn iter(&self) -> IntervalIter<I> {
+        IntervalIter {
+            point_iter: self.points.range(..),
+            elem_iter: None,
+        }
+    }
+
+    pub fn insert(&mut self, item: Arc<I>) {
+        let start_key = item.start_key();
+        let end_key = item.end_key();
+        assert!(start_key < end_key);
+        let bounds = start_key..end_key;
+
+        // Find the starting point and walk forward from there
+        let mut found_start_point = false;
+        let iter = self.points.range_mut(bounds);
+        for (point_key, point) in iter {
+            if *point_key == start_key {
+                found_start_point = true;
+                // It is an error to insert the same item to the tree twice.
+                assert!(
+                    !point.elements.iter().any(|x| Arc::ptr_eq(x, &item)),
+                    "interval is already in the tree"
+                );
+            }
+            point.elements.push(Arc::clone(&item));
+        }
+        if !found_start_point {
+            // Create a new Point for the starting point
+
+            // Look at the previous point, and copy over elements that overlap with this
+            // new point
+            let mut new_elements: Vec<Arc<I>> = Vec::new();
+            if let Some((_, prev_point)) = self.points.range(..start_key).next_back() {
+                let overlapping_prev_elements = prev_point
+                    .elements
+                    .iter()
+                    .filter(|x| x.bounds().contains(&start_key))
+                    .cloned();
+
+                new_elements.extend(overlapping_prev_elements);
+            }
+            new_elements.push(item);
+
+            let new_point = Point {
+                elements: new_elements,
+            };
+            self.points.insert(start_key, new_point);
+        }
+    }
+
+    pub fn remove(&mut self, item: &Arc<I>) {
+        // range search points
+        let start_key = item.start_key();
+        let end_key = item.end_key();
+        let bounds = start_key..end_key;
+
+        let mut points_to_remove: Vec<I::Key> = Vec::new();
+        let mut found_start_point = false;
+        for (point_key, point) in self.points.range_mut(bounds) {
+            if *point_key == start_key {
+                found_start_point = true;
+            }
+            let len_before = point.elements.len();
+            point.elements.retain(|other| !Arc::ptr_eq(other, item));
+            let len_after = point.elements.len();
+            assert_eq!(len_after + 1, len_before);
+            if len_after == 0 {
+                points_to_remove.push(*point_key);
+            }
+        }
+        assert!(found_start_point);
+
+        for k in points_to_remove {
+            self.points.remove(&k).unwrap();
+        }
+    }
+}
+
+pub struct IntervalIter<'a, I: ?Sized>
+where
+    I: IntervalItem,
+{
+    point_iter: std::collections::btree_map::Range<'a, I::Key, Point<I>>,
+    elem_iter: Option<(I::Key, std::slice::Iter<'a, Arc<I>>)>,
+}
+
+impl<'a, I> Iterator for IntervalIter<'a, I>
+where
+    I: IntervalItem + ?Sized,
+{
+    type Item = Arc<I>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // Iterate over all elements in all the points in 'point_iter'. To avoid
+        // returning the same element twice, we only return each element at its
+        // starting point.
+        loop {
+            // Return next remaining element from the current point
+            if let Some((point_key, elem_iter)) = &mut self.elem_iter {
+                for elem in elem_iter {
+                    if elem.start_key() == *point_key {
+                        return Some(Arc::clone(elem));
+                    }
+                }
+            }
+            // No more elements at this point. Move to next point.
+            if let Some((point_key, point)) = self.point_iter.next() {
+                self.elem_iter = Some((*point_key, point.elements.iter()));
+                continue;
+            } else {
+                // No more points, all done
+                return None;
+            }
+        }
+    }
+}
+
+impl<I: ?Sized> Default for IntervalTree<I>
+where
+    I: IntervalItem,
+{
+    fn default() -> Self {
+        IntervalTree {
+            points: BTreeMap::new(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fmt;
+
+    #[derive(Debug)]
+    struct MockItem {
+        start_key: u32,
+        end_key: u32,
+        val: String,
+    }
+    impl IntervalItem for MockItem {
+        type Key = u32;
+
+        fn start_key(&self) -> u32 {
+            self.start_key
+        }
+        fn end_key(&self) -> u32 {
+            self.end_key
+        }
+    }
+    impl MockItem {
+        fn new(start_key: u32, end_key: u32) -> Self {
+            MockItem {
+                start_key,
+                end_key,
+                val: format!("{}-{}", start_key, end_key),
+            }
+        }
+        fn new_str(start_key: u32, end_key: u32, val: &str) -> Self {
+            MockItem {
+                start_key,
+                end_key,
+                val: format!("{}-{}: {}", start_key, end_key, val),
+            }
+        }
+    }
+    impl fmt::Display for MockItem {
+        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            write!(f, "{}", self.val)
+        }
+    }
+    #[rustfmt::skip]
+    fn assert_search(
+        tree: &IntervalTree<MockItem>,
+        key: u32,
+        expected: &[&str],
+    ) -> Option<Arc<MockItem>> {
+        if let Some(v) = tree.search(key) {
+            let vstr = v.to_string();
+
+            assert!(!expected.is_empty(), "search with {} returned {}, expected None", key, v);
+            assert!(
+                expected.contains(&vstr.as_str()),
+                "search with {} returned {}, expected one of: {:?}",
+                key, v, expected,
+            );
+
+            Some(v)
+        } else {
+            assert!(
+                expected.is_empty(),
+                "search with {} returned None, expected one of {:?}",
+                key, expected
+            );
+            None
+        }
+    }
+
+    fn assert_contents(tree: &IntervalTree<MockItem>, expected: &[&str]) {
+        let mut contents: Vec<String> = tree.iter().map(|e| e.to_string()).collect();
+        contents.sort();
+        assert_eq!(contents, expected);
+    }
+
+    fn dump_tree(tree: &IntervalTree<MockItem>) {
+        for (point_key, point) in tree.points.iter() {
+            print!("{}:", point_key);
+            for e in point.elements.iter() {
+                print!(" {}", e);
+            }
+            println!();
+        }
+    }
+
+    #[test]
+    fn test_interval_tree_simple() {
+        let mut tree: IntervalTree<MockItem> = IntervalTree::default();
+
+        // Simple, non-overlapping ranges.
+        tree.insert(Arc::new(MockItem::new(10, 11)));
+        tree.insert(Arc::new(MockItem::new(11, 12)));
+        tree.insert(Arc::new(MockItem::new(12, 13)));
+        tree.insert(Arc::new(MockItem::new(18, 19)));
+        tree.insert(Arc::new(MockItem::new(17, 18)));
+        tree.insert(Arc::new(MockItem::new(15, 16)));
+
+        assert_search(&tree, 9, &[]);
+        assert_search(&tree, 10, &["10-11"]);
+        assert_search(&tree, 11, &["11-12"]);
+        assert_search(&tree, 12, &["12-13"]);
+        assert_search(&tree, 13, &["12-13"]);
+        assert_search(&tree, 14, &["12-13"]);
+        assert_search(&tree, 15, &["15-16"]);
+        assert_search(&tree, 16, &["15-16"]);
+        assert_search(&tree, 17, &["17-18"]);
+        assert_search(&tree, 18, &["18-19"]);
+        assert_search(&tree, 19, &["18-19"]);
+        assert_search(&tree, 20, &["18-19"]);
+
+        // remove a few entries and search around them again
+        tree.remove(&assert_search(&tree, 10, &["10-11"]).unwrap()); // first entry
+        tree.remove(&assert_search(&tree, 12, &["12-13"]).unwrap()); // entry in the middle
+        tree.remove(&assert_search(&tree, 18, &["18-19"]).unwrap()); // last entry
+        assert_search(&tree, 9, &[]);
+        assert_search(&tree, 10, &[]);
+        assert_search(&tree, 11, &["11-12"]);
+        assert_search(&tree, 12, &["11-12"]);
+        assert_search(&tree, 14, &["11-12"]);
+        assert_search(&tree, 15, &["15-16"]);
+        assert_search(&tree, 17, &["17-18"]);
+        assert_search(&tree, 18, &["17-18"]);
+    }
+
+    #[test]
+    fn test_interval_tree_overlap() {
+        let mut tree: IntervalTree<MockItem> = IntervalTree::default();
+
+        // Overlapping items
+        tree.insert(Arc::new(MockItem::new(22, 24)));
+        tree.insert(Arc::new(MockItem::new(23, 25)));
+        let x24_26 = Arc::new(MockItem::new(24, 26));
+        tree.insert(Arc::clone(&x24_26));
+        let x26_28 = Arc::new(MockItem::new(26, 28));
+        tree.insert(Arc::clone(&x26_28));
+        tree.insert(Arc::new(MockItem::new(25, 27)));
+
+        assert_search(&tree, 22, &["22-24"]);
+        assert_search(&tree, 23, &["22-24", "23-25"]);
+        assert_search(&tree, 24, &["23-25", "24-26"]);
+        assert_search(&tree, 25, &["24-26", "25-27"]);
+        assert_search(&tree, 26, &["25-27", "26-28"]);
+        assert_search(&tree, 27, &["26-28"]);
+        assert_search(&tree, 28, &["26-28"]);
+        assert_search(&tree, 29, &["26-28"]);
+
+        tree.remove(&x24_26);
+        tree.remove(&x26_28);
+        assert_search(&tree, 23, &["22-24", "23-25"]);
+        assert_search(&tree, 24, &["23-25"]);
+        assert_search(&tree, 25, &["25-27"]);
+        assert_search(&tree, 26, &["25-27"]);
+        assert_search(&tree, 27, &["25-27"]);
+        assert_search(&tree, 28, &["25-27"]);
+        assert_search(&tree, 29, &["25-27"]);
+    }
+
+    #[test]
+    fn test_interval_tree_nested() {
+        let mut tree: IntervalTree<MockItem> = IntervalTree::default();
+
+        // Items containing other items
+        tree.insert(Arc::new(MockItem::new(31, 39)));
+        tree.insert(Arc::new(MockItem::new(32, 34)));
+        tree.insert(Arc::new(MockItem::new(33, 35)));
+        tree.insert(Arc::new(MockItem::new(30, 40)));
+
+        assert_search(&tree, 30, &["30-40"]);
+        assert_search(&tree, 31, &["30-40", "31-39"]);
+        assert_search(&tree, 32, &["30-40", "32-34", "31-39"]);
+        assert_search(&tree, 33, &["30-40", "32-34", "33-35", "31-39"]);
+        assert_search(&tree, 34, &["30-40", "33-35", "31-39"]);
+        assert_search(&tree, 35, &["30-40", "31-39"]);
+        assert_search(&tree, 36, &["30-40", "31-39"]);
+        assert_search(&tree, 37, &["30-40", "31-39"]);
+        assert_search(&tree, 38, &["30-40", "31-39"]);
+        assert_search(&tree, 39, &["30-40"]);
+        assert_search(&tree, 40, &["30-40"]);
+        assert_search(&tree, 41, &["30-40"]);
+    }
+
+    #[test]
+    fn test_interval_tree_duplicates() {
+        let mut tree: IntervalTree<MockItem> = IntervalTree::default();
+
+        // Duplicate keys
+        let item_a = Arc::new(MockItem::new_str(55, 56, "a"));
+        tree.insert(Arc::clone(&item_a));
+        let item_b = Arc::new(MockItem::new_str(55, 56, "b"));
+        tree.insert(Arc::clone(&item_b));
+        let item_c = Arc::new(MockItem::new_str(55, 56, "c"));
+        tree.insert(Arc::clone(&item_c));
+        let item_d = Arc::new(MockItem::new_str(54, 56, "d"));
+        tree.insert(Arc::clone(&item_d));
+        let item_e = Arc::new(MockItem::new_str(55, 57, "e"));
+        tree.insert(Arc::clone(&item_e));
+
+        dump_tree(&tree);
+
+        assert_search(
+            &tree,
+            55,
+            &["55-56: a", "55-56: b", "55-56: c", "54-56: d", "55-57: e"],
+        );
+        tree.remove(&item_b);
+        dump_tree(&tree);
+
+        assert_contents(&tree, &["54-56: d", "55-56: a", "55-56: c", "55-57: e"]);
+
+        tree.remove(&item_d);
+        dump_tree(&tree);
+        assert_contents(&tree, &["55-56: a", "55-56: c", "55-57: e"]);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_interval_tree_insert_twice() {
+        let mut tree: IntervalTree<MockItem> = IntervalTree::default();
+
+        // Inserting the same item twice is not cool
+        let item = Arc::new(MockItem::new(1, 2));
+        tree.insert(Arc::clone(&item));
+        tree.insert(Arc::clone(&item)); // fails assertion
+    }
+}

pageserver/src/layered_repository/layer_map.rs

@@ -3,26 +3,30 @@
 //!
 //! When the timeline is first accessed, the server lists of all layer files
 //! in the timelines/<timelineid> directory, and populates this map with
-//! ImageLayer and DeltaLayer structs corresponding to each file. When the first
-//! new WAL record is received, we create an InMemoryLayer to hold the incoming
-//! records. Now and then, in the checkpoint() function, the in-memory layer is
-//! are frozen, and it is split up into new image and delta layers and the
-//! corresponding files are written to disk.
+//! ImageLayer and DeltaLayer structs corresponding to each file. When new WAL
+//! is received, we create InMemoryLayers to hold the incoming records. Now and
+//! then, in the checkpoint() function, the in-memory layers are frozen, forming
+//! new image and delta layers and corresponding files are written to disk.
 //!
 
-use crate::layered_repository::storage_layer::Layer;
-use crate::layered_repository::storage_layer::{range_eq, range_overlaps};
+use crate::layered_repository::interval_tree::{IntervalItem, IntervalIter, IntervalTree};
+use crate::layered_repository::storage_layer::{Layer, SegmentTag};
 use crate::layered_repository::InMemoryLayer;
-use crate::repository::Key;
+use crate::relish::*;
 use anyhow::Result;
 use lazy_static::lazy_static;
-use std::ops::Range;
+use std::cmp::Ordering;
+use std::collections::{BinaryHeap, HashMap};
 use std::sync::Arc;
-use tracing::*;
 use zenith_metrics::{register_int_gauge, IntGauge};
 use zenith_utils::lsn::Lsn;
 
+use super::global_layer_map::{LayerId, GLOBAL_LAYER_MAP};
+
 lazy_static! {
+    static ref NUM_INMEMORY_LAYERS: IntGauge =
+        register_int_gauge!("pageserver_inmemory_layers", "Number of layers in memory")
+            .expect("failed to define a metric");
     static ref NUM_ONDISK_LAYERS: IntGauge =
         register_int_gauge!("pageserver_ondisk_layers", "Number of layers on-disk")
             .expect("failed to define a metric");
@@ -33,135 +37,98 @@ lazy_static! {
 ///
 #[derive(Default)]
 pub struct LayerMap {
-    //
-    // 'open_layer' holds the current InMemoryLayer that is accepting new
-    // records. If it is None, 'next_open_layer_at' will be set instead, indicating
-    // where the start LSN of the next InMemoryLayer that is to be created.
-    //
-    pub open_layer: Option<Arc<InMemoryLayer>>,
-    pub next_open_layer_at: Option<Lsn>,
+    /// All the layers keyed by segment tag
+    segs: HashMap<SegmentTag, SegEntry>,
 
-    ///
-    /// The frozen layer, if any, contains WAL older than the current 'open_layer'
-    /// or 'next_open_layer_at', but newer than any historic layer. The frozen
-    /// layer is during checkpointing, when an InMemoryLayer is being written out
-    /// to disk.
-    ///
-    pub frozen_layer: Option<Arc<InMemoryLayer>>,
+    /// All in-memory layers, ordered by 'oldest_lsn' and generation
+    /// of each layer. This allows easy access to the in-memory layer that
+    /// contains the oldest WAL record.
+    open_layers: BinaryHeap<OpenLayerEntry>,
 
-    /// All the historic layers are kept here
-
-    /// TODO: This is a placeholder implementation of a data structure
-    /// to hold information about all the layer files on disk and in
-    /// S3. Currently, it's just a vector and all operations perform a
-    /// linear scan over it.  That obviously becomes slow as the
-    /// number of layers grows. I'm imagining that an R-tree or some
-    /// other 2D data structure would be the long-term solution here.
-    historic_layers: Vec<Arc<dyn Layer>>,
-}
-
-pub struct SearchResult {
-    pub layer: Arc<dyn Layer>,
-    pub lsn_floor: Lsn,
+    /// Generation number, used to distinguish newly inserted entries in the
+    /// binary heap from older entries during checkpoint.
+    current_generation: u64,
 }
 
 impl LayerMap {
-    pub fn search(&self, key: Key, end_lsn: Lsn) -> Result<Option<SearchResult>> {
-        // linear search
-        // Find the latest image layer that covers the given key
-        let mut latest_img: Option<Arc<dyn Layer>> = None;
-        let mut latest_img_lsn: Option<Lsn> = None;
-        for l in self.historic_layers.iter() {
-            if l.is_incremental() {
-                continue;
-            }
-            if !l.get_key_range().contains(&key) {
-                continue;
-            }
-            let img_lsn = l.get_lsn_range().start;
+    ///
+    /// Look up a layer using the given segment tag and LSN. This differs from a
+    /// plain key-value lookup in that if there is any layer that covers the
+    /// given LSN, or precedes the given LSN, it is returned. In other words,
+    /// you don't need to know the exact start LSN of the layer.
+    ///
+    pub fn get(&self, tag: &SegmentTag, lsn: Lsn) -> Option<Arc<dyn Layer>> {
+        let segentry = self.segs.get(tag)?;
 
-            if img_lsn >= end_lsn {
-                // too new
-                continue;
-            }
-            if Lsn(img_lsn.0 + 1) == end_lsn {
-                // found exact match
-                return Ok(Some(SearchResult {
-                    layer: Arc::clone(l),
-                    lsn_floor: img_lsn,
-                }));
-            }
-            if img_lsn > latest_img_lsn.unwrap_or(Lsn(0)) {
-                latest_img = Some(Arc::clone(l));
-                latest_img_lsn = Some(img_lsn);
-            }
-        }
+        segentry.get(lsn)
+    }
 
-        // Search the delta layers
-        let mut latest_delta: Option<Arc<dyn Layer>> = None;
-        for l in self.historic_layers.iter() {
-            if !l.is_incremental() {
-                continue;
-            }
-            if !l.get_key_range().contains(&key) {
-                continue;
-            }
+    ///
+    /// Get the open layer for given segment for writing. Or None if no open
+    /// layer exists.
+    ///
+    pub fn get_open(&self, tag: &SegmentTag) -> Option<Arc<InMemoryLayer>> {
+        let segentry = self.segs.get(tag)?;
 
-            if l.get_lsn_range().start >= end_lsn {
-                // too new
-                continue;
-            }
+        segentry
+            .open_layer_id
+            .and_then(|layer_id| GLOBAL_LAYER_MAP.read().unwrap().get(&layer_id))
+    }
 
-            if l.get_lsn_range().end >= end_lsn {
-                // this layer contains the requested point in the key/lsn space.
-                // No need to search any further
-                trace!(
-                    "found layer {} for request on {} at {}",
-                    l.filename().display(),
-                    key,
-                    end_lsn
-                );
-                latest_delta.replace(Arc::clone(l));
-                break;
-            }
-            // this layer's end LSN is smaller than the requested point. If there's
-            // nothing newer, this is what we need to return. Remember this.
-            if let Some(ref old_candidate) = latest_delta {
-                if l.get_lsn_range().end > old_candidate.get_lsn_range().end {
-                    latest_delta.replace(Arc::clone(l));
-                }
+    ///
+    /// Insert an open in-memory layer
+    ///
+    pub fn insert_open(&mut self, layer: Arc<InMemoryLayer>) {
+        let segentry = self.segs.entry(layer.get_seg_tag()).or_default();
+
+        let layer_id = segentry.update_open(Arc::clone(&layer));
+
+        let oldest_lsn = layer.get_oldest_lsn();
+
+        // After a crash and restart, 'oldest_lsn' of the oldest in-memory
+        // layer becomes the WAL streaming starting point, so it better not point
+        // in the middle of a WAL record.
+        assert!(oldest_lsn.is_aligned());
+
+        // Also add it to the binary heap
+        let open_layer_entry = OpenLayerEntry {
+            oldest_lsn: layer.get_oldest_lsn(),
+            layer_id,
+            generation: self.current_generation,
+        };
+        self.open_layers.push(open_layer_entry);
+
+        NUM_INMEMORY_LAYERS.inc();
+    }
+
+    /// Remove an open in-memory layer
+    pub fn remove_open(&mut self, layer_id: LayerId) {
+        // Note: we don't try to remove the entry from the binary heap.
+        // It will be removed lazily by peek_oldest_open() when it's made it to
+        // the top of the heap.
+
+        let layer_opt = {
+            let mut global_map = GLOBAL_LAYER_MAP.write().unwrap();
+            let layer_opt = global_map.get(&layer_id);
+            global_map.remove(&layer_id);
+            // TODO it's bad that a ref can still exist after being evicted from cache
+            layer_opt
+        };
+
+        if let Some(layer) = layer_opt {
+            let mut segentry = self.segs.get_mut(&layer.get_seg_tag()).unwrap();
+
+            if segentry.open_layer_id == Some(layer_id) {
+                // Also remove it from the SegEntry of this segment
+                segentry.open_layer_id = None;
             } else {
-                latest_delta.replace(Arc::clone(l));
+                // We could have already updated segentry.open for
+                // dropped (non-writeable) layer. This is fine.
+                assert!(!layer.is_writeable());
+                assert!(layer.is_dropped());
             }
-        }
-        if let Some(l) = latest_delta {
-            trace!(
-                "found (old) layer {} for request on {} at {}",
-                l.filename().display(),
-                key,
-                end_lsn
-            );
-            let lsn_floor = std::cmp::max(
-                Lsn(latest_img_lsn.unwrap_or(Lsn(0)).0 + 1),
-                l.get_lsn_range().start,
-            );
-            Ok(Some(SearchResult {
-                lsn_floor,
-                layer: l,
-            }))
-        } else if let Some(l) = latest_img {
-            trace!(
-                "found img layer and no deltas for request on {} at {}",
-                key,
-                end_lsn
-            );
-            Ok(Some(SearchResult {
-                lsn_floor: latest_img_lsn.unwrap(),
-                layer: l,
-            }))
-        } else {
-            trace!("no layer found for request on {} at {}", key, end_lsn);
-            Ok(None)
+
+            NUM_INMEMORY_LAYERS.dec();
         }
     }
 
@@ -169,7 +136,9 @@ impl LayerMap {
     /// Insert an on-disk layer
     ///
     pub fn insert_historic(&mut self, layer: Arc<dyn Layer>) {
-        self.historic_layers.push(layer);
+        let segentry = self.segs.entry(layer.get_seg_tag()).or_default();
+        segentry.insert_historic(layer);
+
         NUM_ONDISK_LAYERS.inc();
     }
 
@@ -178,62 +147,61 @@ impl LayerMap {
     ///
     /// This should be called when the corresponding file on disk has been deleted.
     ///
-    #[allow(dead_code)]
     pub fn remove_historic(&mut self, layer: Arc<dyn Layer>) {
-        let len_before = self.historic_layers.len();
+        let tag = layer.get_seg_tag();
 
-        // FIXME: ptr_eq might fail to return true for 'dyn'
-        // references.  Clippy complains about this. In practice it
-        // seems to work, the assertion below would be triggered
-        // otherwise but this ought to be fixed.
-        #[allow(clippy::vtable_address_comparisons)]
-        self.historic_layers
-            .retain(|other| !Arc::ptr_eq(other, &layer));
-
-        assert_eq!(self.historic_layers.len(), len_before - 1);
+        if let Some(segentry) = self.segs.get_mut(&tag) {
+            segentry.historic.remove(&layer);
+        }
         NUM_ONDISK_LAYERS.dec();
     }
 
+    // List relations along with a flag that marks if they exist at the given lsn.
+    // spcnode 0 and dbnode 0 have special meanings and mean all tabespaces/databases.
+    // Pass Tag if we're only interested in some relations.
+    pub fn list_relishes(&self, tag: Option<RelTag>, lsn: Lsn) -> Result<HashMap<RelishTag, bool>> {
+        let mut rels: HashMap<RelishTag, bool> = HashMap::new();
+
+        for (seg, segentry) in self.segs.iter() {
+            match seg.rel {
+                RelishTag::Relation(reltag) => {
+                    if let Some(request_rel) = tag {
+                        if (request_rel.spcnode == 0 || reltag.spcnode == request_rel.spcnode)
+                            && (request_rel.dbnode == 0 || reltag.dbnode == request_rel.dbnode)
+                        {
+                            if let Some(exists) = segentry.exists_at_lsn(lsn)? {
+                                rels.insert(seg.rel, exists);
+                            }
+                        }
+                    }
+                }
+                _ => {
+                    if tag == None {
+                        if let Some(exists) = segentry.exists_at_lsn(lsn)? {
+                            rels.insert(seg.rel, exists);
+                        }
+                    }
+                }
+            }
+        }
+        Ok(rels)
+    }
+
     /// Is there a newer image layer for given segment?
     ///
     /// This is used for garbage collection, to determine if an old layer can
     /// be deleted.
     /// We ignore segments newer than disk_consistent_lsn because they will be removed at restart
-    /// We also only look at historic layers
-    //#[allow(dead_code)]
     pub fn newer_image_layer_exists(
         &self,
-        key_range: &Range<Key>,
+        seg: SegmentTag,
         lsn: Lsn,
         disk_consistent_lsn: Lsn,
-    ) -> Result<bool> {
-        let mut range_remain = key_range.clone();
-
-        loop {
-            let mut made_progress = false;
-            for l in self.historic_layers.iter() {
-                if l.is_incremental() {
-                    continue;
-                }
-                let img_lsn = l.get_lsn_range().start;
-                if !l.is_incremental()
-                    && l.get_key_range().contains(&range_remain.start)
-                    && img_lsn > lsn
-                    && img_lsn < disk_consistent_lsn
-                {
-                    made_progress = true;
-                    let img_key_end = l.get_key_range().end;
-
-                    if img_key_end >= range_remain.end {
-                        return Ok(true);
-                    }
-                    range_remain.start = img_key_end;
-                }
-            }
-
-            if !made_progress {
-                return Ok(false);
-            }
+    ) -> bool {
+        if let Some(segentry) = self.segs.get(&seg) {
+            segentry.newer_image_layer_exists(lsn, disk_consistent_lsn)
+        } else {
+            false
         }
     }
 
@@ -243,144 +211,284 @@ impl LayerMap {
     /// used for garbage collection, to determine if some alive layer
     /// exists at the lsn. If so, we shouldn't delete a newer dropped layer
     /// to avoid incorrectly making it visible.
-    /*
-        pub fn layer_exists_at_lsn(&self, seg: SegmentTag, lsn: Lsn) -> Result<bool> {
-            Ok(if let Some(segentry) = self.historic_layers.get(&seg) {
-                segentry.exists_at_lsn(seg, lsn)?.unwrap_or(false)
+    pub fn layer_exists_at_lsn(&self, seg: SegmentTag, lsn: Lsn) -> Result<bool> {
+        Ok(if let Some(segentry) = self.segs.get(&seg) {
+            segentry.exists_at_lsn(lsn)?.unwrap_or(false)
+        } else {
+            false
+        })
+    }
+
+    /// Return the oldest in-memory layer, along with its generation number.
+    pub fn peek_oldest_open(&mut self) -> Option<(LayerId, Arc<InMemoryLayer>, u64)> {
+        let global_map = GLOBAL_LAYER_MAP.read().unwrap();
+
+        while let Some(oldest_entry) = self.open_layers.peek() {
+            if let Some(layer) = global_map.get(&oldest_entry.layer_id) {
+                return Some((oldest_entry.layer_id, layer, oldest_entry.generation));
             } else {
-                false
-            })
+                self.open_layers.pop();
+            }
         }
-    */
-
-    pub fn iter_historic_layers(&self) -> std::slice::Iter<Arc<dyn Layer>> {
-        self.historic_layers.iter()
+        None
     }
 
-    fn find_latest_image(&self, key: Key, lsn: Lsn) -> Option<Arc<dyn Layer>> {
-        // Find the last image layer that covers the key
-        let mut candidate_lsn = Lsn(0);
-        let mut candidate = None;
-        for l in self.historic_layers.iter() {
-            if l.is_incremental() {
-                continue;
-            }
-
-            if !l.get_key_range().contains(&key) {
-                continue;
-            }
-
-            let this_lsn = l.get_lsn_range().start;
-            if this_lsn > lsn {
-                continue;
-            }
-            if this_lsn < candidate_lsn {
-                // our previous candidate was better
-                continue;
-            }
-            candidate_lsn = this_lsn;
-            candidate = Some(Arc::clone(l));
-        }
-
-        candidate
+    /// Increment the generation number used to stamp open in-memory layers. Layers
+    /// added with `insert_open` after this call will be associated with the new
+    /// generation. Returns the new generation number.
+    pub fn increment_generation(&mut self) -> u64 {
+        self.current_generation += 1;
+        self.current_generation
     }
 
-    ///
-    /// Divide the whole given range of keys into sub-ranges based on the latest
-    /// image layer that covers each range. (This is used when creating  new
-    /// image layers)
-    ///
-    // FIXME: clippy complains that the result type is very complex. She's probably
-    // right...
-    #[allow(clippy::type_complexity)]
-    pub fn image_coverage(
-        &self,
-        key_range: &Range<Key>,
-        lsn: Lsn,
-    ) -> Result<Vec<(Range<Key>, Option<Arc<dyn Layer>>)>> {
-        let mut points: Vec<Key>;
-
-        points = vec![key_range.start];
-        for l in self.historic_layers.iter() {
-            if l.get_lsn_range().start > lsn {
-                continue;
-            }
-            let range = l.get_key_range();
-            if key_range.contains(&range.start) {
-                points.push(l.get_key_range().start);
-            }
-            if key_range.contains(&range.end) {
-                points.push(l.get_key_range().end);
-            }
+    pub fn iter_historic_layers(&self) -> HistoricLayerIter {
+        HistoricLayerIter {
+            seg_iter: self.segs.iter(),
+            iter: None,
         }
-        points.push(key_range.end);
-
-        points.sort();
-        points.dedup();
-
-        // Ok, we now have a list of "interesting" points in the key space
-
-        // For each range between the points, find the latest image
-        let mut start = *points.first().unwrap();
-        let mut ranges = Vec::new();
-        for end in points[1..].iter() {
-            let img = self.find_latest_image(start, lsn);
-
-            ranges.push((start..*end, img));
-
-            start = *end;
-        }
-        Ok(ranges)
-    }
-
-    pub fn count_deltas(&self, key_range: &Range<Key>, lsn_range: &Range<Lsn>) -> Result<usize> {
-        let mut result = 0;
-        for l in self.historic_layers.iter() {
-            if !l.is_incremental() {
-                continue;
-            }
-            if !range_overlaps(&l.get_lsn_range(), lsn_range) {
-                continue;
-            }
-            if !range_overlaps(&l.get_key_range(), key_range) {
-                continue;
-            }
-
-            // We ignore level0 delta layers. Unless the whole keyspace fits
-            // into one partition
-            if !range_eq(key_range, &(Key::MIN..Key::MAX))
-                && range_eq(&l.get_key_range(), &(Key::MIN..Key::MAX))
-            {
-                continue;
-            }
-
-            result += 1;
-        }
-        Ok(result)
-    }
-
-    pub fn get_level0_deltas(&self) -> Result<Vec<Arc<dyn Layer>>> {
-        let mut deltas = Vec::new();
-        for l in self.historic_layers.iter() {
-            if !l.is_incremental() {
-                continue;
-            }
-            if l.get_key_range() != (Key::MIN..Key::MAX) {
-                continue;
-            }
-            deltas.push(Arc::clone(l));
-        }
-        Ok(deltas)
     }
 
     /// debugging function to print out the contents of the layer map
     #[allow(unused)]
     pub fn dump(&self) -> Result<()> {
         println!("Begin dump LayerMap");
-        for layer in self.historic_layers.iter() {
-            layer.dump()?;
+        for (seg, segentry) in self.segs.iter() {
+            if let Some(open) = &segentry.open_layer_id {
+                if let Some(layer) = GLOBAL_LAYER_MAP.read().unwrap().get(open) {
+                    layer.dump()?;
+                } else {
+                    println!("layer not found in global map");
+                }
+            }
+
+            for layer in segentry.historic.iter() {
+                layer.dump()?;
+            }
         }
         println!("End dump LayerMap");
         Ok(())
     }
 }
+
+impl IntervalItem for dyn Layer {
+    type Key = Lsn;
+
+    fn start_key(&self) -> Lsn {
+        self.get_start_lsn()
+    }
+    fn end_key(&self) -> Lsn {
+        self.get_end_lsn()
+    }
+}
+
+///
+/// Per-segment entry in the LayerMap::segs hash map. Holds all the layers
+/// associated with the segment.
+///
+/// The last layer that is open for writes is always an InMemoryLayer,
+/// and is kept in a separate field, because there can be only one for
+/// each segment. The older layers, stored on disk, are kept in an
+/// IntervalTree.
+#[derive(Default)]
+struct SegEntry {
+    open_layer_id: Option<LayerId>,
+    historic: IntervalTree<dyn Layer>,
+}
+
+impl SegEntry {
+    /// Does the segment exist at given LSN?
+    /// Return None if object is not found in this SegEntry.
+    fn exists_at_lsn(&self, lsn: Lsn) -> Result<Option<bool>> {
+        if let Some(layer) = self.get(lsn) {
+            Ok(Some(layer.get_seg_exists(lsn)?))
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub fn get(&self, lsn: Lsn) -> Option<Arc<dyn Layer>> {
+        if let Some(open_layer_id) = &self.open_layer_id {
+            let open_layer = GLOBAL_LAYER_MAP.read().unwrap().get(open_layer_id)?;
+            if open_layer.get_start_lsn() <= lsn {
+                return Some(open_layer);
+            }
+        }
+
+        self.historic.search(lsn)
+    }
+
+    pub fn newer_image_layer_exists(&self, lsn: Lsn, disk_consistent_lsn: Lsn) -> bool {
+        // We only check on-disk layers, because
+        // in-memory layers are not durable
+
+        // The end-LSN is exclusive, while disk_consistent_lsn is
+        // inclusive. For example, if disk_consistent_lsn is 100, it is
+        // OK for a delta layer to have end LSN 101, but if the end LSN
+        // is 102, then it might not have been fully flushed to disk
+        // before crash.
+        self.historic
+            .iter_newer(lsn)
+            .any(|layer| !layer.is_incremental() && layer.get_end_lsn() <= disk_consistent_lsn + 1)
+    }
+
+    // Set new open layer for a SegEntry.
+    // It's ok to rewrite previous open layer,
+    // but only if it is not writeable anymore.
+    pub fn update_open(&mut self, layer: Arc<InMemoryLayer>) -> LayerId {
+        if let Some(prev_open_layer_id) = &self.open_layer_id {
+            if let Some(prev_open_layer) = GLOBAL_LAYER_MAP.read().unwrap().get(prev_open_layer_id)
+            {
+                assert!(!prev_open_layer.is_writeable());
+            }
+        }
+        let open_layer_id = GLOBAL_LAYER_MAP.write().unwrap().insert(layer);
+        self.open_layer_id = Some(open_layer_id);
+        open_layer_id
+    }
+
+    pub fn insert_historic(&mut self, layer: Arc<dyn Layer>) {
+        self.historic.insert(layer);
+    }
+}
+
+/// Entry held in LayerMap::open_layers, with boilerplate comparison routines
+/// to implement a min-heap ordered by 'oldest_lsn' and 'generation'
+///
+/// The generation number associated with each entry can be used to distinguish
+/// recently-added entries (i.e after last call to increment_generation()) from older
+/// entries with the same 'oldest_lsn'.
+struct OpenLayerEntry {
+    oldest_lsn: Lsn, // copy of layer.get_oldest_lsn()
+    generation: u64,
+    layer_id: LayerId,
+}
+impl Ord for OpenLayerEntry {
+    fn cmp(&self, other: &Self) -> Ordering {
+        // BinaryHeap is a max-heap, and we want a min-heap. Reverse the ordering here
+        // to get that. Entries with identical oldest_lsn are ordered by generation
+        other
+            .oldest_lsn
+            .cmp(&self.oldest_lsn)
+            .then_with(|| other.generation.cmp(&self.generation))
+    }
+}
+impl PartialOrd for OpenLayerEntry {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+impl PartialEq for OpenLayerEntry {
+    fn eq(&self, other: &Self) -> bool {
+        self.cmp(other) == Ordering::Equal
+    }
+}
+impl Eq for OpenLayerEntry {}
+
+/// Iterator returned by LayerMap::iter_historic_layers()
+pub struct HistoricLayerIter<'a> {
+    seg_iter: std::collections::hash_map::Iter<'a, SegmentTag, SegEntry>,
+    iter: Option<IntervalIter<'a, dyn Layer>>,
+}
+
+impl<'a> Iterator for HistoricLayerIter<'a> {
+    type Item = Arc<dyn Layer>;
+
+    fn next(&mut self) -> std::option::Option<<Self as std::iter::Iterator>::Item> {
+        loop {
+            if let Some(x) = &mut self.iter {
+                if let Some(x) = x.next() {
+                    return Some(Arc::clone(&x));
+                }
+            }
+            if let Some((_tag, segentry)) = self.seg_iter.next() {
+                self.iter = Some(segentry.historic.iter());
+                continue;
+            } else {
+                return None;
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::config::PageServerConf;
+    use std::str::FromStr;
+    use zenith_utils::zid::{ZTenantId, ZTimelineId};
+
+    /// Arbitrary relation tag, for testing.
+    const TESTREL_A: RelishTag = RelishTag::Relation(RelTag {
+        spcnode: 0,
+        dbnode: 111,
+        relnode: 1000,
+        forknum: 0,
+    });
+
+    lazy_static! {
+        static ref DUMMY_TIMELINEID: ZTimelineId =
+            ZTimelineId::from_str("00000000000000000000000000000000").unwrap();
+        static ref DUMMY_TENANTID: ZTenantId =
+            ZTenantId::from_str("00000000000000000000000000000000").unwrap();
+    }
+
+    /// Construct a dummy InMemoryLayer for testing
+    fn dummy_inmem_layer(
+        conf: &'static PageServerConf,
+        segno: u32,
+        start_lsn: Lsn,
+        oldest_lsn: Lsn,
+    ) -> Arc<InMemoryLayer> {
+        Arc::new(
+            InMemoryLayer::create(
+                conf,
+                *DUMMY_TIMELINEID,
+                *DUMMY_TENANTID,
+                SegmentTag {
+                    rel: TESTREL_A,
+                    segno,
+                },
+                start_lsn,
+                oldest_lsn,
+            )
+            .unwrap(),
+        )
+    }
+
+    #[test]
+    fn test_open_layers() -> Result<()> {
+        let conf = PageServerConf::dummy_conf(PageServerConf::test_repo_dir("dummy_inmem_layer"));
+        let conf = Box::leak(Box::new(conf));
+        std::fs::create_dir_all(conf.timeline_path(&DUMMY_TIMELINEID, &DUMMY_TENANTID))?;
+
+        let mut layers = LayerMap::default();
+
+        let gen1 = layers.increment_generation();
+        layers.insert_open(dummy_inmem_layer(conf, 0, Lsn(0x100), Lsn(0x100)));
+        layers.insert_open(dummy_inmem_layer(conf, 1, Lsn(0x100), Lsn(0x200)));
+        layers.insert_open(dummy_inmem_layer(conf, 2, Lsn(0x100), Lsn(0x120)));
+        layers.insert_open(dummy_inmem_layer(conf, 3, Lsn(0x100), Lsn(0x110)));
+
+        let gen2 = layers.increment_generation();
+        layers.insert_open(dummy_inmem_layer(conf, 4, Lsn(0x100), Lsn(0x110)));
+        layers.insert_open(dummy_inmem_layer(conf, 5, Lsn(0x100), Lsn(0x100)));
+
+        // A helper function (closure) to pop the next oldest open entry from the layer map,
+        // and assert that it is what we'd expect
+        let mut assert_pop_layer = |expected_segno: u32, expected_generation: u64| {
+            let (layer_id, l, generation) = layers.peek_oldest_open().unwrap();
+            assert!(l.get_seg_tag().segno == expected_segno);
+            assert!(generation == expected_generation);
+            layers.remove_open(layer_id);
+        };
+
+        assert_pop_layer(0, gen1); // 0x100
+        assert_pop_layer(5, gen2); // 0x100
+        assert_pop_layer(3, gen1); // 0x110
+        assert_pop_layer(4, gen2); // 0x110
+        assert_pop_layer(2, gen1); // 0x120
+        assert_pop_layer(1, gen1); // 0x200
+
+        Ok(())
+    }
+}

pageserver/src/layered_repository/page_versions.rs

@@ -0,0 +1,268 @@
+//!
+//! Data structure to ingest incoming WAL into an append-only file.
+//!
+//! - The file is considered temporary, and will be discarded on crash
+//! - based on a B-tree
+//!
+
+use std::os::unix::fs::FileExt;
+use std::{collections::HashMap, ops::RangeBounds, slice};
+
+use anyhow::Result;
+
+use std::cmp::min;
+use std::io::Seek;
+
+use zenith_utils::{lsn::Lsn, vec_map::VecMap};
+
+use super::storage_layer::PageVersion;
+use crate::layered_repository::ephemeral_file::EphemeralFile;
+
+use zenith_utils::bin_ser::BeSer;
+
+const EMPTY_SLICE: &[(Lsn, u64)] = &[];
+
+pub struct PageVersions {
+    map: HashMap<u32, VecMap<Lsn, u64>>,
+
+    /// The PageVersion structs are stored in a serialized format in this file.
+    /// Each serialized PageVersion is preceded by a 'u32' length field.
+    /// The 'map' stores offsets into this file.
+    file: EphemeralFile,
+}
+
+impl PageVersions {
+    pub fn new(file: EphemeralFile) -> PageVersions {
+        PageVersions {
+            map: HashMap::new(),
+            file,
+        }
+    }
+
+    pub fn append_or_update_last(
+        &mut self,
+        blknum: u32,
+        lsn: Lsn,
+        page_version: PageVersion,
+    ) -> Result<Option<u64>> {
+        // remember starting position
+        let pos = self.file.stream_position()?;
+
+        // make room for the 'length' field by writing zeros as a placeholder.
+        self.file.seek(std::io::SeekFrom::Start(pos + 4)).unwrap();
+
+        page_version.ser_into(&mut self.file).unwrap();
+
+        // write the 'length' field.
+        let len = self.file.stream_position()? - pos - 4;
+        let lenbuf = u32::to_ne_bytes(len as u32);
+        self.file.write_all_at(&lenbuf, pos)?;
+
+        let map = self.map.entry(blknum).or_insert_with(VecMap::default);
+        Ok(map.append_or_update_last(lsn, pos as u64).unwrap().0)
+    }
+
+    /// Get all [`PageVersion`]s in a block
+    fn get_block_slice(&self, blknum: u32) -> &[(Lsn, u64)] {
+        self.map
+            .get(&blknum)
+            .map(VecMap::as_slice)
+            .unwrap_or(EMPTY_SLICE)
+    }
+
+    /// Get a range of [`PageVersions`] in a block
+    pub fn get_block_lsn_range<R: RangeBounds<Lsn>>(&self, blknum: u32, range: R) -> &[(Lsn, u64)] {
+        self.map
+            .get(&blknum)
+            .map(|vec_map| vec_map.slice_range(range))
+            .unwrap_or(EMPTY_SLICE)
+    }
+
+    /// Iterate through [`PageVersion`]s in (block, lsn) order.
+    /// If a [`cutoff_lsn`] is set, only show versions with `lsn < cutoff_lsn`
+    pub fn ordered_page_version_iter(&self, cutoff_lsn: Option<Lsn>) -> OrderedPageVersionIter<'_> {
+        let mut ordered_blocks: Vec<u32> = self.map.keys().cloned().collect();
+        ordered_blocks.sort_unstable();
+
+        let slice = ordered_blocks
+            .first()
+            .map(|&blknum| self.get_block_slice(blknum))
+            .unwrap_or(EMPTY_SLICE);
+
+        OrderedPageVersionIter {
+            page_versions: self,
+            ordered_blocks,
+            cur_block_idx: 0,
+            cutoff_lsn,
+            cur_slice_iter: slice.iter(),
+        }
+    }
+
+    ///
+    /// Read a page version.
+    ///
+    pub fn read_pv(&self, off: u64) -> Result<PageVersion> {
+        let mut buf = Vec::new();
+        self.read_pv_bytes(off, &mut buf)?;
+        Ok(PageVersion::des(&buf)?)
+    }
+
+    ///
+    /// Read a page version, as raw bytes, at the given offset. The bytes
+    /// are read into 'buf', which is expanded if necessary. Returns the
+    /// size of the page version.
+    ///
+    pub fn read_pv_bytes(&self, off: u64, buf: &mut Vec<u8>) -> Result<usize> {
+        // read length
+        let mut lenbuf = [0u8; 4];
+        self.file.read_exact_at(&mut lenbuf, off)?;
+        let len = u32::from_ne_bytes(lenbuf) as usize;
+
+        // Resize the buffer to fit the data, if needed.
+        //
+        // We don't shrink the buffer if it's larger than necessary. That avoids
+        // repeatedly shrinking and expanding when you reuse the same buffer to
+        // read multiple page versions. Expanding a Vec requires initializing the
+        // new bytes, which is a waste of time because we're immediately overwriting
+        // it, but there's no way to avoid it without resorting to unsafe code.
+        if buf.len() < len {
+            buf.resize(len, 0);
+        }
+        self.file.read_exact_at(&mut buf[0..len], off + 4)?;
+
+        Ok(len)
+    }
+}
+
+pub struct PageVersionReader<'a> {
+    file: &'a EphemeralFile,
+    pos: u64,
+    end_pos: u64,
+}
+
+impl<'a> std::io::Read for PageVersionReader<'a> {
+    fn read(&mut self, buf: &mut [u8]) -> Result<usize, std::io::Error> {
+        let len = min(buf.len(), (self.end_pos - self.pos) as usize);
+        let n = self.file.read_at(&mut buf[..len], self.pos)?;
+        self.pos += n as u64;
+        Ok(n)
+    }
+}
+
+pub struct OrderedPageVersionIter<'a> {
+    page_versions: &'a PageVersions,
+
+    ordered_blocks: Vec<u32>,
+    cur_block_idx: usize,
+
+    cutoff_lsn: Option<Lsn>,
+
+    cur_slice_iter: slice::Iter<'a, (Lsn, u64)>,
+}
+
+impl OrderedPageVersionIter<'_> {
+    fn is_lsn_before_cutoff(&self, lsn: &Lsn) -> bool {
+        if let Some(cutoff_lsn) = self.cutoff_lsn.as_ref() {
+            lsn < cutoff_lsn
+        } else {
+            true
+        }
+    }
+}
+
+impl<'a> Iterator for OrderedPageVersionIter<'a> {
+    type Item = (u32, Lsn, u64);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            if let Some((lsn, pos)) = self.cur_slice_iter.next() {
+                if self.is_lsn_before_cutoff(lsn) {
+                    let blknum = self.ordered_blocks[self.cur_block_idx];
+                    return Some((blknum, *lsn, *pos));
+                }
+            }
+
+            let next_block_idx = self.cur_block_idx + 1;
+            let blknum: u32 = *self.ordered_blocks.get(next_block_idx)?;
+            self.cur_block_idx = next_block_idx;
+            self.cur_slice_iter = self.page_versions.get_block_slice(blknum).iter();
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use bytes::Bytes;
+
+    use super::*;
+    use crate::config::PageServerConf;
+    use std::fs;
+    use std::str::FromStr;
+    use zenith_utils::zid::{ZTenantId, ZTimelineId};
+
+    fn repo_harness(test_name: &str) -> Result<(&'static PageServerConf, ZTenantId, ZTimelineId)> {
+        let repo_dir = PageServerConf::test_repo_dir(test_name);
+        let _ = fs::remove_dir_all(&repo_dir);
+        let conf = PageServerConf::dummy_conf(repo_dir);
+        // Make a static copy of the config. This can never be free'd, but that's
+        // OK in a test.
+        let conf: &'static PageServerConf = Box::leak(Box::new(conf));
+
+        let tenantid = ZTenantId::from_str("11000000000000000000000000000000").unwrap();
+        let timelineid = ZTimelineId::from_str("22000000000000000000000000000000").unwrap();
+        fs::create_dir_all(conf.timeline_path(&timelineid, &tenantid))?;
+
+        Ok((conf, tenantid, timelineid))
+    }
+
+    #[test]
+    fn test_ordered_iter() -> Result<()> {
+        let (conf, tenantid, timelineid) = repo_harness("test_ordered_iter")?;
+
+        let file = EphemeralFile::create(conf, tenantid, timelineid)?;
+
+        let mut page_versions = PageVersions::new(file);
+
+        const BLOCKS: u32 = 1000;
+        const LSNS: u64 = 50;
+
+        let empty_page = Bytes::from_static(&[0u8; 8192]);
+        let empty_page_version = PageVersion::Page(empty_page);
+
+        for blknum in 0..BLOCKS {
+            for lsn in 0..LSNS {
+                let old = page_versions.append_or_update_last(
+                    blknum,
+                    Lsn(lsn),
+                    empty_page_version.clone(),
+                )?;
+                assert!(old.is_none());
+            }
+        }
+
+        let mut iter = page_versions.ordered_page_version_iter(None);
+        for blknum in 0..BLOCKS {
+            for lsn in 0..LSNS {
+                let (actual_blknum, actual_lsn, _pv) = iter.next().unwrap();
+                assert_eq!(actual_blknum, blknum);
+                assert_eq!(Lsn(lsn), actual_lsn);
+            }
+        }
+        assert!(iter.next().is_none());
+        assert!(iter.next().is_none()); // should be robust against excessive next() calls
+
+        const CUTOFF_LSN: Lsn = Lsn(30);
+        let mut iter = page_versions.ordered_page_version_iter(Some(CUTOFF_LSN));
+        for blknum in 0..BLOCKS {
+            for lsn in 0..CUTOFF_LSN.0 {
+                let (actual_blknum, actual_lsn, _pv) = iter.next().unwrap();
+                assert_eq!(actual_blknum, blknum);
+                assert_eq!(Lsn(lsn), actual_lsn);
+            }
+        }
+        assert!(iter.next().is_none());
+        assert!(iter.next().is_none()); // should be robust against excessive next() calls
+
+        Ok(())
+    }
+}

pageserver/src/layered_repository/storage_layer.rs

@@ -2,84 +2,108 @@
 //! Common traits and structs for layers
 //!
 
-use crate::repository::{Key, Value};
-use crate::walrecord::ZenithWalRecord;
+use crate::relish::RelishTag;
+use crate::repository::{BlockNumber, ZenithWalRecord};
 use crate::{ZTenantId, ZTimelineId};
 use anyhow::Result;
 use bytes::Bytes;
-use std::ops::Range;
+use serde::{Deserialize, Serialize};
+use std::fmt;
 use std::path::PathBuf;
 
 use zenith_utils::lsn::Lsn;
 
-pub fn range_overlaps<T>(a: &Range<T>, b: &Range<T>) -> bool
-where
-    T: PartialOrd<T>,
-{
-    if a.start < b.start {
-        a.end > b.start
-    } else {
-        b.end > a.start
+// Size of one segment in pages (10 MB)
+pub const RELISH_SEG_SIZE: u32 = 10 * 1024 * 1024 / 8192;
+
+///
+/// Each relish stored in the repository is divided into fixed-sized "segments",
+/// with 10 MB of key-space, or 1280 8k pages each.
+///
+#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy, Serialize, Deserialize)]
+pub struct SegmentTag {
+    pub rel: RelishTag,
+    pub segno: u32,
+}
+
+/// SegmentBlk represents a block number within a segment, or the size of segment.
+///
+/// This is separate from BlockNumber, which is used for block number within the
+/// whole relish. Since this is just a type alias, the compiler will let you mix
+/// them freely, but we use the type alias as documentation to make it clear
+/// which one we're dealing with.
+///
+/// (We could turn this into "struct SegmentBlk(u32)" to forbid accidentally
+/// assigning a BlockNumber to SegmentBlk or vice versa, but that makes
+/// operations more verbose).
+pub type SegmentBlk = u32;
+
+impl fmt::Display for SegmentTag {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}.{}", self.rel, self.segno)
     }
 }
 
-pub fn range_eq<T>(a: &Range<T>, b: &Range<T>) -> bool
-where
-    T: PartialEq<T>,
-{
-    a.start == b.start && a.end == b.end
+impl SegmentTag {
+    /// Given a relish and block number, calculate the corresponding segment and
+    /// block number within the segment.
+    pub const fn from_blknum(rel: RelishTag, blknum: BlockNumber) -> (SegmentTag, SegmentBlk) {
+        (
+            SegmentTag {
+                rel,
+                segno: blknum / RELISH_SEG_SIZE,
+            },
+            blknum % RELISH_SEG_SIZE,
+        )
+    }
 }
 
-/// Struct used to communicate across calls to 'get_value_reconstruct_data'.
 ///
-/// Before first call, you can fill in 'page_img' if you have an older cached
-/// version of the page available. That can save work in
-/// 'get_value_reconstruct_data', as it can stop searching for page versions
-/// when all the WAL records going back to the cached image have been collected.
+/// Represents a version of a page at a specific LSN. The LSN is the key of the
+/// entry in the 'page_versions' hash, it is not duplicated here.
 ///
-/// When get_value_reconstruct_data returns Complete, 'img' is set to an image
-/// of the page, or the oldest WAL record in 'records' is a will_init-type
-/// record that initializes the page without requiring a previous image.
+/// A page version can be stored as a full page image, or as WAL record that needs
+/// to be applied over the previous page version to reconstruct this version.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum PageVersion {
+    Page(Bytes),
+    Wal(ZenithWalRecord),
+}
+
 ///
-/// If 'get_page_reconstruct_data' returns Continue, some 'records' may have
-/// been collected, but there are more records outside the current layer. Pass
-/// the same ValueReconstructState struct in the next 'get_value_reconstruct_data'
-/// call, to collect more records.
+/// Data needed to reconstruct a page version
 ///
-#[derive(Debug)]
-pub struct ValueReconstructState {
+/// 'page_img' is the old base image of the page to start the WAL replay with.
+/// It can be None, if the first WAL record initializes the page (will_init)
+/// 'records' contains the records to apply over the base image.
+///
+pub struct PageReconstructData {
     pub records: Vec<(Lsn, ZenithWalRecord)>,
-    pub img: Option<(Lsn, Bytes)>,
+    pub page_img: Option<Bytes>,
 }
 
 /// Return value from Layer::get_page_reconstruct_data
-#[derive(Clone, Copy, Debug)]
-pub enum ValueReconstructResult {
+pub enum PageReconstructResult {
     /// Got all the data needed to reconstruct the requested page
     Complete,
     /// This layer didn't contain all the required data, the caller should look up
     /// the predecessor layer at the returned LSN and collect more data from there.
-    Continue,
-
+    Continue(Lsn),
     /// This layer didn't contain data needed to reconstruct the page version at
     /// the returned LSN. This is usually considered an error, but might be OK
     /// in some circumstances.
-    Missing,
+    Missing(Lsn),
+    /// Use the cached image at `cached_img_lsn` as the base image
+    Cached,
 }
 
-/// A Layer contains all data in a "rectangle" consisting of a range of keys and
-/// range of LSNs.
 ///
+/// A Layer corresponds to one RELISH_SEG_SIZE slice of a relish in a range of LSNs.
 /// There are two kinds of layers, in-memory and on-disk layers. In-memory
-/// layers are used to ingest incoming WAL, and provide fast access to the
-/// recent page versions. On-disk layers are stored as files on disk, and are
-/// immutable. This trait presents the common functionality of in-memory and
-/// on-disk layers.
-///
-/// Furthermore, there are two kinds of on-disk layers: delta and image layers.
-/// A delta layer contains all modifications within a range of LSNs and keys.
-/// An image layer is a snapshot of all the data in a key-range, at a single
-/// LSN
+/// layers are used to ingest incoming WAL, and provide fast access
+/// to the recent page versions. On-disk layers are stored as files on disk, and
+/// are immutable. This trait presents the common functionality of
+/// in-memory and on-disk layers.
 ///
 pub trait Layer: Send + Sync {
     fn get_tenant_id(&self) -> ZTenantId;
@@ -87,16 +111,21 @@ pub trait Layer: Send + Sync {
     /// Identify the timeline this relish belongs to
     fn get_timeline_id(&self) -> ZTimelineId;
 
-    /// Range of segments that this layer covers
-    fn get_key_range(&self) -> Range<Key>;
+    /// Identify the relish segment
+    fn get_seg_tag(&self) -> SegmentTag;
 
     /// Inclusive start bound of the LSN range that this layer holds
+    fn get_start_lsn(&self) -> Lsn;
+
     /// Exclusive end bound of the LSN range that this layer holds.
     ///
     /// - For an open in-memory layer, this is MAX_LSN.
     /// - For a frozen in-memory layer or a delta layer, this is a valid end bound.
     /// - An image layer represents snapshot at one LSN, so end_lsn is always the snapshot LSN + 1
-    fn get_lsn_range(&self) -> Range<Lsn>;
+    fn get_end_lsn(&self) -> Lsn;
+
+    /// Is the segment represented by this layer dropped by PostgreSQL?
+    fn is_dropped(&self) -> bool;
 
     /// Filename used to store this layer on disk. (Even in-memory layers
     /// implement this, to print a handy unique identifier for the layer for
@@ -109,18 +138,27 @@ pub trait Layer: Send + Sync {
     /// It is up to the caller to collect more data from previous layer and
     /// perform WAL redo, if necessary.
     ///
+    /// `cached_img_lsn` should be set to a cached page image's lsn < `lsn`.
+    /// This function will only return data after `cached_img_lsn`.
+    ///
     /// See PageReconstructResult for possible return values. The collected data
     /// is appended to reconstruct_data; the caller should pass an empty struct
-    /// on first call, or a struct with a cached older image of the page if one
-    /// is available. If this returns PageReconstructResult::Continue, look up
-    /// the predecessor layer and call again with the same 'reconstruct_data' to
-    /// collect more data.
-    fn get_value_reconstruct_data(
+    /// on first call. If this returns PageReconstructResult::Continue, look up
+    /// the predecessor layer and call again with the same 'reconstruct_data'
+    /// to collect more data.
+    fn get_page_reconstruct_data(
         &self,
-        key: Key,
-        lsn_range: Range<Lsn>,
-        reconstruct_data: &mut ValueReconstructState,
-    ) -> Result<ValueReconstructResult>;
+        blknum: SegmentBlk,
+        lsn: Lsn,
+        cached_img_lsn: Option<Lsn>,
+        reconstruct_data: &mut PageReconstructData,
+    ) -> Result<PageReconstructResult>;
+
+    /// Return size of the segment at given LSN. (Only for blocky relations.)
+    fn get_seg_size(&self, lsn: Lsn) -> Result<SegmentBlk>;
+
+    /// Does the segment exist at given LSN? Or was it dropped before it.
+    fn get_seg_exists(&self, lsn: Lsn) -> Result<bool>;
 
     /// Does this layer only contain some data for the segment (incremental),
     /// or does it contain a version of every page? This is important to know
@@ -131,9 +169,6 @@ pub trait Layer: Send + Sync {
     /// Returns true for layers that are represented in memory.
     fn is_in_memory(&self) -> bool;
 
-    /// Iterate through all keys and values stored in the layer
-    fn iter(&self) -> Box<dyn Iterator<Item = Result<(Key, Lsn, Value)>> + '_>;
-
     /// Release memory used by this layer. There is no corresponding 'load'
     /// function, that's done implicitly when you call one of the get-functions.
     fn unload(&self) -> Result<()>;

pageserver/src/layered_repository/utils.rs

@@ -1,48 +0,0 @@
-// Utilities for reading and writing Values
-use std::io::{Error, Write};
-use std::os::unix::fs::FileExt;
-
-use bookfile::BoundedReader;
-
-pub fn read_blob<F: FileExt>(file: &F, off: u64) -> Result<Vec<u8>, Error> {
-    // read length
-    let mut len_buf = [0u8; 4];
-    file.read_exact_at(&mut len_buf, off)?;
-
-    let len = u32::from_ne_bytes(len_buf);
-
-    let mut buf: Vec<u8> = Vec::new();
-    buf.resize(len as usize, 0);
-    file.read_exact_at(&mut buf.as_mut_slice(), off + 4)?;
-
-    Ok(buf)
-}
-
-pub fn read_blob_from_chapter<F: FileExt>(
-    file: &BoundedReader<&F>,
-    off: u64,
-) -> Result<Vec<u8>, Error> {
-    // read length
-    let mut len_buf = [0u8; 4];
-    file.read_exact_at(&mut len_buf, off)?;
-
-    let len = u32::from_ne_bytes(len_buf);
-
-    let mut buf: Vec<u8> = Vec::new();
-    buf.resize(len as usize, 0);
-    file.read_exact_at(&mut buf.as_mut_slice(), off + 4)?;
-
-    Ok(buf)
-}
-
-pub fn write_blob<W: Write>(writer: &mut W, buf: &[u8]) -> Result<u64, Error> {
-    let val_len = buf.len() as u32;
-
-    // write the 'length' field and kind byte.
-    let lenbuf = u32::to_ne_bytes(val_len);
-
-    writer.write_all(&lenbuf)?;
-    writer.write_all(buf)?;
-
-    Ok(4 + val_len as u64)
-}

pageserver/src/lib.rs

@@ -3,11 +3,9 @@ pub mod branches;
 pub mod config;
 pub mod http;
 pub mod import_datadir;
-pub mod keyspace;
 pub mod layered_repository;
 pub mod page_cache;
 pub mod page_service;
-pub mod pgdatadir_mapping;
 pub mod relish;
 pub mod remote_storage;
 pub mod repository;
@@ -24,9 +22,6 @@ use lazy_static::lazy_static;
 use zenith_metrics::{register_int_gauge_vec, IntGaugeVec};
 use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
-use layered_repository::LayeredRepository;
-use pgdatadir_mapping::DatadirTimeline;
-
 lazy_static! {
     static ref LIVE_CONNECTIONS_COUNT: IntGaugeVec = register_int_gauge_vec!(
         "pageserver_live_connections_count",
@@ -48,7 +43,3 @@ pub enum CheckpointConfig {
     // Flush all in-memory data and reconstruct all page images
     Forced,
 }
-
-pub type RepositoryImpl = LayeredRepository;
-
-pub type DatadirTimelineImpl = DatadirTimeline<RepositoryImpl>;

pageserver/src/page_service.rs

@@ -32,9 +32,7 @@ use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
 use crate::basebackup;
 use crate::config::PageServerConf;
-use crate::pgdatadir_mapping::DatadirTimeline;
 use crate::relish::*;
-use crate::repository::Repository;
 use crate::repository::Timeline;
 use crate::tenant_mgr;
 use crate::thread_mgr;
@@ -397,8 +395,8 @@ impl PageServerHandler {
     /// In either case, if the page server hasn't received the WAL up to the
     /// requested LSN yet, we will wait for it to arrive. The return value is
     /// the LSN that should be used to look up the page versions.
-    fn wait_or_get_last_lsn<R: Repository>(
-        timeline: &DatadirTimeline<R>,
+    fn wait_or_get_last_lsn(
+        timeline: &dyn Timeline,
         mut lsn: Lsn,
         latest: bool,
         latest_gc_cutoff_lsn: &RwLockReadGuard<Lsn>,
@@ -425,7 +423,7 @@ impl PageServerHandler {
             if lsn <= last_record_lsn {
                 lsn = last_record_lsn;
             } else {
-                timeline.tline.wait_lsn(lsn)?;
+                timeline.wait_lsn(lsn)?;
                 // Since we waited for 'lsn' to arrive, that is now the last
                 // record LSN. (Or close enough for our purposes; the
                 // last-record LSN can advance immediately after we return
@@ -435,7 +433,7 @@ impl PageServerHandler {
             if lsn == Lsn(0) {
                 bail!("invalid LSN(0) in request");
             }
-            timeline.tline.wait_lsn(lsn)?;
+            timeline.wait_lsn(lsn)?;
         }
         ensure!(
             lsn >= **latest_gc_cutoff_lsn,
@@ -445,47 +443,54 @@ impl PageServerHandler {
         Ok(lsn)
     }
 
-    fn handle_get_rel_exists_request<R: Repository>(
+    fn handle_get_rel_exists_request(
         &self,
-        timeline: &DatadirTimeline<R>,
+        timeline: &dyn Timeline,
         req: &PagestreamExistsRequest,
     ) -> Result<PagestreamBeMessage> {
         let _enter = info_span!("get_rel_exists", rel = %req.rel, req_lsn = %req.lsn).entered();
 
-        let latest_gc_cutoff_lsn = timeline.tline.get_latest_gc_cutoff_lsn();
+        let tag = RelishTag::Relation(req.rel);
+        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
         let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn)?;
 
-        let exists = timeline.get_rel_exists(req.rel, lsn)?;
+        let exists = timeline.get_rel_exists(tag, lsn)?;
 
         Ok(PagestreamBeMessage::Exists(PagestreamExistsResponse {
             exists,
         }))
     }
 
-    fn handle_get_nblocks_request<R: Repository>(
+    fn handle_get_nblocks_request(
         &self,
-        timeline: &DatadirTimeline<R>,
+        timeline: &dyn Timeline,
         req: &PagestreamNblocksRequest,
     ) -> Result<PagestreamBeMessage> {
         let _enter = info_span!("get_nblocks", rel = %req.rel, req_lsn = %req.lsn).entered();
-        let latest_gc_cutoff_lsn = timeline.tline.get_latest_gc_cutoff_lsn();
+        let tag = RelishTag::Relation(req.rel);
+        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
         let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn)?;
 
-        let n_blocks = timeline.get_rel_size(req.rel, lsn)?;
+        let n_blocks = timeline.get_relish_size(tag, lsn)?;
+
+        // Return 0 if relation is not found.
+        // This is what postgres smgr expects.
+        let n_blocks = n_blocks.unwrap_or(0);
 
         Ok(PagestreamBeMessage::Nblocks(PagestreamNblocksResponse {
             n_blocks,
         }))
     }
 
-    fn handle_get_page_at_lsn_request<R: Repository>(
+    fn handle_get_page_at_lsn_request(
         &self,
-        timeline: &DatadirTimeline<R>,
+        timeline: &dyn Timeline,
         req: &PagestreamGetPageRequest,
     ) -> Result<PagestreamBeMessage> {
         let _enter = info_span!("get_page", rel = %req.rel, blkno = &req.blkno, req_lsn = %req.lsn)
             .entered();
-        let latest_gc_cutoff_lsn = timeline.tline.get_latest_gc_cutoff_lsn();
+        let tag = RelishTag::Relation(req.rel);
+        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
         let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn)?;
         /*
         // Add a 1s delay to some requests. The delayed causes the requests to
@@ -495,7 +500,7 @@ impl PageServerHandler {
             std::thread::sleep(std::time::Duration::from_millis(1000));
         }
         */
-        let page = timeline.get_rel_page_at_lsn(req.rel, req.blkno, lsn)?;
+        let page = timeline.get_page_at_lsn(tag, req.blkno, lsn)?;
 
         Ok(PagestreamBeMessage::GetPage(PagestreamGetPageResponse {
             page,
@@ -515,7 +520,7 @@ impl PageServerHandler {
         // check that the timeline exists
         let timeline = tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)
             .context("Cannot handle basebackup request for a remote timeline")?;
-        let latest_gc_cutoff_lsn = timeline.tline.get_latest_gc_cutoff_lsn();
+        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
         if let Some(lsn) = lsn {
             timeline
                 .check_lsn_is_in_scope(lsn, &latest_gc_cutoff_lsn)
@@ -694,19 +699,67 @@ impl postgres_backend::Handler for PageServerHandler {
             let repo = tenant_mgr::get_repository_for_tenant(tenantid)?;
             let result = repo.gc_iteration(Some(timelineid), gc_horizon, true)?;
             pgb.write_message_noflush(&BeMessage::RowDescription(&[
-                RowDescriptor::int8_col(b"layers_total"),
-                RowDescriptor::int8_col(b"layers_needed_by_cutoff"),
-                RowDescriptor::int8_col(b"layers_needed_by_branches"),
-                RowDescriptor::int8_col(b"layers_not_updated"),
-                RowDescriptor::int8_col(b"layers_removed"),
+                RowDescriptor::int8_col(b"layer_relfiles_total"),
+                RowDescriptor::int8_col(b"layer_relfiles_needed_by_cutoff"),
+                RowDescriptor::int8_col(b"layer_relfiles_needed_by_branches"),
+                RowDescriptor::int8_col(b"layer_relfiles_not_updated"),
+                RowDescriptor::int8_col(b"layer_relfiles_needed_as_tombstone"),
+                RowDescriptor::int8_col(b"layer_relfiles_removed"),
+                RowDescriptor::int8_col(b"layer_relfiles_dropped"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_total"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_needed_by_cutoff"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_needed_by_branches"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_not_updated"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_needed_as_tombstone"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_removed"),
+                RowDescriptor::int8_col(b"layer_nonrelfiles_dropped"),
                 RowDescriptor::int8_col(b"elapsed"),
             ]))?
             .write_message_noflush(&BeMessage::DataRow(&[
-                Some(result.layers_total.to_string().as_bytes()),
-                Some(result.layers_needed_by_cutoff.to_string().as_bytes()),
-                Some(result.layers_needed_by_branches.to_string().as_bytes()),
-                Some(result.layers_not_updated.to_string().as_bytes()),
-                Some(result.layers_removed.to_string().as_bytes()),
+                Some(result.ondisk_relfiles_total.to_string().as_bytes()),
+                Some(
+                    result
+                        .ondisk_relfiles_needed_by_cutoff
+                        .to_string()
+                        .as_bytes(),
+                ),
+                Some(
+                    result
+                        .ondisk_relfiles_needed_by_branches
+                        .to_string()
+                        .as_bytes(),
+                ),
+                Some(result.ondisk_relfiles_not_updated.to_string().as_bytes()),
+                Some(
+                    result
+                        .ondisk_relfiles_needed_as_tombstone
+                        .to_string()
+                        .as_bytes(),
+                ),
+                Some(result.ondisk_relfiles_removed.to_string().as_bytes()),
+                Some(result.ondisk_relfiles_dropped.to_string().as_bytes()),
+                Some(result.ondisk_nonrelfiles_total.to_string().as_bytes()),
+                Some(
+                    result
+                        .ondisk_nonrelfiles_needed_by_cutoff
+                        .to_string()
+                        .as_bytes(),
+                ),
+                Some(
+                    result
+                        .ondisk_nonrelfiles_needed_by_branches
+                        .to_string()
+                        .as_bytes(),
+                ),
+                Some(result.ondisk_nonrelfiles_not_updated.to_string().as_bytes()),
+                Some(
+                    result
+                        .ondisk_nonrelfiles_needed_as_tombstone
+                        .to_string()
+                        .as_bytes(),
+                ),
+                Some(result.ondisk_nonrelfiles_removed.to_string().as_bytes()),
+                Some(result.ondisk_nonrelfiles_dropped.to_string().as_bytes()),
                 Some(result.elapsed.as_millis().to_string().as_bytes()),
             ]))?
             .write_message(&BeMessage::CommandComplete(b"SELECT 1"))?;
@@ -726,7 +779,7 @@ impl postgres_backend::Handler for PageServerHandler {
             let timeline = tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)
                 .context("Failed to fetch local timeline for checkpoint request")?;
 
-            timeline.tline.checkpoint(CheckpointConfig::Forced)?;
+            timeline.checkpoint(CheckpointConfig::Forced)?;
             pgb.write_message_noflush(&SINGLE_COL_ROWDESC)?
                 .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?;
         } else {

pageserver/src/relish.rs

@@ -1,6 +1,4 @@
 //!
-//! FIXME: relishes are obsolete
-//!
 //! Zenith stores PostgreSQL relations, and some other files, in the
 //! repository.  The relations (i.e. tables and indexes) take up most
 //! of the space in a typical installation, while the other files are
@@ -26,11 +24,110 @@
 //!
 
 use serde::{Deserialize, Serialize};
-use std::cmp::Ordering;
 use std::fmt;
 
 use postgres_ffi::relfile_utils::forknumber_to_name;
-use postgres_ffi::Oid;
+use postgres_ffi::{Oid, TransactionId};
+
+///
+/// RelishTag identifies one relish.
+///
+#[derive(Debug, Clone, Copy, Hash, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub enum RelishTag {
+    // Relations correspond to PostgreSQL relation forks. Each
+    // PostgreSQL relation fork is considered a separate relish.
+    Relation(RelTag),
+
+    // SLRUs include pg_clog, pg_multixact/members, and
+    // pg_multixact/offsets. There are other SLRUs in PostgreSQL, but
+    // they don't need to be stored permanently (e.g. pg_subtrans),
+    // or we do not support them in zenith yet (pg_commit_ts).
+    //
+    // These are currently never requested directly by the compute
+    // nodes, although in principle that would be possible. However,
+    // when a new compute node is created, these are included in the
+    // tarball that we send to the compute node to initialize the
+    // PostgreSQL data directory.
+    //
+    // Each SLRU segment in PostgreSQL is considered a separate
+    // relish. For example, pg_clog/0000, pg_clog/0001, and so forth.
+    //
+    // SLRU segments are divided into blocks, like relations.
+    Slru { slru: SlruKind, segno: u32 },
+
+    // Miscellaneous other files that need to be included in the
+    // tarball at compute node creation. These are non-blocky, and are
+    // expected to be small.
+
+    //
+    // FileNodeMap represents PostgreSQL's 'pg_filenode.map'
+    // files. They are needed to map catalog table OIDs to filenode
+    // numbers. Usually the mapping is done by looking up a relation's
+    // 'relfilenode' field in the 'pg_class' system table, but that
+    // doesn't work for 'pg_class' itself and a few other such system
+    // relations. See PostgreSQL relmapper.c for details.
+    //
+    // Each database has a map file for its local mapped catalogs,
+    // and there is a separate map file for shared catalogs.
+    //
+    // These files are always 512 bytes long (although we don't check
+    // or care about that in the page server).
+    //
+    FileNodeMap { spcnode: Oid, dbnode: Oid },
+
+    //
+    // State files for prepared transactions (e.g pg_twophase/1234)
+    //
+    TwoPhase { xid: TransactionId },
+
+    // The control file, stored in global/pg_control
+    ControlFile,
+
+    // Special entry that represents PostgreSQL checkpoint. It doesn't
+    // correspond to to any physical file in PostgreSQL, but we use it
+    // to track fields needed to restore the checkpoint data in the
+    // control file, when a compute node is created.
+    Checkpoint,
+}
+
+impl RelishTag {
+    pub const fn is_blocky(&self) -> bool {
+        match self {
+            // These relishes work with blocks
+            RelishTag::Relation(_) | RelishTag::Slru { slru: _, segno: _ } => true,
+
+            // and these don't
+            RelishTag::FileNodeMap {
+                spcnode: _,
+                dbnode: _,
+            }
+            | RelishTag::TwoPhase { xid: _ }
+            | RelishTag::ControlFile
+            | RelishTag::Checkpoint => false,
+        }
+    }
+
+    // Physical relishes represent files and use
+    // RelationSizeEntry to track existing and dropped files.
+    // They can be both blocky and non-blocky.
+    pub const fn is_physical(&self) -> bool {
+        match self {
+            // These relishes represent physical files
+            RelishTag::Relation(_)
+            | RelishTag::Slru { .. }
+            | RelishTag::FileNodeMap { .. }
+            | RelishTag::TwoPhase { .. } => true,
+
+            // and these don't
+            RelishTag::ControlFile | RelishTag::Checkpoint => false,
+        }
+    }
+
+    // convenience function to check if this relish is a normal relation.
+    pub const fn is_relation(&self) -> bool {
+        matches!(self, RelishTag::Relation(_))
+    }
+}
 
 ///
 /// Relation data file segment id throughout the Postgres cluster.
@@ -47,10 +144,7 @@ use postgres_ffi::Oid;
 /// are used for the same purpose.
 /// [See more related comments here](https:///github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/relfilenode.h#L57).
 ///
-// FIXME: should move 'forknum' as last field to keep this consistent with Postgres.
-// Then we could replace the custo Ord and PartialOrd implementations below with
-// deriving them.
-#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, Serialize, Deserialize)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy, Serialize, Deserialize)]
 pub struct RelTag {
     pub forknum: u8,
     pub spcnode: Oid,
@@ -58,34 +152,6 @@ pub struct RelTag {
     pub relnode: Oid,
 }
 
-impl PartialOrd for RelTag {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl Ord for RelTag {
-    fn cmp(&self, other: &Self) -> Ordering {
-        let mut cmp;
-
-        cmp = self.spcnode.cmp(&other.spcnode);
-        if cmp != Ordering::Equal {
-            return cmp;
-        }
-        cmp = self.dbnode.cmp(&other.dbnode);
-        if cmp != Ordering::Equal {
-            return cmp;
-        }
-        cmp = self.relnode.cmp(&other.relnode);
-        if cmp != Ordering::Equal {
-            return cmp;
-        }
-        cmp = self.forknum.cmp(&other.forknum);
-
-        cmp
-    }
-}
-
 /// Display RelTag in the same format that's used in most PostgreSQL debug messages:
 ///
 /// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
@@ -104,6 +170,34 @@ impl fmt::Display for RelTag {
     }
 }
 
+/// Display RelTag in the same format that's used in most PostgreSQL debug messages:
+///
+/// <spcnode>/<dbnode>/<relnode>[_fsm|_vm|_init]
+///
+impl fmt::Display for RelishTag {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            RelishTag::Relation(rel) => rel.fmt(f),
+            RelishTag::Slru { slru, segno } => {
+                // e.g. pg_clog/0001
+                write!(f, "{}/{:04X}", slru.to_str(), segno)
+            }
+            RelishTag::FileNodeMap { spcnode, dbnode } => {
+                write!(f, "relmapper file for spc {} db {}", spcnode, dbnode)
+            }
+            RelishTag::TwoPhase { xid } => {
+                write!(f, "pg_twophase/{:08X}", xid)
+            }
+            RelishTag::ControlFile => {
+                write!(f, "control file")
+            }
+            RelishTag::Checkpoint => {
+                write!(f, "checkpoint")
+            }
+        }
+    }
+}
+
 ///
 /// Non-relation transaction status files (clog (a.k.a. pg_xact) and
 /// pg_multixact) in Postgres are handled by SLRU (Simple LRU) buffer,

pageserver/src/remote_storage/local_fs.rs

@@ -641,7 +641,7 @@ mod fs_tests {
     }
 
     async fn upload_dummy_file(
-        harness: &RepoHarness<'_>,
+        harness: &RepoHarness,
         storage: &LocalFs,
         name: &str,
     ) -> anyhow::Result<PathBuf> {

pageserver/src/remote_storage/storage_sync.rs

@@ -881,7 +881,7 @@ mod test_utils {
 
     #[track_caller]
     pub async fn ensure_correct_timeline_upload(
-        harness: &RepoHarness<'_>,
+        harness: &RepoHarness,
         remote_assets: Arc<(LocalFs, RwLock<RemoteTimelineIndex>)>,
         timeline_id: ZTimelineId,
         new_upload: NewCheckpoint,

pageserver/src/tenant_mgr.rs

@@ -4,13 +4,11 @@
 use crate::branches;
 use crate::config::PageServerConf;
 use crate::layered_repository::LayeredRepository;
-use crate::repository::Repository;
-use crate::repository::TimelineSyncState;
+use crate::repository::{Repository, Timeline, TimelineSyncState};
 use crate::thread_mgr;
 use crate::thread_mgr::ThreadKind;
 use crate::walredo::PostgresRedoManager;
 use crate::CheckpointConfig;
-use crate::{DatadirTimelineImpl, RepositoryImpl};
 use anyhow::{bail, Context, Result};
 use lazy_static::lazy_static;
 use log::*;
@@ -26,9 +24,7 @@ lazy_static! {
 
 struct Tenant {
     state: TenantState,
-    repo: Arc<RepositoryImpl>,
-
-    timelines: HashMap<ZTimelineId, Arc<DatadirTimelineImpl>>,
+    repo: Arc<dyn Repository>,
 }
 
 #[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)]
@@ -82,17 +78,15 @@ pub fn set_timeline_states(
             let walredo_mgr = PostgresRedoManager::new(conf, tenant_id);
 
             // Set up an object repository, for actual data storage.
-            let repo = LayeredRepository::new(
+            let repo: Arc<dyn Repository> = Arc::new(LayeredRepository::new(
                 conf,
                 Arc::new(walredo_mgr),
                 tenant_id,
                 conf.remote_storage_config.is_some(),
-            );
-
+            ));
             Tenant {
                 state: TenantState::Idle,
-                repo: Arc::new(repo),
-                timelines: HashMap::new(),
+                repo,
             }
         });
         if let Err(e) = put_timelines_into_tenant(tenant, tenant_id, timeline_states) {
@@ -196,7 +190,6 @@ pub fn create_repository_for_tenant(
             v.insert(Tenant {
                 state: TenantState::Idle,
                 repo,
-                timelines: HashMap::new(),
             });
         }
     }
@@ -255,7 +248,7 @@ pub fn activate_tenant(conf: &'static PageServerConf, tenantid: ZTenantId) -> Re
     Ok(())
 }
 
-pub fn get_repository_for_tenant(tenantid: ZTenantId) -> Result<Arc<RepositoryImpl>> {
+pub fn get_repository_for_tenant(tenantid: ZTenantId) -> Result<Arc<dyn Repository>> {
     let m = access_tenants();
     let tenant = m
         .get(&tenantid)
@@ -267,26 +260,11 @@ pub fn get_repository_for_tenant(tenantid: ZTenantId) -> Result<Arc<RepositoryIm
 pub fn get_timeline_for_tenant(
     tenantid: ZTenantId,
     timelineid: ZTimelineId,
-) -> Result<Arc<DatadirTimelineImpl>> {
-    let mut m = access_tenants();
-    let tenant = m
-        .get_mut(&tenantid)
-        .with_context(|| format!("Tenant not found for tenant {}", tenantid))?;
-
-    if let Some(page_tline) = tenant.timelines.get(&timelineid) {
-        return Ok(Arc::clone(page_tline));
-    }
-    // First access to this timeline. Create a DatadirTimeline wrapper for it
-    let tline = tenant
-        .repo
+) -> Result<Arc<dyn Timeline>> {
+    get_repository_for_tenant(tenantid)?
         .get_timeline(timelineid)?
         .local_timeline()
-        .with_context(|| format!("cannot fetch timeline {}", timelineid))?;
-
-    let page_tline = Arc::new(DatadirTimelineImpl::new(tline));
-    page_tline.init_logical_size()?;
-    tenant.timelines.insert(timelineid, Arc::clone(&page_tline));
-    Ok(page_tline)
+        .with_context(|| format!("cannot fetch timeline {}", timelineid))
 }
 
 #[derive(Serialize, Deserialize, Clone)]

pageserver/src/tenant_threads.rs

@@ -1,7 +1,6 @@
 //! This module contains functions to serve per-tenant background processes,
 //! such as checkpointer and GC
 use crate::config::PageServerConf;
-use crate::repository::Repository;
 use crate::tenant_mgr;
 use crate::tenant_mgr::TenantState;
 use crate::CheckpointConfig;
@@ -14,15 +13,6 @@ use zenith_utils::zid::ZTenantId;
 /// Checkpointer thread's main loop
 ///
 pub fn checkpoint_loop(tenantid: ZTenantId, conf: &'static PageServerConf) -> Result<()> {
-    if let Err(err) = checkpoint_loop_ext(tenantid, conf) {
-        error!("checkpoint loop terminated with error: {:?}", err);
-        Err(err)
-    } else {
-        Ok(())
-    }
-}
-
-fn checkpoint_loop_ext(tenantid: ZTenantId, conf: &'static PageServerConf) -> Result<()> {
     loop {
         if tenant_mgr::get_tenant_state(tenantid) != Some(TenantState::Active) {
             break;

pageserver/src/walreceiver.rs

@@ -6,7 +6,6 @@
 //! We keep one WAL receiver active per timeline.
 
 use crate::config::PageServerConf;
-use crate::repository::Repository;
 use crate::tenant_mgr;
 use crate::thread_mgr;
 use crate::thread_mgr::ThreadKind;
@@ -251,10 +250,11 @@ fn walreceiver_main(
 
                     // It is important to deal with the aligned records as lsn in getPage@LSN is
                     // aligned and can be several bytes bigger. Without this alignment we are
-                    // at risk of hitting a deadlock.
+                    // at risk of hittind a deadlock.
                     assert!(lsn.is_aligned());
 
-                    walingest.ingest_record(&timeline, recdata, lsn)?;
+                    let writer = timeline.writer();
+                    walingest.ingest_record(writer.as_ref(), recdata, lsn)?;
 
                     fail_point!("walreceiver-after-ingest");
 

pageserver/src/walrecord.rs

@@ -10,47 +10,7 @@ use postgres_ffi::{MultiXactId, MultiXactOffset, MultiXactStatus, Oid, Transacti
 use serde::{Deserialize, Serialize};
 use tracing::*;
 
-/// Each update to a page is represented by a ZenithWalRecord. It can be a wrapper
-/// around a PostgreSQL WAL record, or a custom zenith-specific "record".
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub enum ZenithWalRecord {
-    /// Native PostgreSQL WAL record
-    Postgres { will_init: bool, rec: Bytes },
-
-    /// Clear bits in heap visibility map. ('flags' is bitmap of bits to clear)
-    ClearVisibilityMapFlags {
-        new_heap_blkno: Option<u32>,
-        old_heap_blkno: Option<u32>,
-        flags: u8,
-    },
-    /// Mark transaction IDs as committed on a CLOG page
-    ClogSetCommitted { xids: Vec<TransactionId> },
-    /// Mark transaction IDs as aborted on a CLOG page
-    ClogSetAborted { xids: Vec<TransactionId> },
-    /// Extend multixact offsets SLRU
-    MultixactOffsetCreate {
-        mid: MultiXactId,
-        moff: MultiXactOffset,
-    },
-    /// Extend multixact members SLRU.
-    MultixactMembersCreate {
-        moff: MultiXactOffset,
-        members: Vec<MultiXactMember>,
-    },
-}
-
-impl ZenithWalRecord {
-    /// Does replaying this WAL record initialize the page from scratch, or does
-    /// it need to be applied over the previous image of the page?
-    pub fn will_init(&self) -> bool {
-        match self {
-            ZenithWalRecord::Postgres { will_init, rec: _ } => *will_init,
-
-            // None of the special zenith record types currently initialize the page
-            _ => false,
-        }
-    }
-}
+use crate::repository::ZenithWalRecord;
 
 /// DecodedBkpBlock represents per-page data contained in a WAL record.
 #[derive(Default)]
@@ -127,28 +87,6 @@ impl XlRelmapUpdate {
     }
 }
 
-#[repr(C)]
-#[derive(Debug)]
-pub struct XlSmgrCreate {
-    pub rnode: RelFileNode,
-    // FIXME: This is ForkNumber in storage_xlog.h. That's an enum. Does it have
-    // well-defined size?
-    pub forknum: u8,
-}
-
-impl XlSmgrCreate {
-    pub fn decode(buf: &mut Bytes) -> XlSmgrCreate {
-        XlSmgrCreate {
-            rnode: RelFileNode {
-                spcnode: buf.get_u32_le(), /* tablespace */
-                dbnode: buf.get_u32_le(),  /* database */
-                relnode: buf.get_u32_le(), /* relation */
-            },
-            forknum: buf.get_u32_le() as u8,
-        }
-    }
-}
-
 #[repr(C)]
 #[derive(Debug)]
 pub struct XlSmgrTruncate {
@@ -330,11 +268,12 @@ impl XlXactParsedRecord {
         let info = xl_info & pg_constants::XLOG_XACT_OPMASK;
         // The record starts with time of commit/abort
         let xact_time = buf.get_i64_le();
-        let xinfo = if xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 {
-            buf.get_u32_le()
+        let xinfo;
+        if xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 {
+            xinfo = buf.get_u32_le();
         } else {
-            0
-        };
+            xinfo = 0;
+        }
         let db_id;
         let ts_id;
         if xinfo & pg_constants::XACT_XINFO_HAS_DBINFO != 0 {
@@ -563,6 +502,7 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
             0..=pg_constants::XLR_MAX_BLOCK_ID => {
                 /* XLogRecordBlockHeader */
                 let mut blk = DecodedBkpBlock::new();
+                let fork_flags: u8;
 
                 if block_id <= max_block_id {
                     // TODO
@@ -575,7 +515,7 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
                 }
                 max_block_id = block_id;
 
-                let fork_flags: u8 = buf.get_u8();
+                fork_flags = buf.get_u8();
                 blk.forknum = fork_flags & pg_constants::BKPBLOCK_FORK_MASK;
                 blk.flags = fork_flags;
                 blk.has_image = (fork_flags & pg_constants::BKPBLOCK_HAS_IMAGE) != 0;

pageserver/src/walredo.rs

@@ -42,10 +42,8 @@ use zenith_utils::nonblock::set_nonblock;
 use zenith_utils::zid::ZTenantId;
 
 use crate::config::PageServerConf;
-use crate::pgdatadir_mapping::{key_to_rel_block, key_to_slru_block};
 use crate::relish::*;
-use crate::repository::Key;
-use crate::walrecord::ZenithWalRecord;
+use crate::repository::ZenithWalRecord;
 use postgres_ffi::nonrelfile_utils::mx_offset_to_flags_bitshift;
 use postgres_ffi::nonrelfile_utils::mx_offset_to_flags_offset;
 use postgres_ffi::nonrelfile_utils::mx_offset_to_member_offset;
@@ -77,7 +75,8 @@ pub trait WalRedoManager: Send + Sync {
     /// the reords.
     fn request_redo(
         &self,
-        key: Key,
+        rel: RelishTag,
+        blknum: u32,
         lsn: Lsn,
         base_img: Option<Bytes>,
         records: Vec<(Lsn, ZenithWalRecord)>,
@@ -93,7 +92,8 @@ pub struct DummyRedoManager {}
 impl crate::walredo::WalRedoManager for DummyRedoManager {
     fn request_redo(
         &self,
-        _key: Key,
+        _rel: RelishTag,
+        _blknum: u32,
         _lsn: Lsn,
         _base_img: Option<Bytes>,
         _records: Vec<(Lsn, ZenithWalRecord)>,
@@ -102,6 +102,8 @@ impl crate::walredo::WalRedoManager for DummyRedoManager {
     }
 }
 
+static TIMEOUT: Duration = Duration::from_secs(20);
+
 // Metrics collected on WAL redo operations
 //
 // We collect the time spent in actual WAL redo ('redo'), and time waiting
@@ -152,6 +154,28 @@ fn can_apply_in_zenith(rec: &ZenithWalRecord) -> bool {
     }
 }
 
+fn check_forknum(rel: &RelishTag, expected_forknum: u8) -> bool {
+    if let RelishTag::Relation(RelTag {
+        forknum,
+        spcnode: _,
+        dbnode: _,
+        relnode: _,
+    }) = rel
+    {
+        *forknum == expected_forknum
+    } else {
+        false
+    }
+}
+
+fn check_slru_segno(rel: &RelishTag, expected_slru: SlruKind, expected_segno: u32) -> bool {
+    if let RelishTag::Slru { slru, segno } = rel {
+        *slru == expected_slru && *segno == expected_segno
+    } else {
+        false
+    }
+}
+
 /// An error happened in WAL redo
 #[derive(Debug, thiserror::Error)]
 pub enum WalRedoError {
@@ -162,8 +186,6 @@ pub enum WalRedoError {
     InvalidState,
     #[error("cannot perform WAL redo for this request")]
     InvalidRequest,
-    #[error("cannot perform WAL redo for this record")]
-    InvalidRecord,
 }
 
 ///
@@ -178,7 +200,8 @@ impl WalRedoManager for PostgresRedoManager {
     ///
     fn request_redo(
         &self,
-        key: Key,
+        rel: RelishTag,
+        blknum: u32,
         lsn: Lsn,
         base_img: Option<Bytes>,
         records: Vec<(Lsn, ZenithWalRecord)>,
@@ -196,15 +219,9 @@ impl WalRedoManager for PostgresRedoManager {
 
             if rec_zenith != batch_zenith {
                 let result = if batch_zenith {
-                    self.apply_batch_zenith(key, lsn, img, &records[batch_start..i])
+                    self.apply_batch_zenith(rel, blknum, lsn, img, &records[batch_start..i])
                 } else {
-                    self.apply_batch_postgres(
-                        key,
-                        lsn,
-                        img,
-                        &records[batch_start..i],
-                        self.conf.wal_redo_timeout,
-                    )
+                    self.apply_batch_postgres(rel, blknum, lsn, img, &records[batch_start..i])
                 };
                 img = Some(result?);
 
@@ -214,15 +231,9 @@ impl WalRedoManager for PostgresRedoManager {
         }
         // last batch
         if batch_zenith {
-            self.apply_batch_zenith(key, lsn, img, &records[batch_start..])
+            self.apply_batch_zenith(rel, blknum, lsn, img, &records[batch_start..])
         } else {
-            self.apply_batch_postgres(
-                key,
-                lsn,
-                img,
-                &records[batch_start..],
-                self.conf.wal_redo_timeout,
-            )
+            self.apply_batch_postgres(rel, blknum, lsn, img, &records[batch_start..])
         }
     }
 }
@@ -245,16 +256,16 @@ impl PostgresRedoManager {
     ///
     fn apply_batch_postgres(
         &self,
-        key: Key,
+        rel: RelishTag,
+        blknum: u32,
         lsn: Lsn,
         base_img: Option<Bytes>,
         records: &[(Lsn, ZenithWalRecord)],
-        wal_redo_timeout: Duration,
     ) -> Result<Bytes, WalRedoError> {
-        let (rel, blknum) = key_to_rel_block(key).or(Err(WalRedoError::InvalidRecord))?;
-
         let start_time = Instant::now();
 
+        let apply_result: Result<Bytes, Error>;
+
         let mut process_guard = self.process.lock().unwrap();
         let lock_time = Instant::now();
 
@@ -267,11 +278,16 @@ impl PostgresRedoManager {
 
         WAL_REDO_WAIT_TIME.observe(lock_time.duration_since(start_time).as_secs_f64());
 
-        // Relational WAL records are applied using wal-redo-postgres
-        let buf_tag = BufferTag { rel, blknum };
-        let result = process
-            .apply_wal_records(buf_tag, base_img, records, wal_redo_timeout)
-            .map_err(WalRedoError::IoError);
+        let result = if let RelishTag::Relation(rel) = rel {
+            // Relational WAL records are applied using wal-redo-postgres
+            let buf_tag = BufferTag { rel, blknum };
+            apply_result = process.apply_wal_records(buf_tag, base_img, records);
+
+            apply_result.map_err(WalRedoError::IoError)
+        } else {
+            error!("unexpected non-relation relish: {:?}", rel);
+            Err(WalRedoError::InvalidRequest)
+        };
 
         let end_time = Instant::now();
         let duration = end_time.duration_since(lock_time);
@@ -297,7 +313,8 @@ impl PostgresRedoManager {
     ///
     fn apply_batch_zenith(
         &self,
-        key: Key,
+        rel: RelishTag,
+        blknum: u32,
         lsn: Lsn,
         base_img: Option<Bytes>,
         records: &[(Lsn, ZenithWalRecord)],
@@ -316,7 +333,7 @@ impl PostgresRedoManager {
 
         // Apply all the WAL records in the batch
         for (record_lsn, record) in records.iter() {
-            self.apply_record_zenith(key, &mut page, *record_lsn, record)?;
+            self.apply_record_zenith(rel, blknum, &mut page, *record_lsn, record)?;
         }
         // Success!
         let end_time = Instant::now();
@@ -335,7 +352,8 @@ impl PostgresRedoManager {
 
     fn apply_record_zenith(
         &self,
-        key: Key,
+        rel: RelishTag,
+        blknum: u32,
         page: &mut BytesMut,
         _record_lsn: Lsn,
         record: &ZenithWalRecord,
@@ -351,10 +369,9 @@ impl PostgresRedoManager {
                 flags,
             } => {
                 // sanity check that this is modifying the correct relish
-                let (rel, blknum) = key_to_rel_block(key).or(Err(WalRedoError::InvalidRecord))?;
                 assert!(
-                    rel.forknum == pg_constants::VISIBILITYMAP_FORKNUM,
-                    "ClearVisibilityMapFlags record on unexpected rel {}",
+                    check_forknum(&rel, pg_constants::VISIBILITYMAP_FORKNUM),
+                    "ClearVisibilityMapFlags record on unexpected rel {:?}",
                     rel
                 );
                 if let Some(heap_blkno) = *new_heap_blkno {
@@ -388,14 +405,6 @@ impl PostgresRedoManager {
             // Non-relational WAL records are handled here, with custom code that has the
             // same effects as the corresponding Postgres WAL redo function.
             ZenithWalRecord::ClogSetCommitted { xids } => {
-                let (slru_kind, segno, blknum) =
-                    key_to_slru_block(key).or(Err(WalRedoError::InvalidRecord))?;
-                assert_eq!(
-                    slru_kind,
-                    SlruKind::Clog,
-                    "ClogSetCommitted record with unexpected key {}",
-                    key
-                );
                 for &xid in xids {
                     let pageno = xid as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
                     let expected_segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
@@ -403,17 +412,12 @@ impl PostgresRedoManager {
 
                     // Check that we're modifying the correct CLOG block.
                     assert!(
-                        segno == expected_segno,
-                        "ClogSetCommitted record for XID {} with unexpected key {}",
+                        check_slru_segno(&rel, SlruKind::Clog, expected_segno),
+                        "ClogSetCommitted record for XID {} with unexpected rel {:?}",
                         xid,
-                        key
-                    );
-                    assert!(
-                        blknum == expected_blknum,
-                        "ClogSetCommitted record for XID {} with unexpected key {}",
-                        xid,
-                        key
+                        rel
                     );
+                    assert!(blknum == expected_blknum);
 
                     transaction_id_set_status(
                         xid,
@@ -423,14 +427,6 @@ impl PostgresRedoManager {
                 }
             }
             ZenithWalRecord::ClogSetAborted { xids } => {
-                let (slru_kind, segno, blknum) =
-                    key_to_slru_block(key).or(Err(WalRedoError::InvalidRecord))?;
-                assert_eq!(
-                    slru_kind,
-                    SlruKind::Clog,
-                    "ClogSetAborted record with unexpected key {}",
-                    key
-                );
                 for &xid in xids {
                     let pageno = xid as u32 / pg_constants::CLOG_XACTS_PER_PAGE;
                     let expected_segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
@@ -438,30 +434,17 @@ impl PostgresRedoManager {
 
                     // Check that we're modifying the correct CLOG block.
                     assert!(
-                        segno == expected_segno,
-                        "ClogSetAborted record for XID {} with unexpected key {}",
+                        check_slru_segno(&rel, SlruKind::Clog, expected_segno),
+                        "ClogSetCommitted record for XID {} with unexpected rel {:?}",
                         xid,
-                        key
-                    );
-                    assert!(
-                        blknum == expected_blknum,
-                        "ClogSetAborted record for XID {} with unexpected key {}",
-                        xid,
-                        key
+                        rel
                     );
+                    assert!(blknum == expected_blknum);
 
                     transaction_id_set_status(xid, pg_constants::TRANSACTION_STATUS_ABORTED, page);
                 }
             }
             ZenithWalRecord::MultixactOffsetCreate { mid, moff } => {
-                let (slru_kind, segno, blknum) =
-                    key_to_slru_block(key).or(Err(WalRedoError::InvalidRecord))?;
-                assert_eq!(
-                    slru_kind,
-                    SlruKind::MultiXactOffsets,
-                    "MultixactOffsetCreate record with unexpected key {}",
-                    key
-                );
                 // Compute the block and offset to modify.
                 // See RecordNewMultiXact in PostgreSQL sources.
                 let pageno = mid / pg_constants::MULTIXACT_OFFSETS_PER_PAGE as u32;
@@ -472,29 +455,16 @@ impl PostgresRedoManager {
                 let expected_segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
                 let expected_blknum = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
                 assert!(
-                    segno == expected_segno,
-                    "MultiXactOffsetsCreate record for multi-xid {} with unexpected key {}",
+                    check_slru_segno(&rel, SlruKind::MultiXactOffsets, expected_segno),
+                    "MultiXactOffsetsCreate record for multi-xid {} with unexpected rel {:?}",
                     mid,
-                    key
-                );
-                assert!(
-                    blknum == expected_blknum,
-                    "MultiXactOffsetsCreate record for multi-xid {} with unexpected key {}",
-                    mid,
-                    key
+                    rel
                 );
+                assert!(blknum == expected_blknum);
 
                 LittleEndian::write_u32(&mut page[offset..offset + 4], *moff);
             }
             ZenithWalRecord::MultixactMembersCreate { moff, members } => {
-                let (slru_kind, segno, blknum) =
-                    key_to_slru_block(key).or(Err(WalRedoError::InvalidRecord))?;
-                assert_eq!(
-                    slru_kind,
-                    SlruKind::MultiXactMembers,
-                    "MultixactMembersCreate record with unexpected key {}",
-                    key
-                );
                 for (i, member) in members.iter().enumerate() {
                     let offset = moff + i as u32;
 
@@ -509,17 +479,12 @@ impl PostgresRedoManager {
                     let expected_segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT;
                     let expected_blknum = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT;
                     assert!(
-                        segno == expected_segno,
-                        "MultiXactMembersCreate record for offset {} with unexpected key {}",
+                        check_slru_segno(&rel, SlruKind::MultiXactMembers, expected_segno),
+                        "MultiXactMembersCreate record at offset {} with unexpected rel {:?}",
                         moff,
-                        key
-                    );
-                    assert!(
-                        blknum == expected_blknum,
-                        "MultiXactMembersCreate record for offset {} with unexpected key {}",
-                        moff,
-                        key
+                        rel
                     );
+                    assert!(blknum == expected_blknum);
 
                     let mut flagsval = LittleEndian::read_u32(&page[flagsoff..flagsoff + 4]);
                     flagsval &= !(((1 << pg_constants::MXACT_MEMBER_BITS_PER_XACT) - 1) << bshift);
@@ -638,7 +603,6 @@ impl PostgresRedoProcess {
         tag: BufferTag,
         base_img: Option<Bytes>,
         records: &[(Lsn, ZenithWalRecord)],
-        wal_redo_timeout: Duration,
     ) -> Result<Bytes, std::io::Error> {
         // Serialize all the messages to send the WAL redo process first.
         //
@@ -689,7 +653,7 @@ impl PostgresRedoProcess {
             // If we have more data to write, wake up if 'stdin' becomes writeable or
             // we have data to read. Otherwise only wake up if there's data to read.
             let nfds = if nwrite < writebuf.len() { 3 } else { 2 };
-            let n = nix::poll::poll(&mut pollfds[0..nfds], wal_redo_timeout.as_millis() as i32)?;
+            let n = nix::poll::poll(&mut pollfds[0..nfds], TIMEOUT.as_millis() as i32)?;
 
             if n == 0 {
                 return Err(Error::new(ErrorKind::Other, "WAL redo timed out"));

poetry.lock

@@ -91,14 +91,6 @@ botocore = ">=1.11.3"
 future = "*"
 wrapt = "*"
 
-[[package]]
-name = "backoff"
-version = "1.11.1"
-description = "Function decoration for backoff and retry"
-category = "main"
-optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
-
 [[package]]
 name = "boto3"
 version = "1.20.40"
@@ -822,11 +814,11 @@ python-versions = "*"
 
 [[package]]
 name = "moto"
-version = "3.0.4"
+version = "3.0.0"
 description = "A library that allows your python tests to easily mock out the boto library"
 category = "main"
 optional = false
-python-versions = ">=3.6"
+python-versions = "*"
 
 [package.dependencies]
 aws-xray-sdk = {version = ">=0.93,<0.96 || >0.96", optional = true, markers = "extra == \"server\""}
@@ -856,8 +848,7 @@ xmltodict = "*"
 
 [package.extras]
 all = ["PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)", "docker (>=2.5.1)", "graphql-core", "jsondiff (>=1.1.2)", "aws-xray-sdk (>=0.93,!=0.96)", "idna (>=2.5,<4)", "cfn-lint (>=0.4.0)", "sshpubkeys (>=3.1.0)", "setuptools"]
-apigateway = ["PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)"]
-apigatewayv2 = ["PyYAML (>=5.1)"]
+apigateway = ["python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)"]
 appsync = ["graphql-core"]
 awslambda = ["docker (>=2.5.1)"]
 batch = ["docker (>=2.5.1)"]
@@ -1361,7 +1352,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.7"
-content-hash = "58762accad4122026c650fa43421a900546e89f9908e2268410e7b11cc8c6c4e"
+content-hash = "0fa6c9377fbc827240d18d8b7e3742def37e90fc3277fddf8525d82dabd13090"
 
 [metadata.files]
 aiopg = [
@@ -1404,10 +1395,6 @@ aws-xray-sdk = [
     {file = "aws-xray-sdk-2.9.0.tar.gz", hash = "sha256:b0cd972db218d4d8f7b53ad806fc6184626b924c4997ae58fc9f2a8cd1281568"},
     {file = "aws_xray_sdk-2.9.0-py2.py3-none-any.whl", hash = "sha256:98216b3ac8281b51b59a8703f8ec561c460807d9d0679838f5c0179d381d7e58"},
 ]
-backoff = [
-    {file = "backoff-1.11.1-py2.py3-none-any.whl", hash = "sha256:61928f8fa48d52e4faa81875eecf308eccfb1016b018bb6bd21e05b5d90a96c5"},
-    {file = "backoff-1.11.1.tar.gz", hash = "sha256:ccb962a2378418c667b3c979b504fdeb7d9e0d29c0579e3b13b86467177728cb"},
-]
 boto3 = [
     {file = "boto3-1.20.40-py3-none-any.whl", hash = "sha256:cfe85589e4a0a997c7b9ae7432400b03fa6fa5fea29fdc48db3099a903b76998"},
     {file = "boto3-1.20.40.tar.gz", hash = "sha256:66aef9a6d8cad393f69166112ba49e14e2c6766f9278c96134101314a9af2992"},
@@ -1679,8 +1666,8 @@ mccabe = [
     {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"},
 ]
 moto = [
-    {file = "moto-3.0.4-py2.py3-none-any.whl", hash = "sha256:79646213d8438385182f4eea79e28725f94b3d0d3dc9a3eda81db47e0ebef6cc"},
-    {file = "moto-3.0.4.tar.gz", hash = "sha256:168b8a3cb4dd8a6df8e51d582761cefa9657b9f45ac7e1eb24dae394ebc9e000"},
+    {file = "moto-3.0.0-py2.py3-none-any.whl", hash = "sha256:762d33bbad3642c687f6495e69331318bef43f9aa662174397706ec3ad2a3578"},
+    {file = "moto-3.0.0.tar.gz", hash = "sha256:d6b00a2663290e7ebb06823d5ffcb124c8dc9bf526b878539ef7c4a377fd8255"},
 ]
 mypy = [
     {file = "mypy-0.910-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:a155d80ea6cee511a3694b108c4494a39f42de11ee4e61e72bc424c490e46457"},

postgres_ffi/src/pg_constants.rs

@@ -24,9 +24,6 @@ pub const VISIBILITYMAP_FORKNUM: u8 = 2;
 pub const INIT_FORKNUM: u8 = 3;
 
 // From storage_xlog.h
-pub const XLOG_SMGR_CREATE: u8 = 0x10;
-pub const XLOG_SMGR_TRUNCATE: u8 = 0x20;
-
 pub const SMGR_TRUNCATE_HEAP: u32 = 0x0001;
 pub const SMGR_TRUNCATE_VM: u32 = 0x0002;
 pub const SMGR_TRUNCATE_FSM: u32 = 0x0004;
@@ -116,6 +113,7 @@ pub const XACT_XINFO_HAS_TWOPHASE: u32 = 1u32 << 4;
 // From pg_control.h and rmgrlist.h
 pub const XLOG_NEXTOID: u8 = 0x30;
 pub const XLOG_SWITCH: u8 = 0x40;
+pub const XLOG_SMGR_TRUNCATE: u8 = 0x20;
 pub const XLOG_FPI_FOR_HINT: u8 = 0xA0;
 pub const XLOG_FPI: u8 = 0xB0;
 pub const DB_SHUTDOWNED: u32 = 1;

proxy/src/main.rs

@@ -122,7 +122,7 @@ async fn main() -> anyhow::Result<()> {
         None => RouterConfig::Dynamic(auth_method),
         Some(addr) => {
             if let ClientAuthMethod::Password = auth_method {
-                let (host, port) = addr.split_once(':').unwrap();
+                let (host, port) = addr.split_once(":").unwrap();
                 RouterConfig::Static {
                     host: host.to_string(),
                     port: port.parse().unwrap(),

pyproject.toml

@@ -21,7 +21,6 @@ types-psycopg2 = "^2.9.6"
 boto3 = "^1.20.40"
 boto3-stubs = "^1.20.40"
 moto = {version = "^3.0.0", extras = ["server"]}
-backoff = "^1.11.1"
 
 [tool.poetry.dev-dependencies]
 yapf = "==0.31.0"

test_runner/README.md

@@ -89,7 +89,7 @@ def test_foobar(zenith_env_builder: ZenithEnvBuilder):
 
     # Now create the environment. This initializes the repository, and starts
     # up the page server and the safekeepers
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     # Run the test
     ...

test_runner/batch_others/test_auth.py

@@ -8,7 +8,7 @@ import pytest
 
 def test_pageserver_auth(zenith_env_builder: ZenithEnvBuilder):
     zenith_env_builder.pageserver_auth_enabled = True
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     ps = env.pageserver
 
@@ -51,7 +51,7 @@ def test_compute_auth_to_pageserver(zenith_env_builder: ZenithEnvBuilder, with_w
     zenith_env_builder.pageserver_auth_enabled = True
     if with_wal_acceptors:
         zenith_env_builder.num_safekeepers = 3
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     branch = f"test_compute_auth_to_pageserver{with_wal_acceptors}"
     env.zenith_cli.create_branch(branch, "main")

test_runner/batch_others/test_backpressure.py

@@ -93,7 +93,7 @@ def check_backpressure(pg: Postgres, stop_event: threading.Event, polling_interv
 
 def test_backpressure_received_lsn_lag(zenith_env_builder: ZenithEnvBuilder):
     zenith_env_builder.num_safekeepers = 1
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
     # Create a branch for us
     env.zenith_cli.create_branch("test_backpressure", "main")
 

test_runner/batch_others/test_branch_behind.py

@@ -19,7 +19,7 @@ def test_branch_behind(zenith_env_builder: ZenithEnvBuilder):
     #
     # See https://github.com/zenithdb/zenith/issues/1068
     zenith_env_builder.num_safekeepers = 1
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     # Branch at the point where only 100 rows were inserted
     env.zenith_cli.create_branch("test_branch_behind", "main")

test_runner/batch_others/test_next_xid.py

@@ -11,7 +11,7 @@ from fixtures.log_helper import log
 def test_next_xid(zenith_env_builder: ZenithEnvBuilder):
     # One safekeeper is enough for this test.
     zenith_env_builder.num_safekeepers = 1
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     pg = env.postgres.create_start('main')
 

test_runner/batch_others/test_pageserver_api.py

@@ -1,15 +1,8 @@
+import json
 from uuid import uuid4, UUID
-import pytest
-from fixtures.zenith_fixtures import ZenithEnv, ZenithEnvBuilder, ZenithPageserverHttpClient, zenith_binpath
-
-
-# test that we cannot override node id
-def test_pageserver_init_node_id(zenith_env_builder: ZenithEnvBuilder):
-    env = zenith_env_builder.init()
-    with pytest.raises(
-            Exception,
-            match="node id can only be set during pageserver init and cannot be overridden"):
-        env.pageserver.start(overrides=['--pageserver-config-override=id=10'])
+from fixtures.zenith_fixtures import ZenithEnv, ZenithEnvBuilder, ZenithPageserverHttpClient
+from typing import cast
+import pytest, psycopg2
 
 
 def check_client(client: ZenithPageserverHttpClient, initial_tenant: UUID):
@@ -48,7 +41,7 @@ def test_pageserver_http_api_client(zenith_simple_env: ZenithEnv):
 
 def test_pageserver_http_api_client_auth_enabled(zenith_env_builder: ZenithEnvBuilder):
     zenith_env_builder.pageserver_auth_enabled = True
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     management_token = env.auth_keys.generate_management_token()
 

test_runner/batch_others/test_pageserver_catchup.py

@@ -14,7 +14,7 @@ from fixtures.log_helper import log
 # and new compute node contains all data.
 def test_pageserver_catchup_while_compute_down(zenith_env_builder: ZenithEnvBuilder):
     zenith_env_builder.num_safekeepers = 3
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     env.zenith_cli.create_branch("test_pageserver_catchup_while_compute_down", "main")
     pg = env.postgres.create_start('test_pageserver_catchup_while_compute_down')

test_runner/batch_others/test_pageserver_restart.py

@@ -13,7 +13,7 @@ from fixtures.log_helper import log
 def test_pageserver_restart(zenith_env_builder: ZenithEnvBuilder):
     # One safekeeper is enough for this test.
     zenith_env_builder.num_safekeepers = 1
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     env.zenith_cli.create_branch("test_pageserver_restart", "main")
     pg = env.postgres.create_start('test_pageserver_restart')

test_runner/batch_others/test_proxy.py

@@ -1,2 +1,54 @@
+import pytest
+import subprocess
+import signal
+import time
+
+
 def test_proxy_select_1(static_proxy):
     static_proxy.safe_psql("select 1;")
+
+
+def test_proxy_cancel(static_proxy):
+    """Test that we can cancel a big generate_series query."""
+    conn = static_proxy.connect()
+    conn.cancel()
+
+    with conn.cursor() as cur:
+        from psycopg2.errors import QueryCanceled
+        with pytest.raises(QueryCanceled):
+            cur.execute("select * from generate_series(1, 100000000);")
+
+
+def test_proxy_pgbench_cancel(static_proxy, pg_bin):
+    """Test that we can cancel the init phase of pgbench."""
+    start_time = static_proxy.safe_psql("select now();")[0]
+
+    def get_running_queries():
+        magic_string = "fsdsdfhdfhfgbcbfgbfgbf"
+        with static_proxy.connect() as conn:
+            with conn.cursor() as cur:
+                cur.execute(f"""
+                    -- {magic_string}
+                    select query
+                    from pg_stat_activity
+                    where pg_stat_activity.query_start > %s
+                """, start_time)
+                return [
+                    row[0]
+                    for row in cur.fetchall()
+                    if not magic_string in row[0]
+                ]
+
+    # Let pgbench init run for 1 second
+    p = subprocess.Popen(['pgbench', '-s500', '-i', static_proxy.connstr()])
+    time.sleep(1)
+
+    # Make sure something is still running, and that get_running_queries works
+    assert len(get_running_queries()) > 0
+
+    # Send sigint, which would cancel any pgbench queries
+    p.send_signal(signal.SIGINT)
+
+    # Assert that nothing is running
+    time.sleep(1)
+    assert len(get_running_queries()) == 0

test_runner/batch_others/test_remote_storage.py

@@ -42,7 +42,7 @@ def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder,
     data_secret = 'very secret secret'
 
     ##### First start, insert secret data and upload it to the remote storage
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
     pg = env.postgres.create_start()
 
     tenant_id = pg.safe_psql("show zenith.zenith_tenant")[0][0]

test_runner/batch_others/test_restart_compute.py

@@ -13,7 +13,7 @@ def test_restart_compute(zenith_env_builder: ZenithEnvBuilder, with_wal_acceptor
     zenith_env_builder.pageserver_auth_enabled = True
     if with_wal_acceptors:
         zenith_env_builder.num_safekeepers = 3
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     env.zenith_cli.create_branch("test_restart_compute", "main")
 

test_runner/batch_others/test_snapfiles_gc.py

@@ -0,0 +1,131 @@
+from contextlib import closing
+import psycopg2.extras
+import time
+from fixtures.utils import print_gc_result
+from fixtures.zenith_fixtures import ZenithEnv
+from fixtures.log_helper import log
+
+
+#
+# Test Garbage Collection of old layer files
+#
+# This test is pretty tightly coupled with the current implementation of layered
+# storage, in layered_repository.rs.
+#
+def test_layerfiles_gc(zenith_simple_env: ZenithEnv):
+    env = zenith_simple_env
+    env.zenith_cli.create_branch("test_layerfiles_gc", "empty")
+    pg = env.postgres.create_start('test_layerfiles_gc')
+
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            with closing(env.pageserver.connect()) as psconn:
+                with psconn.cursor(cursor_factory=psycopg2.extras.DictCursor) as pscur:
+
+                    # Get the timeline ID of our branch. We need it for the 'do_gc' command
+                    cur.execute("SHOW zenith.zenith_timeline")
+                    timeline = cur.fetchone()[0]
+
+                    # Create a test table
+                    cur.execute("CREATE TABLE foo(x integer)")
+                    cur.execute("INSERT INTO foo VALUES (1)")
+
+                    cur.execute("select relfilenode from pg_class where oid = 'foo'::regclass")
+                    row = cur.fetchone()
+                    log.info(f"relfilenode is {row[0]}")
+
+                    # Run GC, to clear out any garbage left behind in the catalogs by
+                    # the CREATE TABLE command. We want to have a clean slate with no garbage
+                    # before running the actual tests below, otherwise the counts won't match
+                    # what we expect.
+                    #
+                    # Also run vacuum first to make it less likely that autovacuum or pruning
+                    # kicks in and confuses our numbers.
+                    cur.execute("VACUUM")
+
+                    # delete the row, to update the Visibility Map. We don't want the VM
+                    # update to confuse our numbers either.
+                    cur.execute("DELETE FROM foo")
+
+                    log.info("Running GC before test")
+                    pscur.execute(f"do_gc {env.initial_tenant.hex} {timeline} 0")
+                    row = pscur.fetchone()
+                    print_gc_result(row)
+                    # remember the number of files
+                    layer_relfiles_remain = (row['layer_relfiles_total'] -
+                                             row['layer_relfiles_removed'])
+                    assert layer_relfiles_remain > 0
+
+                    # Insert a row and run GC. Checkpoint should freeze the layer
+                    # so that there is only the most recent image layer left for the rel,
+                    # removing the old image and delta layer.
+                    log.info("Inserting one row and running GC")
+                    cur.execute("INSERT INTO foo VALUES (1)")
+                    pscur.execute(f"do_gc {env.initial_tenant.hex} {timeline} 0")
+                    row = pscur.fetchone()
+                    print_gc_result(row)
+                    assert row['layer_relfiles_total'] == layer_relfiles_remain + 2
+                    assert row['layer_relfiles_removed'] == 2
+                    assert row['layer_relfiles_dropped'] == 0
+
+                    # Insert two more rows and run GC.
+                    # This should create new image and delta layer file with the new contents, and
+                    # then remove the old one image and the just-created delta layer.
+                    log.info("Inserting two more rows and running GC")
+                    cur.execute("INSERT INTO foo VALUES (2)")
+                    cur.execute("INSERT INTO foo VALUES (3)")
+
+                    pscur.execute(f"do_gc {env.initial_tenant.hex} {timeline} 0")
+                    row = pscur.fetchone()
+                    print_gc_result(row)
+                    assert row['layer_relfiles_total'] == layer_relfiles_remain + 2
+                    assert row['layer_relfiles_removed'] == 2
+                    assert row['layer_relfiles_dropped'] == 0
+
+                    # Do it again. Should again create two new layer files and remove old ones.
+                    log.info("Inserting two more rows and running GC")
+                    cur.execute("INSERT INTO foo VALUES (2)")
+                    cur.execute("INSERT INTO foo VALUES (3)")
+
+                    pscur.execute(f"do_gc {env.initial_tenant.hex} {timeline} 0")
+                    row = pscur.fetchone()
+                    print_gc_result(row)
+                    assert row['layer_relfiles_total'] == layer_relfiles_remain + 2
+                    assert row['layer_relfiles_removed'] == 2
+                    assert row['layer_relfiles_dropped'] == 0
+
+                    # Run GC again, with no changes in the database. Should not remove anything.
+                    log.info("Run GC again, with nothing to do")
+                    pscur.execute(f"do_gc {env.initial_tenant.hex} {timeline} 0")
+                    row = pscur.fetchone()
+                    print_gc_result(row)
+                    assert row['layer_relfiles_total'] == layer_relfiles_remain
+                    assert row['layer_relfiles_removed'] == 0
+                    assert row['layer_relfiles_dropped'] == 0
+
+                    #
+                    # Test DROP TABLE checks that relation data and metadata was deleted by GC from object storage
+                    #
+                    log.info("Drop table and run GC again")
+                    cur.execute("DROP TABLE foo")
+
+                    pscur.execute(f"do_gc {env.initial_tenant.hex} {timeline} 0")
+                    row = pscur.fetchone()
+                    print_gc_result(row)
+
+                    # We still cannot remove the latest layers
+                    # because they serve as tombstones for earlier layers.
+                    assert row['layer_relfiles_dropped'] == 0
+                    # Each relation fork is counted separately, hence 3.
+                    assert row['layer_relfiles_needed_as_tombstone'] == 3
+
+                    # The catalog updates also create new layer files of the catalogs, which
+                    # are counted as 'removed'
+                    assert row['layer_relfiles_removed'] > 0
+
+                    # TODO Change the test to check actual CG of dropped layers.
+                    # Each relation fork is counted separately, hence 3.
+                    #assert row['layer_relfiles_dropped'] == 3
+
+                    # TODO: perhaps we should count catalog and user relations separately,
+                    # to make this kind of testing more robust

test_runner/batch_others/test_tenant_relocation.py

@@ -122,7 +122,7 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder,
     zenith_env_builder.num_safekeepers = 1
     zenith_env_builder.enable_local_fs_remote_storage()
 
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     # create folder for remote storage mock
     remote_storage_mock_path = env.repo_dir / 'local_fs_remote_storage'

test_runner/batch_others/test_tenants.py

@@ -10,7 +10,7 @@ def test_tenants_normal_work(zenith_env_builder: ZenithEnvBuilder, with_wal_acce
     if with_wal_acceptors:
         zenith_env_builder.num_safekeepers = 3
 
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
     """Tests tenants with and without wal acceptors"""
     tenant_1 = env.create_tenant()
     tenant_2 = env.create_tenant()

test_runner/batch_others/test_timeline_size.py

@@ -67,7 +67,7 @@ def wait_for_pageserver_catchup(pgmain: Postgres, polling_interval=1, timeout=60
 
 def test_timeline_size_quota(zenith_env_builder: ZenithEnvBuilder):
     zenith_env_builder.num_safekeepers = 1
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
     env.zenith_cli.create_branch("test_timeline_size_quota", "main")
 
     client = env.pageserver.http_client()

test_runner/batch_others/test_twophase.py

@@ -78,8 +78,8 @@ def test_twophase(zenith_simple_env: ZenithEnv):
     cur2.execute("ROLLBACK PREPARED 'insert_two'")
 
     cur2.execute('SELECT * FROM foo')
-    assert cur2.fetchall() == [('one', ), ('three', )]
+    assert cur2.fetchall() == [('one', ), ('three', )]  # type: ignore[comparison-overlap]
 
     # Only one committed insert is visible on the original branch
     cur.execute('SELECT * FROM foo')
-    assert cur.fetchall() == [('three', )]
+    assert cur.fetchall() == [('three', )]  # type: ignore[comparison-overlap]

test_runner/batch_others/test_wal_acceptor.py

@@ -22,7 +22,7 @@ from typing import List, Optional, Any
 # succeed and data is written
 def test_normal_work(zenith_env_builder: ZenithEnvBuilder):
     zenith_env_builder.num_safekeepers = 3
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     env.zenith_cli.create_branch("test_wal_acceptors_normal_work", "main")
 
@@ -51,7 +51,7 @@ class BranchMetrics:
 # against different timelines.
 def test_many_timelines(zenith_env_builder: ZenithEnvBuilder):
     zenith_env_builder.num_safekeepers = 3
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     n_timelines = 3
 
@@ -181,7 +181,7 @@ def test_restarts(zenith_env_builder: ZenithEnvBuilder):
     n_acceptors = 3
 
     zenith_env_builder.num_safekeepers = n_acceptors
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     env.zenith_cli.create_branch("test_wal_acceptors_restarts", "main")
     pg = env.postgres.create_start('test_wal_acceptors_restarts')
@@ -218,7 +218,7 @@ def delayed_wal_acceptor_start(wa):
 # When majority of acceptors is offline, commits are expected to be frozen
 def test_unavailability(zenith_env_builder: ZenithEnvBuilder):
     zenith_env_builder.num_safekeepers = 2
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     env.zenith_cli.create_branch("test_wal_acceptors_unavailability", "main")
     pg = env.postgres.create_start('test_wal_acceptors_unavailability')
@@ -289,7 +289,7 @@ def stop_value():
 def test_race_conditions(zenith_env_builder: ZenithEnvBuilder, stop_value):
 
     zenith_env_builder.num_safekeepers = 3
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     env.zenith_cli.create_branch("test_wal_acceptors_race_conditions", "main")
     pg = env.postgres.create_start('test_wal_acceptors_race_conditions')
@@ -404,7 +404,7 @@ def test_sync_safekeepers(zenith_env_builder: ZenithEnvBuilder,
     # We don't really need the full environment for this test, just the
     # safekeepers would be enough.
     zenith_env_builder.num_safekeepers = 3
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     timeline_id = uuid.uuid4()
     tenant_id = uuid.uuid4()
@@ -454,7 +454,7 @@ def test_sync_safekeepers(zenith_env_builder: ZenithEnvBuilder,
 def test_timeline_status(zenith_env_builder: ZenithEnvBuilder):
 
     zenith_env_builder.num_safekeepers = 1
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     env.zenith_cli.create_branch("test_timeline_status", "main")
     pg = env.postgres.create_start('test_timeline_status')
@@ -521,7 +521,12 @@ class SafekeeperEnv:
             http=self.port_distributor.get_port(),
         )
 
-        safekeeper_dir = os.path.join(self.repo_dir, f"sk{i}")
+        if self.num_safekeepers == 1:
+            name = "single"
+        else:
+            name = f"sk{i}"
+
+        safekeeper_dir = os.path.join(self.repo_dir, name)
         mkdir_if_needed(safekeeper_dir)
 
         args = [
@@ -532,8 +537,6 @@ class SafekeeperEnv:
             f"127.0.0.1:{port.http}",
             "-D",
             safekeeper_dir,
-            "--id",
-            str(i),
             "--daemonize"
         ]
 
@@ -601,8 +604,9 @@ def test_safekeeper_without_pageserver(test_output_dir: str,
 
 
 def test_replace_safekeeper(zenith_env_builder: ZenithEnvBuilder):
-    def safekeepers_guc(env: ZenithEnv, sk_names: List[int]) -> str:
-        return ','.join([f'localhost:{sk.port.pg}' for sk in env.safekeepers if sk.id in sk_names])
+    def safekeepers_guc(env: ZenithEnv, sk_names: List[str]) -> str:
+        return ','.join(
+            [f'localhost:{sk.port.pg}' for sk in env.safekeepers if sk.name in sk_names])
 
     def execute_payload(pg: Postgres):
         with closing(pg.connect()) as conn:
@@ -624,17 +628,17 @@ def test_replace_safekeeper(zenith_env_builder: ZenithEnvBuilder):
             http_cli = sk.http_client()
             try:
                 status = http_cli.timeline_status(tenant_id, timeline_id)
-                log.info(f"Safekeeper {sk.id} status: {status}")
+                log.info(f"Safekeeper {sk.name} status: {status}")
             except Exception as e:
-                log.info(f"Safekeeper {sk.id} status error: {e}")
+                log.info(f"Safekeeper {sk.name} status error: {e}")
 
     zenith_env_builder.num_safekeepers = 4
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
     env.zenith_cli.create_branch("test_replace_safekeeper", "main")
 
     log.info("Use only first 3 safekeepers")
     env.safekeepers[3].stop()
-    active_safekeepers = [1, 2, 3]
+    active_safekeepers = ['sk1', 'sk2', 'sk3']
     pg = env.postgres.create('test_replace_safekeeper')
     pg.adjust_for_wal_acceptors(safekeepers_guc(env, active_safekeepers))
     pg.start()
@@ -674,7 +678,7 @@ def test_replace_safekeeper(zenith_env_builder: ZenithEnvBuilder):
 
     log.info("Recreate postgres to replace failed sk1 with new sk4")
     pg.stop_and_destroy().create('test_replace_safekeeper')
-    active_safekeepers = [2, 3, 4]
+    active_safekeepers = ['sk2', 'sk3', 'sk4']
     env.safekeepers[3].start()
     pg.adjust_for_wal_acceptors(safekeepers_guc(env, active_safekeepers))
     pg.start()

test_runner/batch_others/test_wal_acceptor_async.py

@@ -200,7 +200,7 @@ async def run_restarts_under_load(pg: Postgres, acceptors: List[Safekeeper], n_w
 # restart acceptors one by one, while executing and validating bank transactions
 def test_restarts_under_load(zenith_env_builder: ZenithEnvBuilder):
     zenith_env_builder.num_safekeepers = 3
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     env.zenith_cli.create_branch("test_wal_acceptors_restarts_under_load", "main")
     pg = env.postgres.create_start('test_wal_acceptors_restarts_under_load')

test_runner/batch_others/test_zenith_cli.py

@@ -97,7 +97,7 @@ def test_cli_tenant_list(zenith_simple_env: ZenithEnv):
 def test_cli_ipv4_listeners(zenith_env_builder: ZenithEnvBuilder):
     # Start with single sk
     zenith_env_builder.num_safekeepers = 1
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     # Connect to sk port on v4 loopback
     res = requests.get(f'http://127.0.0.1:{env.safekeepers[0].port.http}/v1/status')
@@ -114,7 +114,7 @@ def test_cli_ipv4_listeners(zenith_env_builder: ZenithEnvBuilder):
 def test_cli_start_stop(zenith_env_builder: ZenithEnvBuilder):
     # Start with single sk
     zenith_env_builder.num_safekeepers = 1
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     # Stop default ps/sk
     env.zenith_cli.pageserver_stop()

test_runner/fixtures/utils.py

@@ -74,5 +74,8 @@ def lsn_from_hex(lsn_hex: str) -> int:
 def print_gc_result(row):
     log.info("GC duration {elapsed} ms".format_map(row))
     log.info(
-        "  total: {layers_total}, needed_by_cutoff {layers_needed_by_cutoff}, needed_by_branches: {layers_needed_by_branches}, not_updated: {layers_not_updated}, removed: {layers_removed}"
+        "  REL    total: {layer_relfiles_total}, needed_by_cutoff {layer_relfiles_needed_by_cutoff}, needed_by_branches: {layer_relfiles_needed_by_branches}, not_updated: {layer_relfiles_not_updated}, needed_as_tombstone {layer_relfiles_needed_as_tombstone}, removed: {layer_relfiles_removed}, dropped: {layer_relfiles_dropped}"
+        .format_map(row))
+    log.info(
+        "  NONREL total: {layer_nonrelfiles_total}, needed_by_cutoff {layer_nonrelfiles_needed_by_cutoff}, needed_by_branches: {layer_nonrelfiles_needed_by_branches}, not_updated: {layer_nonrelfiles_not_updated}, needed_as_tombstone {layer_nonrelfiles_needed_as_tombstone}, removed: {layer_nonrelfiles_removed}, dropped: {layer_nonrelfiles_dropped}"
         .format_map(row))

test_runner/fixtures/zenith_fixtures.py

@@ -27,12 +27,11 @@ from dataclasses import dataclass
 
 # Type-related stuff
 from psycopg2.extensions import connection as PgConnection
-from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, TypeVar, cast, Union, Tuple
+from typing import Any, Callable, Dict, Iterator, List, Optional, TypeVar, cast, Union, Tuple
 from typing_extensions import Literal
 import pytest
 
 import requests
-import backoff  # type: ignore
 
 from .utils import (get_self_dir, mkdir_if_needed, subprocess_capture)
 from fixtures.log_helper import log
@@ -238,9 +237,7 @@ def port_distributor(worker_base_port):
 
 class PgProtocol:
     """ Reusable connection logic """
-    def __init__(self,
-                 host: str,
-                 port: int,
+    def __init__(self, host: str, port: int,
                  username: Optional[str] = None,
                  password: Optional[str] = None):
         self.host = host
@@ -408,7 +405,6 @@ class ZenithEnvBuilder:
                  repo_dir: Path,
                  port_distributor: PortDistributor,
                  pageserver_remote_storage: Optional[RemoteStorage] = None,
-                 pageserver_config_override: Optional[str] = None,
                  num_safekeepers: int = 0,
                  pageserver_auth_enabled: bool = False,
                  rust_log_override: Optional[str] = None):
@@ -416,7 +412,6 @@ class ZenithEnvBuilder:
         self.rust_log_override = rust_log_override
         self.port_distributor = port_distributor
         self.pageserver_remote_storage = pageserver_remote_storage
-        self.pageserver_config_override = pageserver_config_override
         self.num_safekeepers = num_safekeepers
         self.pageserver_auth_enabled = pageserver_auth_enabled
         self.env: Optional[ZenithEnv] = None
@@ -434,14 +429,6 @@ class ZenithEnvBuilder:
         self.env = ZenithEnv(self)
         return self.env
 
-    def start(self):
-        self.env.start()
-
-    def init_start(self) -> ZenithEnv:
-        env = self.init()
-        self.start()
-        return env
-
     """
     Sets up the pageserver to use the local fs at the `test_dir/local_fs_remote_storage` path.
     Errors, if the pageserver has some remote storage configuration already, unless `force_enable` is not set to `True`.
@@ -557,7 +544,6 @@ class ZenithEnv:
 
         toml += textwrap.dedent(f"""
             [pageserver]
-            id=1
             listen_pg_addr = 'localhost:{pageserver_port.pg}'
             listen_http_addr = 'localhost:{pageserver_port.http}'
             auth_type = '{pageserver_auth_type}'
@@ -566,8 +552,7 @@ class ZenithEnv:
         # Create a corresponding ZenithPageserver object
         self.pageserver = ZenithPageserver(self,
                                            port=pageserver_port,
-                                           remote_storage=config.pageserver_remote_storage,
-                                           config_override=config.pageserver_config_override)
+                                           remote_storage=config.pageserver_remote_storage)
 
         # Create config and a Safekeeper object for each safekeeper
         for i in range(1, config.num_safekeepers + 1):
@@ -575,22 +560,25 @@ class ZenithEnv:
                 pg=self.port_distributor.get_port(),
                 http=self.port_distributor.get_port(),
             )
-            id = i  # assign ids sequentially
-            toml += textwrap.dedent(f"""
-                [[safekeepers]]
-                id = {id}
-                pg_port = {port.pg}
-                http_port = {port.http}
-                sync = false # Disable fsyncs to make the tests go faster
-            """)
-            safekeeper = Safekeeper(env=self, id=id, port=port)
+
+            if config.num_safekeepers == 1:
+                name = "single"
+            else:
+                name = f"sk{i}"
+            toml += f"""
+[[safekeepers]]
+name = '{name}'
+pg_port = {port.pg}
+http_port = {port.http}
+sync = false # Disable fsyncs to make the tests go faster
+            """
+            safekeeper = Safekeeper(env=self, name=name, port=port)
             self.safekeepers.append(safekeeper)
 
         log.info(f"Config: {toml}")
 
         self.zenith_cli.init(toml)
 
-    def start(self):
         # Start up the page server and all the safekeepers
         self.pageserver.start()
 
@@ -631,7 +619,7 @@ def _shared_simple_env(request: Any, port_distributor) -> Iterator[ZenithEnv]:
 
     with ZenithEnvBuilder(Path(repo_dir), port_distributor) as builder:
 
-        env = builder.init_start()
+        env = builder.init()
 
         # For convenience in tests, create a branch from the freshly-initialized cluster.
         env.zenith_cli.create_branch("empty", "main")
@@ -665,7 +653,7 @@ def zenith_env_builder(test_output_dir, port_distributor) -> Iterator[ZenithEnvB
     To use, define 'zenith_env_builder' fixture in your test to get access to the
     builder object. Set properties on it to describe the environment.
     Finally, initialize and start up the environment by calling
-    zenith_env_builder.init_start().
+    zenith_env_builder.init().
 
     After the initialization, you can launch compute nodes by calling
     the functions in the 'env.postgres' factory object, stop/start the
@@ -847,17 +835,13 @@ class ZenithCli:
             tmp.flush()
 
             cmd = ['init', f'--config={tmp.name}']
-            append_pageserver_param_overrides(cmd,
-                                              self.env.pageserver.remote_storage,
-                                              self.env.pageserver.config_override)
+            append_pageserver_param_overrides(cmd, self.env.pageserver.remote_storage)
 
             return self.raw_cli(cmd)
 
-    def pageserver_start(self, overrides=()) -> 'subprocess.CompletedProcess[str]':
-        start_args = ['pageserver', 'start', *overrides]
-        append_pageserver_param_overrides(start_args,
-                                          self.env.pageserver.remote_storage,
-                                          self.env.pageserver.config_override)
+    def pageserver_start(self) -> 'subprocess.CompletedProcess[str]':
+        start_args = ['pageserver', 'start']
+        append_pageserver_param_overrides(start_args, self.env.pageserver.remote_storage)
         return self.raw_cli(start_args)
 
     def pageserver_stop(self, immediate=False) -> 'subprocess.CompletedProcess[str]':
@@ -868,17 +852,17 @@ class ZenithCli:
         log.info(f"Stopping pageserver with {cmd}")
         return self.raw_cli(cmd)
 
-    def safekeeper_start(self, id: int) -> 'subprocess.CompletedProcess[str]':
-        return self.raw_cli(['safekeeper', 'start', str(id)])
+    def safekeeper_start(self, name: str) -> 'subprocess.CompletedProcess[str]':
+        return self.raw_cli(['safekeeper', 'start', name])
 
     def safekeeper_stop(self,
-                        id: Optional[int] = None,
+                        name: Optional[str] = None,
                         immediate=False) -> 'subprocess.CompletedProcess[str]':
         args = ['safekeeper', 'stop']
-        if id is not None:
-            args.extend(str(id))
         if immediate:
             args.extend(['-m', 'immediate'])
+        if name is not None:
+            args.append(name)
         return self.raw_cli(args)
 
     def pg_create(
@@ -1002,24 +986,21 @@ class ZenithPageserver(PgProtocol):
                  env: ZenithEnv,
                  port: PageserverPort,
                  remote_storage: Optional[RemoteStorage] = None,
-                 config_override: Optional[str] = None,
                  enable_auth=False):
         super().__init__(host='localhost', port=port.pg, username='zenith_admin')
         self.env = env
         self.running = False
         self.service_port = port  # do not shadow PgProtocol.port which is just int
         self.remote_storage = remote_storage
-        self.config_override = config_override
 
-    def start(self, overrides=()) -> 'ZenithPageserver':
+    def start(self) -> 'ZenithPageserver':
         """
         Start the page server.
-        `overrides` allows to add some config to this pageserver start.
         Returns self.
         """
         assert self.running == False
 
-        self.env.zenith_cli.pageserver_start(overrides=overrides)
+        self.env.zenith_cli.pageserver_start()
         self.running = True
         return self
 
@@ -1047,11 +1028,8 @@ class ZenithPageserver(PgProtocol):
         )
 
 
-def append_pageserver_param_overrides(
-    params_to_update: List[str],
-    pageserver_remote_storage: Optional[RemoteStorage],
-    pageserver_config_override: Optional[str] = None,
-):
+def append_pageserver_param_overrides(params_to_update: List[str],
+                                      pageserver_remote_storage: Optional[RemoteStorage]):
     if pageserver_remote_storage is not None:
         if isinstance(pageserver_remote_storage, LocalFsStorage):
             pageserver_storage_override = f"local_path='{pageserver_remote_storage.root}'"
@@ -1077,12 +1055,6 @@ def append_pageserver_param_overrides(
             f'--pageserver-config-override={o.strip()}' for o in env_overrides.split(';')
         ]
 
-    if pageserver_config_override is not None:
-        params_to_update += [
-            f'--pageserver-config-override={o.strip()}'
-            for o in pageserver_config_override.split(';')
-        ]
-
 
 class PgBin:
     """ A helper class for executing postgres binaries """
@@ -1192,39 +1164,48 @@ def vanilla_pg(test_output_dir: str) -> Iterator[VanillaPostgres]:
 class ZenithProxy(PgProtocol):
     def __init__(self, port: int):
         super().__init__(host="127.0.0.1", username="pytest", password="pytest", port=port)
-        self.http_port = 7001
-        self._popen: Optional[subprocess.Popen[bytes]] = None
+        self.running = False
 
     def start_static(self, addr="127.0.0.1:5432") -> None:
-        assert self._popen is None
+        assert not self.running
+        self.running = True
 
-        # Start proxy
-        bin_proxy = os.path.join(str(zenith_binpath), 'proxy')
-        args = [bin_proxy]
-        args.extend(["--http", f"{self.host}:{self.http_port}"])
-        args.extend(["--proxy", f"{self.host}:{self.port}"])
-        args.extend(["--auth-method", "password"])
-        args.extend(["--static-router", addr])
-        self._popen = subprocess.Popen(args)
-        self._wait_until_ready()
+        http_port = "7001"
+        args = [
+            # TODO is cargo run the right thing to do?
+            "cargo",
+            "run",
+            "--bin", "proxy",
+            "--",
+            "--http", http_port,
+            "--proxy", f"{self.host}:{self.port}",
+            "--auth-method", "password",
+            "--static-router", addr,
+        ]
+        self.popen = subprocess.Popen(args)
 
-    @backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_time=10)
-    def _wait_until_ready(self):
-        requests.get(f"http://{self.host}:{self.http_port}/v1/status")
+        # Readiness probe
+        requests.get(f"http://{self.host}:{http_port}/v1/status")
+
+
+    def stop(self) -> None:
+        assert self.running
+        self.running = False
+
+        # NOTE the process will die when we're done with tests anyway, because
+        # it's a child process. This is mostly to clean up in between different tests.
+        self.popen.kill()
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc, tb):
-        if self._popen is not None:
-            # NOTE the process will die when we're done with tests anyway, because
-            # it's a child process. This is mostly to clean up in between different tests.
-            self._popen.kill()
+        if self.running:
+            self.stop()
 
 
 @pytest.fixture(scope='function')
 def static_proxy(vanilla_pg) -> Iterator[ZenithProxy]:
-    """Zenith proxy that routes directly to vanilla postgres."""
     vanilla_pg.start()
     vanilla_pg.safe_psql("create user pytest with password 'pytest';")
 
@@ -1473,14 +1454,12 @@ class Safekeeper:
     """ An object representing a running safekeeper daemon. """
     env: ZenithEnv
     port: SafekeeperPort
-    id: int
+    name: str  # identifier for logging
     auth_token: Optional[str] = None
-    running: bool = False
 
     def start(self) -> 'Safekeeper':
-        assert self.running == False
-        self.env.zenith_cli.safekeeper_start(self.id)
-        self.running = True
+        self.env.zenith_cli.safekeeper_start(self.name)
+
         # wait for wal acceptor start by checking its status
         started_at = time.time()
         while True:
@@ -1498,9 +1477,8 @@ class Safekeeper:
         return self
 
     def stop(self, immediate=False) -> 'Safekeeper':
-        log.info('Stopping safekeeper {}'.format(self.id))
-        self.env.zenith_cli.safekeeper_stop(self.id, immediate)
-        self.running = False
+        log.info('Stopping safekeeper {}'.format(self.name))
+        self.env.zenith_cli.safekeeper_stop(self.name, immediate)
         return self
 
     def append_logical_message(self,

test_runner/performance/test_bulk_tenant_create.py

@@ -23,7 +23,7 @@ def test_bulk_tenant_create(
     """Measure tenant creation time (with and without wal acceptors)"""
     if use_wal_acceptors == 'with_wa':
         zenith_env_builder.num_safekeepers = 3
-    env = zenith_env_builder.init_start()
+    env = zenith_env_builder.init()
 
     time_slices = []
 

test_runner/performance/test_random_writes.py

@@ -26,7 +26,7 @@ def test_random_writes(zenith_with_baseline: PgCompare):
     # but it will take a very long time to run. From what I've seen so far,
     # increasing n_rows doesn't have impact on the (zenith_runtime / vanilla_runtime)
     # performance ratio.
-    n_rows = 10 * 1000 * 1000  # around 360 MB table
+    n_rows = 1 * 1000 * 1000  # around 36 MB table
 
     # Number of writes per 3 segments. A value of 1 should produce a random
     # workload where we almost never write to the same segment twice. Larger
@@ -37,7 +37,7 @@ def test_random_writes(zenith_with_baseline: PgCompare):
 
     # Not sure why but this matters in a weird way (up to 2x difference in perf).
     # TODO look into it
-    n_iterations = 1000
+    n_iterations = 1
 
     with closing(env.pg.connect()) as conn:
         with conn.cursor() as cur:

walkeeper/src/bin/safekeeper.rs

@@ -1,19 +1,17 @@
 //
 // Main entry point for the safekeeper executable
 //
-use anyhow::{bail, Context, Result};
+use anyhow::{Context, Result};
 use clap::{App, Arg};
 use const_format::formatcp;
 use daemonize::Daemonize;
 use fs2::FileExt;
-use std::fs::{self, File};
-use std::io::{ErrorKind, Write};
+use std::fs::File;
 use std::path::{Path, PathBuf};
 use std::thread;
 use tracing::*;
-use walkeeper::control_file::{self};
+use walkeeper::control_file::{self, CreateControlFile};
 use zenith_utils::http::endpoint;
-use zenith_utils::zid::ZNodeId;
 use zenith_utils::{logging, tcp_listener, GIT_VERSION};
 
 use tokio::sync::mpsc;
@@ -27,7 +25,6 @@ use zenith_utils::shutdown::exit_now;
 use zenith_utils::signals;
 
 const LOCK_FILE_NAME: &str = "safekeeper.lock";
-const ID_FILE_NAME: &str = "safekeeper.id";
 
 fn main() -> Result<()> {
     zenith_metrics::set_common_metrics_prefix("safekeeper");
@@ -41,12 +38,6 @@ fn main() -> Result<()> {
                 .takes_value(true)
                 .help("Path to the safekeeper data directory"),
         )
-        .arg(
-            Arg::new("init")
-                .long("init")
-                .takes_value(false)
-                .help("Initialize safekeeper with ID"),
-        )
         .arg(
             Arg::new("listen-pg")
                 .short('l')
@@ -102,13 +93,13 @@ fn main() -> Result<()> {
                 .takes_value(true)
                 .help("Dump control file at path specifed by this argument and exit"),
         )
-        .arg(
-            Arg::new("id").long("id").takes_value(true).help("safekeeper node id: integer")
-        )
         .get_matches();
 
     if let Some(addr) = arg_matches.value_of("dump-control-file") {
-        let state = control_file::FileStorage::load_control_file(Path::new(addr))?;
+        let state = control_file::FileStorage::load_control_file(
+            Path::new(addr),
+            CreateControlFile::False,
+        )?;
         let json = serde_json::to_string(&state)?;
         print!("{}", json);
         return Ok(());
@@ -145,19 +136,10 @@ fn main() -> Result<()> {
         conf.recall_period = humantime::parse_duration(recall)?;
     }
 
-    let mut given_id = None;
-    if let Some(given_id_str) = arg_matches.value_of("id") {
-        given_id = Some(ZNodeId(
-            given_id_str
-                .parse()
-                .context("failed to parse safekeeper id")?,
-        ));
-    }
-
-    start_safekeeper(conf, given_id, arg_matches.is_present("init"))
+    start_safekeeper(conf)
 }
 
-fn start_safekeeper(mut conf: SafeKeeperConf, given_id: Option<ZNodeId>, init: bool) -> Result<()> {
+fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> {
     let log_file = logging::init("safekeeper.log", conf.daemonize)?;
 
     info!("version: {}", GIT_VERSION);
@@ -172,12 +154,6 @@ fn start_safekeeper(mut conf: SafeKeeperConf, given_id: Option<ZNodeId>, init: b
         )
     })?;
 
-    // Set or read our ID.
-    set_id(&mut conf, given_id)?;
-    if init {
-        return Ok(());
-    }
-
     let http_listener = tcp_listener::bind(conf.listen_http_addr.clone()).map_err(|e| {
         error!("failed to bind to address {}: {}", conf.listen_http_addr, e);
         e
@@ -284,49 +260,3 @@ fn start_safekeeper(mut conf: SafeKeeperConf, given_id: Option<ZNodeId>, init: b
         std::process::exit(111);
     })
 }
-
-/// Determine safekeeper id and set it in config.
-fn set_id(conf: &mut SafeKeeperConf, given_id: Option<ZNodeId>) -> Result<()> {
-    let id_file_path = conf.workdir.join(ID_FILE_NAME);
-
-    let my_id: ZNodeId;
-    // If ID exists, read it in; otherwise set one passed
-    match fs::read(&id_file_path) {
-        Ok(id_serialized) => {
-            my_id = ZNodeId(
-                std::str::from_utf8(&id_serialized)
-                    .context("failed to parse safekeeper id")?
-                    .parse()
-                    .context("failed to parse safekeeper id")?,
-            );
-            if let Some(given_id) = given_id {
-                if given_id != my_id {
-                    bail!(
-                        "safekeeper already initialized with id {}, can't set {}",
-                        my_id,
-                        given_id
-                    );
-                }
-            }
-            info!("safekeeper ID {}", my_id);
-        }
-        Err(error) => match error.kind() {
-            ErrorKind::NotFound => {
-                my_id = if let Some(given_id) = given_id {
-                    given_id
-                } else {
-                    bail!("safekeeper id is not specified");
-                };
-                let mut f = File::create(&id_file_path)?;
-                f.write_all(my_id.to_string().as_bytes())?;
-                f.sync_all()?;
-                info!("initialized safekeeper ID {}", my_id);
-            }
-            _ => {
-                return Err(error.into());
-            }
-        },
-    }
-    conf.my_id = my_id;
-    Ok(())
-}

walkeeper/src/control_file.rs

@@ -27,6 +27,13 @@ const CONTROL_FILE_NAME: &str = "safekeeper.control";
 const CONTROL_FILE_NAME_PARTIAL: &str = "safekeeper.control.partial";
 pub const CHECKSUM_SIZE: usize = std::mem::size_of::<u32>();
 
+// A named boolean.
+#[derive(Debug)]
+pub enum CreateControlFile {
+    True,
+    False,
+}
+
 lazy_static! {
     static ref PERSIST_CONTROL_FILE_SECONDS: HistogramVec = register_histogram_vec!(
         "safekeeper_persist_control_file_seconds",
@@ -87,22 +94,28 @@ impl FileStorage {
     pub fn load_control_file_conf(
         conf: &SafeKeeperConf,
         zttid: &ZTenantTimelineId,
+        create: CreateControlFile,
     ) -> Result<SafeKeeperState> {
         let path = conf.timeline_dir(zttid).join(CONTROL_FILE_NAME);
-        Self::load_control_file(path)
+        Self::load_control_file(path, create)
     }
 
     /// Read in the control file.
     /// If create=false and file doesn't exist, bails out.
-    pub fn load_control_file<P: AsRef<Path>>(control_file_path: P) -> Result<SafeKeeperState> {
+    pub fn load_control_file<P: AsRef<Path>>(
+        control_file_path: P,
+        create: CreateControlFile,
+    ) -> Result<SafeKeeperState> {
         info!(
-            "loading control file {}",
+            "loading control file {}, create={:?}",
             control_file_path.as_ref().display(),
+            create,
         );
 
         let mut control_file = OpenOptions::new()
             .read(true)
             .write(true)
+            .create(matches!(create, CreateControlFile::True))
             .open(&control_file_path)
             .with_context(|| {
                 format!(
@@ -111,32 +124,41 @@ impl FileStorage {
                 )
             })?;
 
-        let mut buf = Vec::new();
-        control_file
-            .read_to_end(&mut buf)
-            .context("failed to read control file")?;
+        // Empty file is legit on 'create', don't try to deser from it.
+        let state = if control_file.metadata().unwrap().len() == 0 {
+            if let CreateControlFile::False = create {
+                bail!("control file is empty");
+            }
+            SafeKeeperState::new()
+        } else {
+            let mut buf = Vec::new();
+            control_file
+                .read_to_end(&mut buf)
+                .context("failed to read control file")?;
 
-        let calculated_checksum = crc32c::crc32c(&buf[..buf.len() - CHECKSUM_SIZE]);
+            let calculated_checksum = crc32c::crc32c(&buf[..buf.len() - CHECKSUM_SIZE]);
 
-        let expected_checksum_bytes: &[u8; CHECKSUM_SIZE] =
-            buf[buf.len() - CHECKSUM_SIZE..].try_into()?;
-        let expected_checksum = u32::from_le_bytes(*expected_checksum_bytes);
+            let expected_checksum_bytes: &[u8; CHECKSUM_SIZE] =
+                buf[buf.len() - CHECKSUM_SIZE..].try_into()?;
+            let expected_checksum = u32::from_le_bytes(*expected_checksum_bytes);
 
-        ensure!(
-            calculated_checksum == expected_checksum,
-            format!(
-                "safekeeper control file checksum mismatch: expected {} got {}",
-                expected_checksum, calculated_checksum
-            )
-        );
-
-        let state = FileStorage::deser_sk_state(&mut &buf[..buf.len() - CHECKSUM_SIZE])
-            .with_context(|| {
+            ensure!(
+                calculated_checksum == expected_checksum,
                 format!(
-                    "while reading control file {}",
-                    control_file_path.as_ref().display(),
+                    "safekeeper control file checksum mismatch: expected {} got {}",
+                    expected_checksum, calculated_checksum
                 )
-            })?;
+            );
+
+            FileStorage::deser_sk_state(&mut &buf[..buf.len() - CHECKSUM_SIZE]).with_context(
+                || {
+                    format!(
+                        "while reading control file {}",
+                        control_file_path.as_ref().display(),
+                    )
+                },
+            )?
+        };
         Ok(state)
     }
 }
@@ -172,35 +194,31 @@ impl Storage for FileStorage {
         })?;
 
         // fsync the file
-        if !self.conf.no_sync {
-            control_partial.sync_all().with_context(|| {
-                format!(
-                    "failed to sync partial control file at {}",
-                    control_partial_path.display()
-                )
-            })?;
-        }
+        control_partial.sync_all().with_context(|| {
+            format!(
+                "failed to sync partial control file at {}",
+                control_partial_path.display()
+            )
+        })?;
 
         let control_path = self.timeline_dir.join(CONTROL_FILE_NAME);
 
         // rename should be atomic
         fs::rename(&control_partial_path, &control_path)?;
         // this sync is not required by any standard but postgres does this (see durable_rename)
-        if !self.conf.no_sync {
-            File::open(&control_path)
-                .and_then(|f| f.sync_all())
-                .with_context(|| {
-                    format!(
-                        "failed to sync control file at: {}",
-                        &control_path.display()
-                    )
-                })?;
+        File::open(&control_path)
+            .and_then(|f| f.sync_all())
+            .with_context(|| {
+                format!(
+                    "failed to sync control file at: {}",
+                    &control_path.display()
+                )
+            })?;
 
-            // fsync the directory (linux specific)
-            File::open(&self.timeline_dir)
-                .and_then(|f| f.sync_all())
-                .context("failed to sync control file directory")?;
-        }
+        // fsync the directory (linux specific)
+        File::open(&self.timeline_dir)
+            .and_then(|f| f.sync_all())
+            .context("failed to sync control file directory")?;
         Ok(())
     }
 }
@@ -225,38 +243,31 @@ mod test {
     fn load_from_control_file(
         conf: &SafeKeeperConf,
         zttid: &ZTenantTimelineId,
+        create: CreateControlFile,
     ) -> Result<(FileStorage, SafeKeeperState)> {
         fs::create_dir_all(&conf.timeline_dir(zttid)).expect("failed to create timeline dir");
         Ok((
             FileStorage::new(zttid, conf),
-            FileStorage::load_control_file_conf(conf, zttid)?,
+            FileStorage::load_control_file_conf(conf, zttid, create)?,
         ))
     }
 
-    fn create(
-        conf: &SafeKeeperConf,
-        zttid: &ZTenantTimelineId,
-    ) -> Result<(FileStorage, SafeKeeperState)> {
-        fs::create_dir_all(&conf.timeline_dir(zttid)).expect("failed to create timeline dir");
-        let state = SafeKeeperState::empty();
-        let mut storage = FileStorage::new(zttid, conf);
-        storage.persist(&state)?;
-        Ok((storage, state))
-    }
-
     #[test]
     fn test_read_write_safekeeper_state() {
         let conf = stub_conf();
         let zttid = ZTenantTimelineId::generate();
         {
-            let (mut storage, mut state) = create(&conf, &zttid).expect("failed to create state");
+            let (mut storage, mut state) =
+                load_from_control_file(&conf, &zttid, CreateControlFile::True)
+                    .expect("failed to read state");
             // change something
-            state.commit_lsn = Lsn(42);
+            state.wal_start_lsn = Lsn(42);
             storage.persist(&state).expect("failed to persist state");
         }
 
-        let (_, state) = load_from_control_file(&conf, &zttid).expect("failed to read state");
-        assert_eq!(state.commit_lsn, Lsn(42));
+        let (_, state) = load_from_control_file(&conf, &zttid, CreateControlFile::False)
+            .expect("failed to read state");
+        assert_eq!(state.wal_start_lsn, Lsn(42));
     }
 
     #[test]
@@ -264,10 +275,11 @@ mod test {
         let conf = stub_conf();
         let zttid = ZTenantTimelineId::generate();
         {
-            let (mut storage, mut state) = create(&conf, &zttid).expect("failed to read state");
-
+            let (mut storage, mut state) =
+                load_from_control_file(&conf, &zttid, CreateControlFile::True)
+                    .expect("failed to read state");
             // change something
-            state.commit_lsn = Lsn(42);
+            state.wal_start_lsn = Lsn(42);
             storage.persist(&state).expect("failed to persist state");
         }
         let control_path = conf.timeline_dir(&zttid).join(CONTROL_FILE_NAME);
@@ -275,7 +287,7 @@ mod test {
         data[0] += 1; // change the first byte of the file to fail checksum validation
         fs::write(&control_path, &data).expect("failed to write control file");
 
-        match load_from_control_file(&conf, &zttid) {
+        match load_from_control_file(&conf, &zttid, CreateControlFile::False) {
             Err(err) => assert!(err
                 .to_string()
                 .contains("safekeeper control file checksum mismatch")),

walkeeper/src/control_file_upgrade.rs

@@ -1,6 +1,6 @@
 //! Code to deal with safekeeper control file upgrades
 use crate::safekeeper::{
-    AcceptorState, Peers, PgUuid, SafeKeeperState, ServerInfo, Term, TermHistory, TermSwitchEntry,
+    AcceptorState, PgUuid, SafeKeeperState, ServerInfo, Term, TermHistory, TermSwitchEntry,
 };
 use anyhow::{bail, Result};
 use serde::{Deserialize, Serialize};
@@ -26,7 +26,7 @@ struct SafeKeeperStateV1 {
     /// persistent acceptor state
     acceptor_state: AcceptorStateV1,
     /// information about server
-    server: ServerInfoV2,
+    server: ServerInfo,
     /// Unique id of the last *elected* proposer we dealed with. Not needed
     /// for correctness, exists for monitoring purposes.
     proposer_uuid: PgUuid,
@@ -70,39 +70,6 @@ pub struct SafeKeeperStateV2 {
     pub wal_start_lsn: Lsn,
 }
 
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub struct ServerInfoV3 {
-    /// Postgres server version
-    pub pg_version: u32,
-    pub system_id: SystemId,
-    #[serde(with = "hex")]
-    pub tenant_id: ZTenantId,
-    /// Zenith timelineid
-    #[serde(with = "hex")]
-    pub timeline_id: ZTimelineId,
-    pub wal_seg_size: u32,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct SafeKeeperStateV3 {
-    /// persistent acceptor state
-    pub acceptor_state: AcceptorState,
-    /// information about server
-    pub server: ServerInfoV3,
-    /// Unique id of the last *elected* proposer we dealed with. Not needed
-    /// for correctness, exists for monitoring purposes.
-    #[serde(with = "hex")]
-    pub proposer_uuid: PgUuid,
-    /// part of WAL acknowledged by quorum and available locally
-    pub commit_lsn: Lsn,
-    /// minimal LSN which may be needed for recovery of some safekeeper (end_lsn
-    /// of last record streamed to everyone)
-    pub truncate_lsn: Lsn,
-    // Safekeeper starts receiving WAL from this LSN, zeros before it ought to
-    // be skipped during decoding.
-    pub wal_start_lsn: Lsn,
-}
-
 pub fn upgrade_control_file(buf: &[u8], version: u32) -> Result<SafeKeeperState> {
     // migrate to storing full term history
     if version == 1 {
@@ -116,20 +83,12 @@ pub fn upgrade_control_file(buf: &[u8], version: u32) -> Result<SafeKeeperState>
             }]),
         };
         return Ok(SafeKeeperState {
-            tenant_id: oldstate.server.tenant_id,
-            timeline_id: oldstate.server.ztli,
             acceptor_state: ac,
-            server: ServerInfo {
-                pg_version: oldstate.server.pg_version,
-                system_id: oldstate.server.system_id,
-                wal_seg_size: oldstate.server.wal_seg_size,
-            },
+            server: oldstate.server.clone(),
             proposer_uuid: oldstate.proposer_uuid,
             commit_lsn: oldstate.commit_lsn,
-            s3_wal_lsn: Lsn(0),
-            peer_horizon_lsn: oldstate.truncate_lsn,
-            remote_consistent_lsn: Lsn(0),
-            peers: Peers(vec![]),
+            truncate_lsn: oldstate.truncate_lsn,
+            wal_start_lsn: oldstate.wal_start_lsn,
         });
     // migrate to hexing some zids
     } else if version == 2 {
@@ -138,40 +97,17 @@ pub fn upgrade_control_file(buf: &[u8], version: u32) -> Result<SafeKeeperState>
         let server = ServerInfo {
             pg_version: oldstate.server.pg_version,
             system_id: oldstate.server.system_id,
-            wal_seg_size: oldstate.server.wal_seg_size,
-        };
-        return Ok(SafeKeeperState {
             tenant_id: oldstate.server.tenant_id,
             timeline_id: oldstate.server.ztli,
-            acceptor_state: oldstate.acceptor_state,
-            server,
-            proposer_uuid: oldstate.proposer_uuid,
-            commit_lsn: oldstate.commit_lsn,
-            s3_wal_lsn: Lsn(0),
-            peer_horizon_lsn: oldstate.truncate_lsn,
-            remote_consistent_lsn: Lsn(0),
-            peers: Peers(vec![]),
-        });
-    // migrate to moving ztenantid/ztli to the top and adding some lsns
-    } else if version == 3 {
-        info!("reading safekeeper control file version {}", version);
-        let oldstate = SafeKeeperStateV3::des(&buf[..buf.len()])?;
-        let server = ServerInfo {
-            pg_version: oldstate.server.pg_version,
-            system_id: oldstate.server.system_id,
             wal_seg_size: oldstate.server.wal_seg_size,
         };
         return Ok(SafeKeeperState {
-            tenant_id: oldstate.server.tenant_id,
-            timeline_id: oldstate.server.timeline_id,
             acceptor_state: oldstate.acceptor_state,
             server,
             proposer_uuid: oldstate.proposer_uuid,
             commit_lsn: oldstate.commit_lsn,
-            s3_wal_lsn: Lsn(0),
-            peer_horizon_lsn: oldstate.truncate_lsn,
-            remote_consistent_lsn: Lsn(0),
-            peers: Peers(vec![]),
+            truncate_lsn: oldstate.truncate_lsn,
+            wal_start_lsn: oldstate.wal_start_lsn,
         });
     }
     bail!("unsupported safekeeper control file version {}", version)

walkeeper/src/handler.rs

@@ -3,7 +3,6 @@
 
 use crate::json_ctrl::{handle_json_ctrl, AppendLogicalMessage};
 use crate::receive_wal::ReceiveWalConn;
-use crate::safekeeper::{AcceptorProposerMessage, ProposerAcceptorMessage};
 use crate::send_wal::ReplicationConn;
 use crate::timeline::{Timeline, TimelineTools};
 use crate::SafeKeeperConf;
@@ -13,7 +12,6 @@ use postgres_ffi::xlog_utils::PG_TLI;
 use regex::Regex;
 use std::str::FromStr;
 use std::sync::Arc;
-use tracing::info;
 use zenith_utils::lsn::Lsn;
 use zenith_utils::postgres_backend;
 use zenith_utils::postgres_backend::PostgresBackend;
@@ -21,6 +19,7 @@ use zenith_utils::pq_proto::{BeMessage, FeStartupPacket, RowDescriptor, INT4_OID
 use zenith_utils::zid::{ZTenantId, ZTenantTimelineId, ZTimelineId};
 
 use crate::callmemaybe::CallmeEvent;
+use crate::control_file::CreateControlFile;
 use tokio::sync::mpsc::UnboundedSender;
 
 /// Safekeeper handler of postgres commands
@@ -101,19 +100,29 @@ impl postgres_backend::Handler for SafekeeperPostgresHandler {
     fn process_query(&mut self, pgb: &mut PostgresBackend, query_string: &str) -> Result<()> {
         let cmd = parse_cmd(query_string)?;
 
-        info!("got query {:?}", query_string);
-
-        let create = !(matches!(cmd, SafekeeperPostgresCommand::StartReplication { .. })
-            || matches!(cmd, SafekeeperPostgresCommand::IdentifySystem));
-
-        let tenantid = self.ztenantid.context("tenantid is required")?;
-        let timelineid = self.ztimelineid.context("timelineid is required")?;
-        if self.timeline.is_none() {
-            self.timeline.set(
-                &self.conf,
-                ZTenantTimelineId::new(tenantid, timelineid),
-                create,
-            )?;
+        // Is this command is ztimeline scoped?
+        match cmd {
+            SafekeeperPostgresCommand::StartWalPush { .. }
+            | SafekeeperPostgresCommand::StartReplication { .. }
+            | SafekeeperPostgresCommand::IdentifySystem
+            | SafekeeperPostgresCommand::JSONCtrl { .. } => {
+                let tenantid = self.ztenantid.context("tenantid is required")?;
+                let timelineid = self.ztimelineid.context("timelineid is required")?;
+                if self.timeline.is_none() {
+                    // START_WAL_PUSH is the only command that initializes the timeline in production.
+                    // There is also JSON_CTRL command, which should initialize the timeline for testing.
+                    let create_control_file = match cmd {
+                        SafekeeperPostgresCommand::StartWalPush { .. }
+                        | SafekeeperPostgresCommand::JSONCtrl { .. } => CreateControlFile::True,
+                        _ => CreateControlFile::False,
+                    };
+                    self.timeline.set(
+                        &self.conf,
+                        ZTenantTimelineId::new(tenantid, timelineid),
+                        create_control_file,
+                    )?;
+                }
+            }
         }
 
         match cmd {
@@ -151,17 +160,6 @@ impl SafekeeperPostgresHandler {
         }
     }
 
-    /// Shortcut for calling `process_msg` in the timeline.
-    pub fn process_safekeeper_msg(
-        &self,
-        msg: &ProposerAcceptorMessage,
-    ) -> Result<Option<AcceptorProposerMessage>> {
-        self.timeline
-            .get()
-            .process_msg(msg)
-            .context("failed to process ProposerAcceptorMessage")
-    }
-
     ///
     /// Handle IDENTIFY_SYSTEM replication command
     ///

walkeeper/src/http/mod.rs

@@ -1,3 +1,2 @@
-pub mod models;
 pub mod routes;
 pub use routes::make_router;

walkeeper/src/http/models.rs

@@ -1,9 +0,0 @@
-use serde::{Deserialize, Serialize};
-use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId};
-
-#[derive(Serialize, Deserialize)]
-pub struct TimelineCreateRequest {
-    pub tenant_id: ZTenantId,
-    pub timeline_id: ZTimelineId,
-    pub peer_ids: Vec<ZNodeId>,
-}

walkeeper/src/http/routes.rs

@@ -1,15 +1,13 @@
 use hyper::{Body, Request, Response, StatusCode};
-
 use serde::Serialize;
 use serde::Serializer;
 use std::fmt::Display;
 use std::sync::Arc;
-use zenith_utils::http::json::json_request;
 use zenith_utils::http::{RequestExt, RouterBuilder};
 use zenith_utils::lsn::Lsn;
-use zenith_utils::zid::ZNodeId;
 use zenith_utils::zid::ZTenantTimelineId;
 
+use crate::control_file::CreateControlFile;
 use crate::safekeeper::Term;
 use crate::safekeeper::TermHistory;
 use crate::timeline::GlobalTimelines;
@@ -20,18 +18,9 @@ use zenith_utils::http::json::json_response;
 use zenith_utils::http::request::parse_request_param;
 use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
-use super::models::TimelineCreateRequest;
-
-#[derive(Debug, Serialize)]
-struct SafekeeperStatus {
-    id: ZNodeId,
-}
-
 /// Healthcheck handler.
-async fn status_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    let conf = get_conf(&request);
-    let status = SafekeeperStatus { id: conf.my_id };
-    Ok(json_response(StatusCode::OK, status)?)
+async fn status_handler(_: Request<Body>) -> Result<Response<Body>, ApiError> {
+    Ok(json_response(StatusCode::OK, "")?)
 }
 
 fn get_conf(request: &Request<Body>) -> &SafeKeeperConf {
@@ -69,11 +58,7 @@ struct TimelineStatus {
     #[serde(serialize_with = "display_serialize")]
     commit_lsn: Lsn,
     #[serde(serialize_with = "display_serialize")]
-    s3_wal_lsn: Lsn,
-    #[serde(serialize_with = "display_serialize")]
-    peer_horizon_lsn: Lsn,
-    #[serde(serialize_with = "display_serialize")]
-    remote_consistent_lsn: Lsn,
+    truncate_lsn: Lsn,
     #[serde(serialize_with = "display_serialize")]
     flush_lsn: Lsn,
 }
@@ -85,7 +70,8 @@ async fn timeline_status_handler(request: Request<Body>) -> Result<Response<Body
         parse_request_param(&request, "timeline_id")?,
     );
 
-    let tli = GlobalTimelines::get(get_conf(&request), zttid, false).map_err(ApiError::from_err)?;
+    let tli = GlobalTimelines::get(get_conf(&request), zttid, CreateControlFile::False)
+        .map_err(ApiError::from_err)?;
     let sk_state = tli.get_info();
     let flush_lsn = tli.get_end_of_wal();
 
@@ -100,27 +86,12 @@ async fn timeline_status_handler(request: Request<Body>) -> Result<Response<Body
         timeline_id: zttid.timeline_id,
         acceptor_state: acc_state,
         commit_lsn: sk_state.commit_lsn,
-        s3_wal_lsn: sk_state.s3_wal_lsn,
-        peer_horizon_lsn: sk_state.peer_horizon_lsn,
-        remote_consistent_lsn: sk_state.remote_consistent_lsn,
+        truncate_lsn: sk_state.truncate_lsn,
         flush_lsn,
     };
     Ok(json_response(StatusCode::OK, status)?)
 }
 
-async fn timeline_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    let request_data: TimelineCreateRequest = json_request(&mut request).await?;
-
-    let zttid = ZTenantTimelineId {
-        tenant_id: request_data.tenant_id,
-        timeline_id: request_data.timeline_id,
-    };
-    GlobalTimelines::create(get_conf(&request), zttid, request_data.peer_ids)
-        .map_err(ApiError::from_err)?;
-
-    Ok(json_response(StatusCode::CREATED, ())?)
-}
-
 /// Safekeeper http router.
 pub fn make_router(conf: SafeKeeperConf) -> RouterBuilder<hyper::Body, ApiError> {
     let router = endpoint::make_router();
@@ -131,5 +102,4 @@ pub fn make_router(conf: SafeKeeperConf) -> RouterBuilder<hyper::Body, ApiError>
             "/v1/timeline/:tenant_id/:timeline_id",
             timeline_status_handler,
         )
-        .post("/v1/timeline", timeline_create_handler)
 }

walkeeper/src/lib.rs

@@ -2,7 +2,7 @@
 use std::path::PathBuf;
 use std::time::Duration;
 
-use zenith_utils::zid::{ZNodeId, ZTenantTimelineId};
+use zenith_utils::zid::ZTenantTimelineId;
 
 pub mod callmemaybe;
 pub mod control_file;
@@ -46,7 +46,6 @@ pub struct SafeKeeperConf {
     pub listen_http_addr: String,
     pub ttl: Option<Duration>,
     pub recall_period: Duration,
-    pub my_id: ZNodeId,
 }
 
 impl SafeKeeperConf {
@@ -70,7 +69,6 @@ impl Default for SafeKeeperConf {
             listen_http_addr: defaults::DEFAULT_HTTP_LISTEN_ADDR.to_string(),
             ttl: None,
             recall_period: defaults::DEFAULT_RECALL_PERIOD,
-            my_id: ZNodeId(0),
         }
     }
 }

walkeeper/src/receive_wal.rs

@@ -2,21 +2,15 @@
 //! Gets messages from the network, passes them down to consensus module and
 //! sends replies back.
 
-use anyhow::{anyhow, bail, Result};
-
+use anyhow::{bail, Context, Result};
+use bytes::Bytes;
 use bytes::BytesMut;
 use tokio::sync::mpsc::UnboundedSender;
 use tracing::*;
-use zenith_utils::sock_split::ReadStream;
 
 use crate::timeline::Timeline;
-
 use std::net::SocketAddr;
-use std::sync::mpsc::channel;
-use std::sync::mpsc::Receiver;
-
 use std::sync::Arc;
-use std::thread;
 
 use crate::safekeeper::AcceptorProposerMessage;
 use crate::safekeeper::ProposerAcceptorMessage;
@@ -52,6 +46,21 @@ impl<'pg> ReceiveWalConn<'pg> {
         }
     }
 
+    // Read and extract the bytes of a `CopyData` message from the postgres instance
+    fn read_msg_bytes(&mut self) -> Result<Bytes> {
+        match self.pg_backend.read_message()? {
+            Some(FeMessage::CopyData(bytes)) => Ok(bytes),
+            Some(msg) => bail!("expected `CopyData` message, found {:?}", msg),
+            None => bail!("connection closed unexpectedly"),
+        }
+    }
+
+    // Read and parse message sent from the postgres instance
+    fn read_msg(&mut self) -> Result<ProposerAcceptorMessage> {
+        let data = self.read_msg_bytes()?;
+        ProposerAcceptorMessage::parse(data)
+    }
+
     // Send message to the postgres
     fn write_msg(&mut self, msg: &AcceptorProposerMessage) -> Result<()> {
         let mut buf = BytesMut::with_capacity(128);
@@ -68,22 +77,18 @@ impl<'pg> ReceiveWalConn<'pg> {
         self.pg_backend
             .write_message(&BeMessage::CopyBothResponse)?;
 
-        let r = self
-            .pg_backend
-            .take_stream_in()
-            .ok_or_else(|| anyhow!("failed to take read stream from pgbackend"))?;
-        let mut poll_reader = ProposerPollStream::new(r)?;
-
         // Receive information about server
-        let next_msg = poll_reader.recv_msg()?;
-        match next_msg {
+        let mut msg = self
+            .read_msg()
+            .context("failed to receive proposer greeting")?;
+        match msg {
             ProposerAcceptorMessage::Greeting(ref greeting) => {
                 info!(
                     "start handshake with wal proposer {} sysid {} timeline {}",
                     self.peer_addr, greeting.system_id, greeting.tli,
                 );
             }
-            _ => bail!("unexpected message {:?} instead of greeting", next_msg),
+            _ => bail!("unexpected message {:?} instead of greeting", msg),
         }
 
         // Register the connection and defer unregister.
@@ -95,97 +100,16 @@ impl<'pg> ReceiveWalConn<'pg> {
             callmemaybe_tx: spg.tx.clone(),
         };
 
-        let mut next_msg = Some(next_msg);
-
         loop {
-            if matches!(next_msg, Some(ProposerAcceptorMessage::AppendRequest(_))) {
-                // poll AppendRequest's without blocking and write WAL to disk without flushing,
-                // while it's readily available
-                while let Some(ProposerAcceptorMessage::AppendRequest(append_request)) = next_msg {
-                    let msg = ProposerAcceptorMessage::NoFlushAppendRequest(append_request);
-
-                    let reply = spg.process_safekeeper_msg(&msg)?;
-                    if let Some(reply) = reply {
-                        self.write_msg(&reply)?;
-                    }
-
-                    next_msg = poll_reader.poll_msg();
-                }
-
-                // flush all written WAL to the disk
-                let reply = spg.process_safekeeper_msg(&ProposerAcceptorMessage::FlushWAL)?;
-                if let Some(reply) = reply {
-                    self.write_msg(&reply)?;
-                }
-            } else if let Some(msg) = next_msg.take() {
-                // process other message
-                let reply = spg.process_safekeeper_msg(&msg)?;
-                if let Some(reply) = reply {
-                    self.write_msg(&reply)?;
-                }
+            let reply = spg
+                .timeline
+                .get()
+                .process_msg(&msg)
+                .context("failed to process ProposerAcceptorMessage")?;
+            if let Some(reply) = reply {
+                self.write_msg(&reply)?;
             }
-
-            // blocking wait for the next message
-            if next_msg.is_none() {
-                next_msg = Some(poll_reader.recv_msg()?);
-            }
-        }
-    }
-}
-
-struct ProposerPollStream {
-    msg_rx: Receiver<ProposerAcceptorMessage>,
-    read_thread: Option<thread::JoinHandle<Result<()>>>,
-}
-
-impl ProposerPollStream {
-    fn new(mut r: ReadStream) -> Result<Self> {
-        let (msg_tx, msg_rx) = channel();
-
-        let read_thread = thread::Builder::new()
-            .name("Read WAL thread".into())
-            .spawn(move || -> Result<()> {
-                loop {
-                    let copy_data = match FeMessage::read(&mut r)? {
-                        Some(FeMessage::CopyData(bytes)) => bytes,
-                        Some(msg) => bail!("expected `CopyData` message, found {:?}", msg),
-                        None => bail!("connection closed unexpectedly"),
-                    };
-
-                    let msg = ProposerAcceptorMessage::parse(copy_data)?;
-                    msg_tx.send(msg)?;
-                }
-                // msg_tx will be dropped here, this will also close msg_rx
-            })?;
-
-        Ok(Self {
-            msg_rx,
-            read_thread: Some(read_thread),
-        })
-    }
-
-    fn recv_msg(&mut self) -> Result<ProposerAcceptorMessage> {
-        self.msg_rx.recv().map_err(|_| {
-            // return error from the read thread
-            let res = match self.read_thread.take() {
-                Some(thread) => thread.join(),
-                None => return anyhow!("read thread is gone"),
-            };
-
-            match res {
-                Ok(Ok(())) => anyhow!("unexpected result from read thread"),
-                Err(err) => anyhow!("read thread panicked: {:?}", err),
-                Ok(Err(err)) => err,
-            }
-        })
-    }
-
-    fn poll_msg(&mut self) -> Option<ProposerAcceptorMessage> {
-        let res = self.msg_rx.try_recv();
-
-        match res {
-            Err(_) => None,
-            Ok(msg) => Some(msg),
+            msg = self.read_msg()?;
         }
     }
 }

walkeeper/src/safekeeper.rs

@@ -10,8 +10,6 @@ use std::cmp::min;
 use std::fmt;
 use std::io::Read;
 use tracing::*;
-use zenith_utils::zid::ZNodeId;
-use zenith_utils::zid::ZTenantTimelineId;
 
 use lazy_static::lazy_static;
 
@@ -27,13 +25,12 @@ use zenith_utils::pq_proto::ZenithFeedback;
 use zenith_utils::zid::{ZTenantId, ZTimelineId};
 
 pub const SK_MAGIC: u32 = 0xcafeceefu32;
-pub const SK_FORMAT_VERSION: u32 = 4;
+pub const SK_FORMAT_VERSION: u32 = 3;
 const SK_PROTOCOL_VERSION: u32 = 1;
 const UNKNOWN_SERVER_VERSION: u32 = 0;
 
 /// Consensus logical timestamp.
 pub type Term = u64;
-const INVALID_TERM: Term = 0;
 
 #[derive(Debug, Clone, Copy, Serialize, Deserialize)]
 pub struct TermSwitchEntry {
@@ -131,47 +128,18 @@ pub struct ServerInfo {
     /// Postgres server version
     pub pg_version: u32,
     pub system_id: SystemId,
-    pub wal_seg_size: u32,
-}
-
-/// Data published by safekeeper to the peers
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct PeerInfo {
-    /// LSN up to which safekeeper offloaded WAL to s3.
-    s3_wal_lsn: Lsn,
-    /// Term of the last entry.
-    term: Term,
-    /// LSN of the last record.
-    flush_lsn: Lsn,
-    /// Up to which LSN safekeeper regards its WAL as committed.
-    commit_lsn: Lsn,
-}
-
-impl PeerInfo {
-    fn new() -> Self {
-        Self {
-            s3_wal_lsn: Lsn(0),
-            term: INVALID_TERM,
-            flush_lsn: Lsn(0),
-            commit_lsn: Lsn(0),
-        }
-    }
-}
-
-// vector-based node id -> peer state map with very limited functionality we
-// need/
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct Peers(pub Vec<(ZNodeId, PeerInfo)>);
-
-/// Persistent information stored on safekeeper node
-/// On disk data is prefixed by magic and format version and followed by checksum.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct SafeKeeperState {
     #[serde(with = "hex")]
     pub tenant_id: ZTenantId,
     /// Zenith timelineid
     #[serde(with = "hex")]
     pub timeline_id: ZTimelineId,
+    pub wal_seg_size: u32,
+}
+
+/// Persistent information stored on safekeeper node
+/// On disk data is prefixed by magic and format version and followed by checksum.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SafeKeeperState {
     /// persistent acceptor state
     pub acceptor_state: AcceptorState,
     /// information about server
@@ -180,33 +148,19 @@ pub struct SafeKeeperState {
     /// for correctness, exists for monitoring purposes.
     #[serde(with = "hex")]
     pub proposer_uuid: PgUuid,
-    /// Part of WAL acknowledged by quorum and available locally. Always points
-    /// to record boundary.
+    /// part of WAL acknowledged by quorum and available locally
     pub commit_lsn: Lsn,
-    /// First LSN not yet offloaded to s3. Useful to persist to avoid finding
-    /// out offloading progress on boot.
-    pub s3_wal_lsn: Lsn,
-    /// Minimal LSN which may be needed for recovery of some safekeeper (end_lsn
-    /// of last record streamed to everyone). Persisting it helps skipping
-    /// recovery in walproposer, generally we compute it from peers. In
-    /// walproposer proto called 'truncate_lsn'.
-    pub peer_horizon_lsn: Lsn,
-    /// LSN of the oldest known checkpoint made by pageserver and successfully
-    /// pushed to s3. We don't remove WAL beyond it. Persisted only for
-    /// informational purposes, we receive it from pageserver.
-    pub remote_consistent_lsn: Lsn,
-    // Peers and their state as we remember it. Knowing peers themselves is
-    // fundamental; but state is saved here only for informational purposes and
-    // obviously can be stale. (Currently not saved at all, but let's provision
-    // place to have less file version upgrades).
-    pub peers: Peers,
+    /// minimal LSN which may be needed for recovery of some safekeeper (end_lsn
+    /// of last record streamed to everyone)
+    pub truncate_lsn: Lsn,
+    // Safekeeper starts receiving WAL from this LSN, zeros before it ought to
+    // be skipped during decoding.
+    pub wal_start_lsn: Lsn,
 }
 
 impl SafeKeeperState {
-    pub fn new(zttid: &ZTenantTimelineId, peers: Vec<ZNodeId>) -> SafeKeeperState {
+    pub fn new() -> SafeKeeperState {
         SafeKeeperState {
-            tenant_id: zttid.tenant_id,
-            timeline_id: zttid.timeline_id,
             acceptor_state: AcceptorState {
                 term: 0,
                 term_history: TermHistory::empty(),
@@ -214,20 +168,21 @@ impl SafeKeeperState {
             server: ServerInfo {
                 pg_version: UNKNOWN_SERVER_VERSION, /* Postgres server version */
                 system_id: 0,                       /* Postgres system identifier */
+                tenant_id: ZTenantId::from([0u8; 16]),
+                timeline_id: ZTimelineId::from([0u8; 16]),
                 wal_seg_size: 0,
             },
             proposer_uuid: [0; 16],
-            commit_lsn: Lsn(0),
-            s3_wal_lsn: Lsn(0),
-            peer_horizon_lsn: Lsn(0),
-            remote_consistent_lsn: Lsn(0),
-            peers: Peers(peers.iter().map(|p| (*p, PeerInfo::new())).collect()),
+            commit_lsn: Lsn(0),   /* part of WAL acknowledged by quorum */
+            truncate_lsn: Lsn(0), /* minimal LSN which may be needed for recovery of some safekeeper */
+            wal_start_lsn: Lsn(0),
         }
     }
+}
 
-    #[cfg(test)]
-    pub fn empty() -> Self {
-        SafeKeeperState::new(&ZTenantTimelineId::empty(), vec![])
+impl Default for SafeKeeperState {
+    fn default() -> Self {
+        Self::new()
     }
 }
 
@@ -346,8 +301,6 @@ pub enum ProposerAcceptorMessage {
     VoteRequest(VoteRequest),
     Elected(ProposerElected),
     AppendRequest(AppendRequest),
-    NoFlushAppendRequest(AppendRequest),
-    FlushWAL,
 }
 
 impl ProposerAcceptorMessage {
@@ -466,7 +419,6 @@ lazy_static! {
 
 struct SafeKeeperMetrics {
     commit_lsn: Gauge,
-    // WAL-related metrics are in WalStorageMetrics
 }
 
 impl SafeKeeperMetrics {
@@ -489,7 +441,7 @@ pub struct SafeKeeper<CTRL: control_file::Storage, WAL: wal_storage::Storage> {
 
     /// not-yet-flushed pairs of same named fields in s.*
     pub commit_lsn: Lsn,
-    pub peer_horizon_lsn: Lsn,
+    pub truncate_lsn: Lsn,
     pub s: SafeKeeperState, // persistent part
 
     pub control_store: CTRL,
@@ -508,14 +460,16 @@ where
         wal_store: WAL,
         state: SafeKeeperState,
     ) -> SafeKeeper<CTRL, WAL> {
-        if state.timeline_id != ZTimelineId::from([0u8; 16]) && ztli != state.timeline_id {
-            panic!("Calling SafeKeeper::new with inconsistent ztli ({}) and SafeKeeperState.server.timeline_id ({})", ztli, state.timeline_id);
+        if state.server.timeline_id != ZTimelineId::from([0u8; 16])
+            && ztli != state.server.timeline_id
+        {
+            panic!("Calling SafeKeeper::new with inconsistent ztli ({}) and SafeKeeperState.server.timeline_id ({})", ztli, state.server.timeline_id);
         }
 
         SafeKeeper {
-            metrics: SafeKeeperMetrics::new(state.tenant_id, ztli, state.commit_lsn),
+            metrics: SafeKeeperMetrics::new(state.server.tenant_id, ztli, state.commit_lsn),
             commit_lsn: state.commit_lsn,
-            peer_horizon_lsn: state.peer_horizon_lsn,
+            truncate_lsn: state.truncate_lsn,
             s: state,
             control_store,
             wal_store,
@@ -545,11 +499,7 @@ where
             ProposerAcceptorMessage::Greeting(msg) => self.handle_greeting(msg),
             ProposerAcceptorMessage::VoteRequest(msg) => self.handle_vote_request(msg),
             ProposerAcceptorMessage::Elected(msg) => self.handle_elected(msg),
-            ProposerAcceptorMessage::AppendRequest(msg) => self.handle_append_request(msg, true),
-            ProposerAcceptorMessage::NoFlushAppendRequest(msg) => {
-                self.handle_append_request(msg, false)
-            }
-            ProposerAcceptorMessage::FlushWAL => self.handle_flush(),
+            ProposerAcceptorMessage::AppendRequest(msg) => self.handle_append_request(msg),
         }
     }
 
@@ -576,24 +526,12 @@ where
                 msg.pg_version, self.s.server.pg_version
             );
         }
-        if msg.tenant_id != self.s.tenant_id {
-            bail!(
-                "invalid tenant ID, got {}, expected {}",
-                msg.tenant_id,
-                self.s.tenant_id
-            );
-        }
-        if msg.ztli != self.s.timeline_id {
-            bail!(
-                "invalid timeline ID, got {}, expected {}",
-                msg.ztli,
-                self.s.timeline_id
-            );
-        }
 
         // set basic info about server, if not yet
         // TODO: verify that is doesn't change after
         self.s.server.system_id = msg.system_id;
+        self.s.server.tenant_id = msg.tenant_id;
+        self.s.server.timeline_id = msg.ztli;
         self.s.server.wal_seg_size = msg.wal_seg_size;
         self.control_store
             .persist(&self.s)
@@ -624,7 +562,7 @@ where
             term: self.s.acceptor_state.term,
             vote_given: false as u64,
             flush_lsn: self.wal_store.flush_lsn(),
-            truncate_lsn: self.s.peer_horizon_lsn,
+            truncate_lsn: self.s.truncate_lsn,
             term_history: self.get_term_history(),
         };
         if self.s.acceptor_state.term < msg.term {
@@ -649,16 +587,14 @@ where
 
     /// Form AppendResponse from current state.
     fn append_response(&self) -> AppendResponse {
-        let ar = AppendResponse {
+        AppendResponse {
             term: self.s.acceptor_state.term,
             flush_lsn: self.wal_store.flush_lsn(),
             commit_lsn: self.s.commit_lsn,
             // will be filled by the upper code to avoid bothering safekeeper
             hs_feedback: HotStandbyFeedback::empty(),
             zenith_feedback: ZenithFeedback::empty(),
-        };
-        trace!("formed AppendResponse {:?}", ar);
-        ar
+        }
     }
 
     fn handle_elected(&mut self, msg: &ProposerElected) -> Result<Option<AcceptorProposerMessage>> {
@@ -669,10 +605,7 @@ where
             return Ok(None);
         }
 
-        // TODO: cross check divergence point, check if msg.start_streaming_at corresponds to
-        // intersection of our history and history from msg
-
-        // truncate wal, update the LSNs
+        // truncate wal, update the lsns
         self.wal_store.truncate_wal(msg.start_streaming_at)?;
 
         // and now adopt term history from proposer
@@ -689,7 +622,6 @@ where
     fn handle_append_request(
         &mut self,
         msg: &AppendRequest,
-        mut require_flush: bool,
     ) -> Result<Option<AcceptorProposerMessage>> {
         if self.s.acceptor_state.term < msg.h.term {
             bail!("got AppendRequest before ProposerElected");
@@ -713,21 +645,14 @@ where
         if !msg.wal_data.is_empty() {
             self.wal_store.write_wal(msg.h.begin_lsn, &msg.wal_data)?;
 
-            // If this was the first record we ever receieved, initialize
-            // commit_lsn to help find_end_of_wal skip the hole in the
-            // beginning.
-            if self.s.commit_lsn == Lsn(0) {
-                self.s.commit_lsn = msg.h.begin_lsn;
+            // If this was the first record we ever receieved, remember LSN to help
+            // find_end_of_wal skip the hole in the beginning.
+            if self.s.wal_start_lsn == Lsn(0) {
+                self.s.wal_start_lsn = msg.h.begin_lsn;
                 sync_control_file = true;
-                require_flush = true;
             }
         }
 
-        // flush wal to the disk, if required
-        if require_flush {
-            self.wal_store.flush_wal()?;
-        }
-
         // Advance commit_lsn taking into account what we have locally.
         // commit_lsn can be 0, being unknown to new walproposer while he hasn't
         // collected majority of its epoch acks yet, ignore it in this case.
@@ -744,46 +669,34 @@ where
                 .set(u64::from(self.commit_lsn) as f64);
         }
 
-        self.peer_horizon_lsn = msg.h.truncate_lsn;
-        // Update truncate and commit LSN in control file.
-        // To avoid negative impact on performance of extra fsync, do it only
-        // when truncate_lsn delta exceeds WAL segment size.
+        self.truncate_lsn = msg.h.truncate_lsn;
+        /*
+         * Update truncate and commit LSN in control file.
+         * To avoid negative impact on performance of extra fsync, do it only
+         * when truncate_lsn delta exceeds WAL segment size.
+         */
         sync_control_file |=
-            self.s.peer_horizon_lsn + (self.s.server.wal_seg_size as u64) < self.peer_horizon_lsn;
+            self.s.truncate_lsn + (self.s.server.wal_seg_size as u64) < self.truncate_lsn;
         if sync_control_file {
             self.s.commit_lsn = self.commit_lsn;
-            self.s.peer_horizon_lsn = self.peer_horizon_lsn;
+            self.s.truncate_lsn = self.truncate_lsn;
         }
 
         if sync_control_file {
             self.control_store.persist(&self.s)?;
         }
 
+        let resp = self.append_response();
         trace!(
-            "processed AppendRequest of len {}, end_lsn={:?}, commit_lsn={:?}, truncate_lsn={:?}, flushed={:?}",
+            "processed AppendRequest of len {}, end_lsn={:?}, commit_lsn={:?}, truncate_lsn={:?}, resp {:?}",
             msg.wal_data.len(),
             msg.h.end_lsn,
             msg.h.commit_lsn,
             msg.h.truncate_lsn,
-            require_flush,
+            &resp,
         );
-
-        // If flush_lsn hasn't updated, AppendResponse is not very useful.
-        if !require_flush {
-            return Ok(None);
-        }
-
-        let resp = self.append_response();
         Ok(Some(AcceptorProposerMessage::AppendResponse(resp)))
     }
-
-    /// Flush WAL to disk. Return AppendResponse with latest LSNs.
-    fn handle_flush(&mut self) -> Result<Option<AcceptorProposerMessage>> {
-        self.wal_store.flush_wal()?;
-        Ok(Some(AcceptorProposerMessage::AppendResponse(
-            self.append_response(),
-        )))
-    }
 }
 
 #[cfg(test)]
@@ -825,20 +738,16 @@ mod tests {
             self.lsn = end_pos;
             Ok(())
         }
-
-        fn flush_wal(&mut self) -> Result<()> {
-            Ok(())
-        }
     }
 
     #[test]
     fn test_voting() {
         let storage = InMemoryState {
-            persisted_state: SafeKeeperState::empty(),
+            persisted_state: SafeKeeperState::new(),
         };
         let wal_store = DummyWalStore { lsn: Lsn(0) };
         let ztli = ZTimelineId::from([0u8; 16]);
-        let mut sk = SafeKeeper::new(ztli, storage, wal_store, SafeKeeperState::empty());
+        let mut sk = SafeKeeper::new(ztli, storage, wal_store, SafeKeeperState::new());
 
         // check voting for 1 is ok
         let vote_request = ProposerAcceptorMessage::VoteRequest(VoteRequest { term: 1 });
@@ -866,11 +775,11 @@ mod tests {
     #[test]
     fn test_epoch_switch() {
         let storage = InMemoryState {
-            persisted_state: SafeKeeperState::empty(),
+            persisted_state: SafeKeeperState::new(),
         };
         let wal_store = DummyWalStore { lsn: Lsn(0) };
         let ztli = ZTimelineId::from([0u8; 16]);
-        let mut sk = SafeKeeper::new(ztli, storage, wal_store, SafeKeeperState::empty());
+        let mut sk = SafeKeeper::new(ztli, storage, wal_store, SafeKeeperState::new());
 
         let mut ar_hdr = AppendRequestHeader {
             term: 1,

walkeeper/src/timeline.rs

@@ -1,7 +1,7 @@
 //! This module contains timeline id -> safekeeper state map with file-backed
 //! persistence and support for interaction between sending and receiving wal.
 
-use anyhow::{bail, Context, Result};
+use anyhow::{Context, Result};
 
 use lazy_static::lazy_static;
 
@@ -9,24 +9,22 @@ use std::cmp::{max, min};
 use std::collections::HashMap;
 use std::fs::{self};
 
-use std::sync::{Arc, Condvar, Mutex, MutexGuard};
+use std::sync::{Arc, Condvar, Mutex};
 use std::time::Duration;
 use tokio::sync::mpsc::UnboundedSender;
 use tracing::*;
 
 use zenith_utils::lsn::Lsn;
-use zenith_utils::zid::{ZNodeId, ZTenantTimelineId};
+use zenith_utils::zid::ZTenantTimelineId;
 
 use crate::callmemaybe::{CallmeEvent, SubscriptionStateKey};
+use crate::control_file::{self, CreateControlFile};
 
-use crate::control_file;
-use crate::control_file::Storage as cf_storage;
 use crate::safekeeper::{
     AcceptorProposerMessage, ProposerAcceptorMessage, SafeKeeper, SafeKeeperState,
 };
 use crate::send_wal::HotStandbyFeedback;
-use crate::wal_storage;
-use crate::wal_storage::Storage as wal_storage_iface;
+use crate::wal_storage::{self, Storage};
 use crate::SafeKeeperConf;
 
 use zenith_utils::pq_proto::ZenithFeedback;
@@ -89,39 +87,21 @@ struct SharedState {
 }
 
 impl SharedState {
-    /// Initialize timeline state, creating control file
-    fn create(
+    /// Restore SharedState from control file.
+    /// If create=false and file doesn't exist, bails out.
+    fn create_restore(
         conf: &SafeKeeperConf,
         zttid: &ZTenantTimelineId,
-        peer_ids: Vec<ZNodeId>,
+        create: CreateControlFile,
     ) -> Result<Self> {
-        let state = SafeKeeperState::new(zttid, peer_ids);
-        let control_store = control_file::FileStorage::new(zttid, conf);
-        let wal_store = wal_storage::PhysicalStorage::new(zttid, conf);
-        let mut sk = SafeKeeper::new(zttid.timeline_id, control_store, wal_store, state);
-        sk.control_store.persist(&sk.s)?;
-
-        Ok(Self {
-            notified_commit_lsn: Lsn(0),
-            sk,
-            replicas: Vec::new(),
-            active: false,
-            num_computes: 0,
-            pageserver_connstr: None,
-        })
-    }
-
-    /// Restore SharedState from control file.
-    /// If file doesn't exist, bails out.
-    fn restore(conf: &SafeKeeperConf, zttid: &ZTenantTimelineId) -> Result<Self> {
-        let state = control_file::FileStorage::load_control_file_conf(conf, zttid)
+        let state = control_file::FileStorage::load_control_file_conf(conf, zttid, create)
             .context("failed to load from control file")?;
 
         let control_store = control_file::FileStorage::new(zttid, conf);
 
         let wal_store = wal_storage::PhysicalStorage::new(zttid, conf);
 
-        info!("timeline {} restored", zttid.timeline_id);
+        info!("timeline {} created or restored", zttid.timeline_id);
 
         Ok(Self {
             notified_commit_lsn: Lsn(0),
@@ -438,13 +418,26 @@ impl Timeline {
 
 // Utilities needed by various Connection-like objects
 pub trait TimelineTools {
-    fn set(&mut self, conf: &SafeKeeperConf, zttid: ZTenantTimelineId, create: bool) -> Result<()>;
+    fn set(
+        &mut self,
+        conf: &SafeKeeperConf,
+        zttid: ZTenantTimelineId,
+        create: CreateControlFile,
+    ) -> Result<()>;
 
     fn get(&self) -> &Arc<Timeline>;
 }
 
 impl TimelineTools for Option<Arc<Timeline>> {
-    fn set(&mut self, conf: &SafeKeeperConf, zttid: ZTenantTimelineId, create: bool) -> Result<()> {
+    fn set(
+        &mut self,
+        conf: &SafeKeeperConf,
+        zttid: ZTenantTimelineId,
+        create: CreateControlFile,
+    ) -> Result<()> {
+        // We will only set the timeline once. If it were to ever change,
+        // anyone who cloned the Arc would be out of date.
+        assert!(self.is_none());
         *self = Some(GlobalTimelines::get(conf, zttid, create)?);
         Ok(())
     }
@@ -463,73 +456,30 @@ lazy_static! {
 pub struct GlobalTimelines;
 
 impl GlobalTimelines {
-    fn create_internal(
-        mut timelines: MutexGuard<HashMap<ZTenantTimelineId, Arc<Timeline>>>,
-        conf: &SafeKeeperConf,
-        zttid: ZTenantTimelineId,
-        peer_ids: Vec<ZNodeId>,
-    ) -> Result<Arc<Timeline>> {
-        match timelines.get(&zttid) {
-            Some(_) => bail!("timeline {} already exists", zttid),
-            None => {
-                // TODO: check directory existence
-                let dir = conf.timeline_dir(&zttid);
-                fs::create_dir_all(dir)?;
-                let shared_state = SharedState::create(conf, &zttid, peer_ids)
-                    .context("failed to create shared state")?;
-
-                let new_tli = Arc::new(Timeline::new(zttid, shared_state));
-                timelines.insert(zttid, Arc::clone(&new_tli));
-                Ok(new_tli)
-            }
-        }
-    }
-
-    pub fn create(
-        conf: &SafeKeeperConf,
-        zttid: ZTenantTimelineId,
-        peer_ids: Vec<ZNodeId>,
-    ) -> Result<Arc<Timeline>> {
-        let timelines = TIMELINES.lock().unwrap();
-        GlobalTimelines::create_internal(timelines, conf, zttid, peer_ids)
-    }
-
     /// Get a timeline with control file loaded from the global TIMELINES map.
-    /// If control file doesn't exist, bails out.
+    /// If control file doesn't exist and create=false, bails out.
     pub fn get(
         conf: &SafeKeeperConf,
         zttid: ZTenantTimelineId,
-        create: bool,
+        create: CreateControlFile,
     ) -> Result<Arc<Timeline>> {
         let mut timelines = TIMELINES.lock().unwrap();
 
         match timelines.get(&zttid) {
             Some(result) => Ok(Arc::clone(result)),
             None => {
-                let shared_state =
-                    SharedState::restore(conf, &zttid).context("failed to restore shared state");
+                if let CreateControlFile::True = create {
+                    let dir = conf.timeline_dir(&zttid);
+                    info!(
+                        "creating timeline dir {}, create is {:?}",
+                        dir.display(),
+                        create
+                    );
+                    fs::create_dir_all(dir)?;
+                }
 
-                let shared_state = match shared_state {
-                    Ok(shared_state) => shared_state,
-                    Err(error) => {
-                        // TODO: always create timeline explicitly
-                        if error
-                            .root_cause()
-                            .to_string()
-                            .contains("No such file or directory")
-                            && create
-                        {
-                            return GlobalTimelines::create_internal(
-                                timelines,
-                                conf,
-                                zttid,
-                                vec![],
-                            );
-                        } else {
-                            return Err(error);
-                        }
-                    }
-                };
+                let shared_state = SharedState::create_restore(conf, &zttid, create)
+                    .context("failed to restore shared state")?;
 
                 let new_tli = Arc::new(Timeline::new(zttid, shared_state));
                 timelines.insert(zttid, Arc::clone(&new_tli));

walkeeper/src/wal_storage.rs

@@ -7,7 +7,7 @@
 //!
 //! Note that last file has `.partial` suffix, that's different from postgres.
 
-use anyhow::{anyhow, bail, Context, Result};
+use anyhow::{anyhow, Context, Result};
 use std::io::{Read, Seek, SeekFrom};
 
 use lazy_static::lazy_static;
@@ -58,20 +58,12 @@ lazy_static! {
         DISK_WRITE_SECONDS_BUCKETS.to_vec()
     )
     .expect("Failed to register safekeeper_write_wal_seconds histogram vec");
-    static ref FLUSH_WAL_SECONDS: HistogramVec = register_histogram_vec!(
-        "safekeeper_flush_wal_seconds",
-        "Seconds spent syncing WAL to a disk, grouped by timeline",
-        &["tenant_id", "timeline_id"],
-        DISK_WRITE_SECONDS_BUCKETS.to_vec()
-    )
-    .expect("Failed to register safekeeper_flush_wal_seconds histogram vec");
 }
 
 struct WalStorageMetrics {
     flush_lsn: Gauge,
     write_wal_bytes: Histogram,
     write_wal_seconds: Histogram,
-    flush_wal_seconds: Histogram,
 }
 
 impl WalStorageMetrics {
@@ -82,38 +74,24 @@ impl WalStorageMetrics {
             flush_lsn: FLUSH_LSN_GAUGE.with_label_values(&[&tenant_id, &timeline_id]),
             write_wal_bytes: WRITE_WAL_BYTES.with_label_values(&[&tenant_id, &timeline_id]),
             write_wal_seconds: WRITE_WAL_SECONDS.with_label_values(&[&tenant_id, &timeline_id]),
-            flush_wal_seconds: FLUSH_WAL_SECONDS.with_label_values(&[&tenant_id, &timeline_id]),
         }
     }
 }
 
 pub trait Storage {
-    /// LSN of last durably stored WAL record.
+    /// lsn of last durably stored WAL record.
     fn flush_lsn(&self) -> Lsn;
 
-    /// Init storage with wal_seg_size and read WAL from disk to get latest LSN.
+    /// Init storage with wal_seg_size and read WAL from disk to get latest lsn.
     fn init_storage(&mut self, state: &SafeKeeperState) -> Result<()>;
 
-    /// Write piece of WAL from buf to disk, but not necessarily sync it.
+    /// Write piece of wal in buf to disk and sync it.
     fn write_wal(&mut self, startpos: Lsn, buf: &[u8]) -> Result<()>;
 
-    /// Truncate WAL at specified LSN, which must be the end of WAL record.
+    // Truncate WAL at specified LSN.
     fn truncate_wal(&mut self, end_pos: Lsn) -> Result<()>;
-
-    /// Durably store WAL on disk, up to the last written WAL record.
-    fn flush_wal(&mut self) -> Result<()>;
 }
 
-/// PhysicalStorage is a storage that stores WAL on disk. Writes are separated from flushes
-/// for better performance. Storage must be initialized before use.
-///
-/// WAL is stored in segments, each segment is a file. Last segment has ".partial" suffix in
-/// its filename and may be not fully flushed.
-///
-/// Relationship of LSNs:
-/// `write_lsn` >= `write_record_lsn` >= `flush_record_lsn`
-///
-/// When storage is just created, all LSNs are zeroes and there are no segments on disk.
 pub struct PhysicalStorage {
     metrics: WalStorageMetrics,
     zttid: ZTenantTimelineId,
@@ -121,29 +99,27 @@ pub struct PhysicalStorage {
     conf: SafeKeeperConf,
 
     // fields below are filled upon initialization
-    /// None if unitialized, Some(usize) if storage is initialized.
+
+    // None if unitialized, Some(lsn) if storage is initialized
     wal_seg_size: Option<usize>,
 
-    /// Written to disk, but possibly still in the cache and not fully persisted.
-    /// Also can be ahead of record_lsn, if happen to be in the middle of a WAL record.
+    // Relationship of lsns:
+    // `write_lsn` >= `write_record_lsn` >= `flush_record_lsn`
+    //
+    // All lsns are zeroes, if storage is just created, and there are no segments on disk.
+
+    // Written to disk, but possibly still in the cache and not fully persisted.
+    // Also can be ahead of record_lsn, if happen to be in the middle of a WAL record.
     write_lsn: Lsn,
 
-    /// The LSN of the last WAL record written to disk. Still can be not fully flushed.
+    // The LSN of the last WAL record written to disk. Still can be not fully flushed.
     write_record_lsn: Lsn,
 
-    /// The LSN of the last WAL record flushed to disk.
+    // The LSN of the last WAL record flushed to disk.
     flush_record_lsn: Lsn,
 
-    /// Decoder is required for detecting boundaries of WAL records.
+    // Decoder is required for detecting boundaries of WAL records.
     decoder: WalStreamDecoder,
-
-    /// Cached open file for the last segment.
-    ///
-    /// If Some(file) is open, then it always:
-    /// - has ".partial" suffix
-    /// - points to write_lsn, so no seek is needed for writing
-    /// - doesn't point to the end of the segment
-    file: Option<File>,
 }
 
 impl PhysicalStorage {
@@ -159,150 +135,131 @@ impl PhysicalStorage {
             write_record_lsn: Lsn(0),
             flush_record_lsn: Lsn(0),
             decoder: WalStreamDecoder::new(Lsn(0)),
-            file: None,
         }
     }
 
-    /// Wrapper for flush_lsn updates that also updates metrics.
+    // wrapper for flush_lsn updates that also updates metrics
     fn update_flush_lsn(&mut self) {
         self.flush_record_lsn = self.write_record_lsn;
         self.metrics.flush_lsn.set(self.flush_record_lsn.0 as f64);
     }
 
-    /// Call fdatasync if config requires so.
-    fn fdatasync_file(&self, file: &mut File) -> Result<()> {
-        if !self.conf.no_sync {
-            self.metrics
-                .flush_wal_seconds
-                .observe_closure_duration(|| file.sync_data())?;
-        }
-        Ok(())
-    }
-
-    /// Call fsync if config requires so.
-    fn fsync_file(&self, file: &mut File) -> Result<()> {
-        if !self.conf.no_sync {
-            self.metrics
-                .flush_wal_seconds
-                .observe_closure_duration(|| file.sync_all())?;
-        }
-        Ok(())
-    }
-
-    /// Open or create WAL segment file. Caller must call seek to the wanted position.
-    /// Returns `file` and `is_partial`.
-    fn open_or_create(&self, segno: XLogSegNo, wal_seg_size: usize) -> Result<(File, bool)> {
-        let (wal_file_path, wal_file_partial_path) =
-            wal_file_paths(&self.timeline_dir, segno, wal_seg_size)?;
-
-        // Try to open already completed segment
-        if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_path) {
-            Ok((file, false))
-        } else if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_partial_path) {
-            // Try to open existing partial file
-            Ok((file, true))
-        } else {
-            // Create and fill new partial file
-            let mut file = OpenOptions::new()
-                .create(true)
-                .write(true)
-                .open(&wal_file_partial_path)
-                .with_context(|| format!("Failed to open log file {:?}", &wal_file_path))?;
-
-            write_zeroes(&mut file, wal_seg_size)?;
-            self.fsync_file(&mut file)?;
-            Ok((file, true))
-        }
-    }
-
-    /// Write WAL bytes, which are known to be located in a single WAL segment.
-    fn write_in_segment(
-        &mut self,
-        segno: u64,
-        xlogoff: usize,
-        buf: &[u8],
-        wal_seg_size: usize,
-    ) -> Result<()> {
-        let mut file = if let Some(file) = self.file.take() {
-            file
-        } else {
-            let (mut file, is_partial) = self.open_or_create(segno, wal_seg_size)?;
-            assert!(is_partial, "unexpected write into non-partial segment file");
-            file.seek(SeekFrom::Start(xlogoff as u64))?;
-            file
-        };
-
-        file.write_all(buf)?;
-
-        if xlogoff + buf.len() == wal_seg_size {
-            // If we reached the end of a WAL segment, flush and close it.
-            self.fdatasync_file(&mut file)?;
-
-            // Rename partial file to completed file
-            let (wal_file_path, wal_file_partial_path) =
-                wal_file_paths(&self.timeline_dir, segno, wal_seg_size)?;
-            fs::rename(&wal_file_partial_path, &wal_file_path)?;
-        } else {
-            // otherwise, file can be reused later
-            self.file = Some(file);
-        }
-
-        Ok(())
-    }
-
-    /// Writes WAL to the segment files, until everything is writed. If some segments
-    /// are fully written, they are flushed to disk. The last (partial) segment can
-    /// be flushed separately later.
-    ///
-    /// Updates `write_lsn`.
-    fn write_exact(&mut self, pos: Lsn, mut buf: &[u8]) -> Result<()> {
+    /// Helper returning full path to WAL segment file and its .partial brother.
+    fn wal_file_paths(&self, segno: XLogSegNo) -> Result<(PathBuf, PathBuf)> {
         let wal_seg_size = self
             .wal_seg_size
             .ok_or_else(|| anyhow!("wal_seg_size is not initialized"))?;
 
-        if self.write_lsn != pos {
-            // need to flush the file before discarding it
-            if let Some(mut file) = self.file.take() {
-                self.fdatasync_file(&mut file)?;
+        let wal_file_name = XLogFileName(PG_TLI, segno, wal_seg_size);
+        let wal_file_path = self.timeline_dir.join(wal_file_name.clone());
+        let wal_file_partial_path = self.timeline_dir.join(wal_file_name + ".partial");
+        Ok((wal_file_path, wal_file_partial_path))
+    }
+
+    // TODO: this function is going to be refactored soon, what will change:
+    //      - flush will be called separately from write_wal, this function
+    //        will only write bytes to disk
+    //      - File will be cached in PhysicalStorage, to remove extra syscalls,
+    //        such as open(), seek(), close()
+    fn write_and_flush(&mut self, startpos: Lsn, buf: &[u8]) -> Result<()> {
+        let wal_seg_size = self
+            .wal_seg_size
+            .ok_or_else(|| anyhow!("wal_seg_size is not initialized"))?;
+
+        let mut bytes_left: usize = buf.len();
+        let mut bytes_written: usize = 0;
+        let mut partial;
+        let mut start_pos = startpos;
+        const ZERO_BLOCK: &[u8] = &[0u8; XLOG_BLCKSZ];
+
+        /* Extract WAL location for this block */
+        let mut xlogoff = start_pos.segment_offset(wal_seg_size) as usize;
+
+        while bytes_left != 0 {
+            let bytes_to_write;
+
+            /*
+             * If crossing a WAL boundary, only write up until we reach wal
+             * segment size.
+             */
+            if xlogoff + bytes_left > wal_seg_size {
+                bytes_to_write = wal_seg_size - xlogoff;
+            } else {
+                bytes_to_write = bytes_left;
             }
 
-            self.write_lsn = pos;
+            /* Open file */
+            let segno = start_pos.segment_number(wal_seg_size);
+            let (wal_file_path, wal_file_partial_path) = self.wal_file_paths(segno)?;
+            {
+                let mut wal_file: File;
+                /* Try to open already completed segment */
+                if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_path) {
+                    wal_file = file;
+                    partial = false;
+                } else if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_partial_path)
+                {
+                    /* Try to open existed partial file */
+                    wal_file = file;
+                    partial = true;
+                } else {
+                    /* Create and fill new partial file */
+                    partial = true;
+                    match OpenOptions::new()
+                        .create(true)
+                        .write(true)
+                        .open(&wal_file_partial_path)
+                    {
+                        Ok(mut file) => {
+                            for _ in 0..(wal_seg_size / XLOG_BLCKSZ) {
+                                file.write_all(ZERO_BLOCK)?;
+                            }
+                            wal_file = file;
+                        }
+                        Err(e) => {
+                            error!("Failed to open log file {:?}: {}", &wal_file_path, e);
+                            return Err(e.into());
+                        }
+                    }
+                }
+                wal_file.seek(SeekFrom::Start(xlogoff as u64))?;
+                wal_file.write_all(&buf[bytes_written..(bytes_written + bytes_to_write)])?;
+
+                // Flush file, if not said otherwise
+                if !self.conf.no_sync {
+                    wal_file.sync_all()?;
+                }
+            }
+            /* Write was successful, advance our position */
+            bytes_written += bytes_to_write;
+            bytes_left -= bytes_to_write;
+            start_pos += bytes_to_write as u64;
+            xlogoff += bytes_to_write;
+
+            /* Did we reach the end of a WAL segment? */
+            if start_pos.segment_offset(wal_seg_size) == 0 {
+                xlogoff = 0;
+                if partial {
+                    fs::rename(&wal_file_partial_path, &wal_file_path)?;
+                }
+            }
         }
-
-        while !buf.is_empty() {
-            // Extract WAL location for this block
-            let xlogoff = self.write_lsn.segment_offset(wal_seg_size) as usize;
-            let segno = self.write_lsn.segment_number(wal_seg_size);
-
-            // If crossing a WAL boundary, only write up until we reach wal segment size.
-            let bytes_write = if xlogoff + buf.len() > wal_seg_size {
-                wal_seg_size - xlogoff
-            } else {
-                buf.len()
-            };
-
-            self.write_in_segment(segno, xlogoff, &buf[..bytes_write], wal_seg_size)?;
-            self.write_lsn += bytes_write as u64;
-            buf = &buf[bytes_write..];
-        }
-
         Ok(())
     }
 }
 
 impl Storage for PhysicalStorage {
-    /// flush_lsn returns LSN of last durably stored WAL record.
+    // flush_lsn returns lsn of last durably stored WAL record.
     fn flush_lsn(&self) -> Lsn {
         self.flush_record_lsn
     }
 
-    /// Storage needs to know wal_seg_size to know which segment to read/write, but
-    /// wal_seg_size is not always known at the moment of storage creation. This method
-    /// allows to postpone its initialization.
+    // Storage needs to know wal_seg_size to know which segment to read/write, but
+    // wal_seg_size is not always known at the moment of storage creation. This method
+    // allows to postpone its initialization.
     fn init_storage(&mut self, state: &SafeKeeperState) -> Result<()> {
         if state.server.wal_seg_size == 0 {
-            // wal_seg_size is still unknown. This is dead path normally, should
-            // be used only in tests.
+            // wal_seg_size is still unknown
             return Ok(());
         }
 
@@ -316,13 +273,9 @@ impl Storage for PhysicalStorage {
         let wal_seg_size = state.server.wal_seg_size as usize;
         self.wal_seg_size = Some(wal_seg_size);
 
-        // Find out where stored WAL ends, starting at commit_lsn which is a
-        // known recent record boundary (unless we don't have WAL at all).
-        self.write_lsn = if state.commit_lsn == Lsn(0) {
-            Lsn(0)
-        } else {
-            Lsn(find_end_of_wal(&self.timeline_dir, wal_seg_size, true, state.commit_lsn)?.0)
-        };
+        // we need to read WAL from disk to know which LSNs are stored on disk
+        self.write_lsn =
+            Lsn(find_end_of_wal(&self.timeline_dir, wal_seg_size, true, state.wal_start_lsn)?.0);
 
         self.write_record_lsn = self.write_lsn;
 
@@ -331,43 +284,39 @@ impl Storage for PhysicalStorage {
         self.update_flush_lsn();
 
         info!(
-            "initialized storage for timeline {}, flush_lsn={}, commit_lsn={}, peer_horizon_lsn={}",
-            self.zttid.timeline_id, self.flush_record_lsn, state.commit_lsn, state.peer_horizon_lsn,
+            "initialized storage for timeline {}, flush_lsn={}, commit_lsn={}, truncate_lsn={}",
+            self.zttid.timeline_id, self.flush_record_lsn, state.commit_lsn, state.truncate_lsn,
         );
-        if self.flush_record_lsn < state.commit_lsn
-            || self.flush_record_lsn < state.peer_horizon_lsn
-        {
-            warn!("timeline {} potential data loss: flush_lsn by find_end_of_wal is less than either commit_lsn or peer_horizon_lsn from control file", self.zttid.timeline_id);
+        if self.flush_record_lsn < state.commit_lsn || self.flush_record_lsn < state.truncate_lsn {
+            warn!("timeline {} potential data loss: flush_lsn by find_end_of_wal is less than either commit_lsn or truncate_lsn from control file", self.zttid.timeline_id);
         }
 
         Ok(())
     }
 
-    /// Write WAL to disk.
+    // Write and flush WAL to disk.
     fn write_wal(&mut self, startpos: Lsn, buf: &[u8]) -> Result<()> {
-        // Disallow any non-sequential writes, which can result in gaps or overwrites.
-        // If we need to move the pointer, use truncate_wal() instead.
         if self.write_lsn > startpos {
-            bail!(
+            warn!(
                 "write_wal rewrites WAL written before, write_lsn={}, startpos={}",
-                self.write_lsn,
-                startpos
+                self.write_lsn, startpos
             );
         }
-        if self.write_lsn < startpos && self.write_lsn != Lsn(0) {
-            bail!(
+        if self.write_lsn < startpos {
+            warn!(
                 "write_wal creates gap in written WAL, write_lsn={}, startpos={}",
-                self.write_lsn,
-                startpos
+                self.write_lsn, startpos
             );
+            // TODO: return error if write_lsn is not zero
         }
 
         {
             let _timer = self.metrics.write_wal_seconds.start_timer();
-            self.write_exact(startpos, buf)?;
+            self.write_and_flush(startpos, buf)?;
         }
 
-        // WAL is written, updating write metrics
+        // WAL is written and flushed, updating lsns
+        self.write_lsn = startpos + buf.len() as u64;
         self.metrics.write_wal_bytes.observe(buf.len() as f64);
 
         // figure out last record's end lsn for reporting (if we got the
@@ -390,67 +339,69 @@ impl Storage for PhysicalStorage {
             }
         }
 
-        Ok(())
-    }
-
-    fn flush_wal(&mut self) -> Result<()> {
-        if self.flush_record_lsn == self.write_record_lsn {
-            // no need to do extra flush
-            return Ok(());
-        }
-
-        if let Some(mut unflushed_file) = self.file.take() {
-            self.fdatasync_file(&mut unflushed_file)?;
-            self.file = Some(unflushed_file);
-        } else {
-            // We have unflushed data (write_lsn != flush_lsn), but no file.
-            // This should only happen if last file was fully written and flushed,
-            // but haven't updated flush_lsn yet.
-            assert!(self.write_lsn.segment_offset(self.wal_seg_size.unwrap()) == 0);
-        }
-
-        // everything is flushed now, let's update flush_lsn
         self.update_flush_lsn();
         Ok(())
     }
 
-    /// Truncate written WAL by removing all WAL segments after the given LSN.
-    /// end_pos must point to the end of the WAL record.
+    // Truncate written WAL by removing all WAL segments after the given LSN.
+    // end_pos must point to the end of the WAL record.
     fn truncate_wal(&mut self, end_pos: Lsn) -> Result<()> {
         let wal_seg_size = self
             .wal_seg_size
             .ok_or_else(|| anyhow!("wal_seg_size is not initialized"))?;
 
-        // Streaming must not create a hole, so truncate cannot be called on non-written lsn
-        assert!(self.write_lsn == Lsn(0) || self.write_lsn >= end_pos);
+        // TODO: cross check divergence point
 
-        // Close previously opened file, if any
-        if let Some(mut unflushed_file) = self.file.take() {
-            self.fdatasync_file(&mut unflushed_file)?;
+        // nothing to truncate
+        if self.write_lsn == Lsn(0) {
+            return Ok(());
         }
 
-        let xlogoff = end_pos.segment_offset(wal_seg_size) as usize;
-        let segno = end_pos.segment_number(wal_seg_size);
-        let (mut file, is_partial) = self.open_or_create(segno, wal_seg_size)?;
+        // Streaming must not create a hole, so truncate cannot be called on non-written lsn
+        assert!(self.write_lsn >= end_pos);
 
-        // Fill end with zeroes
-        file.seek(SeekFrom::Start(xlogoff as u64))?;
-        write_zeroes(&mut file, wal_seg_size - xlogoff)?;
-        self.fdatasync_file(&mut file)?;
+        // open segment files and delete or fill end with zeroes
 
-        if !is_partial {
+        let partial;
+        const ZERO_BLOCK: &[u8] = &[0u8; XLOG_BLCKSZ];
+
+        /* Extract WAL location for this block */
+        let mut xlogoff = end_pos.segment_offset(wal_seg_size) as usize;
+
+        /* Open file */
+        let mut segno = end_pos.segment_number(wal_seg_size);
+        let (wal_file_path, wal_file_partial_path) = self.wal_file_paths(segno)?;
+        {
+            let mut wal_file: File;
+            /* Try to open already completed segment */
+            if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_path) {
+                wal_file = file;
+                partial = false;
+            } else {
+                wal_file = OpenOptions::new()
+                    .write(true)
+                    .open(&wal_file_partial_path)?;
+                partial = true;
+            }
+            wal_file.seek(SeekFrom::Start(xlogoff as u64))?;
+            while xlogoff < wal_seg_size {
+                let bytes_to_write = min(XLOG_BLCKSZ, wal_seg_size - xlogoff);
+                wal_file.write_all(&ZERO_BLOCK[0..bytes_to_write])?;
+                xlogoff += bytes_to_write;
+            }
+            // Flush file, if not said otherwise
+            if !self.conf.no_sync {
+                wal_file.sync_all()?;
+            }
+        }
+        if !partial {
             // Make segment partial once again
-            let (wal_file_path, wal_file_partial_path) =
-                wal_file_paths(&self.timeline_dir, segno, wal_seg_size)?;
             fs::rename(&wal_file_path, &wal_file_partial_path)?;
         }
-
         // Remove all subsequent segments
-        let mut segno = segno;
         loop {
             segno += 1;
-            let (wal_file_path, wal_file_partial_path) =
-                wal_file_paths(&self.timeline_dir, segno, wal_seg_size)?;
+            let (wal_file_path, wal_file_partial_path) = self.wal_file_paths(segno)?;
             // TODO: better use fs::try_exists which is currenty avaialble only in nightly build
             if wal_file_path.exists() {
                 fs::remove_file(&wal_file_path)?;
@@ -461,7 +412,7 @@ impl Storage for PhysicalStorage {
             }
         }
 
-        // Update LSNs
+        // Update lsns
         self.write_lsn = end_pos;
         self.write_record_lsn = end_pos;
         self.update_flush_lsn();
@@ -540,28 +491,3 @@ impl WalReader {
             })
     }
 }
-
-/// Zero block for filling created WAL segments.
-const ZERO_BLOCK: &[u8] = &[0u8; XLOG_BLCKSZ];
-
-/// Helper for filling file with zeroes.
-fn write_zeroes(file: &mut File, mut count: usize) -> Result<()> {
-    while count >= XLOG_BLCKSZ {
-        file.write_all(ZERO_BLOCK)?;
-        count -= XLOG_BLCKSZ;
-    }
-    file.write_all(&ZERO_BLOCK[0..count])?;
-    Ok(())
-}
-
-/// Helper returning full path to WAL segment file and its .partial brother.
-fn wal_file_paths(
-    timeline_dir: &Path,
-    segno: XLogSegNo,
-    wal_seg_size: usize,
-) -> Result<(PathBuf, PathBuf)> {
-    let wal_file_name = XLogFileName(PG_TLI, segno, wal_seg_size);
-    let wal_file_path = timeline_dir.join(wal_file_name.clone());
-    let wal_file_partial_path = timeline_dir.join(wal_file_name + ".partial");
-    Ok((wal_file_path, wal_file_partial_path))
-}

zenith/src/main.rs

@@ -18,35 +18,32 @@ use walkeeper::defaults::{
 };
 use zenith_utils::auth::{Claims, Scope};
 use zenith_utils::postgres_backend::AuthType;
-use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId};
+use zenith_utils::zid::{ZTenantId, ZTimelineId};
 use zenith_utils::GIT_VERSION;
 
 use pageserver::branches::BranchInfo;
 
-// Default id of a safekeeper node, if not specified on the command line.
-const DEFAULT_SAFEKEEPER_ID: ZNodeId = ZNodeId(1);
-const DEFAULT_PAGESERVER_ID: ZNodeId = ZNodeId(1);
+// Default name of a safekeeper node, if not specified on the command line.
+const DEFAULT_SAFEKEEPER_NAME: &str = "single";
 
 fn default_conf() -> String {
     format!(
         r#"
 # Default built-in configuration, defined in main.rs
 [pageserver]
-id = {pageserver_id}
 listen_pg_addr = '{pageserver_pg_addr}'
 listen_http_addr = '{pageserver_http_addr}'
 auth_type = '{pageserver_auth_type}'
 
 [[safekeepers]]
-id = {safekeeper_id}
+name = '{safekeeper_name}'
 pg_port = {safekeeper_pg_port}
 http_port = {safekeeper_http_port}
 "#,
-        pageserver_id = DEFAULT_PAGESERVER_ID,
         pageserver_pg_addr = DEFAULT_PAGESERVER_PG_ADDR,
         pageserver_http_addr = DEFAULT_PAGESERVER_HTTP_ADDR,
         pageserver_auth_type = AuthType::Trust,
-        safekeeper_id = DEFAULT_SAFEKEEPER_ID,
+        safekeeper_name = DEFAULT_SAFEKEEPER_NAME,
         safekeeper_pg_port = DEFAULT_SAFEKEEPER_PG_PORT,
         safekeeper_http_port = DEFAULT_SAFEKEEPER_HTTP_PORT,
     )
@@ -77,9 +74,9 @@ fn main() -> Result<()> {
         .required(true);
 
     #[rustfmt::skip]
-    let safekeeper_id_arg = Arg::new("id")
+    let safekeeper_node_arg = Arg::new("node")
         .index(1)
-        .help("safekeeper id")
+        .help("Node name")
         .required(false);
 
     let timeline_arg = Arg::new("timeline")
@@ -157,16 +154,16 @@ fn main() -> Result<()> {
                 .about("Manage safekeepers")
                 .subcommand(App::new("start")
                             .about("Start local safekeeper")
-                            .arg(safekeeper_id_arg.clone())
+                            .arg(safekeeper_node_arg.clone())
                 )
                 .subcommand(App::new("stop")
                             .about("Stop local safekeeper")
-                            .arg(safekeeper_id_arg.clone())
+                            .arg(safekeeper_node_arg.clone())
                             .arg(stop_mode_arg.clone())
                 )
                 .subcommand(App::new("restart")
                             .about("Restart local safekeeper")
-                            .arg(safekeeper_id_arg.clone())
+                            .arg(safekeeper_node_arg.clone())
                             .arg(stop_mode_arg.clone())
                 )
         )
@@ -392,7 +389,7 @@ fn get_tenantid(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<ZTe
     if let Some(tenantid_cmd) = sub_match.value_of("tenantid") {
         Ok(ZTenantId::from_str(tenantid_cmd)?)
     } else if let Some(tenantid_conf) = env.default_tenantid {
-        Ok(ZTenantId::from(tenantid_conf))
+        Ok(tenantid_conf)
     } else {
         bail!("No tenantid. Use --tenantid, or set 'default_tenantid' in the config file");
     }
@@ -418,7 +415,7 @@ fn handle_init(init_match: &ArgMatches) -> Result<()> {
     let pageserver = PageServerNode::from_env(&env);
     if let Err(e) = pageserver.init(
         // default_tenantid was generated by the `env.init()` call above
-        Some(&ZTenantId::from(env.default_tenantid.unwrap()).to_string()),
+        Some(&env.default_tenantid.unwrap().to_string()),
         &pageserver_config_overrides(init_match),
     ) {
         eprintln!("pageserver init failed: {}", e);
@@ -631,11 +628,11 @@ fn handle_pageserver(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul
     Ok(())
 }
 
-fn get_safekeeper(env: &local_env::LocalEnv, id: ZNodeId) -> Result<SafekeeperNode> {
-    if let Some(node) = env.safekeepers.iter().find(|node| node.id == id) {
+fn get_safekeeper(env: &local_env::LocalEnv, name: &str) -> Result<SafekeeperNode> {
+    if let Some(node) = env.safekeepers.iter().find(|node| node.name == name) {
         Ok(SafekeeperNode::from_env(env, node))
     } else {
-        bail!("could not find safekeeper '{}'", id)
+        bail!("could not find safekeeper '{}'", name)
     }
 }
 
@@ -646,12 +643,8 @@ fn handle_safekeeper(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul
     };
 
     // All the commands take an optional safekeeper name argument
-    let sk_id = if let Some(id_str) = sub_args.value_of("id") {
-        ZNodeId(id_str.parse().context("while parsing safekeeper id")?)
-    } else {
-        DEFAULT_SAFEKEEPER_ID
-    };
-    let safekeeper = get_safekeeper(env, sk_id)?;
+    let node_name = sub_args.value_of("node").unwrap_or(DEFAULT_SAFEKEEPER_NAME);
+    let safekeeper = get_safekeeper(env, node_name)?;
 
     match sub_name {
         "start" => {
@@ -704,7 +697,7 @@ fn handle_start_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result
     for node in env.safekeepers.iter() {
         let safekeeper = SafekeeperNode::from_env(env, node);
         if let Err(e) = safekeeper.start() {
-            eprintln!("safekeeper '{}' start failed: {}", safekeeper.id, e);
+            eprintln!("safekeeper '{}' start failed: {}", safekeeper.name, e);
             exit(1);
         }
     }
@@ -731,7 +724,7 @@ fn handle_stop_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<
     for node in env.safekeepers.iter() {
         let safekeeper = SafekeeperNode::from_env(env, node);
         if let Err(e) = safekeeper.stop(immediate) {
-            eprintln!("safekeeper '{}' stop failed: {}", safekeeper.id, e);
+            eprintln!("safekeeper '{}' stop failed: {}", safekeeper.name, e);
         }
     }
     Ok(())