unset current dir for safekeeper

follow-up of https://github.com/neondatabase/neon/pull/2615
which is neon.git: 538876650a

must be deployed after cloud.git change
https://github.com/neondatabase/cloud/issues/3232

fixes https://github.com/neondatabase/neon/issues/3041

.github/actions/neon-branch-create/action.yml

@@ -5,12 +5,12 @@ inputs:
   api_key:
     desctiption: 'Neon API key'
     required: true
-  environment:
-    desctiption: 'dev (aka captest) or staging'
-    required: true
   project_id:
     desctiption: 'ID of the Project to create Branch in'
     required: true
+  api_host:
+    desctiption: 'Neon API host'
+    default: console.stage.neon.tech
 outputs:
   dsn:
     description: 'Created Branch DSN (for main database)'
@@ -22,27 +22,6 @@ outputs:
 runs:
   using: "composite"
   steps:
-    - name: Parse Input
-      id: parse-input
-      shell: bash -euxo pipefail {0}
-      run: |
-        case "${ENVIRONMENT}" in
-          dev)
-            API_HOST=console.dev.neon.tech
-            ;;
-          staging)
-            API_HOST=console.stage.neon.tech
-            ;;
-          *)
-            echo 2>&1 "Unknown environment=${ENVIRONMENT}. Allowed 'dev' or 'staging' only"
-            exit 1
-            ;;
-        esac
-
-        echo "api_host=${API_HOST}" >> $GITHUB_OUTPUT
-      env:
-        ENVIRONMENT: ${{ inputs.environment }}
-
     - name: Create New Branch
       id: create-branch
       shell: bash -euxo pipefail {0}
@@ -56,7 +35,12 @@ runs:
             --data "{
               \"branch\": {
                 \"name\": \"Created by actions/neon-branch-create; GITHUB_RUN_ID=${GITHUB_RUN_ID} at $(date +%s)\"
-              }
+              },
+              \"endpoints\": [
+                {
+                  \"type\": \"read_write\"
+                }
+              ]
             }")
 
           if [ -z "${branch}" ]; then
@@ -84,8 +68,8 @@ runs:
         host=$(echo $branch | jq --raw-output '.endpoints[0].host')
         echo "host=${host}" >> $GITHUB_OUTPUT
       env:
+        API_HOST: ${{ inputs.api_host }}
         API_KEY: ${{ inputs.api_key }}
-        API_HOST: ${{ steps.parse-input.outputs.api_host }}
         PROJECT_ID: ${{ inputs.project_id }}
 
     - name: Get Role name
@@ -103,8 +87,8 @@ runs:
         role_name=$(echo $roles | jq --raw-output '.roles[] | select(.protected == false) | .name')
         echo "role_name=${role_name}" >> $GITHUB_OUTPUT
       env:
+        API_HOST: ${{ inputs.api_host }}
         API_KEY: ${{ inputs.api_key }}
-        API_HOST: ${{ steps.parse-input.outputs.api_host }}
         PROJECT_ID: ${{ inputs.project_id }}
         BRANCH_ID: ${{ steps.create-branch.outputs.branch_id }}
 
@@ -146,8 +130,8 @@ runs:
         echo "::add-mask::${dsn}"
         echo "dsn=${dsn}" >> $GITHUB_OUTPUT
       env:
+        API_HOST: ${{ inputs.api_host }}
         API_KEY: ${{ inputs.api_key }}
-        API_HOST: ${{ steps.parse-input.outputs.api_host }}
         PROJECT_ID: ${{ inputs.project_id }}
         BRANCH_ID: ${{ steps.create-branch.outputs.branch_id }}
         ROLE_NAME: ${{ steps.role-name.outputs.role_name }}

.github/actions/neon-branch-delete/action.yml

@@ -5,40 +5,19 @@ inputs:
   api_key:
     desctiption: 'Neon API key'
     required: true
-  environment:
-    desctiption: 'dev (aka captest) or staging'
-    required: true
   project_id:
     desctiption: 'ID of the Project which should be deleted'
     required: true
   branch_id:
     desctiption: 'ID of the branch to delete'
     required: true
+  api_host:
+    desctiption: 'Neon API host'
+    default: console.stage.neon.tech
 
 runs:
   using: "composite"
   steps:
-    - name: Parse Input
-      id: parse-input
-      shell: bash -euxo pipefail {0}
-      run: |
-        case "${ENVIRONMENT}" in
-          dev)
-            API_HOST=console.dev.neon.tech
-            ;;
-          staging)
-            API_HOST=console.stage.neon.tech
-            ;;
-          *)
-            echo 2>&1 "Unknown environment=${ENVIRONMENT}. Allowed 'dev' or 'staging' only"
-            exit 1
-            ;;
-        esac
-
-        echo "api_host=${API_HOST}" >> $GITHUB_OUTPUT
-      env:
-        ENVIRONMENT: ${{ inputs.environment }}
-
     - name: Delete Branch
       # Do not try to delete a branch if .github/actions/neon-project-create
       # or .github/actions/neon-branch-create failed before
@@ -73,7 +52,7 @@ runs:
           exit 1
         fi
       env:
+        API_HOST: ${{ inputs.api_host }}
         API_KEY: ${{ inputs.api_key }}
         PROJECT_ID: ${{ inputs.project_id }}
         BRANCH_ID: ${{ inputs.branch_id }}
-        API_HOST: ${{ steps.parse-input.outputs.api_host }}

.github/actions/neon-project-create/action.yml

@@ -5,12 +5,16 @@ inputs:
   api_key:
     desctiption: 'Neon API key'
     required: true
-  environment:
-    desctiption: 'dev (aka captest) or staging'
-    required: true
   region_id:
     desctiption: 'Region ID, if not set the project will be created in the default region'
-    required: false
+    default: aws-us-east-2
+  postgres_version:
+    desctiption: 'Postgres version; default is 15'
+    default: 15
+  api_host:
+    desctiption: 'Neon API host'
+    default: console.stage.neon.tech
+
 outputs:
   dsn:
     description: 'Created Project DSN (for main database)'
@@ -22,31 +26,6 @@ outputs:
 runs:
   using: "composite"
   steps:
-    - name: Parse Input
-      id: parse-input
-      shell: bash -euxo pipefail {0}
-      run: |
-        case "${ENVIRONMENT}" in
-          dev)
-            API_HOST=console.dev.neon.tech
-            REGION_ID=${REGION_ID:-aws-eu-west-1}
-            ;;
-          staging)
-            API_HOST=console.stage.neon.tech
-            REGION_ID=${REGION_ID:-aws-us-east-2}
-            ;;
-          *)
-            echo 2>&1 "Unknown environment=${ENVIRONMENT}. Allowed 'dev' or 'staging' only"
-            exit 1
-            ;;
-        esac
-
-        echo "api_host=${API_HOST}" >> $GITHUB_OUTPUT
-        echo "region_id=${REGION_ID}" >> $GITHUB_OUTPUT
-      env:
-        ENVIRONMENT: ${{ inputs.environment }}
-        REGION_ID: ${{ inputs.region_id }}
-
     - name: Create Neon Project
       id: create-neon-project
       # A shell without `set -x` to not to expose password/dsn in logs
@@ -61,6 +40,7 @@ runs:
           --data "{
             \"project\": {
               \"name\": \"Created by actions/neon-project-create; GITHUB_RUN_ID=${GITHUB_RUN_ID}\",
+              \"pg_version\": ${POSTGRES_VERSION},
               \"region_id\": \"${REGION_ID}\",
               \"settings\": { }
             }
@@ -75,7 +55,10 @@ runs:
 
         project_id=$(echo $project | jq --raw-output '.project.id')
         echo "project_id=${project_id}" >> $GITHUB_OUTPUT
+
+        echo "Project ${project_id} has been created"
       env:
+        API_HOST: ${{ inputs.api_host }}
         API_KEY: ${{ inputs.api_key }}
-        API_HOST: ${{ steps.parse-input.outputs.api_host }}
-        REGION_ID: ${{ steps.parse-input.outputs.region_id }}
+        REGION_ID: ${{ inputs.region_id }}
+        POSTGRES_VERSION: ${{ inputs.postgres_version }}

.github/actions/neon-project-delete/action.yml

@@ -5,37 +5,16 @@ inputs:
   api_key:
     desctiption: 'Neon API key'
     required: true
-  environment:
-    desctiption: 'dev (aka captest) or staging'
-    required: true
   project_id:
     desctiption: 'ID of the Project to delete'
     required: true
+  api_host:
+    desctiption: 'Neon API host'
+    default: console.stage.neon.tech
 
 runs:
   using: "composite"
   steps:
-    - name: Parse Input
-      id: parse-input
-      shell: bash -euxo pipefail {0}
-      run: |
-        case "${ENVIRONMENT}" in
-          dev)
-            API_HOST=console.dev.neon.tech
-            ;;
-          staging)
-            API_HOST=console.stage.neon.tech
-            ;;
-          *)
-            echo 2>&1 "Unknown environment=${ENVIRONMENT}. Allowed 'dev' or 'staging' only"
-            exit 1
-            ;;
-        esac
-
-        echo "api_host=${API_HOST}" >> $GITHUB_OUTPUT
-      env:
-        ENVIRONMENT: ${{ inputs.environment }}
-
     - name: Delete Neon Project
       # Do not try to delete a project if .github/actions/neon-project-create failed before
       if: ${{ inputs.project_id != '' }}
@@ -48,7 +27,9 @@ runs:
           --header "Accept: application/json" \
           --header "Content-Type: application/json" \
           --header "Authorization: Bearer ${API_KEY}"
+
+        echo "Project ${PROJECT_ID} has been deleted"
       env:
+        API_HOST: ${{ inputs.api_host }}
         API_KEY: ${{ inputs.api_key }}
         PROJECT_ID: ${{ inputs.project_id }}
-        API_HOST: ${{ steps.parse-input.outputs.api_host }}

.github/ansible/ansible.cfg

@@ -3,7 +3,6 @@
 localhost_warning = False
 host_key_checking = False
 timeout = 30
-collections_paths = ./collections
 
 [ssh_connection]
 ssh_args   = -F ./ansible.ssh.cfg

.github/ansible/neon-stress.hosts.yaml

@@ -3,7 +3,7 @@ storage:
     bucket_name: neon-storage-ireland
     bucket_region: eu-west-1
     console_mgmt_base_url: http://neon-stress-console.local
-    etcd_endpoints: neon-stress-etcd.local:2379
+    broker_endpoint: http://storage-broker.neon-stress.local:50051
     safekeeper_enable_s3_offload: 'false'
     pageserver_config_stub:
       pg_distrib_dir: /usr/local
@@ -14,6 +14,7 @@ storage:
     safekeeper_s3_prefix: neon-stress/wal
     hostname_suffix: ".local"
     remote_user: admin
+    sentry_environment: development
   children:
     pageservers:
       hosts:

.github/ansible/prod.ap-southeast-1.hosts.yaml

@@ -3,7 +3,7 @@ storage:
     bucket_name: neon-prod-storage-ap-southeast-1
     bucket_region: ap-southeast-1
     console_mgmt_base_url: http://console-release.local
-    etcd_endpoints: etcd-0.ap-southeast-1.aws.neon.tech:2379
+    broker_endpoint: http://storage-broker-lb.epsilon.ap-southeast-1.internal.aws.neon.tech:50051
     pageserver_config_stub:
       pg_distrib_dir: /usr/local
       remote_storage:
@@ -16,6 +16,7 @@ storage:
     ansible_aws_ssm_region: ap-southeast-1
     ansible_aws_ssm_bucket_name: neon-prod-storage-ap-southeast-1
     console_region_id: aws-ap-southeast-1
+    sentry_environment: production
 
   children:
     pageservers:

.github/ansible/prod.eu-central-1.hosts.yaml

@@ -3,7 +3,7 @@ storage:
     bucket_name: neon-prod-storage-eu-central-1
     bucket_region: eu-central-1
     console_mgmt_base_url: http://console-release.local
-    etcd_endpoints: etcd-0.eu-central-1.aws.neon.tech:2379
+    broker_endpoint: http://storage-broker-lb.gamma.eu-central-1.internal.aws.neon.tech:50051
     pageserver_config_stub:
       pg_distrib_dir: /usr/local
       remote_storage:
@@ -16,6 +16,7 @@ storage:
     ansible_aws_ssm_region: eu-central-1
     ansible_aws_ssm_bucket_name: neon-prod-storage-eu-central-1
     console_region_id: aws-eu-central-1
+    sentry_environment: production
 
   children:
     pageservers:

.github/ansible/prod.us-east-2.hosts.yaml

@@ -3,7 +3,7 @@ storage:
     bucket_name: neon-prod-storage-us-east-2
     bucket_region: us-east-2
     console_mgmt_base_url: http://console-release.local
-    etcd_endpoints: etcd-0.us-east-2.aws.neon.tech:2379
+    broker_endpoint: http://storage-broker-lb.delta.us-east-2.internal.aws.neon.tech:50051
     pageserver_config_stub:
       pg_distrib_dir: /usr/local
       remote_storage:
@@ -16,6 +16,7 @@ storage:
     ansible_aws_ssm_region: us-east-2
     ansible_aws_ssm_bucket_name: neon-prod-storage-us-east-2
     console_region_id: aws-us-east-2
+    sentry_environment: production
 
   children:
     pageservers:

.github/ansible/prod.us-west-2.hosts.yaml

@@ -0,0 +1,37 @@
+storage:
+  vars:
+    bucket_name: neon-prod-storage-us-west-2
+    bucket_region: us-west-2
+    console_mgmt_base_url: http://console-release.local
+    broker_endpoint: http://storage-broker-lb.eta.us-west-2.internal.aws.neon.tech:50051
+    pageserver_config_stub:
+      pg_distrib_dir: /usr/local
+      remote_storage:
+        bucket_name: "{{ bucket_name }}"
+        bucket_region: "{{ bucket_region }}"
+        prefix_in_bucket: "pageserver/v1"
+    safekeeper_s3_prefix: safekeeper/v1/wal
+    hostname_suffix: ""
+    remote_user: ssm-user
+    ansible_aws_ssm_region: us-west-2
+    ansible_aws_ssm_bucket_name: neon-prod-storage-us-west-2
+    console_region_id: aws-us-west-2-new
+    sentry_environment: production
+
+  children:
+    pageservers:
+      hosts:
+        pageserver-0.us-west-2.aws.neon.tech:
+          ansible_host: i-0d9f6dfae0e1c780d 
+        pageserver-1.us-west-2.aws.neon.tech:
+          ansible_host: i-0c834be1dddba8b3f
+
+    safekeepers:
+      hosts:
+        safekeeper-0.us-west-2.aws.neon.tech:
+          ansible_host: i-00719d8a74986fda6
+        safekeeper-1.us-west-2.aws.neon.tech:
+          ansible_host: i-074682f9d3c712e7c
+        safekeeper-2.us-west-2.aws.neon.tech:
+          ansible_host: i-042b7efb1729d7966 
+          

.github/ansible/production.hosts.yaml

@@ -4,7 +4,7 @@ storage:
     console_mgmt_base_url: http://console-release.local
     bucket_name: zenith-storage-oregon
     bucket_region: us-west-2
-    etcd_endpoints: zenith-1-etcd.local:2379
+    broker_endpoint: http://storage-broker.prod.local:50051
     pageserver_config_stub:
       pg_distrib_dir: /usr/local
       remote_storage:
@@ -14,6 +14,7 @@ storage:
     safekeeper_s3_prefix: prod-1/wal
     hostname_suffix: ".local"
     remote_user: admin
+    sentry_environment: production
 
   children:
     pageservers:

.github/ansible/scripts/init_pageserver.sh

@@ -1,7 +1,8 @@
 #!/bin/sh
 
-# get instance id from meta-data service
+# fetch params from meta-data service
 INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
+AZ_ID=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)
 
 # store fqdn hostname in var
 HOST=$(hostname -f)
@@ -16,7 +17,8 @@ cat <<EOF | tee /tmp/payload
   "instance_id": "${INSTANCE_ID}",
   "http_host": "${HOST}",
   "http_port": 9898,
-  "active": false
+  "active": false,
+  "availability_zone_id": "${AZ_ID}"
 }
 EOF
 

.github/ansible/staging.eu-west-1.hosts.yaml

@@ -3,7 +3,7 @@ storage:
     bucket_name: neon-dev-storage-eu-west-1
     bucket_region: eu-west-1
     console_mgmt_base_url: http://console-staging.local
-    etcd_endpoints: etcd-0.eu-west-1.aws.neon.build:2379
+    broker_endpoint: http://storage-broker-lb.zeta.eu-west-1.internal.aws.neon.build:50051
     pageserver_config_stub:
       pg_distrib_dir: /usr/local
       remote_storage:
@@ -16,6 +16,7 @@ storage:
     ansible_aws_ssm_region: eu-west-1
     ansible_aws_ssm_bucket_name: neon-dev-storage-eu-west-1
     console_region_id: aws-eu-west-1
+    sentry_environment: development
 
   children:
     pageservers:

.github/ansible/staging.hosts.yaml

@@ -3,7 +3,7 @@ storage:
     bucket_name: zenith-staging-storage-us-east-1
     bucket_region: us-east-1
     console_mgmt_base_url: http://console-staging.local
-    etcd_endpoints: etcd-0.us-east-2.aws.neon.build:2379
+    broker_endpoint: http://storage-broker.staging.local:50051
     pageserver_config_stub:
       pg_distrib_dir: /usr/local
       remote_storage:
@@ -13,6 +13,7 @@ storage:
     safekeeper_s3_prefix: us-stage/wal
     hostname_suffix: ".local"
     remote_user: admin
+    sentry_environment: development
 
   children:
     pageservers:

.github/ansible/staging.us-east-2.hosts.yaml

@@ -3,7 +3,7 @@ storage:
     bucket_name: neon-staging-storage-us-east-2
     bucket_region: us-east-2
     console_mgmt_base_url: http://console-staging.local
-    etcd_endpoints: etcd-0.us-east-2.aws.neon.build:2379
+    broker_endpoint: http://storage-broker-lb.beta.us-east-2.internal.aws.neon.build:50051
     pageserver_config_stub:
       pg_distrib_dir: /usr/local
       remote_storage:
@@ -16,6 +16,7 @@ storage:
     ansible_aws_ssm_region: us-east-2
     ansible_aws_ssm_bucket_name: neon-staging-storage-us-east-2
     console_region_id: aws-us-east-2
+    sentry_environment: development
 
   children:
     pageservers:

.github/ansible/systemd/pageserver.service

@@ -5,8 +5,8 @@ After=network.target auditd.service
 [Service]
 Type=simple
 User=pageserver
-Environment=RUST_BACKTRACE=1 NEON_REPO_DIR=/storage/pageserver LD_LIBRARY_PATH=/usr/local/v14/lib
-ExecStart=/usr/local/bin/pageserver -c "pg_distrib_dir='/usr/local'" -c "listen_pg_addr='0.0.0.0:6400'" -c "listen_http_addr='0.0.0.0:9898'" -c "broker_endpoints=['{{ etcd_endpoints }}']" -D /storage/pageserver/data
+Environment=RUST_BACKTRACE=1 NEON_REPO_DIR=/storage/pageserver LD_LIBRARY_PATH=/usr/local/v14/lib SENTRY_DSN={{ SENTRY_URL_PAGESERVER }} SENTRY_ENVIRONMENT={{ sentry_environment }}
+ExecStart=/usr/local/bin/pageserver -c "pg_distrib_dir='/usr/local'" -c "listen_pg_addr='0.0.0.0:6400'" -c "listen_http_addr='0.0.0.0:9898'" -c "broker_endpoint='{{ broker_endpoint }}'" -D /storage/pageserver/data
 ExecReload=/bin/kill -HUP $MAINPID
 KillMode=mixed
 KillSignal=SIGINT

.github/ansible/systemd/safekeeper.service

@@ -5,8 +5,8 @@ After=network.target auditd.service
 [Service]
 Type=simple
 User=safekeeper
-Environment=RUST_BACKTRACE=1 NEON_REPO_DIR=/storage/safekeeper/data LD_LIBRARY_PATH=/usr/local/v14/lib
-ExecStart=/usr/local/bin/safekeeper -l {{ inventory_hostname }}{{ hostname_suffix }}:6500 --listen-http {{ inventory_hostname }}{{ hostname_suffix }}:7676 -D /storage/safekeeper/data --broker-endpoints={{ etcd_endpoints }} --remote-storage='{bucket_name="{{bucket_name}}", bucket_region="{{bucket_region}}", prefix_in_bucket="{{ safekeeper_s3_prefix }}"}'
+Environment=RUST_BACKTRACE=1 NEON_REPO_DIR=/storage/safekeeper/data LD_LIBRARY_PATH=/usr/local/v14/lib SENTRY_DSN={{ SENTRY_URL_SAFEKEEPER }} SENTRY_ENVIRONMENT={{ sentry_environment }}
+ExecStart=/usr/local/bin/safekeeper -l {{ inventory_hostname }}{{ hostname_suffix }}:6500 --listen-http {{ inventory_hostname }}{{ hostname_suffix }}:7676 -D /storage/safekeeper/data --broker-endpoint={{ broker_endpoint }} --remote-storage='{bucket_name="{{bucket_name}}", bucket_region="{{bucket_region}}", prefix_in_bucket="{{ safekeeper_s3_prefix }}"}'
 ExecReload=/bin/kill -HUP $MAINPID
 KillMode=mixed
 KillSignal=SIGINT

.github/helm-values/dev-eu-west-1-zeta.neon-proxy-scram.yaml

@@ -8,6 +8,7 @@ settings:
   authBackend: "console"
   authEndpoint: "http://console-staging.local/management/api/v2"
   domain: "*.eu-west-1.aws.neon.build"
+  sentryEnvironment: "development"
 
 # -- Additional labels for neon-proxy pods
 podLabels:
@@ -29,3 +30,28 @@ exposedService:
 #    enabled: true
 #    selector:
 #      release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-proxy.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-proxy-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-proxy
+        app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}"
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-proxy"
+      endpoints:
+        - port: http
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"

.github/helm-values/dev-eu-west-1-zeta.neon-storage-broker.yaml

@@ -0,0 +1,52 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: staging
+  neon_service: storage-broker
+
+# Use L4 LB
+service:
+  # service.annotations -- Annotations to add to the service
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: external  # use newer AWS Load Balancer Controller
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+    service.beta.kubernetes.io/aws-load-balancer-scheme: internal  # deploy LB to private subnet
+    # assign service to this name at external-dns
+    external-dns.alpha.kubernetes.io/hostname: storage-broker-lb.zeta.eu-west-1.internal.aws.neon.build
+  # service.type -- Service type
+  type: LoadBalancer
+  # service.port -- broker listen port
+  port: 50051
+
+ingress:
+  enabled: false
+
+metrics:
+  enabled: false
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+
+settings:
+  sentryEnvironment: "development"

.github/helm-values/dev-us-east-2-beta.neon-proxy-link.yaml

@@ -8,6 +8,7 @@ settings:
   authBackend: "link"
   authEndpoint: "https://console.stage.neon.tech/authenticate_proxy_request/"
   uri: "https://console.stage.neon.tech/psql_session/"
+  sentryEnvironment: "development"
 
 # -- Additional labels for neon-proxy-link pods
 podLabels:
@@ -37,3 +38,28 @@ exposedService:
 #    enabled: true
 #    selector:
 #      release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-proxy.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-proxy-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-proxy
+        app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}"
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-proxy"
+      endpoints:
+        - port: http
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"

.github/helm-values/dev-us-east-2-beta.neon-proxy-scram-legacy.yaml

@@ -8,6 +8,7 @@ settings:
   authBackend: "console"
   authEndpoint: "http://console-staging.local/management/api/v2"
   domain: "*.cloud.stage.neon.tech"
+  sentryEnvironment: "development"
 
 # -- Additional labels for neon-proxy pods
 podLabels:
@@ -29,3 +30,28 @@ exposedService:
 #    enabled: true
 #    selector:
 #      release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-proxy.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-proxy-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-proxy
+        app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}"
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-proxy"
+      endpoints:
+        - port: http
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"

.github/helm-values/dev-us-east-2-beta.neon-proxy-scram.yaml

@@ -8,6 +8,7 @@ settings:
   authBackend: "console"
   authEndpoint: "http://console-staging.local/management/api/v2"
   domain: "*.us-east-2.aws.neon.build"
+  sentryEnvironment: "development"
 
 # -- Additional labels for neon-proxy pods
 podLabels:
@@ -29,3 +30,28 @@ exposedService:
 #    enabled: true
 #    selector:
 #      release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-proxy.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-proxy-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-proxy
+        app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}"
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-proxy"
+      endpoints:
+        - port: http
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"

.github/helm-values/dev-us-east-2-beta.neon-storage-broker.yaml

@@ -0,0 +1,52 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: staging
+  neon_service: storage-broker
+
+# Use L4 LB
+service:
+  # service.annotations -- Annotations to add to the service
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: external  # use newer AWS Load Balancer Controller
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+    service.beta.kubernetes.io/aws-load-balancer-scheme: internal  # deploy LB to private subnet
+    # assign service to this name at external-dns
+    external-dns.alpha.kubernetes.io/hostname: storage-broker-lb.beta.us-east-2.internal.aws.neon.build
+  # service.type -- Service type
+  type: LoadBalancer
+  # service.port -- broker listen port
+  port: 50051
+
+ingress:
+  enabled: false
+
+metrics:
+  enabled: false
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+
+settings:
+  sentryEnvironment: "development"

.github/helm-values/neon-stress.neon-storage-broker.yaml

@@ -0,0 +1,56 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: neon-stress
+  neon_service: storage-broker
+
+# Use L4 LB
+service:
+  # service.annotations -- Annotations to add to the service
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: external  # use newer AWS Load Balancer Controller
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+    service.beta.kubernetes.io/aws-load-balancer-scheme: internal  # deploy LB to private subnet
+    # assign service to this name at external-dns
+    external-dns.alpha.kubernetes.io/hostname: storage-broker.neon-stress.local
+  # service.type -- Service type
+  type: LoadBalancer
+  # service.port -- broker listen port
+  port: 50051
+
+ingress:
+  enabled: false
+
+metrics:
+  enabled: true
+  serviceMonitor:
+    enabled: true
+    selector:
+      release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+
+settings:
+  sentryEnvironment: "development"

.github/helm-values/neon-stress.proxy-scram.yaml

@@ -4,6 +4,7 @@ settings:
   authBackend: "console"
   authEndpoint: "http://neon-stress-console.local/management/api/v2"
   domain: "*.stress.neon.tech"
+  sentryEnvironment: "development"
 
 podLabels:
   zenith_service: proxy-scram
@@ -24,3 +25,28 @@ metrics:
     enabled: true
     selector:
       release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-proxy.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-proxy-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-proxy
+        app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}"
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-proxy"
+      endpoints:
+        - port: http
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"

.github/helm-values/neon-stress.proxy.yaml

@@ -4,6 +4,7 @@ settings:
   authBackend: "link"
   authEndpoint: "https://console.dev.neon.tech/authenticate_proxy_request/"
   uri: "https://console.dev.neon.tech/psql_session/"
+  sentryEnvironment: "development"
 
 # -- Additional labels for zenith-proxy pods
 podLabels:
@@ -33,3 +34,28 @@ metrics:
     enabled: true
     selector:
       release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-proxy.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-proxy-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-proxy
+        app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}"
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-proxy"
+      endpoints:
+        - port: http
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"

.github/helm-values/prod-ap-southeast-1-epsilon.neon-proxy-scram.yaml

@@ -8,6 +8,7 @@ settings:
   authBackend: "console"
   authEndpoint: "http://console-release.local/management/api/v2"
   domain: "*.ap-southeast-1.aws.neon.tech"
+  sentryEnvironment: "production"
 
 # -- Additional labels for neon-proxy pods
 podLabels:
@@ -29,3 +30,28 @@ exposedService:
 #    enabled: true
 #    selector:
 #      release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-proxy.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-proxy-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-proxy
+        app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}"
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-proxy"
+      endpoints:
+        - port: http
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"

.github/helm-values/prod-ap-southeast-1-epsilon.neon-storage-broker.yaml

@@ -0,0 +1,52 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: production
+  neon_service: storage-broker
+
+# Use L4 LB
+service:
+  # service.annotations -- Annotations to add to the service
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: external  # use newer AWS Load Balancer Controller
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+    service.beta.kubernetes.io/aws-load-balancer-scheme: internal  # deploy LB to private subnet
+    # assign service to this name at external-dns
+    external-dns.alpha.kubernetes.io/hostname: storage-broker-lb.epsilon.ap-southeast-1.internal.aws.neon.tech
+  # service.type -- Service type
+  type: LoadBalancer
+  # service.port -- broker listen port
+  port: 50051
+
+ingress:
+  enabled: false
+
+metrics:
+  enabled: false
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+
+settings:
+  sentryEnvironment: "production"

.github/helm-values/prod-eu-central-1-gamma.neon-proxy-scram.yaml

@@ -8,6 +8,7 @@ settings:
   authBackend: "console"
   authEndpoint: "http://console-release.local/management/api/v2"
   domain: "*.eu-central-1.aws.neon.tech"
+  sentryEnvironment: "production"
 
 # -- Additional labels for neon-proxy pods
 podLabels:
@@ -29,3 +30,28 @@ exposedService:
 #    enabled: true
 #    selector:
 #      release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-proxy.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-proxy-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-proxy
+        app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}"
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-proxy"
+      endpoints:
+        - port: http
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"

.github/helm-values/prod-eu-central-1-gamma.neon-storage-broker.yaml

@@ -0,0 +1,52 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: production
+  neon_service: storage-broker
+
+# Use L4 LB
+service:
+  # service.annotations -- Annotations to add to the service
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: external  # use newer AWS Load Balancer Controller
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+    service.beta.kubernetes.io/aws-load-balancer-scheme: internal  # deploy LB to private subnet
+    # assign service to this name at external-dns
+    external-dns.alpha.kubernetes.io/hostname: storage-broker-lb.gamma.eu-central-1.internal.aws.neon.tech
+  # service.type -- Service type
+  type: LoadBalancer
+  # service.port -- broker listen port
+  port: 50051
+
+ingress:
+  enabled: false
+
+metrics:
+  enabled: false
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+
+settings:
+  sentryEnvironment: "production"

.github/helm-values/prod-us-east-2-delta.neon-proxy-scram.yaml

@@ -8,6 +8,7 @@ settings:
   authBackend: "console"
   authEndpoint: "http://console-release.local/management/api/v2"
   domain: "*.us-east-2.aws.neon.tech"
+  sentryEnvironment: "production"
 
 # -- Additional labels for neon-proxy pods
 podLabels:
@@ -29,3 +30,28 @@ exposedService:
 #    enabled: true
 #    selector:
 #      release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-proxy.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-proxy-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-proxy
+        app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}"
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-proxy"
+      endpoints:
+        - port: http
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"

.github/helm-values/prod-us-east-2-delta.neon-storage-broker.yaml

@@ -0,0 +1,52 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: production
+  neon_service: storage-broker
+
+# Use L4 LB
+service:
+  # service.annotations -- Annotations to add to the service
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: external  # use newer AWS Load Balancer Controller
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+    service.beta.kubernetes.io/aws-load-balancer-scheme: internal  # deploy LB to private subnet
+    # assign service to this name at external-dns
+    external-dns.alpha.kubernetes.io/hostname: storage-broker-lb.delta.us-east-2.internal.aws.neon.tech
+  # service.type -- Service type
+  type: LoadBalancer
+  # service.port -- broker listen port
+  port: 50051
+
+ingress:
+  enabled: false
+
+metrics:
+  enabled: false
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+
+settings:
+  sentryEnvironment: "production"

.github/helm-values/prod-us-west-2-eta.neon-proxy-scram.yaml

@@ -0,0 +1,57 @@
+# Helm chart values for neon-proxy-scram.
+# This is a YAML-formatted file.
+
+image:
+  repository: neondatabase/neon
+
+settings:
+  authBackend: "console"
+  authEndpoint: "http://console-release.local/management/api/v2"
+  domain: "*.us-west-2.aws.neon.tech"
+  sentryEnvironment: "production"
+
+# -- Additional labels for neon-proxy pods
+podLabels:
+  zenith_service: proxy-scram
+  zenith_env: prod
+  zenith_region: us-west-2
+  zenith_region_slug: us-west-2
+
+exposedService:
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: external
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
+    external-dns.alpha.kubernetes.io/hostname: us-west-2.aws.neon.tech
+
+#metrics:
+#  enabled: true
+#  serviceMonitor:
+#    enabled: true
+#    selector:
+#      release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-proxy.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-proxy-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-proxy
+        app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}"
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-proxy"
+      endpoints:
+        - port: http
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"

.github/helm-values/prod-us-west-2-eta.neon-storage-broker.yaml

@@ -0,0 +1,52 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: production
+  neon_service: storage-broker
+
+# Use L4 LB
+service:
+  # service.annotations -- Annotations to add to the service
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: external  # use newer AWS Load Balancer Controller
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+    service.beta.kubernetes.io/aws-load-balancer-scheme: internal  # deploy LB to private subnet
+    # assign service to this name at external-dns
+    external-dns.alpha.kubernetes.io/hostname: storage-broker-lb.eta.us-west-2.internal.aws.neon.tech
+  # service.type -- Service type
+  type: LoadBalancer
+  # service.port -- broker listen port
+  port: 50051
+
+ingress:
+  enabled: false
+
+metrics:
+  enabled: false
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+
+settings:
+  sentryEnvironment: "production"

.github/helm-values/production.neon-storage-broker.yaml

@@ -0,0 +1,56 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: production
+  neon_service: storage-broker
+
+# Use L4 LB
+service:
+  # service.annotations -- Annotations to add to the service
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: external  # use newer AWS Load Balancer Controller
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+    service.beta.kubernetes.io/aws-load-balancer-scheme: internal  # deploy LB to private subnet
+    # assign service to this name at external-dns
+    external-dns.alpha.kubernetes.io/hostname: storage-broker.prod.local
+  # service.type -- Service type
+  type: LoadBalancer
+  # service.port -- broker listen port
+  port: 50051
+
+ingress:
+  enabled: false
+
+metrics:
+  enabled: true
+  serviceMonitor:
+    enabled: true
+    selector:
+      release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+
+settings:
+  sentryEnvironment: "production"

.github/helm-values/production.proxy-scram.yaml

@@ -2,6 +2,7 @@ settings:
   authBackend: "console"
   authEndpoint: "http://console-release.local/management/api/v2"
   domain: "*.cloud.neon.tech"
+  sentryEnvironment: "production"
 
 podLabels:
   zenith_service: proxy-scram
@@ -22,3 +23,28 @@ metrics:
     enabled: true
     selector:
       release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-proxy.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-proxy-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-proxy
+        app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}"
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-proxy"
+      endpoints:
+        - port: http
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"

.github/helm-values/production.proxy.yaml

@@ -2,6 +2,7 @@ settings:
   authBackend: "link"
   authEndpoint: "https://console.neon.tech/authenticate_proxy_request/"
   uri: "https://console.neon.tech/psql_session/"
+  sentryEnvironment: "production"
 
 # -- Additional labels for zenith-proxy pods
 podLabels:
@@ -31,3 +32,28 @@ metrics:
     enabled: true
     selector:
       release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-proxy.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-proxy-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-proxy
+        app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}"
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-proxy"
+      endpoints:
+        - port: http
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"

.github/helm-values/staging.neon-storage-broker.yaml

@@ -0,0 +1,56 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: staging
+  neon_service: storage-broker
+
+# Use L4 LB
+service:
+  # service.annotations -- Annotations to add to the service
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: external  # use newer AWS Load Balancer Controller
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+    service.beta.kubernetes.io/aws-load-balancer-scheme: internal  # deploy LB to private subnet
+    # assign service to this name at external-dns
+    external-dns.alpha.kubernetes.io/hostname: storage-broker.staging.local
+  # service.type -- Service type
+  type: LoadBalancer
+  # service.port -- broker listen port
+  port: 50051
+
+ingress:
+  enabled: false
+
+metrics:
+  enabled: true
+  serviceMonitor:
+    enabled: true
+    selector:
+      release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+
+settings:
+  sentryEnvironment: "development"

.github/helm-values/staging.proxy-scram.yaml

@@ -8,6 +8,7 @@ settings:
   authBackend: "console"
   authEndpoint: "http://console-staging.local/management/api/v2"
   domain: "*.cloud.stage.neon.tech"
+  sentryEnvironment: "development"
 
 # -- Additional labels for zenith-proxy pods
 podLabels:
@@ -29,3 +30,28 @@ metrics:
     enabled: true
     selector:
       release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-proxy.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-proxy-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-proxy
+        app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}"
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-proxy"
+      endpoints:
+        - port: http
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"

.github/helm-values/staging.proxy.yaml

@@ -8,6 +8,7 @@ settings:
   authBackend: "link"
   authEndpoint: "https://console.stage.neon.tech/authenticate_proxy_request/"
   uri: "https://console.stage.neon.tech/psql_session/"
+  sentryEnvironment: "development"
 
 # -- Additional labels for zenith-proxy pods
 podLabels:
@@ -29,3 +30,28 @@ metrics:
     enabled: true
     selector:
       release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-proxy.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-proxy-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-proxy
+        app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}"
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-proxy"
+      endpoints:
+        - port: http
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"

.github/workflows/benchmarking.yml

@@ -15,12 +15,10 @@ on:
 
   workflow_dispatch: # adds ability to run this manually
     inputs:
-      environment:
-        description: 'Environment to run remote tests on (dev or staging)'
-        required: false
       region_id:
         description: 'Use a particular region. If not set the default region will be used'
         required: false
+        default: 'aws-us-east-2'
       save_perf_report:
         type: boolean
         description: 'Publish perf report or not. If not set, the report is published only for the main branch'
@@ -37,103 +35,69 @@ concurrency:
 
 jobs:
   bench:
-    # this workflow runs on self hosteed runner
-    # it's environment is quite different from usual guthub runner
-    # probably the most important difference is that it doesn't start from clean workspace each time
-    # e g if you install system packages they are not cleaned up since you install them directly in host machine
-    # not a container or something
-    # See documentation for more info: https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners
-    runs-on: [self-hosted, zenith-benchmarker]
-
     env:
-      POSTGRES_DISTRIB_DIR: /usr/pgsql
+      TEST_PG_BENCH_DURATIONS_MATRIX: "300"
+      TEST_PG_BENCH_SCALES_MATRIX: "10,100"
+      POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
       DEFAULT_PG_VERSION: 14
+      TEST_OUTPUT: /tmp/test_output
+      BUILD_TYPE: remote
+      SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref == 'refs/heads/main' ) }}
+      PLATFORM: "neon-staging"
+
+    runs-on: [ self-hosted, us-east-2, x64 ]
+    container:
+      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
+      options: --init
 
     steps:
-    - name: Checkout zenith repo
-      uses: actions/checkout@v3
+    - uses: actions/checkout@v3
 
-    # actions/setup-python@v2 is not working correctly on self-hosted runners
-    # see https://github.com/actions/setup-python/issues/162
-    # and probably https://github.com/actions/setup-python/issues/162#issuecomment-865387976 in particular
-    # so the simplest solution to me is to use already installed system python and spin virtualenvs for job runs.
-    # there is Python 3.7.10 already installed on the machine so use it to install poetry and then use poetry's virtuealenvs
-    - name: Install poetry & deps
-      run: |
-        python3 -m pip install --upgrade poetry wheel
-        # since pip/poetry caches are reused there shouldn't be any troubles with install every time
-        ./scripts/pysync
-
-    - name: Show versions
-      run: |
-        echo Python
-        python3 --version
-        poetry run python3 --version
-        echo Poetry
-        poetry --version
-        echo Pgbench
-        ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version
+    - name: Download Neon artifact
+      uses: ./.github/actions/download
+      with:
+        name: neon-${{ runner.os }}-release-artifact
+        path: /tmp/neon/
+        prefix: latest
 
     - name: Create Neon Project
       id: create-neon-project
       uses: ./.github/actions/neon-project-create
       with:
-        environment: ${{ github.event.inputs.environment || 'staging' }}
-        api_key: ${{ ( github.event.inputs.environment || 'staging' ) == 'staging' && secrets.NEON_STAGING_API_KEY  || secrets.NEON_CAPTEST_API_KEY }}
+        region_id: ${{ github.event.inputs.region_id || 'aws-us-east-2' }}
+        postgres_version: ${{ env.DEFAULT_PG_VERSION }}
+        api_key: ${{ secrets.NEON_STAGING_API_KEY }}
 
     - name: Run benchmark
-      # pgbench is installed system wide from official repo
-      # https://download.postgresql.org/pub/repos/yum/13/redhat/rhel-7-x86_64/
-      # via
-      # sudo tee /etc/yum.repos.d/pgdg.repo<<EOF
-      # [pgdg13]
-      # name=PostgreSQL 13 for RHEL/CentOS 7 - x86_64
-      # baseurl=https://download.postgresql.org/pub/repos/yum/13/redhat/rhel-7-x86_64/
-      # enabled=1
-      # gpgcheck=0
-      # EOF
-      # sudo yum makecache
-      # sudo yum install postgresql13-contrib
-      # actual binaries are located in /usr/pgsql-13/bin/
+      uses: ./.github/actions/run-python-test-set
+      with:
+        build_type: ${{ env.BUILD_TYPE }}
+        test_selection: performance
+        run_in_parallel: false
+        save_perf_report: ${{ env.SAVE_PERF_REPORT }}
+        # Set --sparse-ordering option of pytest-order plugin
+        # to ensure tests are running in order of appears in the file.
+        # It's important for test_perf_pgbench.py::test_pgbench_remote_* tests
+        extra_params: -m remote_cluster --sparse-ordering --timeout 5400 --ignore test_runner/performance/test_perf_olap.py
       env:
-        # The pgbench test runs two tests of given duration against each scale.
-        # So the total runtime with these parameters is 2 * 2 * 300 = 1200, or 20 minutes.
-        # Plus time needed to initialize the test databases.
-        TEST_PG_BENCH_DURATIONS_MATRIX: "300"
-        TEST_PG_BENCH_SCALES_MATRIX: "10,100"
-        PLATFORM: "neon-staging"
         BENCHMARK_CONNSTR: ${{ steps.create-neon-project.outputs.dsn }}
-        REMOTE_ENV: "1" # indicate to test harness that we do not have zenith binaries locally
-      run: |
-        # just to be sure that no data was cached on self hosted runner
-        # since it might generate duplicates when calling ingest_perf_test_result.py
-        rm -rf perf-report-staging
-        mkdir -p perf-report-staging
-        # Set --sparse-ordering option of pytest-order plugin to ensure tests are running in order of appears in the file,
-        # it's important for test_perf_pgbench.py::test_pgbench_remote_* tests.
-        # Do not run tests from test_runner/performance/test_perf_olap.py because they require a prepared DB. We run them separately in `clickbench-compare` job.
-        ./scripts/pytest test_runner/performance/ -v \
-          -m "remote_cluster" \
-          --sparse-ordering \
-          --out-dir perf-report-staging \
-          --timeout 5400 \
-          --ignore test_runner/performance/test_perf_olap.py
-
-    - name: Submit result
-      env:
         VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
         PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
-      run: |
-        REPORT_FROM=$(realpath perf-report-staging) REPORT_TO=staging scripts/generate_and_push_perf_report.sh
 
     - name: Delete Neon Project
       if: ${{ always() }}
       uses: ./.github/actions/neon-project-delete
       with:
-        environment: staging
         project_id: ${{ steps.create-neon-project.outputs.project_id }}
         api_key: ${{ secrets.NEON_STAGING_API_KEY }}
 
+    - name: Create Allure report
+      if: success() || failure()
+      uses: ./.github/actions/allure-report
+      with:
+        action: generate
+        build_type: ${{ env.BUILD_TYPE }}
+
     - name: Post to a Slack channel
       if: ${{ github.event.schedule && failure() }}
       uses: slackapi/slack-github-action@v1
@@ -152,15 +116,16 @@ jobs:
         # neon-captest-prefetch: Same, with prefetching enabled (new project)
         # rds-aurora: Aurora Postgres Serverless v2 with autoscaling from 0.5 to 2 ACUs
         # rds-postgres: RDS Postgres db.m5.large instance (2 vCPU, 8 GiB) with gp3 EBS storage
-        platform: [ neon-captest-new, neon-captest-reuse, neon-captest-prefetch, rds-postgres ]
+        platform: [ neon-captest-reuse, neon-captest-prefetch, rds-postgres ]
         db_size: [ 10gb ]
+        runner: [ us-east-2 ]
         include:
-          - platform: neon-captest-new
-            db_size: 50gb
           - platform: neon-captest-prefetch
             db_size: 50gb
+            runner: us-east-2
           - platform: rds-aurora
             db_size: 50gb
+            runner: us-east-2
 
     env:
       TEST_PG_BENCH_DURATIONS_MATRIX: "60m"
@@ -172,9 +137,9 @@ jobs:
       SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref == 'refs/heads/main' ) }}
       PLATFORM: ${{ matrix.platform }}
 
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: [ self-hosted, "${{ matrix.runner }}", x64 ]
     container:
-      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rustlegacy:pinned
+      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
 
     timeout-minutes: 360 # 6h
@@ -199,8 +164,9 @@ jobs:
       id: create-neon-project
       uses: ./.github/actions/neon-project-create
       with:
-        environment: ${{ github.event.inputs.environment || 'dev' }}
-        api_key: ${{ ( github.event.inputs.environment || 'dev' ) == 'staging' && secrets.NEON_STAGING_API_KEY  || secrets.NEON_CAPTEST_API_KEY }}
+        region_id: ${{ github.event.inputs.region_id || 'aws-us-east-2' }}
+        postgres_version: ${{ env.DEFAULT_PG_VERSION }}
+        api_key: ${{ secrets.NEON_STAGING_API_KEY }}
 
     - name: Set up Connection String
       id: set-up-connstr
@@ -231,8 +197,11 @@ jobs:
     - name: Set database options
       if: matrix.platform == 'neon-captest-prefetch'
       run: |
-        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE neondb SET enable_seqscan_prefetch=on"
-        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE neondb SET seqscan_prefetch_buffers=10"
+        DB_NAME=$(psql ${BENCHMARK_CONNSTR} --no-align --quiet -t -c "SELECT current_database()")
+
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET enable_seqscan_prefetch=on"
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET effective_io_concurrency=32"
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET maintenance_io_concurrency=32"
       env:
         BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
 
@@ -275,6 +244,13 @@ jobs:
         VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
         PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
 
+    - name: Delete Neon Project
+      if: ${{ steps.create-neon-project.outputs.project_id && always() }}
+      uses: ./.github/actions/neon-project-delete
+      with:
+        project_id: ${{ steps.create-neon-project.outputs.project_id }}
+        api_key: ${{ secrets.NEON_STAGING_API_KEY }}
+
     - name: Create Allure report
       if: success() || failure()
       uses: ./.github/actions/allure-report
@@ -282,14 +258,6 @@ jobs:
         action: generate
         build_type: ${{ env.BUILD_TYPE }}
 
-    - name: Delete Neon Project
-      if: ${{ steps.create-neon-project.outputs.project_id && always() }}
-      uses: ./.github/actions/neon-project-delete
-      with:
-        environment: dev
-        project_id: ${{ steps.create-neon-project.outputs.project_id }}
-        api_key: ${{ secrets.NEON_CAPTEST_API_KEY }}
-
     - name: Post to a Slack channel
       if: ${{ github.event.schedule && failure() }}
       uses: slackapi/slack-github-action@v1
@@ -326,9 +294,9 @@ jobs:
       SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref == 'refs/heads/main' ) }}
       PLATFORM: ${{ matrix.platform }}
 
-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: [ self-hosted, us-east-2, x64 ]
     container:
-      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rustlegacy:pinned
+      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
 
     timeout-minutes: 360 # 6h
@@ -374,12 +342,15 @@ jobs:
     - name: Set database options
       if: matrix.platform == 'neon-captest-prefetch'
       run: |
-        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE main SET enable_seqscan_prefetch=on"
-        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE main SET seqscan_prefetch_buffers=10"
+        DB_NAME=$(psql ${BENCHMARK_CONNSTR} --no-align --quiet -t -c "SELECT current_database()")
+
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET enable_seqscan_prefetch=on"
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET effective_io_concurrency=32"
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET maintenance_io_concurrency=32"
       env:
         BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
 
-    - name: Benchmark clickbench
+    - name: ClickBench benchmark
       uses: ./.github/actions/run-python-test-set
       with:
         build_type: ${{ env.BUILD_TYPE }}
@@ -407,3 +378,114 @@ jobs:
         slack-message: "Periodic OLAP perf testing ${{ matrix.platform }}: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
       env:
         SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+
+  tpch-compare:
+    # TCP-H DB for rds-aurora and rds-Postgres deployed to the same clusters
+    # we use for performance testing in pgbench-compare & clickbench-compare.
+    # Run this job only when clickbench-compare is finished to avoid the intersection.
+    # We might change it after https://github.com/neondatabase/neon/issues/2900.
+    #
+    # *_TPCH_S10_CONNSTR: DB generated with scale factor 10 (~10 GB)
+    if: success() || failure()
+    needs: [ clickbench-compare ]
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # neon-captest-prefetch: We have pre-created projects with prefetch enabled
+        # rds-aurora: Aurora Postgres Serverless v2 with autoscaling from 0.5 to 2 ACUs
+        # rds-postgres: RDS Postgres db.m5.large instance (2 vCPU, 8 GiB) with gp3 EBS storage
+        platform: [ neon-captest-prefetch, rds-postgres, rds-aurora ]
+
+    env:
+      POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
+      DEFAULT_PG_VERSION: 14
+      TEST_OUTPUT: /tmp/test_output
+      BUILD_TYPE: remote
+      SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref == 'refs/heads/main' ) }}
+      PLATFORM: ${{ matrix.platform }}
+
+    runs-on: [ self-hosted, us-east-2, x64 ]
+    container:
+      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rustlegacy:pinned
+      options: --init
+
+    timeout-minutes: 360 # 6h
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Download Neon artifact
+      uses: ./.github/actions/download
+      with:
+        name: neon-${{ runner.os }}-release-artifact
+        path: /tmp/neon/
+        prefix: latest
+
+    - name: Add Postgres binaries to PATH
+      run: |
+        ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version
+        echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH
+
+    - name: Set up Connection String
+      id: set-up-connstr
+      run: |
+        case "${PLATFORM}" in
+          neon-captest-prefetch)
+            CONNSTR=${{ secrets.BENCHMARK_CAPTEST_TPCH_S10_CONNSTR }}
+            ;;
+          rds-aurora)
+            CONNSTR=${{ secrets.BENCHMARK_RDS_AURORA_TPCH_S10_CONNSTR }}
+            ;;
+          rds-postgres)
+            CONNSTR=${{ secrets.BENCHMARK_RDS_POSTGRES_TPCH_S10_CONNSTR }}
+            ;;
+          *)
+            echo 2>&1 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neon-captest-prefetch', 'rds-aurora', or 'rds-postgres'"
+            exit 1
+            ;;
+        esac
+
+        echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
+
+        psql ${CONNSTR} -c "SELECT version();"
+
+    - name: Set database options
+      if: matrix.platform == 'neon-captest-prefetch'
+      run: |
+        DB_NAME=$(psql ${BENCHMARK_CONNSTR} --no-align --quiet -t -c "SELECT current_database()")
+
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET enable_seqscan_prefetch=on"
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET effective_io_concurrency=32"
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET maintenance_io_concurrency=32"
+      env:
+        BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
+
+    - name: Run TPC-H benchmark
+      uses: ./.github/actions/run-python-test-set
+      with:
+        build_type: ${{ env.BUILD_TYPE }}
+        test_selection: performance/test_perf_olap.py
+        run_in_parallel: false
+        save_perf_report: ${{ env.SAVE_PERF_REPORT }}
+        extra_params: -m remote_cluster --timeout 21600 -k test_tpch
+      env:
+        VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
+        PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
+        BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
+
+    - name: Create Allure report
+      if: success() || failure()
+      uses: ./.github/actions/allure-report
+      with:
+        action: generate
+        build_type: ${{ env.BUILD_TYPE }}
+
+    - name: Post to a Slack channel
+      if: ${{ github.event.schedule && failure() }}
+      uses: slackapi/slack-github-action@v1
+      with:
+        channel-id: "C033QLM5P7D" # dev-staging-stream
+        slack-message: "Periodic TPC-H perf testing ${{ matrix.platform }}: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+      env:
+        SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

.github/workflows/build_and_test.yml

@@ -7,6 +7,10 @@ on:
       - release
   pull_request:
 
+defaults:
+  run:
+    shell: bash -euxo pipefail {0}
+
 concurrency:
   # Allow only one workflow per any non-`main` branch.
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref == 'refs/heads/main' && github.sha || 'anysha' }}
@@ -45,6 +49,83 @@ jobs:
         shell: bash
         id: build-tag
 
+  check-codestyle-python:
+    runs-on: [ self-hosted, dev, x64 ]
+    container:
+      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cloud:pinned
+      options: --init
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: false
+          fetch-depth: 1
+
+      - name: Cache poetry deps
+        id: cache_poetry
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pypoetry/virtualenvs
+          key: v1-codestyle-python-deps-${{ hashFiles('poetry.lock') }}
+
+      - name: Install Python deps
+        run: ./scripts/pysync
+
+      - name: Run isort to ensure code format
+        run: poetry run isort --diff --check .
+
+      - name: Run black to ensure code format
+        run: poetry run black --diff --check .
+
+      - name: Run flake8 to ensure code format
+        run: poetry run flake8 .
+
+      - name: Run mypy to check types
+        run: poetry run mypy .
+
+  check-codestyle-rust:
+    runs-on: [ self-hosted, dev, x64 ]
+    container:
+      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
+      options: --init
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Restore cargo deps cache
+        id: cache_cargo
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/registry/
+            !~/.cargo/registry/src
+            ~/.cargo/git/
+            target/
+          key: v1-${{ runner.os }}-cargo-clippy-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('Cargo.lock') }}
+
+      # Some of our rust modules use FFI and need those to be checked
+      - name: Get postgres headers
+        run: make postgres-headers -j$(nproc)
+      - name: Run cargo clippy
+        run: ./run_clippy.sh
+
+      # Use `${{ !cancelled() }}` to run quck tests after the longer clippy run
+      - name: Check formatting
+        if: ${{ !cancelled() }}
+        run: cargo fmt --all -- --check
+
+      # https://github.com/facebookincubator/cargo-guppy/tree/bec4e0eb29dcd1faac70b1b5360267fc02bf830e/tools/cargo-hakari#2-keep-the-workspace-hack-up-to-date-in-ci
+      - name: Check rust dependencies
+        if: ${{ !cancelled() }}
+        run: |
+          cargo hakari generate --diff  # workspace-hack Cargo.toml is up-to-date
+          cargo hakari manage-deps --dry-run  # all workspace crates depend on workspace-hack
+
   build-neon:
     runs-on: [ self-hosted, dev, x64 ]
     container:
@@ -79,12 +160,10 @@ jobs:
       - name: Set pg 14 revision for caching
         id: pg_v14_rev
         run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v14) >> $GITHUB_OUTPUT
-        shell: bash -euxo pipefail {0}
 
       - name: Set pg 15 revision for caching
         id: pg_v15_rev
         run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v15) >> $GITHUB_OUTPUT
-        shell: bash -euxo pipefail {0}
 
       # Set some environment variables used by all the steps.
       #
@@ -101,16 +180,15 @@ jobs:
           if [[ $BUILD_TYPE == "debug" ]]; then
             cov_prefix="scripts/coverage --profraw-prefix=$GITHUB_JOB --dir=/tmp/coverage run"
             CARGO_FEATURES="--features testing"
-            CARGO_FLAGS="--locked --timings $CARGO_FEATURES"
+            CARGO_FLAGS="--locked $CARGO_FEATURES"
           elif [[ $BUILD_TYPE == "release" ]]; then
             cov_prefix=""
             CARGO_FEATURES="--features testing,profiling"
-            CARGO_FLAGS="--locked --timings --release $CARGO_FEATURES"
+            CARGO_FLAGS="--locked --release $CARGO_FEATURES"
           fi
           echo "cov_prefix=${cov_prefix}" >> $GITHUB_ENV
           echo "CARGO_FEATURES=${CARGO_FEATURES}" >> $GITHUB_ENV
           echo "CARGO_FLAGS=${CARGO_FLAGS}" >> $GITHUB_ENV
-        shell: bash -euxo pipefail {0}
 
       # Don't include the ~/.cargo/registry/src directory. It contains just
       # uncompressed versions of the crates in ~/.cargo/registry/cache
@@ -127,8 +205,8 @@ jobs:
             target/
           # Fall back to older versions of the key, if no cache for current Cargo.lock was found
           key: |
-            v10-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('Cargo.lock') }}
-            v10-${{ runner.os }}-${{ matrix.build_type }}-cargo-
+            v1-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('Cargo.lock') }}
+            v1-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('rust-toolchain.toml') }}-
 
       - name: Cache postgres v14 build
         id: cache_pg_14
@@ -147,26 +225,21 @@ jobs:
       - name: Build postgres v14
         if: steps.cache_pg_14.outputs.cache-hit != 'true'
         run: mold -run make postgres-v14 -j$(nproc)
-        shell: bash -euxo pipefail {0}
 
       - name: Build postgres v15
         if: steps.cache_pg_15.outputs.cache-hit != 'true'
         run: mold -run make postgres-v15 -j$(nproc)
-        shell: bash -euxo pipefail {0}
 
       - name: Build neon extensions
         run: mold -run make neon-pg-ext -j$(nproc)
-        shell: bash -euxo pipefail {0}
 
       - name: Run cargo build
         run: |
           ${cov_prefix} mold -run cargo build $CARGO_FLAGS --bins --tests
-        shell: bash -euxo pipefail {0}
 
       - name: Run cargo test
         run: |
           ${cov_prefix} cargo test $CARGO_FLAGS
-        shell: bash -euxo pipefail {0}
 
       - name: Install rust binaries
         run: |
@@ -207,11 +280,9 @@ jobs:
               echo "/tmp/neon/bin/$bin" >> /tmp/coverage/binaries.list
             done
           fi
-        shell: bash -euxo pipefail {0}
 
       - name: Install postgres binaries
         run: cp -a pg_install /tmp/neon/pg_install
-        shell: bash -euxo pipefail {0}
 
       - name: Upload Neon artifact
         uses: ./.github/actions/upload
@@ -219,17 +290,6 @@ jobs:
           name: neon-${{ runner.os }}-${{ matrix.build_type }}-artifact
           path: /tmp/neon
 
-      - name: Prepare cargo build timing stats for storing
-        run: |
-          mkdir -p "/tmp/neon/cargo-timings/$BUILD_TYPE/"
-          cp -r ./target/cargo-timings/* "/tmp/neon/cargo-timings/$BUILD_TYPE/"
-        shell: bash -euxo pipefail {0}
-      - name: Upload cargo build stats
-        uses: ./.github/actions/upload
-        with:
-          name: neon-${{ runner.os }}-${{ matrix.build_type }}-build-stats
-          path: /tmp/neon/cargo-timings/
-
       # XXX: keep this after the binaries.list is formed, so the coverage can properly work later
       - name: Merge and upload coverage data
         if: matrix.build_type == 'debug'
@@ -250,7 +310,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: true
-          fetch-depth: 2
+          fetch-depth: 1
 
       - name: Pytest regression tests
         uses: ./.github/actions/run-python-test-set
@@ -284,7 +344,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: true
-          fetch-depth: 2
+          fetch-depth: 1
 
       - name: Pytest benchmarks
         uses: ./.github/actions/run-python-test-set
@@ -305,7 +365,7 @@ jobs:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
     needs: [ regress-tests, benchmarks ]
-    if: success() || failure()
+    if: ${{ !cancelled() }}
     strategy:
       fail-fast: false
       matrix:
@@ -330,7 +390,6 @@ jobs:
           SHA: ${{ github.event.pull_request.head.sha || github.sha }}
           REPORT_URL: ${{ steps.create-allure-report.outputs.report-url }}
           TEST_RESULT_CONNSTR: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR }}
-        shell: bash -euxo pipefail {0}
         run: |
           curl --fail --output suites.json ${REPORT_URL%/index.html}/data/suites.json
           ./scripts/pysync
@@ -363,7 +422,7 @@ jobs:
             !~/.cargo/registry/src
             ~/.cargo/git/
             target/
-          key: v10-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('Cargo.lock') }}
+          key: v1-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('Cargo.lock') }}
 
       - name: Get Neon artifact
         uses: ./.github/actions/download
@@ -379,7 +438,6 @@ jobs:
 
       - name: Merge coverage data
         run: scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage merge
-        shell: bash -euxo pipefail {0}
 
       - name: Build and upload coverage report
         run: |
@@ -412,7 +470,6 @@ jobs:
               \"description\": \"Coverage report is ready\",
               \"target_url\": \"$REPORT_URL\"
             }"
-        shell: bash -euxo pipefail {0}
 
   trigger-e2e-tests:
     runs-on: [ self-hosted, dev, x64 ]
@@ -463,6 +520,9 @@ jobs:
     runs-on: [ self-hosted, dev, x64 ]
     needs: [ tag ]
     container: gcr.io/kaniko-project/executor:v1.9.0-debug
+    defaults:
+      run:
+        shell: sh -eu {0}
 
     steps:
       - name: Checkout
@@ -481,6 +541,9 @@ jobs:
     runs-on: [ self-hosted, dev, x64 ]
     needs: [ tag ]
     container: gcr.io/kaniko-project/executor:v1.9.0-debug
+    defaults:
+      run:
+        shell: sh -eu {0}
 
     steps:
       - name: Checkout
@@ -496,6 +559,10 @@ jobs:
     runs-on: [ self-hosted, dev, x64 ]
     container: gcr.io/kaniko-project/executor:v1.9.0-debug
     needs: [ tag ]
+    defaults:
+      run:
+        shell: sh -eu {0}
+
     steps:
       - name: Checkout
         uses: actions/checkout@v1 # v3 won't work with kaniko
@@ -513,6 +580,10 @@ jobs:
     runs-on: [ self-hosted, dev, x64 ]
     container: gcr.io/kaniko-project/executor:v1.9.0-debug
     needs: [ tag ]
+    defaults:
+      run:
+        shell: sh -eu {0}
+
     steps:
       - name: Checkout
         uses: actions/checkout@v1 # v3 won't work with kaniko
@@ -658,7 +729,7 @@ jobs:
           crane tag neondatabase/compute-node-v15:${{needs.tag.outputs.build-tag}} latest
 
   calculate-deploy-targets:
-    runs-on: [ self-hosted, Linux, k8s-runner ]
+    runs-on: [ self-hosted, dev, x64 ]
     if: |
       (github.ref_name == 'main' || github.ref_name == 'release') &&
       github.event_name != 'workflow_dispatch'
@@ -668,11 +739,10 @@ jobs:
       - id: set-matrix
         run: |
           if [[ "$GITHUB_REF_NAME" == "main" ]]; then
-            STAGING='{"env_name": "staging", "proxy_job": "neon-proxy", "proxy_config": "staging.proxy", "kubeconfig_secret": "STAGING_KUBECONFIG_DATA", "console_api_key_secret": "NEON_STAGING_API_KEY"}'
-            NEON_STRESS='{"env_name": "neon-stress", "proxy_job": "neon-stress-proxy", "proxy_config": "neon-stress.proxy", "kubeconfig_secret": "NEON_STRESS_KUBECONFIG_DATA", "console_api_key_secret": "NEON_CAPTEST_API_KEY"}'
-            echo "include=[$STAGING, $NEON_STRESS]" >> $GITHUB_OUTPUT
+            STAGING='{"env_name": "staging", "proxy_job": "neon-proxy", "proxy_config": "staging.proxy", "storage_broker_ns": "neon-storage-broker", "storage_broker_config": "staging.neon-storage-broker", "kubeconfig_secret": "STAGING_KUBECONFIG_DATA", "console_api_key_secret": "NEON_STAGING_API_KEY"}'
+            echo "include=[$STAGING]" >> $GITHUB_OUTPUT
           elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
-            PRODUCTION='{"env_name": "production", "proxy_job": "neon-proxy", "proxy_config": "production.proxy", "kubeconfig_secret": "PRODUCTION_KUBECONFIG_DATA", "console_api_key_secret": "NEON_PRODUCTION_API_KEY"}'
+            PRODUCTION='{"env_name": "production", "proxy_job": "neon-proxy", "proxy_config": "production.proxy", "storage_broker_ns": "neon-storage-broker", "storage_broker_config": "production.neon-storage-broker", "kubeconfig_secret": "PRODUCTION_KUBECONFIG_DATA", "console_api_key_secret": "NEON_PRODUCTION_API_KEY"}'
             echo "include=[$PRODUCTION]" >> $GITHUB_OUTPUT
           else
             echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
@@ -680,8 +750,8 @@ jobs:
           fi
 
   deploy:
-    runs-on: [ self-hosted, Linux, k8s-runner ]
-    #container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:latest
+    runs-on: [ self-hosted, dev, x64 ]
+    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
     # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version.
     # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly
     needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
@@ -701,16 +771,6 @@ jobs:
           submodules: true
           fetch-depth: 0
 
-      - name: Setup python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-
-      - name: Setup ansible
-        run: |
-          export PATH="/root/.local/bin:$PATH"
-          pip install --progress-bar off --user ansible boto3 toml
-
       - name: Redeploy
         run: |
           export DOCKER_TAG=${{needs.tag.outputs.build-tag}}
@@ -731,8 +791,8 @@ jobs:
           chmod 0600 ssh-key
           ssh-add ssh-key
           rm -f ssh-key ssh-key-cert.pub
-          ansible-galaxy collection install sivel.toiletwater
-          ansible-playbook deploy.yaml -i ${{ matrix.env_name }}.hosts.yaml -e CONSOLE_API_TOKEN=${{ secrets[matrix.console_api_key_secret] }}
+          ANSIBLE_CONFIG=./ansible.cfg ansible-galaxy collection install sivel.toiletwater
+          ANSIBLE_CONFIG=./ansible.cfg ansible-playbook deploy.yaml -i ${{ matrix.env_name }}.hosts.yaml -e CONSOLE_API_TOKEN=${{ secrets[matrix.console_api_key_secret] }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }}
           rm -f neon_install.tar.gz .neon_current_version
 
   deploy-new:
@@ -770,7 +830,7 @@ jobs:
             exit 1
           fi
           ansible-galaxy collection install sivel.toiletwater
-          ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_STAGING_API_KEY}}
+          ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_STAGING_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }}
           rm -f neon_install.tar.gz .neon_current_version
 
   deploy-pr-test-new:
@@ -780,7 +840,7 @@ jobs:
     # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly
     needs: [ push-docker-hub, tag, regress-tests ]
     if: |
-      contains(github.event.pull_request.labels.*.name, 'deploy-test-storage') && 
+      contains(github.event.pull_request.labels.*.name, 'deploy-test-storage') &&
       github.event_name != 'workflow_dispatch'
     defaults:
       run:
@@ -803,7 +863,7 @@ jobs:
           ./get_binaries.sh
 
           ansible-galaxy collection install sivel.toiletwater
-          ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_STAGING_API_KEY}}
+          ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_STAGING_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }}
           rm -f neon_install.tar.gz .neon_current_version
 
   deploy-prod-new:
@@ -820,7 +880,7 @@ jobs:
         shell: bash
     strategy:
       matrix:
-        target_region: [ us-east-2, eu-central-1, ap-southeast-1 ]
+        target_region: [ us-east-2, us-west-2, eu-central-1, ap-southeast-1 ]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -843,12 +903,12 @@ jobs:
           fi
 
           ansible-galaxy collection install sivel.toiletwater
-          ansible-playbook deploy.yaml -i prod.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_PRODUCTION_API_KEY}}
+          ansible-playbook deploy.yaml -i prod.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_PRODUCTION_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }}
           rm -f neon_install.tar.gz .neon_current_version
 
   deploy-proxy:
     runs-on: [ self-hosted, dev, x64 ]
-    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:latest
+    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned
     # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
     needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
     if: |
@@ -885,8 +945,49 @@ jobs:
       - name: Re-deploy proxy
         run: |
           DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade ${{ matrix.proxy_job }}       neondatabase/neon-proxy --namespace neon-proxy --install -f .github/helm-values/${{ matrix.proxy_config }}.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
-          helm upgrade ${{ matrix.proxy_job }}-scram neondatabase/neon-proxy --namespace neon-proxy --install -f .github/helm-values/${{ matrix.proxy_config }}-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
+          helm upgrade ${{ matrix.proxy_job }}       neondatabase/neon-proxy --namespace neon-proxy --install -f .github/helm-values/${{ matrix.proxy_config }}.yaml       --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+          helm upgrade ${{ matrix.proxy_job }}-scram neondatabase/neon-proxy --namespace neon-proxy --install -f .github/helm-values/${{ matrix.proxy_config }}-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+
+  deploy-storage-broker:
+    name: deploy storage broker on old staging and old prod
+    runs-on: [ self-hosted, dev, x64 ]
+    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned
+    # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
+    needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
+    if: |
+      (github.ref_name == 'main' || github.ref_name == 'release') &&
+      github.event_name != 'workflow_dispatch'
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        include: ${{fromJSON(needs.calculate-deploy-targets.outputs.matrix-include)}}
+    env:
+      KUBECONFIG: .kubeconfig
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Add curl
+        run: apt update && apt install curl -y
+
+      - name: Store kubeconfig file
+        run: |
+          echo "${{ secrets[matrix.kubeconfig_secret] }}" | base64 --decode > ${KUBECONFIG}
+          chmod 0600 ${KUBECONFIG}
+
+      - name: Setup helm v3
+        run: |
+          curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+          helm repo add neondatabase https://neondatabase.github.io/helm-charts
+
+      - name: Deploy storage-broker
+        run:
+          helm upgrade neon-storage-broker neondatabase/neon-storage-broker --namespace ${{ matrix.storage_broker_ns }} --create-namespace --install --atomic -f .github/helm-values/${{ matrix.storage_broker_config }}.yaml --set image.tag=${{ needs.tag.outputs.build-tag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s
 
   deploy-proxy-new:
     runs-on: [ self-hosted, dev, x64 ]
@@ -925,19 +1026,53 @@ jobs:
       - name: Re-deploy scram proxy
         run: |
           DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
+          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
 
       - name: Re-deploy link proxy
         if: matrix.deploy_link_proxy
         run: |
           DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
+          helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
 
       - name: Re-deploy legacy scram proxy
         if: matrix.deploy_legacy_scram_proxy
         run: |
           DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
+          helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+
+  deploy-storage-broker-dev-new:
+    runs-on: [ self-hosted, dev, x64 ]
+    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
+    # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
+    needs: [ push-docker-hub, tag, regress-tests ]
+    if: |
+      (github.ref_name == 'main') &&
+      github.event_name != 'workflow_dispatch'
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        include:
+          - target_region:  us-east-2
+            target_cluster: dev-us-east-2-beta
+          - target_region:  eu-west-1
+            target_cluster: dev-eu-west-1-zeta
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Configure environment
+        run: |
+          helm repo add neondatabase https://neondatabase.github.io/helm-charts
+          aws --region ${{ matrix.target_region }} eks update-kubeconfig --name  ${{ matrix.target_cluster }}
+
+      - name: Deploy storage-broker
+        run:
+          helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ needs.tag.outputs.build-tag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s
 
   deploy-proxy-prod-new:
     runs-on: prod
@@ -955,6 +1090,8 @@ jobs:
         include:
           - target_region:  us-east-2
             target_cluster: prod-us-east-2-delta
+          - target_region:  us-west-2
+            target_cluster: prod-us-west-2-eta
           - target_region: eu-central-1
             target_cluster: prod-eu-central-1-gamma
           - target_region: ap-southeast-1
@@ -974,7 +1111,45 @@ jobs:
       - name: Re-deploy proxy
         run: |
           DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
+          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+
+  deploy-storage-broker-prod-new:
+    runs-on: prod
+    container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
+    # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
+    needs: [ push-docker-hub, tag, regress-tests ]
+    if: |
+      (github.ref_name == 'release') &&
+      github.event_name != 'workflow_dispatch'
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        include:
+          - target_region:  us-east-2
+            target_cluster: prod-us-east-2-delta
+          - target_region:  us-west-2
+            target_cluster: prod-us-west-2-eta
+          - target_region: eu-central-1
+            target_cluster: prod-eu-central-1-gamma
+          - target_region: ap-southeast-1
+            target_cluster: prod-ap-southeast-1-epsilon
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Configure environment
+        run: |
+          helm repo add neondatabase https://neondatabase.github.io/helm-charts
+          aws --region ${{ matrix.target_region }} eks update-kubeconfig --name  ${{ matrix.target_cluster }}
+
+      - name: Deploy storage-broker
+        run:
+          helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ needs.tag.outputs.build-tag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s
 
   promote-compatibility-data:
     runs-on: [ self-hosted, dev, x64 ]
@@ -985,7 +1160,6 @@ jobs:
     if: github.ref_name == 'release' && github.event_name != 'workflow_dispatch'
     steps:
       - name: Promote compatibility snapshot for the release
-        shell: bash -euxo pipefail {0}
         env:
           BUCKET: neon-github-public-dev
           PREFIX: artifacts/latest

.github/workflows/codestyle.yml

@@ -1,166 +0,0 @@
-name: Check code style and build
-
-on:
-  push:
-    branches:
-    - main
-  pull_request:
-
-defaults:
-  run:
-    shell: bash -euxo pipefail {0}
-
-concurrency:
-  # Allow only one workflow per any non-`main` branch.
-  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref == 'refs/heads/main' && github.sha || 'anysha' }}
-  cancel-in-progress: true
-
-env:
-  RUST_BACKTRACE: 1
-  COPT: '-Werror'
-
-jobs:
-  check-codestyle-rust:
-    strategy:
-      fail-fast: false
-      matrix:
-        # XXX: both OSes have rustup
-        #   * https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md#rust-tools
-        #   * https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2204-Readme.md#rust-tools
-        # this is all we need to install our toolchain later via rust-toolchain.toml
-        # so don't install any toolchain explicitly.
-        os: [ubuntu-latest, macos-latest]
-    timeout-minutes: 90
-    name: check codestyle rust and postgres
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          fetch-depth: 2
-
-      - name: Check formatting
-        run: cargo fmt --all -- --check
-
-      - name: Install Ubuntu postgres dependencies
-        if: matrix.os == 'ubuntu-latest'
-        run: |
-          sudo apt update
-          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev libssl-dev protobuf-compiler
-
-      - name: Install macOS postgres dependencies
-        if: matrix.os == 'macos-latest'
-        run: brew install flex bison openssl protobuf
-
-      - name: Set pg 14 revision for caching
-        id: pg_v14_rev
-        run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v14) >> $GITHUB_OUTPUT
-        shell: bash -euxo pipefail {0}
-
-      - name: Set pg 15 revision for caching
-        id: pg_v15_rev
-        run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v15) >> $GITHUB_OUTPUT
-        shell: bash -euxo pipefail {0}
-
-      - name: Cache postgres v14 build
-        id: cache_pg_14
-        uses: actions/cache@v3
-        with:
-          path: pg_install/v14
-          key: v1-${{ runner.os }}-${{ matrix.build_type }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
-
-      - name: Cache postgres v15 build
-        id: cache_pg_15
-        uses: actions/cache@v3
-        with:
-          path: pg_install/v15
-          key: v1-${{ runner.os }}-${{ matrix.build_type }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
-
-      - name: Set extra env for macOS
-        if: matrix.os == 'macos-latest'
-        run: |
-          echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV
-          echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV
-
-      - name: Build postgres v14
-        if: steps.cache_pg_14.outputs.cache-hit != 'true'
-        run: make postgres-v14
-        shell: bash -euxo pipefail {0}
-
-      - name: Build postgres v15
-        if: steps.cache_pg_15.outputs.cache-hit != 'true'
-        run: make postgres-v15
-        shell: bash -euxo pipefail {0}
-
-      - name: Build neon extensions
-        run: make neon-pg-ext
-
-      - name: Cache cargo deps
-        id: cache_cargo
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/registry
-            !~/.cargo/registry/src
-            ~/.cargo/git
-            target
-          key: v6-${{ runner.os }}-cargo-${{ hashFiles('./Cargo.lock') }}-rust
-
-      - name: Run cargo clippy
-        run: ./run_clippy.sh
-
-      - name: Ensure all project builds
-        run: cargo build --locked --all --all-targets
-
-  check-rust-dependencies:
-    runs-on: [ self-hosted, dev, x64 ]
-    container:
-      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
-      options: --init
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: false
-          fetch-depth: 1
-
-      # https://github.com/facebookincubator/cargo-guppy/tree/bec4e0eb29dcd1faac70b1b5360267fc02bf830e/tools/cargo-hakari#2-keep-the-workspace-hack-up-to-date-in-ci
-      - name: Check every project module is covered by Hakari
-        run: |
-          cargo hakari generate --diff  # workspace-hack Cargo.toml is up-to-date
-          cargo hakari manage-deps --dry-run  # all workspace crates depend on workspace-hack
-        shell: bash -euxo pipefail {0}
-
-  check-codestyle-python:
-    runs-on: [ self-hosted, Linux, k8s-runner ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: false
-          fetch-depth: 1
-
-      - name: Cache poetry deps
-        id: cache_poetry
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: v1-codestyle-python-deps-${{ hashFiles('poetry.lock') }}
-
-      - name: Install Python deps
-        run: ./scripts/pysync
-
-      - name: Run isort to ensure code format
-        run: poetry run isort --diff --check .
-
-      - name: Run black to ensure code format
-        run: poetry run black --diff --check .
-
-      - name: Run flake8 to ensure code format
-        run: poetry run flake8 .
-
-      - name: Run mypy to check types
-        run: poetry run mypy .

.github/workflows/neon_extra_builds.yml

@@ -0,0 +1,128 @@
+name: Check neon with extra platform builds
+
+on:
+  push:
+    branches:
+    - main
+
+defaults:
+  run:
+    shell: bash -euxo pipefail {0}
+
+concurrency:
+  # Allow only one workflow per any non-`main` branch.
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref == 'refs/heads/main' && github.sha || 'anysha' }}
+  cancel-in-progress: true
+
+env:
+  RUST_BACKTRACE: 1
+  COPT: '-Werror'
+
+jobs:
+  check-macos-build:
+    timeout-minutes: 90
+    runs-on: macos-latest
+
+    env:
+      # Use release build only, to have less debug info around
+      # Hence keeping target/ (and general cache size) smaller
+      BUILD_TYPE: release
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Install macOS postgres dependencies
+        run: brew install flex bison openssl protobuf
+
+      - name: Set pg 14 revision for caching
+        id: pg_v14_rev
+        run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v14) >> $GITHUB_OUTPUT
+
+      - name: Set pg 15 revision for caching
+        id: pg_v15_rev
+        run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v15) >> $GITHUB_OUTPUT
+
+      - name: Cache postgres v14 build
+        id: cache_pg_14
+        uses: actions/cache@v3
+        with:
+          path: pg_install/v14
+          key: v1-${{ runner.os }}-${{ matrix.build_type }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
+
+      - name: Cache postgres v15 build
+        id: cache_pg_15
+        uses: actions/cache@v3
+        with:
+          path: pg_install/v15
+          key: v1-${{ runner.os }}-${{ matrix.build_type }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
+
+      - name: Set extra env for macOS
+        run: |
+          echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV
+          echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV
+
+      - name: Cache cargo deps
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/registry
+            !~/.cargo/registry/src
+            ~/.cargo/git
+            target
+          key: v1-${{ runner.os }}-cargo-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./rust-toolchain.toml') }}-rust
+
+      - name: Build postgres v14
+        if: steps.cache_pg_14.outputs.cache-hit != 'true'
+        run: make postgres-v14 -j$(nproc)
+
+      - name: Build postgres v15
+        if: steps.cache_pg_15.outputs.cache-hit != 'true'
+        run: make postgres-v15 -j$(nproc)
+
+      - name: Build neon extensions
+        run: make neon-pg-ext -j$(nproc)
+
+      - name: Run cargo build
+        run: cargo build --all --release
+
+      - name: Check that no warnings are produced
+        run: ./run_clippy.sh
+
+  gather-rust-build-stats:
+    timeout-minutes: 90
+    runs-on: ubuntu-latest
+
+    env:
+      BUILD_TYPE: release
+      # build with incremental compilation produce partial results
+      # so do not attempt to cache this build, also disable the incremental compilation
+      CARGO_INCREMENTAL: 0
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Install Ubuntu postgres dependencies
+        run: |
+          sudo apt update
+          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev libssl-dev protobuf-compiler
+
+      # Some of our rust modules use FFI and need those to be checked
+      - name: Get postgres headers
+        run: make postgres-headers -j$(nproc)
+
+      - name: Produce the build stats
+        run: cargo build --all --release --timings
+
+      - name: Upload the build stats
+        uses: actions/upload-artifact@v3
+        with:
+          name: neon-${{ runner.os }}-release-build-stats
+          path: ./target/cargo-timings/

.github/workflows/pg_clients.yml

@@ -23,6 +23,7 @@ jobs:
     runs-on: [ ubuntu-latest ]
 
     env:
+      DEFAULT_PG_VERSION: 14
       TEST_OUTPUT: /tmp/test_output
 
     steps:
@@ -51,8 +52,8 @@ jobs:
       id: create-neon-project
       uses: ./.github/actions/neon-project-create
       with:
-        environment: staging
         api_key: ${{ secrets.NEON_STAGING_API_KEY }}
+        postgres_version: ${{ env.DEFAULT_PG_VERSION }}
 
     - name: Run pytest
       env:
@@ -63,7 +64,7 @@ jobs:
       run: |
         # Test framework expects we have psql binary;
         # but since we don't really need it in this test, let's mock it
-        mkdir -p "$POSTGRES_DISTRIB_DIR/v14/bin" && touch "$POSTGRES_DISTRIB_DIR/v14/bin/psql";
+        mkdir -p "$POSTGRES_DISTRIB_DIR/v${DEFAULT_PG_VERSION}/bin" && touch "$POSTGRES_DISTRIB_DIR/v${DEFAULT_PG_VERSION}/bin/psql";
         ./scripts/pytest \
           --junitxml=$TEST_OUTPUT/junit.xml \
           --tb=short \
@@ -75,7 +76,6 @@ jobs:
       if: ${{ always() }}
       uses: ./.github/actions/neon-project-delete
       with:
-        environment: staging
         project_id: ${{ steps.create-neon-project.outputs.project_id }}
         api_key: ${{ secrets.NEON_STAGING_API_KEY }}
 

Cargo.toml

@@ -86,4 +86,4 @@ lto = true
 # This is only needed for proxy's tests.
 # TODO: we should probably fork `tokio-postgres-rustls` instead.
 [patch.crates-io]
-tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }

Dockerfile

@@ -79,7 +79,7 @@ COPY --from=pg-build /home/nonroot/postgres_install.tar.gz /data/
 RUN mkdir -p /data/.neon/ && chown -R neon:neon /data/.neon/ \
     && /usr/local/bin/pageserver -D /data/.neon/ --init \
        -c "id=1234" \
-       -c "broker_endpoints=['http://etcd:2379']" \
+       -c "broker_endpoint='http://storage_broker:50051'" \
        -c "pg_distrib_dir='/usr/local/'" \
        -c "listen_pg_addr='0.0.0.0:6400'" \
        -c "listen_http_addr='0.0.0.0:9898'"

Makefile

@@ -61,146 +61,115 @@ all: neon postgres neon-pg-ext
 #
 # The 'postgres_ffi' depends on the Postgres headers.
 .PHONY: neon
-neon: postgres-v14-headers postgres-v15-headers
+neon: postgres-headers
 	+@echo "Compiling Neon"
 	$(CARGO_CMD_PREFIX) cargo build $(CARGO_BUILD_FLAGS)
 
 ### PostgreSQL parts
-# The rules are duplicated for Postgres v14 and 15. We may want to refactor
+# Some rules are duplicated for Postgres v14 and 15. We may want to refactor
 # to avoid the duplication in the future, but it's tolerable for now.
 #
-$(POSTGRES_INSTALL_DIR)/build/v14/config.status:
-	+@echo "Configuring Postgres v14 build"
-	mkdir -p $(POSTGRES_INSTALL_DIR)/build/v14
-	(cd $(POSTGRES_INSTALL_DIR)/build/v14 && \
-	env PATH="$(EXTRA_PATH_OVERRIDES):$$PATH" $(ROOT_PROJECT_DIR)/vendor/postgres-v14/configure \
+$(POSTGRES_INSTALL_DIR)/build/%/config.status:
+	+@echo "Configuring Postgres $* build"
+	mkdir -p $(POSTGRES_INSTALL_DIR)/build/$*
+	(cd $(POSTGRES_INSTALL_DIR)/build/$* && \
+	env PATH="$(EXTRA_PATH_OVERRIDES):$$PATH" $(ROOT_PROJECT_DIR)/vendor/postgres-$*/configure \
 		CFLAGS='$(PG_CFLAGS)' \
 		$(PG_CONFIGURE_OPTS) \
-		--prefix=$(abspath $(POSTGRES_INSTALL_DIR))/v14 > configure.log)
-
-$(POSTGRES_INSTALL_DIR)/build/v15/config.status:
-	+@echo "Configuring Postgres v15 build"
-	mkdir -p $(POSTGRES_INSTALL_DIR)/build/v15
-	(cd $(POSTGRES_INSTALL_DIR)/build/v15 && \
-	env PATH="$(EXTRA_PATH_OVERRIDES):$$PATH" $(ROOT_PROJECT_DIR)/vendor/postgres-v15/configure \
-		CFLAGS='$(PG_CFLAGS)' \
-		$(PG_CONFIGURE_OPTS) \
-		--prefix=$(abspath $(POSTGRES_INSTALL_DIR))/v15 > configure.log)
+		--prefix=$(abspath $(POSTGRES_INSTALL_DIR))/$* > configure.log)
 
 # nicer alias to run 'configure'
-.PHONY: postgres-v14-configure
-postgres-v14-configure: $(POSTGRES_INSTALL_DIR)/build/v14/config.status
-
-.PHONY: postgres-v15-configure
-postgres-v15-configure: $(POSTGRES_INSTALL_DIR)/build/v15/config.status
+# Note: I've been unable to use templates for this part of our configuration.
+# I'm not sure why it wouldn't work, but this is the only place (apart from
+# the "build-all-versions" entry points) where direct mention of PostgreSQL
+# versions is used.
+.PHONY: postgres-configure-v15
+postgres-configure-v15: $(POSTGRES_INSTALL_DIR)/build/v15/config.status
+.PHONY: postgres-configure-v14
+postgres-configure-v14: $(POSTGRES_INSTALL_DIR)/build/v14/config.status
 
 # Install the PostgreSQL header files into $(POSTGRES_INSTALL_DIR)/<version>/include
-.PHONY: postgres-v14-headers
-postgres-v14-headers: postgres-v14-configure
-	+@echo "Installing PostgreSQL v14 headers"
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/src/include MAKELEVEL=0 install
-
-.PHONY: postgres-v15-headers
-postgres-v15-headers: postgres-v15-configure
-	+@echo "Installing PostgreSQL v15 headers"
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/src/include MAKELEVEL=0 install
+.PHONY: postgres-headers-%
+postgres-headers-%: postgres-configure-%
+	+@echo "Installing PostgreSQL $* headers"
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/$*/src/include MAKELEVEL=0 install
 
 # Compile and install PostgreSQL
-.PHONY: postgres-v14
-postgres-v14: postgres-v14-configure \
-		  postgres-v14-headers # to prevent `make install` conflicts with neon's `postgres-headers`
-	+@echo "Compiling PostgreSQL v14"
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14 MAKELEVEL=0 install
-	+@echo "Compiling libpq v14"
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/src/interfaces/libpq install
-	+@echo "Compiling pg_prewarm v14"
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pg_prewarm install
-	+@echo "Compiling pg_buffercache v14"
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pg_buffercache install
-	+@echo "Compiling pageinspect v14"
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pageinspect install
+.PHONY: postgres-%
+postgres-%: postgres-configure-% \
+		  postgres-headers-% # to prevent `make install` conflicts with neon's `postgres-headers`
+	+@echo "Compiling PostgreSQL $*"
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/$* MAKELEVEL=0 install
+	+@echo "Compiling libpq $*"
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/$*/src/interfaces/libpq install
+	+@echo "Compiling pg_prewarm $*"
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/$*/contrib/pg_prewarm install
+	+@echo "Compiling pg_buffercache $*"
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/$*/contrib/pg_buffercache install
+	+@echo "Compiling pageinspect $*"
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/$*/contrib/pageinspect install
 
-.PHONY: postgres-v15
-postgres-v15: postgres-v15-configure \
-		  postgres-v15-headers # to prevent `make install` conflicts with neon's `postgres-headers`
-	+@echo "Compiling PostgreSQL v15"
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15 MAKELEVEL=0 install
-	+@echo "Compiling libpq v15"
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/src/interfaces/libpq install
-	+@echo "Compiling pg_prewarm v15"
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/contrib/pg_prewarm install
-	+@echo "Compiling pg_buffercache v15"
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/contrib/pg_buffercache install
-	+@echo "Compiling pageinspect v15"
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/contrib/pageinspect install
+.PHONY: postgres-clean-%
+postgres-clean-%:
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/$* MAKELEVEL=0 clean
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/$*/contrib/pg_buffercache clean
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/$*/contrib/pageinspect clean
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/$*/src/interfaces/libpq clean
 
-# shorthand to build all Postgres versions
-postgres: postgres-v14 postgres-v15
+.PHONY: neon-pg-ext-%
+neon-pg-ext-%: postgres-%
+	+@echo "Compiling neon $*"
+	mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-$*
+	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/$*/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \
+		-C $(POSTGRES_INSTALL_DIR)/build/neon-$* \
+		-f $(ROOT_PROJECT_DIR)/pgxn/neon/Makefile install
+	+@echo "Compiling neon_walredo $*"
+	mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-walredo-$*
+	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/$*/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \
+		-C $(POSTGRES_INSTALL_DIR)/build/neon-walredo-$* \
+		-f $(ROOT_PROJECT_DIR)/pgxn/neon_walredo/Makefile install
+	+@echo "Compiling neon_test_utils $*"
+	mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-$*
+	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/$*/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \
+		-C $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-$* \
+		-f $(ROOT_PROJECT_DIR)/pgxn/neon_test_utils/Makefile install
 
-.PHONY: postgres-v14-clean
-postgres-v14-clean:
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14 MAKELEVEL=0 clean
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pg_buffercache clean
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pageinspect clean
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/src/interfaces/libpq clean
+.PHONY: neon-pg-ext-clean-%
+neon-pg-ext-clean-%:
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/pgxn/neon-$* -f $(ROOT_PROJECT_DIR)/pgxn/neon/Makefile clean
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/pgxn/neon_walredo-$* -f $(ROOT_PROJECT_DIR)/pgxn/neon_walredo/Makefile clean
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/pgxn/neon_test_utils-$* -f $(ROOT_PROJECT_DIR)/pgxn/neon_test_utils/Makefile clean
 
-.PHONY: postgres-v15-clean
-postgres-v15-clean:
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15 MAKELEVEL=0 clean
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/contrib/pg_buffercache clean
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/contrib/pageinspect clean
-	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/src/interfaces/libpq clean
-
-neon-pg-ext-v14: postgres-v14
-	+@echo "Compiling neon v14"
-	mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-v14
-	(cd $(POSTGRES_INSTALL_DIR)/build/neon-v14 && \
-	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v14/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \
-		-f $(ROOT_PROJECT_DIR)/pgxn/neon/Makefile install)
-	+@echo "Compiling neon_walredo v14"
-	mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-walredo-v14
-	(cd $(POSTGRES_INSTALL_DIR)/build/neon-walredo-v14 && \
-	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v14/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \
-		-f $(ROOT_PROJECT_DIR)/pgxn/neon_walredo/Makefile install)
-	+@echo "Compiling neon_test_utils" v14
-	mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-v14
-	(cd $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-v14 && \
-	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v14/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \
-		-f $(ROOT_PROJECT_DIR)/pgxn/neon_test_utils/Makefile install)
-
-neon-pg-ext-v15: postgres-v15
-	+@echo "Compiling neon v15"
-	mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-v15
-	(cd $(POSTGRES_INSTALL_DIR)/build/neon-v15 && \
-	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v15/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \
-		-f $(ROOT_PROJECT_DIR)/pgxn/neon/Makefile install)
-	+@echo "Compiling neon_walredo v15"
-	mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-walredo-v15
-	(cd $(POSTGRES_INSTALL_DIR)/build/neon-walredo-v15 && \
-	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v15/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \
-		-f $(ROOT_PROJECT_DIR)/pgxn/neon_walredo/Makefile install)
-	+@echo "Compiling neon_test_utils" v15
-	mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-v15
-	(cd $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-v15 && \
-	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v15/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \
-		-f $(ROOT_PROJECT_DIR)/pgxn/neon_test_utils/Makefile install)
+.PHONY: neon-pg-ext
+neon-pg-ext: \
+	neon-pg-ext-v14 \
+	neon-pg-ext-v15
 
 .PHONY: neon-pg-ext-clean
-	$(MAKE) -C $(ROOT_PROJECT_DIR)/pgxn/neon clean
-	$(MAKE) -C $(ROOT_PROJECT_DIR)/pgxn/neon_test_utils clean
+neon-pg-ext-clean: \
+	neon-pg-ext-clean-v14 \
+	neon-pg-ext-clean-v15
 
-neon-pg-ext: neon-pg-ext-v14 neon-pg-ext-v15
-postgres-headers: postgres-v14-headers postgres-v15-headers
-postgres-clean: postgres-v14-clean postgres-v15-clean
+# shorthand to build all Postgres versions
+.PHONY: postgres
+postgres: \
+	postgres-v14 \
+	postgres-v15
+
+.PHONY: postgres-headers
+postgres-headers: \
+	postgres-headers-v14 \
+	postgres-headers-v15
+
+.PHONY: postgres-clean
+postgres-clean: \
+	postgres-clean-v14 \
+	postgres-clean-v15
 
 # This doesn't remove the effects of 'configure'.
 .PHONY: clean
-clean:
-	cd $(POSTGRES_INSTALL_DIR)/build/v14 && $(MAKE) clean
-	cd $(POSTGRES_INSTALL_DIR)/build/v15 && $(MAKE) clean
+clean: postgres-clean neon-pg-ext-clean
 	$(CARGO_CMD_PREFIX) cargo clean
-	cd pgxn/neon && $(MAKE) clean
-	cd pgxn/neon_test_utils && $(MAKE) clean
 
 # This removes everything
 .PHONY: distclean

README.md

@@ -2,29 +2,20 @@
 
 Neon is a serverless open-source alternative to AWS Aurora Postgres. It separates storage and compute and substitutes the PostgreSQL storage layer by redistributing data across a cluster of nodes.
 
-The project used to be called "Zenith". Many of the commands and code comments
-still refer to "zenith", but we are in the process of renaming things.
-
 ## Quick start
-[Join the waitlist](https://neon.tech/) for our free tier to receive your serverless postgres instance. Then connect to it with your preferred postgres client (psql, dbeaver, etc) or use the online SQL editor.
+Try the [Neon Free Tier](https://neon.tech/docs/introduction/technical-preview-free-tier/) to create a serverless Postgres instance. Then connect to it with your preferred Postgres client (psql, dbeaver, etc) or use the online [SQL Editor](https://neon.tech/docs/get-started-with-neon/query-with-neon-sql-editor/). See [Connect from any application](https://neon.tech/docs/connect/connect-from-any-app/) for connection instructions.
 
 Alternatively, compile and run the project [locally](#running-local-installation).
 
 ## Architecture overview
 
-A Neon installation consists of compute nodes and a Neon storage engine.
-
-Compute nodes are stateless PostgreSQL nodes backed by the Neon storage engine.
+A Neon installation consists of compute nodes and the Neon storage engine. Compute nodes are stateless PostgreSQL nodes backed by the Neon storage engine.
 
 The Neon storage engine consists of two major components:
 - Pageserver. Scalable storage backend for the compute nodes.
-- WAL service. The service receives WAL from the compute node and ensures that it is stored durably.
+- Safekeepers. The safekeepers form a redundant WAL service that received WAL from the compute node, and stores it durably until it has been processed by the pageserver and uploaded to cloud storage.
 
-Pageserver consists of:
-- Repository - Neon storage implementation.
-- WAL receiver - service that receives WAL from WAL service and stores it in the repository.
-- Page service - service that communicates with compute nodes and responds with pages from the repository.
-- WAL redo - service that builds pages from base images and WAL records on Page service request
+See developer documentation in [/docs/SUMMARY.md](/docs/SUMMARY.md) for more information.
 
 ## Running local installation
 
@@ -35,12 +26,12 @@ Pageserver consists of:
 * On Ubuntu or Debian, this set of packages should be sufficient to build the code:
 ```bash
 apt install build-essential libtool libreadline-dev zlib1g-dev flex bison libseccomp-dev \
-libssl-dev clang pkg-config libpq-dev etcd cmake postgresql-client protobuf-compiler
+libssl-dev clang pkg-config libpq-dev cmake postgresql-client protobuf-compiler
 ```
 * On Fedora, these packages are needed:
 ```bash
 dnf install flex bison readline-devel zlib-devel openssl-devel \
-  libseccomp-devel perl clang cmake etcd postgresql postgresql-contrib protobuf-compiler
+  libseccomp-devel perl clang cmake postgresql postgresql-contrib protobuf-compiler
 ```
 
 2. [Install Rust](https://www.rust-lang.org/tools/install)
@@ -53,7 +44,7 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
 1. Install XCode and dependencies
 ```
 xcode-select --install
-brew install protobuf etcd openssl flex bison
+brew install protobuf openssl flex bison
 ```
 
 2. [Install Rust](https://www.rust-lang.org/tools/install)
@@ -132,12 +123,12 @@ Stopped pageserver 1 process with pid 2545906
 
 # start pageserver and safekeeper
 > ./target/debug/neon_local start
-Starting etcd broker using "/usr/bin/etcd"
-etcd started, pid: 2545996
+Starting neon broker at 127.0.0.1:50051
+storage_broker started, pid: 2918372
 Starting pageserver at '127.0.0.1:64000' in '.neon'.
-pageserver started, pid: 2546005
+pageserver started, pid: 2918386
 Starting safekeeper at '127.0.0.1:5454' in '.neon/safekeepers/sk1'.
-safekeeper 1 started, pid: 2546041
+safekeeper 1 started, pid: 2918437
 
 # start postgres compute node
 > ./target/debug/neon_local pg start main
@@ -229,12 +220,20 @@ CARGO_BUILD_FLAGS="--features=testing" make
 
 ## Documentation
 
-Now we use README files to cover design ideas and overall architecture for each module and `rustdoc` style documentation comments. See also [/docs/](/docs/) a top-level overview of all available markdown documentation.
+[/docs/](/docs/) Contains a top-level overview of all available markdown documentation.
 
 - [/docs/sourcetree.md](/docs/sourcetree.md) contains overview of source tree layout.
 
 To view your `rustdoc` documentation in a browser, try running `cargo doc --no-deps --open`
 
+See also README files in some source directories, and `rustdoc` style documentation comments.
+
+Other resources:
+
+- [SELECT 'Hello, World'](https://neon.tech/blog/hello-world/): Blog post by Nikita Shamgunov on the high level architecture
+- [Architecture decisions in Neon](https://neon.tech/blog/architecture-decisions-in-neon/): Blog post by Heikki Linnakangas
+- [Neon: Serverless PostgreSQL!](https://www.youtube.com/watch?v=rES0yzeERns): Presentation on storage system by Heikki Linnakangas in the CMU Database Group seminar series
+
 ### Postgres-specific terms
 
 Due to Neon's very close relation with PostgreSQL internals, numerous specific terms are used.

compute_tools/Cargo.toml

@@ -5,19 +5,19 @@ edition = "2021"
 
 [dependencies]
 anyhow = "1.0"
-chrono = "0.4"
+chrono = { version = "0.4", default-features = false, features = ["clock"] }
 clap = "4.0"
 env_logger = "0.9"
 futures = "0.3.13"
 hyper = { version = "0.14", features = ["full"] }
 log = { version = "0.4", features = ["std", "serde"] }
 notify = "5.0.0"
-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
 regex = "1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
 tar = "0.4"
 tokio = { version = "1.17", features = ["macros", "rt", "rt-multi-thread"] }
-tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
 url = "2.2.2"
 workspace_hack = { version = "0.1", path = "../workspace_hack" }

control_plane/Cargo.toml

@@ -10,7 +10,7 @@ comfy-table = "6.1"
 git-version = "0.3.5"
 nix = "0.25"
 once_cell = "1.13.0"
-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev = "d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev = "43e6db254a97fdecbce33d8bc0890accfd74495e" }
 regex = "1"
 reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "rustls-tls"] }
 serde = { version = "1.0", features = ["derive"] }
@@ -25,5 +25,7 @@ url = "2.2.2"
 pageserver_api = { path = "../libs/pageserver_api" }
 postgres_connection = { path = "../libs/postgres_connection" }
 safekeeper_api = { path = "../libs/safekeeper_api" }
+# Note: main broker code is inside the binary crate, so linking with the library shouldn't be heavy.
+storage_broker = { version = "0.1", path = "../storage_broker" }
 utils = { path = "../libs/utils" }
 workspace_hack = { version = "0.1", path = "../workspace_hack" }

control_plane/simple.conf

@@ -10,5 +10,5 @@ id = 1
 pg_port = 5454
 http_port = 7676
 
-[etcd_broker]
-broker_endpoints = ['http://127.0.0.1:2379']
+[broker]
+listen_addr = '127.0.0.1:50051'

control_plane/src/background_process.rs

@@ -14,17 +14,19 @@
 
 use std::ffi::OsStr;
 use std::io::Write;
-use std::path::Path;
+use std::os::unix::prelude::AsRawFd;
+use std::os::unix::process::CommandExt;
+use std::path::{Path, PathBuf};
 use std::process::{Child, Command};
 use std::time::Duration;
 use std::{fs, io, thread};
 
-use anyhow::{anyhow, bail, Context, Result};
+use anyhow::Context;
 use nix::errno::Errno;
+use nix::fcntl::{FcntlArg, FdFlag};
 use nix::sys::signal::{kill, Signal};
 use nix::unistd::Pid;
-
-use utils::lock_file;
+use utils::pid_file::{self, PidFileRead};
 
 // These constants control the loop used to poll for process start / stop.
 //
@@ -49,21 +51,21 @@ pub enum InitialPidFile<'t> {
 }
 
 /// Start a background child process using the parameters given.
-pub fn start_process<
-    F,
-    S: AsRef<OsStr>,
-    EI: IntoIterator<Item = (String, String)>, // Not generic AsRef<OsStr>, otherwise empty `envs` prevents type inference
->(
+pub fn start_process<F, AI, A, EI>(
     process_name: &str,
     datadir: &Path,
     command: &Path,
-    args: &[S],
+    args: AI,
     envs: EI,
     initial_pid_file: InitialPidFile,
     process_status_check: F,
 ) -> anyhow::Result<Child>
 where
     F: Fn() -> anyhow::Result<bool>,
+    AI: IntoIterator<Item = A>,
+    A: AsRef<OsStr>,
+    // Not generic AsRef<OsStr>, otherwise empty `envs` prevents type inference
+    EI: IntoIterator<Item = (String, String)>,
 {
     let log_path = datadir.join(format!("{process_name}.log"));
     let process_log_file = fs::OpenOptions::new()
@@ -86,6 +88,14 @@ where
     let filled_cmd = fill_aws_secrets_vars(fill_rust_env_vars(background_command));
     filled_cmd.envs(envs);
 
+    let pid_file_to_check = match initial_pid_file {
+        InitialPidFile::Create(path) => {
+            pre_exec_create_pidfile(filled_cmd, path);
+            path
+        }
+        InitialPidFile::Expect(path) => path,
+    };
+
     let mut spawned_process = filled_cmd.spawn().with_context(|| {
         format!("Could not spawn {process_name}, see console output and log files for details.")
     })?;
@@ -95,29 +105,8 @@ where
             .with_context(|| format!("Subprocess {process_name} has invalid pid {pid}"))?,
     );
 
-    let pid_file_to_check = match initial_pid_file {
-        InitialPidFile::Create(target_pid_file_path) => {
-            match lock_file::create_lock_file(target_pid_file_path, pid.to_string()) {
-                lock_file::LockCreationResult::Created { .. } => {
-                    // We use "lock" file here only to create the pid file. The lock on the pidfile will be dropped as soon
-                    // as this CLI invocation exits, so it's a bit useless, but doesn't any harm either.
-                }
-                lock_file::LockCreationResult::AlreadyLocked { .. } => {
-                    anyhow::bail!("Cannot write pid file for {process_name} at path {target_pid_file_path:?}: file is already locked by another process")
-                }
-                lock_file::LockCreationResult::CreationFailed(e) => {
-                    return Err(e.context(format!(
-                    "Failed to create pid file for {process_name} at path {target_pid_file_path:?}"
-                )))
-                }
-            }
-            None
-        }
-        InitialPidFile::Expect(pid_file_path) => Some(pid_file_path),
-    };
-
     for retries in 0..RETRIES {
-        match process_started(pid, pid_file_to_check, &process_status_check) {
+        match process_started(pid, Some(pid_file_to_check), &process_status_check) {
             Ok(true) => {
                 println!("\n{process_name} started, pid: {pid}");
                 return Ok(spawned_process);
@@ -165,12 +154,27 @@ pub fn send_stop_child_process(child: &std::process::Child) -> anyhow::Result<()
 
 /// Stops the process, using the pid file given. Returns Ok also if the process is already not running.
 pub fn stop_process(immediate: bool, process_name: &str, pid_file: &Path) -> anyhow::Result<()> {
-    if !pid_file.exists() {
-        println!("{process_name} is already stopped: no pid file {pid_file:?} is present");
-        return Ok(());
-    }
-    let pid = read_pidfile(pid_file)?;
+    let pid = match pid_file::read(pid_file)
+        .with_context(|| format!("read pid_file {pid_file:?}"))?
+    {
+        PidFileRead::NotExist => {
+            println!("{process_name} is already stopped: no pid file present at {pid_file:?}");
+            return Ok(());
+        }
+        PidFileRead::NotHeldByAnyProcess(_) => {
+            // Don't try to kill according to file contents beacuse the pid might have been re-used by another process.
+            // Don't delete the file either, it can race with new pid file creation.
+            // Read `pid_file` module comment for details.
+            println!(
+                "No process is holding the pidfile. The process must have already exited. Leave in place to avoid race conditions: {pid_file:?}"
+            );
+            return Ok(());
+        }
+        PidFileRead::LockedByOtherProcess(pid) => pid,
+    };
+    // XXX the pid could become invalid (and recycled) at any time before the kill() below.
 
+    // send signal
     let sig = if immediate {
         print!("Stopping {process_name} with pid {pid} immediately..");
         Signal::SIGQUIT
@@ -182,8 +186,9 @@ pub fn stop_process(immediate: bool, process_name: &str, pid_file: &Path) -> any
     match kill(pid, sig) {
         Ok(()) => (),
         Err(Errno::ESRCH) => {
+            // Again, don't delete the pid file. The unlink can race with a new pid file being created.
             println!(
-                "{process_name} with pid {pid} does not exist, but a pid file {pid_file:?} was found"
+                "{process_name} with pid {pid} does not exist, but a pid file {pid_file:?} was found. Likely the pid got recycled. Lucky we didn't harm anyone."
             );
             return Ok(());
         }
@@ -252,6 +257,69 @@ fn fill_aws_secrets_vars(mut cmd: &mut Command) -> &mut Command {
     cmd
 }
 
+/// Add a `pre_exec` to the cmd that, inbetween fork() and exec(),
+/// 1. Claims a pidfile with a fcntl lock on it and
+/// 2. Sets up the pidfile's file descriptor so that it (and the lock)
+///    will remain held until the cmd exits.
+fn pre_exec_create_pidfile<P>(cmd: &mut Command, path: P) -> &mut Command
+where
+    P: Into<PathBuf>,
+{
+    let path: PathBuf = path.into();
+    // SAFETY
+    // pre_exec is marked unsafe because it runs between fork and exec.
+    // Why is that dangerous in various ways?
+    // Long answer:  https://github.com/rust-lang/rust/issues/39575
+    // Short answer: in a multi-threaded program, other threads may have
+    // been inside of critical sections at the time of fork. In the
+    // original process, that was allright, assuming they protected
+    // the critical sections appropriately, e.g., through locks.
+    // Fork adds another process to the mix that
+    //   1. Has a single thread T
+    //   2. In an exact copy of the address space at the time of fork.
+    // A variety of problems scan occur now:
+    //   1. T tries to grab a lock that was locked at the time of fork.
+    //      It will wait forever since in its address space, the lock
+    //      is in state 'taken' but the thread that would unlock it is
+    //      not there.
+    //   2. A rust object that represented some external resource in the
+    //      parent now got implicitly copied by the the fork, even though
+    //      the object's type is not `Copy`. The parent program may use
+    //      non-copyability as way to enforce unique ownership of an
+    //      external resource in the typesystem. The fork breaks that
+    //      assumption, as now both parent and child process have an
+    //      owned instance of the object that represents the same
+    //      underlying resource.
+    // While these seem like niche problems, (1) in particular is
+    // highly relevant. For example, `malloc()` may grab a mutex internally,
+    // and so, if we forked while another thread was mallocing' and our
+    // pre_exec closure allocates as well, it will block on the malloc
+    // mutex forever
+    //
+    // The proper solution is to only use C library functions that are marked
+    // "async-signal-safe": https://man7.org/linux/man-pages/man7/signal-safety.7.html
+    //
+    // With this specific pre_exec() closure, the non-error path doesn't allocate.
+    // The error path uses `anyhow`, and hence does allocate.
+    // We take our chances there, hoping that any potential disaster is constrained
+    // to the child process (e.g., malloc has no state ourside of the child process).
+    // Last, `expect` prints to stderr, and stdio is not async-signal-safe.
+    // Again, we take our chances, making the same assumptions as for malloc.
+    unsafe {
+        cmd.pre_exec(move || {
+            let file = pid_file::claim_for_current_process(&path).expect("claim pid file");
+            // Remove the FD_CLOEXEC flag on the pidfile descriptor so that the pidfile
+            // remains locked after exec.
+            nix::fcntl::fcntl(file.as_raw_fd(), FcntlArg::F_SETFD(FdFlag::empty()))
+                .expect("remove FD_CLOEXEC");
+            // Don't run drop(file), it would close the file before we actually exec.
+            std::mem::forget(file);
+            Ok(())
+        });
+    }
+    cmd
+}
+
 fn process_started<F>(
     pid: Pid,
     pid_file_to_check: Option<&Path>,
@@ -262,14 +330,11 @@ where
 {
     match status_check() {
         Ok(true) => match pid_file_to_check {
-            Some(pid_file_path) => {
-                if pid_file_path.exists() {
-                    let pid_in_file = read_pidfile(pid_file_path)?;
-                    Ok(pid_in_file == pid)
-                } else {
-                    Ok(false)
-                }
-            }
+            Some(pid_file_path) => match pid_file::read(pid_file_path)? {
+                PidFileRead::NotExist => Ok(false),
+                PidFileRead::LockedByOtherProcess(pid_in_file) => Ok(pid_in_file == pid),
+                PidFileRead::NotHeldByAnyProcess(_) => Ok(false),
+            },
             None => Ok(true),
         },
         Ok(false) => Ok(false),
@@ -277,21 +342,6 @@ where
     }
 }
 
-/// Read a PID file
-///
-/// We expect a file that contains a single integer.
-fn read_pidfile(pidfile: &Path) -> Result<Pid> {
-    let pid_str = fs::read_to_string(pidfile)
-        .with_context(|| format!("failed to read pidfile {pidfile:?}"))?;
-    let pid: i32 = pid_str
-        .parse()
-        .map_err(|_| anyhow!("failed to parse pidfile {pidfile:?}"))?;
-    if pid < 1 {
-        bail!("pidfile {pidfile:?} contained bad value '{pid}'");
-    }
-    Ok(Pid::from_raw(pid))
-}
-
 fn process_has_stopped(pid: Pid) -> anyhow::Result<bool> {
     match kill(pid, None) {
         // Process exists, keep waiting

control_plane/src/bin/neon_local.rs

@@ -8,10 +8,10 @@
 use anyhow::{anyhow, bail, Context, Result};
 use clap::{value_parser, Arg, ArgAction, ArgMatches, Command};
 use control_plane::compute::ComputeControlPlane;
-use control_plane::local_env::{EtcdBroker, LocalEnv};
+use control_plane::local_env::LocalEnv;
 use control_plane::pageserver::PageServerNode;
 use control_plane::safekeeper::SafekeeperNode;
-use control_plane::{etcd, local_env};
+use control_plane::{broker, local_env};
 use pageserver_api::models::TimelineInfo;
 use pageserver_api::{
     DEFAULT_HTTP_LISTEN_ADDR as DEFAULT_PAGESERVER_HTTP_ADDR,
@@ -22,9 +22,10 @@ use safekeeper_api::{
     DEFAULT_PG_LISTEN_PORT as DEFAULT_SAFEKEEPER_PG_PORT,
 };
 use std::collections::{BTreeSet, HashMap};
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::process::exit;
 use std::str::FromStr;
+use storage_broker::DEFAULT_LISTEN_ADDR as DEFAULT_BROKER_ADDR;
 use utils::{
     auth::{Claims, Scope},
     id::{NodeId, TenantId, TenantTimelineId, TimelineId},
@@ -41,13 +42,12 @@ project_git_version!(GIT_VERSION);
 
 const DEFAULT_PG_VERSION: &str = "14";
 
-fn default_conf(etcd_binary_path: &Path) -> String {
+fn default_conf() -> String {
     format!(
         r#"
 # Default built-in configuration, defined in main.rs
-[etcd_broker]
-broker_endpoints = ['http://localhost:2379']
-etcd_binary_path = '{etcd_binary_path}'
+[broker]
+listen_addr = '{DEFAULT_BROKER_ADDR}'
 
 [pageserver]
 id = {DEFAULT_PAGESERVER_ID}
@@ -60,7 +60,6 @@ id = {DEFAULT_SAFEKEEPER_ID}
 pg_port = {DEFAULT_SAFEKEEPER_PG_PORT}
 http_port = {DEFAULT_SAFEKEEPER_HTTP_PORT}
 "#,
-        etcd_binary_path = etcd_binary_path.display(),
         pageserver_auth_type = AuthType::Trust,
     )
 }
@@ -298,7 +297,7 @@ fn handle_init(init_match: &ArgMatches) -> anyhow::Result<LocalEnv> {
         })?
     } else {
         // Built-in default config
-        default_conf(&EtcdBroker::locate_etcd()?)
+        default_conf()
     };
 
     let pg_version = init_match
@@ -324,7 +323,7 @@ fn handle_init(init_match: &ArgMatches) -> anyhow::Result<LocalEnv> {
             pg_version,
         )
         .unwrap_or_else(|e| {
-            eprintln!("pageserver init failed: {e}");
+            eprintln!("pageserver init failed: {e:?}");
             exit(1);
         });
 
@@ -342,7 +341,7 @@ fn pageserver_config_overrides(init_match: &ArgMatches) -> Vec<&str> {
         .get_many::<String>("pageserver-config-override")
         .into_iter()
         .flatten()
-        .map(|s| s.as_str())
+        .map(String::as_str)
         .collect()
 }
 
@@ -807,14 +806,14 @@ fn handle_safekeeper(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul
 }
 
 fn handle_start_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow::Result<()> {
-    etcd::start_etcd_process(env)?;
+    broker::start_broker_process(env)?;
     let pageserver = PageServerNode::from_env(env);
 
     // Postgres nodes are not started automatically
 
     if let Err(e) = pageserver.start(&pageserver_config_overrides(sub_match)) {
         eprintln!("pageserver start failed: {e}");
-        try_stop_etcd_process(env);
+        try_stop_storage_broker_process(env);
         exit(1);
     }
 
@@ -822,7 +821,7 @@ fn handle_start_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow
         let safekeeper = SafekeeperNode::from_env(env, node);
         if let Err(e) = safekeeper.start() {
             eprintln!("safekeeper '{}' start failed: {e}", safekeeper.id);
-            try_stop_etcd_process(env);
+            try_stop_storage_broker_process(env);
             exit(1);
         }
     }
@@ -854,14 +853,14 @@ fn handle_stop_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<
         }
     }
 
-    try_stop_etcd_process(env);
+    try_stop_storage_broker_process(env);
 
     Ok(())
 }
 
-fn try_stop_etcd_process(env: &local_env::LocalEnv) {
-    if let Err(e) = etcd::stop_etcd_process(env) {
-        eprintln!("etcd stop failed: {e}");
+fn try_stop_storage_broker_process(env: &local_env::LocalEnv) {
+    if let Err(e) = broker::stop_broker_process(env) {
+        eprintln!("neon broker stop failed: {e}");
     }
 }
 

control_plane/src/broker.rs

@@ -0,0 +1,48 @@
+use anyhow::Context;
+
+use std::path::PathBuf;
+
+use crate::{background_process, local_env};
+
+pub fn start_broker_process(env: &local_env::LocalEnv) -> anyhow::Result<()> {
+    let broker = &env.broker;
+    let listen_addr = &broker.listen_addr;
+
+    print!("Starting neon broker at {}", listen_addr);
+
+    let args = [format!("--listen-addr={listen_addr}")];
+
+    let client = reqwest::blocking::Client::new();
+    background_process::start_process(
+        "storage_broker",
+        &env.base_data_dir,
+        &env.storage_broker_bin(),
+        &args,
+        [],
+        background_process::InitialPidFile::Create(&storage_broker_pid_file_path(env)),
+        || {
+            let url = broker.client_url();
+            let status_url = url.join("status").with_context(|| {
+                format!("Failed to append /status path to broker endpoint {url}",)
+            })?;
+            let request = client
+                .get(status_url)
+                .build()
+                .with_context(|| format!("Failed to construct request to broker endpoint {url}"))?;
+            match client.execute(request) {
+                Ok(resp) => Ok(resp.status().is_success()),
+                Err(_) => Ok(false),
+            }
+        },
+    )
+    .context("Failed to spawn storage_broker subprocess")?;
+    Ok(())
+}
+
+pub fn stop_broker_process(env: &local_env::LocalEnv) -> anyhow::Result<()> {
+    background_process::stop_process(true, "storage_broker", &storage_broker_pid_file_path(env))
+}
+
+fn storage_broker_pid_file_path(env: &local_env::LocalEnv) -> PathBuf {
+    env.base_data_dir.join("storage_broker.pid")
+}

control_plane/src/etcd.rs

@@ -1,78 +0,0 @@
-use std::{fs, path::PathBuf};
-
-use anyhow::Context;
-
-use crate::{background_process, local_env};
-
-pub fn start_etcd_process(env: &local_env::LocalEnv) -> anyhow::Result<()> {
-    let etcd_broker = &env.etcd_broker;
-    print!(
-        "Starting etcd broker using {:?}",
-        etcd_broker.etcd_binary_path
-    );
-
-    let etcd_data_dir = env.base_data_dir.join("etcd");
-    fs::create_dir_all(&etcd_data_dir)
-        .with_context(|| format!("Failed to create etcd data dir {etcd_data_dir:?}"))?;
-
-    let client_urls = etcd_broker.comma_separated_endpoints();
-    let args = [
-        format!("--data-dir={}", etcd_data_dir.display()),
-        format!("--listen-client-urls={client_urls}"),
-        format!("--advertise-client-urls={client_urls}"),
-        // Set --quota-backend-bytes to keep the etcd virtual memory
-        // size smaller. Our test etcd clusters are very small.
-        // See https://github.com/etcd-io/etcd/issues/7910
-        "--quota-backend-bytes=100000000".to_string(),
-        // etcd doesn't compact (vacuum) with default settings,
-        // enable it to prevent space exhaustion.
-        "--auto-compaction-mode=revision".to_string(),
-        "--auto-compaction-retention=1".to_string(),
-    ];
-
-    let pid_file_path = etcd_pid_file_path(env);
-
-    let client = reqwest::blocking::Client::new();
-
-    background_process::start_process(
-        "etcd",
-        &etcd_data_dir,
-        &etcd_broker.etcd_binary_path,
-        &args,
-        [],
-        background_process::InitialPidFile::Create(&pid_file_path),
-        || {
-            for broker_endpoint in &etcd_broker.broker_endpoints {
-                let request = broker_endpoint
-                    .join("health")
-                    .with_context(|| {
-                        format!(
-                            "Failed to append /health path to broker endopint {}",
-                            broker_endpoint
-                        )
-                    })
-                    .and_then(|url| {
-                        client.get(&url.to_string()).build().with_context(|| {
-                            format!("Failed to construct request to etcd endpoint {url}")
-                        })
-                    })?;
-                if client.execute(request).is_ok() {
-                    return Ok(true);
-                }
-            }
-
-            Ok(false)
-        },
-    )
-    .context("Failed to spawn etcd subprocess")?;
-
-    Ok(())
-}
-
-pub fn stop_etcd_process(env: &local_env::LocalEnv) -> anyhow::Result<()> {
-    background_process::stop_process(true, "etcd", &etcd_pid_file_path(env))
-}
-
-fn etcd_pid_file_path(env: &local_env::LocalEnv) -> PathBuf {
-    env.base_data_dir.join("etcd.pid")
-}

control_plane/src/lib.rs

@@ -8,8 +8,8 @@
 //
 
 mod background_process;
+pub mod broker;
 pub mod compute;
-pub mod etcd;
 pub mod local_env;
 pub mod pageserver;
 pub mod postgresql_conf;

control_plane/src/local_env.rs

@@ -4,12 +4,16 @@
 //! script which will use local paths.
 
 use anyhow::{bail, ensure, Context};
+
 use reqwest::Url;
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
 use std::collections::HashMap;
 use std::env;
 use std::fs;
+use std::net::IpAddr;
+use std::net::Ipv4Addr;
+use std::net::SocketAddr;
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
 use utils::{
@@ -62,7 +66,7 @@ pub struct LocalEnv {
     #[serde(default)]
     pub private_key_path: PathBuf,
 
-    pub etcd_broker: EtcdBroker,
+    pub broker: NeonBroker,
 
     pub pageserver: PageServerConf,
 
@@ -78,67 +82,26 @@ pub struct LocalEnv {
     branch_name_mappings: HashMap<String, Vec<(TenantId, TimelineId)>>,
 }
 
-/// Etcd broker config for cluster internal communication.
-#[serde_as]
+/// Broker config for cluster internal communication.
 #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)]
-pub struct EtcdBroker {
-    /// A prefix to all to any key when pushing/polling etcd from a node.
-    #[serde(default)]
-    pub broker_etcd_prefix: Option<String>,
-
-    /// Broker (etcd) endpoints for storage nodes coordination, e.g. 'http://127.0.0.1:2379'.
-    #[serde(default)]
-    #[serde_as(as = "Vec<DisplayFromStr>")]
-    pub broker_endpoints: Vec<Url>,
-
-    /// Etcd binary path to use.
-    #[serde(default)]
-    pub etcd_binary_path: PathBuf,
+#[serde(default)]
+pub struct NeonBroker {
+    /// Broker listen address for storage nodes coordination, e.g. '127.0.0.1:50051'.
+    pub listen_addr: SocketAddr,
 }
 
-impl EtcdBroker {
-    pub fn locate_etcd() -> anyhow::Result<PathBuf> {
-        let which_output = Command::new("which")
-            .arg("etcd")
-            .output()
-            .context("Failed to run 'which etcd' command")?;
-        let stdout = String::from_utf8_lossy(&which_output.stdout);
-        ensure!(
-            which_output.status.success(),
-            "'which etcd' invocation failed. Status: {}, stdout: {stdout}, stderr: {}",
-            which_output.status,
-            String::from_utf8_lossy(&which_output.stderr)
-        );
-
-        let etcd_path = PathBuf::from(stdout.trim());
-        ensure!(
-            etcd_path.is_file(),
-            "'which etcd' invocation was successful, but the path it returned is not a file or does not exist: {}",
-            etcd_path.display()
-        );
-
-        Ok(etcd_path)
+// Dummy Default impl to satisfy Deserialize derive.
+impl Default for NeonBroker {
+    fn default() -> Self {
+        NeonBroker {
+            listen_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0),
+        }
     }
+}
 
-    pub fn comma_separated_endpoints(&self) -> String {
-        self.broker_endpoints
-            .iter()
-            .map(|url| {
-                // URL by default adds a '/' path at the end, which is not what etcd CLI wants.
-                let url_string = url.as_str();
-                if url_string.ends_with('/') {
-                    &url_string[0..url_string.len() - 1]
-                } else {
-                    url_string
-                }
-            })
-            .fold(String::new(), |mut comma_separated_urls, url| {
-                if !comma_separated_urls.is_empty() {
-                    comma_separated_urls.push(',');
-                }
-                comma_separated_urls.push_str(url);
-                comma_separated_urls
-            })
+impl NeonBroker {
+    pub fn client_url(&self) -> Url {
+        Url::parse(&format!("http://{}", self.listen_addr)).expect("failed to construct url")
     }
 }
 
@@ -234,6 +197,10 @@ impl LocalEnv {
         self.neon_distrib_dir.join("safekeeper")
     }
 
+    pub fn storage_broker_bin(&self) -> PathBuf {
+        self.neon_distrib_dir.join("storage_broker")
+    }
+
     pub fn pg_data_dirs_path(&self) -> PathBuf {
         self.base_data_dir.join("pgdatadirs").join("tenants")
     }
@@ -511,8 +478,8 @@ mod tests {
             "failed to parse simple config {simple_conf_toml}, reason: {simple_conf_parse_result:?}"
         );
 
-        let string_to_replace = "broker_endpoints = ['http://127.0.0.1:2379']";
-        let spoiled_url_str = "broker_endpoints = ['!@$XOXO%^&']";
+        let string_to_replace = "listen_addr = '127.0.0.1:50051'";
+        let spoiled_url_str = "listen_addr = '!@$XOXO%^&'";
         let spoiled_url_toml = simple_conf_toml.replace(string_to_replace, spoiled_url_str);
         assert!(
             spoiled_url_toml.contains(spoiled_url_str),

control_plane/src/pageserver.rs

@@ -1,9 +1,10 @@
+use std::borrow::Cow;
 use std::collections::HashMap;
 use std::fs::File;
 use std::io::{BufReader, Write};
 use std::num::NonZeroU64;
-use std::path::{Path, PathBuf};
-use std::process::Child;
+use std::path::PathBuf;
+use std::process::{Child, Command};
 use std::{io, result};
 
 use anyhow::{bail, ensure, Context};
@@ -96,13 +97,8 @@ impl PageServerNode {
         }
     }
 
-    pub fn initialize(
-        &self,
-        create_tenant: Option<TenantId>,
-        initial_timeline_id: Option<TimelineId>,
-        config_overrides: &[&str],
-        pg_version: u32,
-    ) -> anyhow::Result<TimelineId> {
+    // pageserver conf overrides defined by neon_local configuration.
+    fn neon_local_overrides(&self) -> Vec<String> {
         let id = format!("id={}", self.env.pageserver.id);
         // FIXME: the paths should be shell-escaped to handle paths with spaces, quotas etc.
         let pg_distrib_dir_param = format!(
@@ -117,44 +113,54 @@ impl PageServerNode {
         );
         let listen_pg_addr_param =
             format!("listen_pg_addr='{}'", self.env.pageserver.listen_pg_addr);
-        let broker_endpoints_param = format!(
-            "broker_endpoints=[{}]",
-            self.env
-                .etcd_broker
-                .broker_endpoints
-                .iter()
-                .map(|url| format!("'{url}'"))
-                .collect::<Vec<_>>()
-                .join(",")
-        );
-        let broker_etcd_prefix_param = self
-            .env
-            .etcd_broker
-            .broker_etcd_prefix
-            .as_ref()
-            .map(|prefix| format!("broker_etcd_prefix='{prefix}'"));
+        let broker_endpoint_param = format!("broker_endpoint='{}'", self.env.broker.client_url());
 
-        let mut init_config_overrides = config_overrides.to_vec();
-        init_config_overrides.push(&id);
-        init_config_overrides.push(&pg_distrib_dir_param);
-        init_config_overrides.push(&authg_type_param);
-        init_config_overrides.push(&listen_http_addr_param);
-        init_config_overrides.push(&listen_pg_addr_param);
-        init_config_overrides.push(&broker_endpoints_param);
-
-        if let Some(broker_etcd_prefix_param) = broker_etcd_prefix_param.as_deref() {
-            init_config_overrides.push(broker_etcd_prefix_param);
-        }
+        let mut overrides = vec![
+            id,
+            pg_distrib_dir_param,
+            authg_type_param,
+            listen_http_addr_param,
+            listen_pg_addr_param,
+            broker_endpoint_param,
+        ];
 
         if self.env.pageserver.auth_type != AuthType::Trust {
-            init_config_overrides.push("auth_validation_public_key_path='auth_public_key.pem'");
+            overrides.push("auth_validation_public_key_path='auth_public_key.pem'".to_owned());
         }
+        overrides
+    }
 
+    /// Initializes a pageserver node by creating its config with the overrides provided,
+    /// and creating an initial tenant and timeline afterwards.
+    pub fn initialize(
+        &self,
+        create_tenant: Option<TenantId>,
+        initial_timeline_id: Option<TimelineId>,
+        config_overrides: &[&str],
+        pg_version: u32,
+    ) -> anyhow::Result<TimelineId> {
+        // First, run `pageserver --init` and wait for it to write a config into FS and exit.
+        self.pageserver_init(config_overrides).with_context(|| {
+            format!(
+                "Failed to run init for pageserver node {}",
+                self.env.pageserver.id,
+            )
+        })?;
+
+        // Then, briefly start it fully to run HTTP commands on it,
+        // to create initial tenant and timeline.
+        // We disable the remote storage, since we stop pageserver right after the timeline creation,
+        // hence most of the uploads will either aborted or not started: no point to start them at all.
+        let disabled_remote_storage_override = "remote_storage={}";
         let mut pageserver_process = self
-            .start_node(&init_config_overrides, &self.env.base_data_dir, true)
+            .start_node(
+                &[disabled_remote_storage_override],
+                // Previous overrides will be taken from the config created before, don't overwrite them.
+                false,
+            )
             .with_context(|| {
                 format!(
-                    "Failed to start a process for pageserver {}",
+                    "Failed to start a process for pageserver node {}",
                     self.env.pageserver.id,
                 )
             })?;
@@ -215,52 +221,73 @@ impl PageServerNode {
     }
 
     pub fn start(&self, config_overrides: &[&str]) -> anyhow::Result<Child> {
-        self.start_node(config_overrides, &self.repo_path(), false)
+        self.start_node(config_overrides, false)
     }
 
-    fn start_node(
-        &self,
-        config_overrides: &[&str],
-        datadir: &Path,
-        update_config: bool,
-    ) -> anyhow::Result<Child> {
-        print!(
-            "Starting pageserver at '{}' in '{}'",
+    fn pageserver_init(&self, config_overrides: &[&str]) -> anyhow::Result<()> {
+        let datadir = self.repo_path();
+        let node_id = self.env.pageserver.id;
+        println!(
+            "Initializing pageserver node {} at '{}' in {:?}",
+            node_id,
             self.pg_connection_config.raw_address(),
-            datadir.display()
+            datadir
         );
         io::stdout().flush()?;
 
-        let mut args = vec![
-            "-D",
-            datadir.to_str().with_context(|| {
-                format!("Datadir path {datadir:?} cannot be represented as a unicode string")
-            })?,
-        ];
+        let datadir_path_str = datadir.to_str().with_context(|| {
+            format!("Cannot start pageserver node {node_id} in path that has no string representation: {datadir:?}")
+        })?;
+        let mut args = self.pageserver_basic_args(config_overrides, datadir_path_str);
+        args.push(Cow::Borrowed("--init"));
 
+        let init_output = Command::new(&self.env.pageserver_bin())
+            .args(args.iter().map(Cow::as_ref))
+            .envs(self.pageserver_env_variables()?)
+            .output()
+            .with_context(|| format!("Failed to run pageserver init for node {node_id}"))?;
+
+        anyhow::ensure!(
+            init_output.status.success(),
+            "Pageserver init for node {} did not finish successfully, stdout: {}, stderr: {}",
+            node_id,
+            String::from_utf8_lossy(&init_output.stdout),
+            String::from_utf8_lossy(&init_output.stderr),
+        );
+
+        Ok(())
+    }
+
+    fn start_node(&self, config_overrides: &[&str], update_config: bool) -> anyhow::Result<Child> {
+        let mut overrides = self.neon_local_overrides();
+        overrides.extend(config_overrides.iter().map(|&c| c.to_owned()));
+
+        let datadir = self.repo_path();
+        print!(
+            "Starting pageserver node {} at '{}' in {:?}",
+            self.env.pageserver.id,
+            self.pg_connection_config.raw_address(),
+            datadir
+        );
+        io::stdout().flush()?;
+
+        let datadir_path_str = datadir.to_str().with_context(|| {
+            format!(
+                "Cannot start pageserver node {} in path that has no string representation: {:?}",
+                self.env.pageserver.id, datadir,
+            )
+        })?;
+        let mut args = self.pageserver_basic_args(config_overrides, datadir_path_str);
         if update_config {
-            args.push("--update-config");
+            args.push(Cow::Borrowed("--update-config"));
         }
 
-        for config_override in config_overrides {
-            args.extend(["-c", config_override]);
-        }
-
-        let envs = if self.env.pageserver.auth_type != AuthType::Trust {
-            // Generate a token to connect from the pageserver to a safekeeper
-            let token = self
-                .env
-                .generate_auth_token(&Claims::new(None, Scope::SafekeeperData))?;
-            vec![("ZENITH_AUTH_TOKEN".to_owned(), token)]
-        } else {
-            vec![]
-        };
         background_process::start_process(
             "pageserver",
-            datadir,
+            &datadir,
             &self.env.pageserver_bin(),
-            &args,
-            envs,
+            args.iter().map(Cow::as_ref),
+            self.pageserver_env_variables()?,
             background_process::InitialPidFile::Expect(&self.pid_file()),
             || match self.check_status() {
                 Ok(()) => Ok(true),
@@ -270,6 +297,35 @@ impl PageServerNode {
         )
     }
 
+    fn pageserver_basic_args<'a>(
+        &self,
+        config_overrides: &'a [&'a str],
+        datadir_path_str: &'a str,
+    ) -> Vec<Cow<'a, str>> {
+        let mut args = vec![Cow::Borrowed("-D"), Cow::Borrowed(datadir_path_str)];
+
+        let mut overrides = self.neon_local_overrides();
+        overrides.extend(config_overrides.iter().map(|&c| c.to_owned()));
+        for config_override in overrides {
+            args.push(Cow::Borrowed("-c"));
+            args.push(Cow::Owned(config_override));
+        }
+
+        args
+    }
+
+    fn pageserver_env_variables(&self) -> anyhow::Result<Vec<(String, String)>> {
+        Ok(if self.env.pageserver.auth_type != AuthType::Trust {
+            // Generate a token to connect from the pageserver to a safekeeper
+            let token = self
+                .env
+                .generate_auth_token(&Claims::new(None, Scope::SafekeeperData))?;
+            vec![("ZENITH_AUTH_TOKEN".to_owned(), token)]
+        } else {
+            Vec::new()
+        })
+    }
+
     ///
     /// Stop the server.
     ///

control_plane/src/safekeeper.rs

@@ -131,13 +131,8 @@ impl SafekeeperNode {
             args.push("--no-sync");
         }
 
-        let comma_separated_endpoints = self.env.etcd_broker.comma_separated_endpoints();
-        if !comma_separated_endpoints.is_empty() {
-            args.extend(["--broker-endpoints", &comma_separated_endpoints]);
-        }
-        if let Some(prefix) = self.env.etcd_broker.broker_etcd_prefix.as_deref() {
-            args.extend(["--broker-etcd-prefix", prefix]);
-        }
+        let broker_endpoint = format!("{}", self.env.broker.client_url());
+        args.extend(["--broker-endpoint", &broker_endpoint]);
 
         let mut backup_threads = String::new();
         if let Some(threads) = self.conf.backup_threads {

docker-compose/docker-compose.yml

@@ -1,29 +1,6 @@
 version: '3'
 
 services:
-  etcd:
-    restart: always
-    image: quay.io/coreos/etcd:v3.5.4
-    ports:
-      - 2379:2379
-      - 2380:2380
-    environment:
-      # This signifficantly speeds up etcd and we anyway don't data persistency there.
-      ETCD_UNSAFE_NO_FSYNC: "1"
-    command:
-      - "etcd"
-      - "--auto-compaction-mode=revision"
-      - "--auto-compaction-retention=1"
-      - "--name=etcd-cluster"
-      - "--initial-cluster-state=new"
-      - "--initial-cluster-token=etcd-cluster-1"
-      - "--initial-cluster=etcd-cluster=http://etcd:2380"
-      - "--initial-advertise-peer-urls=http://etcd:2380"
-      - "--advertise-client-urls=http://etcd:2379"
-      - "--listen-client-urls=http://0.0.0.0:2379"
-      - "--listen-peer-urls=http://0.0.0.0:2380"
-      - "--quota-backend-bytes=134217728" # 128 MB
-
   minio:
     restart: always
     image: quay.io/minio/minio:RELEASE.2022-10-20T00-55-09Z
@@ -56,7 +33,7 @@ services:
     restart: always
     image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
     environment:
-      - BROKER_ENDPOINT='http://etcd:2379'
+      - BROKER_ENDPOINT='http://storage_broker:50051'
       - AWS_ACCESS_KEY_ID=minio
       - AWS_SECRET_ACCESS_KEY=password
       #- RUST_BACKTRACE=1
@@ -68,7 +45,7 @@ services:
       - "-c"
     command:
       - "/usr/local/bin/pageserver -D /data/.neon/
-                                   -c \"broker_endpoints=[$$BROKER_ENDPOINT]\"
+                                   -c \"broker_endpoint=$$BROKER_ENDPOINT\"
                                    -c \"listen_pg_addr='0.0.0.0:6400'\"
                                    -c \"listen_http_addr='0.0.0.0:9898'\"
                                    -c \"remote_storage={endpoint='http://minio:9000',
@@ -76,7 +53,7 @@ services:
                                                         bucket_region='eu-north-1',
                                                         prefix_in_bucket='/pageserver/'}\""
     depends_on:
-      - etcd
+      - storage_broker
       - minio_create_buckets
 
   safekeeper1:
@@ -85,7 +62,7 @@ services:
     environment:
       - SAFEKEEPER_ADVERTISE_URL=safekeeper1:5454
       - SAFEKEEPER_ID=1
-      - BROKER_ENDPOINT=http://etcd:2379
+      - BROKER_ENDPOINT=http://storage_broker:50051
       - AWS_ACCESS_KEY_ID=minio
       - AWS_SECRET_ACCESS_KEY=password
       #- RUST_BACKTRACE=1
@@ -99,14 +76,14 @@ services:
       - "safekeeper --listen-pg=$$SAFEKEEPER_ADVERTISE_URL
                     --listen-http='0.0.0.0:7676'
                     --id=$$SAFEKEEPER_ID
-                    --broker-endpoints=$$BROKER_ENDPOINT
+                    --broker-endpoint=$$BROKER_ENDPOINT
                     -D /data
                     --remote-storage=\"{endpoint='http://minio:9000',
                                         bucket_name='neon',
                                         bucket_region='eu-north-1',
                                         prefix_in_bucket='/safekeeper/'}\""
     depends_on:
-      - etcd
+      - storage_broker
       - minio_create_buckets
 
   safekeeper2:
@@ -115,7 +92,7 @@ services:
     environment:
       - SAFEKEEPER_ADVERTISE_URL=safekeeper2:5454
       - SAFEKEEPER_ID=2
-      - BROKER_ENDPOINT=http://etcd:2379
+      - BROKER_ENDPOINT=http://storage_broker:50051
       - AWS_ACCESS_KEY_ID=minio
       - AWS_SECRET_ACCESS_KEY=password
       #- RUST_BACKTRACE=1
@@ -129,14 +106,14 @@ services:
       - "safekeeper --listen-pg=$$SAFEKEEPER_ADVERTISE_URL
                     --listen-http='0.0.0.0:7676'
                     --id=$$SAFEKEEPER_ID
-                    --broker-endpoints=$$BROKER_ENDPOINT
+                    --broker-endpoint=$$BROKER_ENDPOINT
                     -D /data
                     --remote-storage=\"{endpoint='http://minio:9000',
                                         bucket_name='neon',
                                         bucket_region='eu-north-1',
                                         prefix_in_bucket='/safekeeper/'}\""
     depends_on:
-      - etcd
+      - storage_broker
       - minio_create_buckets
 
   safekeeper3:
@@ -145,7 +122,7 @@ services:
     environment:
       - SAFEKEEPER_ADVERTISE_URL=safekeeper3:5454
       - SAFEKEEPER_ID=3
-      - BROKER_ENDPOINT=http://etcd:2379
+      - BROKER_ENDPOINT=http://storage_broker:50051
       - AWS_ACCESS_KEY_ID=minio
       - AWS_SECRET_ACCESS_KEY=password
       #- RUST_BACKTRACE=1
@@ -159,16 +136,25 @@ services:
       - "safekeeper --listen-pg=$$SAFEKEEPER_ADVERTISE_URL
                     --listen-http='0.0.0.0:7676'
                     --id=$$SAFEKEEPER_ID
-                    --broker-endpoints=$$BROKER_ENDPOINT
+                    --broker-endpoint=$$BROKER_ENDPOINT
                     -D /data
                     --remote-storage=\"{endpoint='http://minio:9000',
                                         bucket_name='neon',
                                         bucket_region='eu-north-1',
                                         prefix_in_bucket='/safekeeper/'}\""
     depends_on:
-      - etcd
+      - storage_broker
       - minio_create_buckets
 
+  storage_broker:
+    restart: always
+    image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
+    ports:
+      - 50051:50051
+    command:
+      - "storage_broker"
+      - "--listen-addr=0.0.0.0:50051"
+
   compute:
     restart: always
     build:

docs/authentication.md

@@ -2,7 +2,7 @@
 
 ### Overview
 We use JWT tokens in communication between almost all components (compute, pageserver, safekeeper, CLI) regardless of the protocol used (HTTP/PostgreSQL).
-Etcd currently has no authentication.
+storage_broker currently has no authentication.
 Authentication is optional and is disabled by default for easier debugging.
 It is used in some tests, though.
 Note that we do not cover authentication with `pg.neon.tech` here.
@@ -84,7 +84,7 @@ the scope is the tenant and the token is usually passed through the
 Pageserver keeps track of multiple tenants, each having multiple timelines.
 For each timeline, it connects to the corresponding Safekeeper.
 Information about "corresponding Safekeeper" is published by Safekeepers
-in the Etcd, but they do not publish access tokens, otherwise what is
+in the storage_broker, but they do not publish access tokens, otherwise what is
 the point of authentication.
 
 Pageserver keeps a connection to some set of Safekeepers, which

docs/docker.md

@@ -23,9 +23,9 @@ We build all images after a successful `release` tests run and push automaticall
 
 You can see a [docker compose](https://docs.docker.com/compose/) example to create a neon cluster in [/docker-compose/docker-compose.yml](/docker-compose/docker-compose.yml). It creates the following conatainers.
 
-- etcd x 1
 - pageserver x 1
 - safekeeper x 3
+- storage_broker x 1
 - compute x 1
 - MinIO x 1        # This is Amazon S3 compatible object storage
 
@@ -41,7 +41,7 @@ $ cd docker-compose/docker-compose.yml
 $ docker-compose down   # remove the conainers if exists
 $ PG_VERSION=15 TAG=2221 docker-compose up --build -d  # You can specify the postgres and image version
 Creating network "dockercompose_default" with the default driver
-Creating dockercompose_etcd3_1 ...
+Creating docker-compose_storage_broker_1       ... done
 (...omit...)
 ```
 

docs/settings.md

@@ -10,7 +10,6 @@ the values in the config file, if any are specified for the same key and get int
 
 ```toml
 # Initial configuration file created by 'pageserver --init'
-
 listen_pg_addr = '127.0.0.1:64000'
 listen_http_addr = '127.0.0.1:9898'
 
@@ -25,13 +24,12 @@ max_file_descriptors = '100'
 # initial superuser role name to use when creating a new tenant
 initial_superuser_name = 'cloud_admin'
 
-broker_etcd_prefix = 'neon'
-broker_endpoints = ['some://etcd']
+broker_endpoint = 'http://127.0.0.1:50051'
 
 # [remote_storage]
 ```
 
-The config above shows default values for all basic pageserver settings, besides `broker_endpoints`: that one has to be set by the user,
+The config above shows default values for all basic pageserver settings, besides `broker_endpoint`: that one has to be set by the user,
 see the corresponding section below.
 Pageserver uses default values for all files that are missing in the config, so it's not a hard error to leave the config blank.
 Yet, it validates the config values it can (e.g. postgres install dir) and errors if the validation fails, refusing to start.
@@ -50,16 +48,10 @@ Example: `${PAGESERVER_BIN} -c "checkpoint_timeout = '10 m'" -c "remote_storage=
 
 Note that TOML distinguishes between strings and integers, the former require single or double quotes around them.
 
-#### broker_endpoints
+#### broker_endpoint
 
-A list of endpoints (etcd currently) to connect and pull the information from.
-Mandatory, does not have a default, since requires etcd to be started as a separate process,
-and its connection url should be specified separately.
-
-#### broker_etcd_prefix
-
-A prefix to add for every etcd key used, to separate one group of related instances from another, in the same cluster.
-Default is `neon`.
+A storage broker endpoint to connect and pull the information from. Default is
+`'http://127.0.0.1:50051'`. 
 
 #### checkpoint_distance
 

docs/sourcetree.md

@@ -45,9 +45,9 @@ and create new databases and accounts (control plane API in our case).
 
 Integration tests, written in Python using the `pytest` framework.
 
-`/vendor/postgres-v14`:
+`/vendor/postgres-v14` and `/vendor/postgres-v15`:
 
-PostgreSQL source tree, with the modifications needed for Neon.
+PostgreSQL source tree per version, with the modifications needed for Neon.
 
 `/pgxn/neon`:
 

libs/etcd_broker/Cargo.toml

@@ -1,18 +0,0 @@
-[package]
- name = "etcd_broker"
- version = "0.1.0"
- edition = "2021"
-
- [dependencies]
- etcd-client = "0.9.0"
- regex = "1.4.5"
- serde = { version = "1.0", features = ["derive"] }
- serde_json = "1"
- serde_with = "2.0"
- once_cell = "1.13.0"
-
- utils = { path = "../utils" }
- workspace_hack = { version = "0.1", path = "../../workspace_hack" }
- tokio = "1"
- tracing = "0.1"
- thiserror = "1"

libs/etcd_broker/src/lib.rs

@@ -1,209 +0,0 @@
-//! A set of primitives to access a shared data/updates, propagated via etcd broker (not persistent).
-//! Intended to connect services to each other, not to store their data.
-
-/// All broker keys, that are used when dealing with etcd.
-pub mod subscription_key;
-/// All broker values, possible to use when dealing with etcd.
-pub mod subscription_value;
-
-use std::str::FromStr;
-
-use serde::de::DeserializeOwned;
-
-use subscription_key::SubscriptionKey;
-use tokio::{sync::mpsc, task::JoinHandle};
-use tracing::*;
-
-use crate::subscription_key::SubscriptionFullKey;
-
-pub use etcd_client::*;
-
-/// Default value to use for prefixing to all etcd keys with.
-/// This way allows isolating safekeeper/pageserver groups in the same etcd cluster.
-pub const DEFAULT_NEON_BROKER_ETCD_PREFIX: &str = "neon";
-
-/// A way to control the data retrieval from a certain subscription.
-pub struct BrokerSubscription<V> {
-    /// An unbounded channel to fetch the relevant etcd updates from.
-    pub value_updates: mpsc::UnboundedReceiver<BrokerUpdate<V>>,
-    key: SubscriptionKey,
-    /// A subscription task handle, to allow waiting on it for the task to complete.
-    /// Both the updates channel and the handle require `&mut`, so it's better to keep
-    /// both `pub` to allow using both in the same structures without borrow checker complaining.
-    pub watcher_handle: JoinHandle<Result<(), BrokerError>>,
-    watcher: Watcher,
-}
-
-impl<V> BrokerSubscription<V> {
-    /// Cancels the subscription, stopping the data poller and waiting for it to shut down.
-    pub async fn cancel(mut self) -> Result<(), BrokerError> {
-        self.watcher.cancel().await.map_err(|e| {
-            BrokerError::EtcdClient(
-                e,
-                format!("Failed to cancel broker subscription, kind: {:?}", self.key),
-            )
-        })?;
-        match (&mut self.watcher_handle).await {
-            Ok(res) => res,
-            Err(e) => {
-                if e.is_cancelled() {
-                    // don't error on the tasks that are cancelled already
-                    Ok(())
-                } else {
-                    Err(BrokerError::InternalError(format!(
-                        "Panicked during broker subscription task, kind: {:?}, error: {e}",
-                        self.key
-                    )))
-                }
-            }
-        }
-    }
-}
-
-impl<V> Drop for BrokerSubscription<V> {
-    fn drop(&mut self) {
-        // we poll data from etcd into the channel in the same struct, so if the whole struct gets dropped,
-        // no more data is used by the receiver and it's safe to cancel and drop the whole etcd subscription task.
-        self.watcher_handle.abort();
-    }
-}
-
-/// An update from the etcd broker.
-pub struct BrokerUpdate<V> {
-    /// Etcd generation version, the bigger the more actual the data is.
-    pub etcd_version: i64,
-    /// Etcd key for the corresponding value, parsed from the broker KV.
-    pub key: SubscriptionFullKey,
-    /// Current etcd value, parsed from the broker KV.
-    pub value: V,
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum BrokerError {
-    #[error("Etcd client error: {0}. Context: {1}")]
-    EtcdClient(etcd_client::Error, String),
-    #[error("Error during parsing etcd key: {0}")]
-    KeyNotParsed(String),
-    #[error("Internal error: {0}")]
-    InternalError(String),
-}
-
-/// Creates a background task to poll etcd for timeline updates from safekeepers.
-/// Stops and returns `Err` on any error during etcd communication.
-/// Watches the key changes until either the watcher is cancelled via etcd or the subscription cancellation handle,
-/// exiting normally in such cases.
-/// Etcd values are parsed as json fukes into a type, specified in the generic patameter.
-pub async fn subscribe_for_json_values<V>(
-    client: &mut Client,
-    key: SubscriptionKey,
-) -> Result<BrokerSubscription<V>, BrokerError>
-where
-    V: DeserializeOwned + Send + 'static,
-{
-    subscribe_for_values(client, key, |_, value_str| {
-        match serde_json::from_str::<V>(value_str) {
-            Ok(value) => Some(value),
-            Err(e) => {
-                error!("Failed to parse value str '{value_str}': {e}");
-                None
-            }
-        }
-    })
-    .await
-}
-
-/// Same as [`subscribe_for_json_values`], but allows to specify a custom parser of a etcd value string.
-pub async fn subscribe_for_values<P, V>(
-    client: &mut Client,
-    key: SubscriptionKey,
-    value_parser: P,
-) -> Result<BrokerSubscription<V>, BrokerError>
-where
-    V: Send + 'static,
-    P: Fn(SubscriptionFullKey, &str) -> Option<V> + Send + 'static,
-{
-    info!("Subscribing to broker value updates, key: {key:?}");
-    let subscription_key = key.clone();
-
-    let (watcher, mut stream) = client
-        .watch(key.watch_key(), Some(WatchOptions::new().with_prefix()))
-        .await
-        .map_err(|e| {
-            BrokerError::EtcdClient(
-                e,
-                format!("Failed to init the watch for subscription {key:?}"),
-            )
-        })?;
-
-    let (value_updates_sender, value_updates_receiver) = mpsc::unbounded_channel();
-    let watcher_handle = tokio::spawn(async move {
-        while let Some(resp) = stream.message().await.map_err(|e| BrokerError::InternalError(format!(
-            "Failed to get messages from the subscription stream, kind: {:?}, error: {e}", key.kind
-        )))? {
-            if resp.canceled() {
-                info!("Watch for timeline updates subscription was canceled, exiting");
-                break;
-            }
-
-            let events = resp.events();
-            debug!("Processing {} events", events.len());
-
-            for event in events {
-                if EventType::Put == event.event_type() {
-                    if let Some(new_etcd_kv) = event.kv() {
-                        match parse_etcd_kv(new_etcd_kv, &value_parser, &key.cluster_prefix) {
-                            Ok(Some((key, value))) => if let Err(e) = value_updates_sender.send(BrokerUpdate {
-                                etcd_version: new_etcd_kv.version(),
-                                key,
-                                value,
-                            }) {
-                                info!("Broker value updates for key {key:?} sender got dropped, exiting: {e}");
-                                break;
-                            },
-                            Ok(None) => debug!("Ignoring key {key:?} : no value was returned by the parser"),
-                            Err(BrokerError::KeyNotParsed(e)) => debug!("Unexpected key {key:?} for timeline update: {e}"),
-                            Err(e) => error!("Failed to represent etcd KV {new_etcd_kv:?}: {e}"),
-                        };
-                    }
-                }
-            }
-        }
-
-        Ok(())
-    }.instrument(info_span!("etcd_broker")));
-
-    Ok(BrokerSubscription {
-        key: subscription_key,
-        value_updates: value_updates_receiver,
-        watcher_handle,
-        watcher,
-    })
-}
-
-fn parse_etcd_kv<P, V>(
-    kv: &KeyValue,
-    value_parser: &P,
-    cluster_prefix: &str,
-) -> Result<Option<(SubscriptionFullKey, V)>, BrokerError>
-where
-    P: Fn(SubscriptionFullKey, &str) -> Option<V>,
-{
-    let key_str = kv.key_str().map_err(|e| {
-        BrokerError::EtcdClient(e, "Failed to extract key str out of etcd KV".to_string())
-    })?;
-    let value_str = kv.value_str().map_err(|e| {
-        BrokerError::EtcdClient(e, "Failed to extract value str out of etcd KV".to_string())
-    })?;
-
-    if !key_str.starts_with(cluster_prefix) {
-        return Err(BrokerError::KeyNotParsed(format!(
-            "KV has unexpected key '{key_str}' that does not start with cluster prefix {cluster_prefix}"
-        )));
-    }
-
-    let key = SubscriptionFullKey::from_str(&key_str[cluster_prefix.len()..]).map_err(|e| {
-        BrokerError::KeyNotParsed(format!("Failed to parse KV key '{key_str}': {e}"))
-    })?;
-
-    Ok(value_parser(key, value_str).map(|value| (key, value)))
-}

libs/etcd_broker/src/subscription_key.rs

@@ -1,310 +0,0 @@
-//! Etcd broker keys, used in the project and shared between instances.
-//! The keys are split into two categories:
-//!
-//! * [`SubscriptionFullKey`] full key format: `<cluster_prefix>/<tenant>/<timeline>/<node_kind>/<operation>/<node_id>`
-//! Always returned from etcd in this form, always start with the user key provided.
-//!
-//! * [`SubscriptionKey`] user input key format: always partial, since it's unknown which `node_id`'s are available.
-//! Full key always starts with the user input one, due to etcd subscription properties.
-
-use std::{fmt::Display, str::FromStr};
-
-use once_cell::sync::Lazy;
-use regex::{Captures, Regex};
-use utils::id::{NodeId, TenantId, TenantTimelineId};
-
-/// The subscription kind to the timeline updates from safekeeper.
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct SubscriptionKey {
-    /// Generic cluster prefix, allowing to use the same etcd instance by multiple logic groups.
-    pub cluster_prefix: String,
-    /// The subscription kind.
-    pub kind: SubscriptionKind,
-}
-
-/// All currently possible key kinds of a etcd broker subscription.
-/// Etcd works so, that every key that starts with the subbscription key given is considered matching and
-/// returned as part of the subscrption.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum SubscriptionKind {
-    /// Get every update in etcd.
-    All,
-    /// Get etcd updates for any timeiline of a certain tenant, affected by any operation from any node kind.
-    TenantTimelines(TenantId),
-    /// Get etcd updates for a certain timeline of a tenant, affected by any operation from any node kind.
-    Timeline(TenantTimelineId),
-    /// Get etcd timeline updates, specific to a certain node kind.
-    Node(TenantTimelineId, NodeKind),
-    /// Get etcd timeline updates for a certain operation on specific nodes.
-    Operation(TenantTimelineId, NodeKind, OperationKind),
-}
-
-/// All kinds of nodes, able to write into etcd.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum NodeKind {
-    Safekeeper,
-    Pageserver,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum OperationKind {
-    Safekeeper(SkOperationKind),
-}
-
-/// Current operations, running inside the safekeeper node.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum SkOperationKind {
-    TimelineInfo,
-    WalBackup,
-}
-
-static SUBSCRIPTION_FULL_KEY_REGEX: Lazy<Regex> = Lazy::new(|| {
-    Regex::new("/([[:xdigit:]]+)/([[:xdigit:]]+)/([^/]+)/([^/]+)/([[:digit:]]+)$")
-        .expect("wrong subscription full etcd key regex")
-});
-
-/// Full key, received from etcd during any of the component's work.
-/// No other etcd keys are considered during system's work.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct SubscriptionFullKey {
-    pub id: TenantTimelineId,
-    pub node_kind: NodeKind,
-    pub operation: OperationKind,
-    pub node_id: NodeId,
-}
-
-impl SubscriptionKey {
-    /// Subscribes for all etcd updates.
-    pub fn all(cluster_prefix: String) -> Self {
-        SubscriptionKey {
-            cluster_prefix,
-            kind: SubscriptionKind::All,
-        }
-    }
-
-    /// Subscribes to a given timeline info updates from safekeepers.
-    pub fn sk_timeline_info(cluster_prefix: String, timeline: TenantTimelineId) -> Self {
-        Self {
-            cluster_prefix,
-            kind: SubscriptionKind::Operation(
-                timeline,
-                NodeKind::Safekeeper,
-                OperationKind::Safekeeper(SkOperationKind::TimelineInfo),
-            ),
-        }
-    }
-
-    /// Subscribes to all timeine updates during specific operations, running on the corresponding nodes.
-    pub fn operation(
-        cluster_prefix: String,
-        timeline: TenantTimelineId,
-        node_kind: NodeKind,
-        operation: OperationKind,
-    ) -> Self {
-        Self {
-            cluster_prefix,
-            kind: SubscriptionKind::Operation(timeline, node_kind, operation),
-        }
-    }
-
-    /// Etcd key to use for watching a certain timeline updates from safekeepers.
-    pub fn watch_key(&self) -> String {
-        let cluster_prefix = &self.cluster_prefix;
-        match self.kind {
-            SubscriptionKind::All => cluster_prefix.to_string(),
-            SubscriptionKind::TenantTimelines(tenant_id) => {
-                format!("{cluster_prefix}/{tenant_id}")
-            }
-            SubscriptionKind::Timeline(id) => {
-                format!("{cluster_prefix}/{id}")
-            }
-            SubscriptionKind::Node(id, node_kind) => {
-                format!("{cluster_prefix}/{id}/{node_kind}")
-            }
-            SubscriptionKind::Operation(id, node_kind, operation_kind) => {
-                format!("{cluster_prefix}/{id}/{node_kind}/{operation_kind}")
-            }
-        }
-    }
-}
-
-impl Display for OperationKind {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            OperationKind::Safekeeper(o) => o.fmt(f),
-        }
-    }
-}
-
-impl FromStr for OperationKind {
-    type Err = String;
-
-    fn from_str(operation_kind_str: &str) -> Result<Self, Self::Err> {
-        match operation_kind_str {
-            "timeline_info" => Ok(OperationKind::Safekeeper(SkOperationKind::TimelineInfo)),
-            "wal_backup" => Ok(OperationKind::Safekeeper(SkOperationKind::WalBackup)),
-            _ => Err(format!("Unknown operation kind: {operation_kind_str}")),
-        }
-    }
-}
-
-impl Display for SubscriptionFullKey {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let Self {
-            id,
-            node_kind,
-            operation,
-            node_id,
-        } = self;
-        write!(f, "{id}/{node_kind}/{operation}/{node_id}")
-    }
-}
-
-impl FromStr for SubscriptionFullKey {
-    type Err = String;
-
-    fn from_str(subscription_kind_str: &str) -> Result<Self, Self::Err> {
-        let key_captures = match SUBSCRIPTION_FULL_KEY_REGEX.captures(subscription_kind_str) {
-            Some(captures) => captures,
-            None => {
-                return Err(format!(
-                    "Subscription kind str does not match a subscription full key regex {}",
-                    SUBSCRIPTION_FULL_KEY_REGEX.as_str()
-                ));
-            }
-        };
-
-        Ok(Self {
-            id: TenantTimelineId::new(
-                parse_capture(&key_captures, 1)?,
-                parse_capture(&key_captures, 2)?,
-            ),
-            node_kind: parse_capture(&key_captures, 3)?,
-            operation: parse_capture(&key_captures, 4)?,
-            node_id: NodeId(parse_capture(&key_captures, 5)?),
-        })
-    }
-}
-
-fn parse_capture<T>(caps: &Captures, index: usize) -> Result<T, String>
-where
-    T: FromStr,
-    <T as FromStr>::Err: Display,
-{
-    let capture_match = caps
-        .get(index)
-        .ok_or_else(|| format!("Failed to get capture match at index {index}"))?
-        .as_str();
-    capture_match.parse().map_err(|e| {
-        format!(
-            "Failed to parse {} from {capture_match}: {e}",
-            std::any::type_name::<T>()
-        )
-    })
-}
-
-impl Display for NodeKind {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Self::Safekeeper => write!(f, "safekeeper"),
-            Self::Pageserver => write!(f, "pageserver"),
-        }
-    }
-}
-
-impl FromStr for NodeKind {
-    type Err = String;
-
-    fn from_str(node_kind_str: &str) -> Result<Self, Self::Err> {
-        match node_kind_str {
-            "safekeeper" => Ok(Self::Safekeeper),
-            "pageserver" => Ok(Self::Pageserver),
-            _ => Err(format!("Invalid node kind: {node_kind_str}")),
-        }
-    }
-}
-
-impl Display for SkOperationKind {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Self::TimelineInfo => write!(f, "timeline_info"),
-            Self::WalBackup => write!(f, "wal_backup"),
-        }
-    }
-}
-
-impl FromStr for SkOperationKind {
-    type Err = String;
-
-    fn from_str(operation_str: &str) -> Result<Self, Self::Err> {
-        match operation_str {
-            "timeline_info" => Ok(Self::TimelineInfo),
-            "wal_backup" => Ok(Self::WalBackup),
-            _ => Err(format!("Invalid operation: {operation_str}")),
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use utils::id::TimelineId;
-
-    use super::*;
-
-    #[test]
-    fn full_cluster_key_parsing() {
-        let prefix = "neon";
-        let node_kind = NodeKind::Safekeeper;
-        let operation_kind = OperationKind::Safekeeper(SkOperationKind::WalBackup);
-        let tenant_id = TenantId::generate();
-        let timeline_id = TimelineId::generate();
-        let id = TenantTimelineId::new(tenant_id, timeline_id);
-        let node_id = NodeId(1);
-
-        let timeline_subscription_keys = [
-            SubscriptionKey {
-                cluster_prefix: prefix.to_string(),
-                kind: SubscriptionKind::All,
-            },
-            SubscriptionKey {
-                cluster_prefix: prefix.to_string(),
-                kind: SubscriptionKind::TenantTimelines(tenant_id),
-            },
-            SubscriptionKey {
-                cluster_prefix: prefix.to_string(),
-                kind: SubscriptionKind::Timeline(id),
-            },
-            SubscriptionKey {
-                cluster_prefix: prefix.to_string(),
-                kind: SubscriptionKind::Node(id, node_kind),
-            },
-            SubscriptionKey {
-                cluster_prefix: prefix.to_string(),
-                kind: SubscriptionKind::Operation(id, node_kind, operation_kind),
-            },
-        ];
-
-        let full_key_string = format!(
-            "{}/{node_id}",
-            timeline_subscription_keys.last().unwrap().watch_key()
-        );
-
-        for key in timeline_subscription_keys {
-            assert!(full_key_string.starts_with(&key.watch_key()), "Full key '{full_key_string}' should start with any of the keys, keys, but {key:?} did not match");
-        }
-
-        let full_key = SubscriptionFullKey::from_str(&full_key_string).unwrap_or_else(|e| {
-            panic!("Failed to parse {full_key_string} as a subscription full key: {e}")
-        });
-
-        assert_eq!(
-            full_key,
-            SubscriptionFullKey {
-                id,
-                node_kind,
-                operation: operation_kind,
-                node_id
-            }
-        )
-    }
-}

libs/etcd_broker/src/subscription_value.rs

@@ -1,38 +0,0 @@
-//! Module for the values to put into etcd.
-
-use serde::{Deserialize, Serialize};
-use serde_with::{serde_as, DisplayFromStr};
-use utils::lsn::Lsn;
-
-/// Data about safekeeper's timeline. Fields made optional for easy migrations.
-#[serde_as]
-#[derive(Debug, Clone, Deserialize, Serialize)]
-pub struct SkTimelineInfo {
-    /// Term of the last entry.
-    pub last_log_term: Option<u64>,
-    /// LSN of the last record.
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    #[serde(default)]
-    pub flush_lsn: Option<Lsn>,
-    /// Up to which LSN safekeeper regards its WAL as committed.
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    #[serde(default)]
-    pub commit_lsn: Option<Lsn>,
-    /// LSN up to which safekeeper has backed WAL.
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    #[serde(default)]
-    pub backup_lsn: Option<Lsn>,
-    /// LSN of last checkpoint uploaded by pageserver.
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    #[serde(default)]
-    pub remote_consistent_lsn: Option<Lsn>,
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    #[serde(default)]
-    pub peer_horizon_lsn: Option<Lsn>,
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    #[serde(default)]
-    pub local_start_lsn: Option<Lsn>,
-    /// A connection string to use for WAL receiving.
-    #[serde(default)]
-    pub safekeeper_connstr: Option<String>,
-}

libs/pageserver_api/src/models.rs

@@ -117,6 +117,7 @@ impl TenantCreateRequest {
 #[serde_as]
 #[derive(Serialize, Deserialize)]
 pub struct TenantConfigRequest {
+    #[serde_as(as = "DisplayFromStr")]
     pub tenant_id: TenantId,
     #[serde(default)]
     #[serde_as(as = "Option<DisplayFromStr>")]
@@ -201,32 +202,7 @@ pub struct TimelineInfo {
     pub last_received_msg_ts: Option<u128>,
     pub pg_version: u32,
 
-    pub awaits_download: bool,
-
     pub state: TimelineState,
-
-    // Some of the above fields are duplicated in 'local' and 'remote', for backwards-
-    // compatility with older clients.
-    pub local: LocalTimelineInfo,
-    pub remote: RemoteTimelineInfo,
-}
-
-#[serde_as]
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct LocalTimelineInfo {
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub ancestor_timeline_id: Option<TimelineId>,
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub ancestor_lsn: Option<Lsn>,
-    pub current_logical_size: Option<u64>, // is None when timeline is Unloaded
-    pub current_physical_size: Option<u64>, // is None when timeline is Unloaded
-}
-
-#[serde_as]
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct RemoteTimelineInfo {
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub remote_consistent_lsn: Option<Lsn>,
 }
 
 pub type ConfigureFailpointsRequest = Vec<FailpointConfig>;

libs/postgres_connection/Cargo.toml

@@ -8,8 +8,8 @@ edition = "2021"
 [dependencies]
 anyhow = "1.0"
 itertools = "0.10.3"
-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev = "d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev = "43e6db254a97fdecbce33d8bc0890accfd74495e" }
+tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
 url = "2.2.2"
 workspace_hack = { version = "0.1", path = "../../workspace_hack" }
 

libs/postgres_ffi/Cargo.toml

@@ -21,7 +21,7 @@ workspace_hack = { version = "0.1", path = "../../workspace_hack" }
 
 [dev-dependencies]
 env_logger = "0.9"
-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
 wal_craft = { path = "wal_craft" }
 
 [build-dependencies]

libs/postgres_ffi/wal_craft/Cargo.toml

@@ -11,7 +11,7 @@ clap = "4.0"
 env_logger = "0.9"
 log = "0.4"
 once_cell = "1.13.0"
-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
 postgres_ffi = { path = "../" }
 tempfile = "3.2"
 workspace_hack = { version = "0.1", path = "../../../workspace_hack" }

libs/postgres_ffi/wal_craft/src/lib.rs

@@ -1,7 +1,6 @@
 use anyhow::*;
 use core::time::Duration;
 use log::*;
-use once_cell::sync::Lazy;
 use postgres::types::PgLsn;
 use postgres::Client;
 use postgres_ffi::{WAL_SEGMENT_SIZE, XLOG_BLCKSZ};
@@ -26,15 +25,13 @@ pub struct PostgresServer {
     client_config: postgres::Config,
 }
 
-pub static REQUIRED_POSTGRES_CONFIG: Lazy<Vec<&'static str>> = Lazy::new(|| {
-    vec![
-        "wal_keep_size=50MB",            // Ensure old WAL is not removed
-        "shared_preload_libraries=neon", // can only be loaded at startup
-        // Disable background processes as much as possible
-        "wal_writer_delay=10s",
-        "autovacuum=off",
-    ]
-});
+pub static REQUIRED_POSTGRES_CONFIG: [&str; 4] = [
+    "wal_keep_size=50MB",            // Ensure old WAL is not removed
+    "shared_preload_libraries=neon", // can only be loaded at startup
+    // Disable background processes as much as possible
+    "wal_writer_delay=10s",
+    "autovacuum=off",
+];
 
 impl Conf {
     pub fn pg_distrib_dir(&self) -> anyhow::Result<PathBuf> {

libs/pq_proto/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 anyhow = "1.0"
 bytes = "1.0.1"
 pin-project-lite = "0.2.7"
-postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
 rand = "0.8.3"
 serde = { version = "1.0", features = ["derive"] }
 tokio = { version = "1.17", features = ["macros"] }

libs/pq_proto/src/lib.rs

@@ -463,7 +463,10 @@ pub enum BeMessage<'a> {
     EncryptionResponse(bool),
     NoData,
     ParameterDescription,
-    ParameterStatus(BeParameterStatusMessage<'a>),
+    ParameterStatus {
+        name: &'a [u8],
+        value: &'a [u8],
+    },
     ParseComplete,
     ReadyForQuery,
     RowDescription(&'a [RowDescriptor<'a>]),
@@ -472,6 +475,28 @@ pub enum BeMessage<'a> {
     KeepAlive(WalSndKeepAlive),
 }
 
+/// Common shorthands.
+impl<'a> BeMessage<'a> {
+    /// A [`BeMessage::ParameterStatus`] holding the client encoding, i.e. UTF-8.
+    /// This is a sensible default, given that:
+    ///  * rust strings only support this encoding out of the box.
+    ///  * tokio-postgres, postgres-jdbc (and probably more) mandate it.
+    ///
+    /// TODO: do we need to report `server_encoding` as well?
+    pub const CLIENT_ENCODING: Self = Self::ParameterStatus {
+        name: b"client_encoding",
+        value: b"UTF8",
+    };
+
+    /// Build a [`BeMessage::ParameterStatus`] holding the server version.
+    pub fn server_version(version: &'a str) -> Self {
+        Self::ParameterStatus {
+            name: b"server_version",
+            value: version.as_bytes(),
+        }
+    }
+}
+
 #[derive(Debug)]
 pub enum BeAuthenticationSaslMessage<'a> {
     Methods(&'a [&'a str]),
@@ -485,12 +510,6 @@ pub enum BeParameterStatusMessage<'a> {
     ServerVersion(&'a str),
 }
 
-impl BeParameterStatusMessage<'static> {
-    pub fn encoding() -> BeMessage<'static> {
-        BeMessage::ParameterStatus(Self::Encoding("UTF8"))
-    }
-}
-
 // One row description in RowDescription packet.
 #[derive(Debug)]
 pub struct RowDescriptor<'a> {
@@ -587,14 +606,15 @@ fn write_body<R>(buf: &mut BytesMut, f: impl FnOnce(&mut BytesMut) -> R) -> R {
 }
 
 /// Safe write of s into buf as cstring (String in the protocol).
-fn write_cstr(s: &[u8], buf: &mut BytesMut) -> Result<(), io::Error> {
-    if s.contains(&0) {
+fn write_cstr(s: impl AsRef<[u8]>, buf: &mut BytesMut) -> Result<(), io::Error> {
+    let bytes = s.as_ref();
+    if bytes.contains(&0) {
         return Err(io::Error::new(
             io::ErrorKind::InvalidInput,
             "string contains embedded null",
         ));
     }
-    buf.put_slice(s);
+    buf.put_slice(bytes);
     buf.put_u8(0);
     Ok(())
 }
@@ -644,7 +664,7 @@ impl<'a> BeMessage<'a> {
                         Methods(methods) => {
                             buf.put_i32(10); // Specifies that SASL auth method is used.
                             for method in methods.iter() {
-                                write_cstr(method.as_bytes(), buf)?;
+                                write_cstr(method, buf)?;
                             }
                             buf.put_u8(0); // zero terminator for the list
                         }
@@ -759,7 +779,7 @@ impl<'a> BeMessage<'a> {
                     buf.put_slice(b"CXX000\0");
 
                     buf.put_u8(b'M'); // the message
-                    write_cstr(error_msg.as_bytes(), buf)?;
+                    write_cstr(error_msg, buf)?;
 
                     buf.put_u8(0); // terminator
                     Ok::<_, io::Error>(())
@@ -799,24 +819,12 @@ impl<'a> BeMessage<'a> {
                 buf.put_u8(response);
             }
 
-            BeMessage::ParameterStatus(param) => {
-                use std::io::{IoSlice, Write};
-                use BeParameterStatusMessage::*;
-
-                let [name, value] = match param {
-                    Encoding(name) => [b"client_encoding", name.as_bytes()],
-                    ServerVersion(version) => [b"server_version", version.as_bytes()],
-                };
-
-                // Parameter names and values are passed as null-terminated strings
-                let iov = &mut [name, b"\0", value, b"\0"].map(IoSlice::new);
-                let mut buffer = [0u8; 64]; // this should be enough
-                let cnt = buffer.as_mut().write_vectored(iov).unwrap();
-
+            BeMessage::ParameterStatus { name, value } => {
                 buf.put_u8(b'S');
                 write_body(buf, |buf| {
-                    buf.put_slice(&buffer[..cnt]);
-                });
+                    write_cstr(name, buf)?;
+                    write_cstr(value, buf)
+                })?;
             }
 
             BeMessage::ParameterDescription => {

libs/remote_storage/src/lib.rs

@@ -10,7 +10,7 @@ mod s3_bucket;
 
 use std::{
     collections::HashMap,
-    fmt::{Debug, Display},
+    fmt::Debug,
     num::{NonZeroU32, NonZeroUsize},
     ops::Deref,
     path::{Path, PathBuf},
@@ -41,44 +41,27 @@ pub const DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT: usize = 100;
 
 const REMOTE_STORAGE_PREFIX_SEPARATOR: char = '/';
 
-#[derive(Clone, PartialEq, Eq)]
-pub struct RemoteObjectId(String);
+/// Path on the remote storage, relative to some inner prefix.
+/// The prefix is an implementation detail, that allows representing local paths
+/// as the remote ones, stripping the local storage prefix away.
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct RemotePath(PathBuf);
+
+impl RemotePath {
+    pub fn new(relative_path: &Path) -> anyhow::Result<Self> {
+        anyhow::ensure!(
+            relative_path.is_relative(),
+            "Path {relative_path:?} is not relative"
+        );
+        Ok(Self(relative_path.to_path_buf()))
+    }
+
+    pub fn with_base(&self, base_path: &Path) -> PathBuf {
+        base_path.join(&self.0)
+    }
 
-///
-/// A key that refers to an object in remote storage. It works much like a Path,
-/// but it's a separate datatype so that you don't accidentally mix local paths
-/// and remote keys.
-///
-impl RemoteObjectId {
-    // Needed to retrieve last component for RemoteObjectId.
-    // In other words a file name
-    /// Turn a/b/c or a/b/c/ into c
     pub fn object_name(&self) -> Option<&str> {
-        // corner case, char::to_string is not const, thats why this is more verbose than it needs to be
-        // see https://github.com/rust-lang/rust/issues/88674
-        if self.0.len() == 1 && self.0.chars().next().unwrap() == REMOTE_STORAGE_PREFIX_SEPARATOR {
-            return None;
-        }
-
-        if self.0.ends_with(REMOTE_STORAGE_PREFIX_SEPARATOR) {
-            self.0.rsplit(REMOTE_STORAGE_PREFIX_SEPARATOR).nth(1)
-        } else {
-            self.0
-                .rsplit_once(REMOTE_STORAGE_PREFIX_SEPARATOR)
-                .map(|(_, last)| last)
-        }
-    }
-}
-
-impl Debug for RemoteObjectId {
-    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
-        Debug::fmt(&self.0, fmt)
-    }
-}
-
-impl Display for RemoteObjectId {
-    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        Display::fmt(&self.0, fmt)
+        self.0.file_name().and_then(|os_str| os_str.to_str())
     }
 }
 
@@ -87,54 +70,40 @@ impl Display for RemoteObjectId {
 /// providing basic CRUD operations for storage files.
 #[async_trait::async_trait]
 pub trait RemoteStorage: Send + Sync + 'static {
-    /// Attempts to derive the storage path out of the local path, if the latter is correct.
-    fn remote_object_id(&self, local_path: &Path) -> anyhow::Result<RemoteObjectId>;
-
-    /// Gets the download path of the given storage file.
-    fn local_path(&self, remote_object_id: &RemoteObjectId) -> anyhow::Result<PathBuf>;
-
     /// Lists all items the storage has right now.
-    async fn list(&self) -> anyhow::Result<Vec<RemoteObjectId>>;
+    async fn list(&self) -> anyhow::Result<Vec<RemotePath>>;
 
     /// Lists all top level subdirectories for a given prefix
     /// Note: here we assume that if the prefix is passed it was obtained via remote_object_id
     /// which already takes into account any kind of global prefix (prefix_in_bucket for S3 or storage_root for LocalFS)
     /// so this method doesnt need to.
-    async fn list_prefixes(
-        &self,
-        prefix: Option<&RemoteObjectId>,
-    ) -> anyhow::Result<Vec<RemoteObjectId>>;
+    async fn list_prefixes(&self, prefix: Option<&RemotePath>) -> anyhow::Result<Vec<RemotePath>>;
 
     /// Streams the local file contents into remote into the remote storage entry.
     async fn upload(
         &self,
-        from: Box<(dyn io::AsyncRead + Unpin + Send + Sync + 'static)>,
+        data: Box<(dyn io::AsyncRead + Unpin + Send + Sync + 'static)>,
         // S3 PUT request requires the content length to be specified,
         // otherwise it starts to fail with the concurrent connection count increasing.
-        from_size_bytes: usize,
-        to: &RemoteObjectId,
+        data_size_bytes: usize,
+        to: &RemotePath,
         metadata: Option<StorageMetadata>,
     ) -> anyhow::Result<()>;
 
     /// Streams the remote storage entry contents into the buffered writer given, returns the filled writer.
     /// Returns the metadata, if any was stored with the file previously.
-    async fn download(&self, from: &RemoteObjectId) -> Result<Download, DownloadError>;
+    async fn download(&self, from: &RemotePath) -> Result<Download, DownloadError>;
 
     /// Streams a given byte range of the remote storage entry contents into the buffered writer given, returns the filled writer.
     /// Returns the metadata, if any was stored with the file previously.
     async fn download_byte_range(
         &self,
-        from: &RemoteObjectId,
+        from: &RemotePath,
         start_inclusive: u64,
         end_exclusive: Option<u64>,
     ) -> Result<Download, DownloadError>;
 
-    async fn delete(&self, path: &RemoteObjectId) -> anyhow::Result<()>;
-
-    /// Downcast to LocalFs implementation. For tests.
-    fn as_local(&self) -> Option<&LocalFs> {
-        None
-    }
+    async fn delete(&self, path: &RemotePath) -> anyhow::Result<()>;
 }
 
 pub struct Download {
@@ -168,7 +137,7 @@ impl std::fmt::Display for DownloadError {
                 write!(f, "Failed to download a remote file due to user input: {e}")
             }
             DownloadError::NotFound => write!(f, "No file found for the remote object id given"),
-            DownloadError::Other(e) => write!(f, "Failed to download a remote file: {e}"),
+            DownloadError::Other(e) => write!(f, "Failed to download a remote file: {e:?}"),
         }
     }
 }
@@ -178,34 +147,35 @@ impl std::error::Error for DownloadError {}
 /// Every storage, currently supported.
 /// Serves as a simple way to pass around the [`RemoteStorage`] without dealing with generics.
 #[derive(Clone)]
-pub struct GenericRemoteStorage(Arc<dyn RemoteStorage>);
+pub enum GenericRemoteStorage {
+    LocalFs(LocalFs),
+    AwsS3(Arc<S3Bucket>),
+}
 
 impl Deref for GenericRemoteStorage {
     type Target = dyn RemoteStorage;
 
     fn deref(&self) -> &Self::Target {
-        self.0.as_ref()
+        match self {
+            GenericRemoteStorage::LocalFs(local_fs) => local_fs,
+            GenericRemoteStorage::AwsS3(s3_bucket) => s3_bucket.as_ref(),
+        }
     }
 }
 
 impl GenericRemoteStorage {
-    pub fn new(storage: impl RemoteStorage) -> Self {
-        Self(Arc::new(storage))
-    }
-
     pub fn from_config(
-        working_directory: PathBuf,
         storage_config: &RemoteStorageConfig,
     ) -> anyhow::Result<GenericRemoteStorage> {
         Ok(match &storage_config.storage {
             RemoteStorageKind::LocalFs(root) => {
                 info!("Using fs root '{}' as a remote storage", root.display());
-                GenericRemoteStorage::new(LocalFs::new(root.clone(), working_directory)?)
+                GenericRemoteStorage::LocalFs(LocalFs::new(root.clone())?)
             }
             RemoteStorageKind::AwsS3(s3_config) => {
                 info!("Using s3 bucket '{}' in region '{}' as a remote storage, prefix in bucket: '{:?}', bucket endpoint: '{:?}'",
                       s3_config.bucket_name, s3_config.bucket_region, s3_config.prefix_in_bucket, s3_config.endpoint);
-                GenericRemoteStorage::new(S3Bucket::new(s3_config, working_directory)?)
+                GenericRemoteStorage::AwsS3(Arc::new(S3Bucket::new(s3_config)?))
             }
         })
     }
@@ -219,23 +189,12 @@ impl GenericRemoteStorage {
         &self,
         from: Box<dyn tokio::io::AsyncRead + Unpin + Send + Sync + 'static>,
         from_size_bytes: usize,
-        from_path: &Path,
+        to: &RemotePath,
     ) -> anyhow::Result<()> {
-        let target_storage_path = self.remote_object_id(from_path).with_context(|| {
-            format!(
-                "Failed to get the storage path for source local path '{}'",
-                from_path.display()
-            )
-        })?;
-
-        self.upload(from, from_size_bytes, &target_storage_path, None)
+        self.upload(from, from_size_bytes, to, None)
             .await
             .with_context(|| {
-                format!(
-                    "Failed to upload from '{}' to storage path '{:?}'",
-                    from_path.display(),
-                    target_storage_path
-                )
+                format!("Failed to upload data of length {from_size_bytes} to storage path {to:?}")
             })
     }
 
@@ -244,24 +203,11 @@ impl GenericRemoteStorage {
     pub async fn download_storage_object(
         &self,
         byte_range: Option<(u64, Option<u64>)>,
-        to_path: &Path,
+        from: &RemotePath,
     ) -> Result<Download, DownloadError> {
-        let remote_object_path = self
-            .remote_object_id(to_path)
-            .with_context(|| {
-                format!(
-                    "Failed to get the storage path for target local path '{}'",
-                    to_path.display()
-                )
-            })
-            .map_err(DownloadError::BadInput)?;
-
         match byte_range {
-            Some((start, end)) => {
-                self.download_byte_range(&remote_object_path, start, end)
-                    .await
-            }
-            None => self.download(&remote_object_path).await,
+            Some((start, end)) => self.download_byte_range(from, start, end).await,
+            None => self.download(from).await,
         }
     }
 }
@@ -271,23 +217,6 @@ impl GenericRemoteStorage {
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct StorageMetadata(HashMap<String, String>);
 
-fn strip_path_prefix<'a>(prefix: &'a Path, path: &'a Path) -> anyhow::Result<&'a Path> {
-    if prefix == path {
-        anyhow::bail!(
-            "Prefix and the path are equal, cannot strip: '{}'",
-            prefix.display()
-        )
-    } else {
-        path.strip_prefix(prefix).with_context(|| {
-            format!(
-                "Path '{}' is not prefixed with '{}'",
-                path.display(),
-                prefix.display(),
-            )
-        })
-    }
-}
-
 /// External backup storage configuration, enough for creating a client for that storage.
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct RemoteStorageConfig {
@@ -343,7 +272,7 @@ impl Debug for S3Config {
 }
 
 impl RemoteStorageConfig {
-    pub fn from_toml(toml: &toml_edit::Item) -> anyhow::Result<RemoteStorageConfig> {
+    pub fn from_toml(toml: &toml_edit::Item) -> anyhow::Result<Option<RemoteStorageConfig>> {
         let local_path = toml.get("local_path");
         let bucket_name = toml.get("bucket_name");
         let bucket_region = toml.get("bucket_region");
@@ -367,7 +296,8 @@ impl RemoteStorageConfig {
         .context("Failed to parse 'concurrency_limit' as a positive integer")?;
 
         let storage = match (local_path, bucket_name, bucket_region) {
-            (None, None, None) => bail!("no 'local_path' nor 'bucket_name' option"),
+            // no 'local_path' nor 'bucket_name' options are provided, consider this remote storage disabled
+            (None, None, None) => return Ok(None),
             (_, Some(_), None) => {
                 bail!("'bucket_region' option is mandatory if 'bucket_name' is given ")
             }
@@ -393,11 +323,11 @@ impl RemoteStorageConfig {
             (Some(_), Some(_), _) => bail!("local_path and bucket_name are mutually exclusive"),
         };
 
-        Ok(RemoteStorageConfig {
+        Ok(Some(RemoteStorageConfig {
             max_concurrent_syncs,
             max_sync_errors,
             storage,
-        })
+        }))
     }
 }
 
@@ -431,21 +361,24 @@ mod tests {
     use super::*;
 
     #[test]
-    fn object_name() {
-        let k = RemoteObjectId("a/b/c".to_owned());
+    fn test_object_name() {
+        let k = RemotePath::new(Path::new("a/b/c")).unwrap();
         assert_eq!(k.object_name(), Some("c"));
 
-        let k = RemoteObjectId("a/b/c/".to_owned());
+        let k = RemotePath::new(Path::new("a/b/c/")).unwrap();
         assert_eq!(k.object_name(), Some("c"));
 
-        let k = RemoteObjectId("a/".to_owned());
+        let k = RemotePath::new(Path::new("a/")).unwrap();
         assert_eq!(k.object_name(), Some("a"));
 
         // XXX is it impossible to have an empty key?
-        let k = RemoteObjectId("".to_owned());
-        assert_eq!(k.object_name(), None);
-
-        let k = RemoteObjectId("/".to_owned());
+        let k = RemotePath::new(Path::new("")).unwrap();
         assert_eq!(k.object_name(), None);
     }
+
+    #[test]
+    fn rempte_path_cannot_be_created_from_absolute_ones() {
+        let err = RemotePath::new(Path::new("/")).expect_err("Should fail on absolute paths");
+        assert_eq!(err.to_string(), "Path \"/\" is not relative");
+    }
 }

libs/remote_storage/src/local_fs.rs

@@ -5,6 +5,7 @@
 //! volume is mounted to the local FS.
 
 use std::{
+    borrow::Cow,
     future::Future,
     path::{Path, PathBuf},
     pin::Pin,
@@ -18,60 +19,33 @@ use tokio::{
 use tracing::*;
 use utils::crashsafe::path_with_suffix_extension;
 
-use crate::{Download, DownloadError, RemoteObjectId};
+use crate::{Download, DownloadError, RemotePath};
 
-use super::{strip_path_prefix, RemoteStorage, StorageMetadata};
+use super::{RemoteStorage, StorageMetadata};
 
 const LOCAL_FS_TEMP_FILE_SUFFIX: &str = "___temp";
 
-/// Convert a Path in the remote storage into a RemoteObjectId
-fn remote_object_id_from_path(path: &Path) -> anyhow::Result<RemoteObjectId> {
-    Ok(RemoteObjectId(
-        path.to_str()
-            .ok_or_else(|| anyhow::anyhow!("unexpected characters found in path"))?
-            .to_string(),
-    ))
-}
-
+#[derive(Debug, Clone)]
 pub struct LocalFs {
-    working_directory: PathBuf,
     storage_root: PathBuf,
 }
 
 impl LocalFs {
     /// Attempts to create local FS storage, along with its root directory.
-    pub fn new(root: PathBuf, working_directory: PathBuf) -> anyhow::Result<Self> {
-        if !root.exists() {
-            std::fs::create_dir_all(&root).with_context(|| {
-                format!(
-                    "Failed to create all directories in the given root path '{}'",
-                    root.display(),
-                )
+    /// Storage root will be created (if does not exist) and transformed into an absolute path (if passed as relative).
+    pub fn new(mut storage_root: PathBuf) -> anyhow::Result<Self> {
+        if !storage_root.exists() {
+            std::fs::create_dir_all(&storage_root).with_context(|| {
+                format!("Failed to create all directories in the given root path {storage_root:?}")
             })?;
         }
-        Ok(Self {
-            working_directory,
-            storage_root: root,
-        })
-    }
-
-    ///
-    /// Get the absolute path in the local filesystem to given remote object.
-    ///
-    /// This is public so that it can be used in tests. Should not be used elsewhere.
-    ///
-    pub fn resolve_in_storage(&self, remote_object_id: &RemoteObjectId) -> anyhow::Result<PathBuf> {
-        let path = PathBuf::from(&remote_object_id.0);
-        if path.is_relative() {
-            Ok(self.storage_root.join(path))
-        } else if path.starts_with(&self.storage_root) {
-            Ok(path)
-        } else {
-            bail!(
-                "Path '{}' does not belong to the current storage",
-                path.display()
-            )
+        if !storage_root.is_absolute() {
+            storage_root = storage_root.canonicalize().with_context(|| {
+                format!("Failed to represent path {storage_root:?} as an absolute path")
+            })?;
         }
+
+        Ok(Self { storage_root })
     }
 
     async fn read_storage_metadata(
@@ -103,45 +77,48 @@ impl LocalFs {
 
 #[async_trait::async_trait]
 impl RemoteStorage for LocalFs {
-    /// Convert a "local" path into a "remote path"
-    fn remote_object_id(&self, local_path: &Path) -> anyhow::Result<RemoteObjectId> {
-        let path = self.storage_root.join(
-            strip_path_prefix(&self.working_directory, local_path)
-                .context("local path does not belong to this storage")?,
-        );
-        remote_object_id_from_path(&path)
+    async fn list(&self) -> anyhow::Result<Vec<RemotePath>> {
+        Ok(get_all_files(&self.storage_root, true)
+            .await?
+            .into_iter()
+            .map(|path| {
+                path.strip_prefix(&self.storage_root)
+                    .context("Failed to strip storage root prefix")
+                    .and_then(RemotePath::new)
+                    .expect(
+                        "We list files for storage root, hence should be able to remote the prefix",
+                    )
+            })
+            .collect())
     }
 
-    fn local_path(&self, remote_object_id: &RemoteObjectId) -> anyhow::Result<PathBuf> {
-        let storage_path = PathBuf::from(&remote_object_id.0);
-        let relative_path = strip_path_prefix(&self.storage_root, &storage_path)
-            .context("local path does not belong to this storage")?;
-        Ok(self.working_directory.join(relative_path))
-    }
-
-    async fn list(&self) -> anyhow::Result<Vec<RemoteObjectId>> {
-        get_all_files(&self.storage_root, true).await
-    }
-
-    async fn list_prefixes(
-        &self,
-        prefix: Option<&RemoteObjectId>,
-    ) -> anyhow::Result<Vec<RemoteObjectId>> {
+    async fn list_prefixes(&self, prefix: Option<&RemotePath>) -> anyhow::Result<Vec<RemotePath>> {
         let path = match prefix {
-            Some(prefix) => Path::new(&prefix.0),
-            None => &self.storage_root,
+            Some(prefix) => Cow::Owned(prefix.with_base(&self.storage_root)),
+            None => Cow::Borrowed(&self.storage_root),
         };
-        get_all_files(path, false).await
+        Ok(get_all_files(path.as_ref(), false)
+            .await?
+            .into_iter()
+            .map(|path| {
+                path.strip_prefix(&self.storage_root)
+                    .context("Failed to strip preifix")
+                    .and_then(RemotePath::new)
+                    .expect(
+                        "We list files for storage root, hence should be able to remote the prefix",
+                    )
+            })
+            .collect())
     }
 
     async fn upload(
         &self,
-        from: Box<(dyn io::AsyncRead + Unpin + Send + Sync + 'static)>,
-        from_size_bytes: usize,
-        to: &RemoteObjectId,
+        data: Box<(dyn io::AsyncRead + Unpin + Send + Sync + 'static)>,
+        data_size_bytes: usize,
+        to: &RemotePath,
         metadata: Option<StorageMetadata>,
     ) -> anyhow::Result<()> {
-        let target_file_path = self.resolve_in_storage(to)?;
+        let target_file_path = to.with_base(&self.storage_root);
         create_target_directory(&target_file_path).await?;
         // We need this dance with sort of durable rename (without fsyncs)
         // to prevent partial uploads. This was really hit when pageserver shutdown
@@ -162,8 +139,8 @@ impl RemoteStorage for LocalFs {
                 })?,
         );
 
-        let from_size_bytes = from_size_bytes as u64;
-        let mut buffer_to_read = from.take(from_size_bytes);
+        let from_size_bytes = data_size_bytes as u64;
+        let mut buffer_to_read = data.take(from_size_bytes);
 
         let bytes_read = io::copy(&mut buffer_to_read, &mut destination)
             .await
@@ -220,27 +197,22 @@ impl RemoteStorage for LocalFs {
         Ok(())
     }
 
-    async fn download(&self, from: &RemoteObjectId) -> Result<Download, DownloadError> {
-        let file_path = self
-            .resolve_in_storage(from)
-            .map_err(DownloadError::BadInput)?;
-        if file_exists(&file_path).map_err(DownloadError::BadInput)? {
+    async fn download(&self, from: &RemotePath) -> Result<Download, DownloadError> {
+        let target_path = from.with_base(&self.storage_root);
+        if file_exists(&target_path).map_err(DownloadError::BadInput)? {
             let source = io::BufReader::new(
                 fs::OpenOptions::new()
                     .read(true)
-                    .open(&file_path)
+                    .open(&target_path)
                     .await
                     .with_context(|| {
-                        format!(
-                            "Failed to open source file '{}' to use in the download",
-                            file_path.display()
-                        )
+                        format!("Failed to open source file {target_path:?} to use in the download")
                     })
                     .map_err(DownloadError::Other)?,
             );
 
             let metadata = self
-                .read_storage_metadata(&file_path)
+                .read_storage_metadata(&target_path)
                 .await
                 .map_err(DownloadError::Other)?;
             Ok(Download {
@@ -254,7 +226,7 @@ impl RemoteStorage for LocalFs {
 
     async fn download_byte_range(
         &self,
-        from: &RemoteObjectId,
+        from: &RemotePath,
         start_inclusive: u64,
         end_exclusive: Option<u64>,
     ) -> Result<Download, DownloadError> {
@@ -266,20 +238,15 @@ impl RemoteStorage for LocalFs {
                 return Err(DownloadError::Other(anyhow::anyhow!("Invalid range, start ({start_inclusive}) and end_exclusive ({end_exclusive:?}) difference is zero bytes")));
             }
         }
-        let file_path = self
-            .resolve_in_storage(from)
-            .map_err(DownloadError::BadInput)?;
-        if file_exists(&file_path).map_err(DownloadError::BadInput)? {
+        let target_path = from.with_base(&self.storage_root);
+        if file_exists(&target_path).map_err(DownloadError::BadInput)? {
             let mut source = io::BufReader::new(
                 fs::OpenOptions::new()
                     .read(true)
-                    .open(&file_path)
+                    .open(&target_path)
                     .await
                     .with_context(|| {
-                        format!(
-                            "Failed to open source file '{}' to use in the download",
-                            file_path.display()
-                        )
+                        format!("Failed to open source file {target_path:?} to use in the download")
                     })
                     .map_err(DownloadError::Other)?,
             );
@@ -289,7 +256,7 @@ impl RemoteStorage for LocalFs {
                 .context("Failed to seek to the range start in a local storage file")
                 .map_err(DownloadError::Other)?;
             let metadata = self
-                .read_storage_metadata(&file_path)
+                .read_storage_metadata(&target_path)
                 .await
                 .map_err(DownloadError::Other)?;
 
@@ -308,21 +275,14 @@ impl RemoteStorage for LocalFs {
         }
     }
 
-    async fn delete(&self, path: &RemoteObjectId) -> anyhow::Result<()> {
-        let file_path = self.resolve_in_storage(path)?;
+    async fn delete(&self, path: &RemotePath) -> anyhow::Result<()> {
+        let file_path = path.with_base(&self.storage_root);
         if file_path.exists() && file_path.is_file() {
             Ok(fs::remove_file(file_path).await?)
         } else {
-            bail!(
-                "File '{}' either does not exist or is not a file",
-                file_path.display()
-            )
+            bail!("File {file_path:?} either does not exist or is not a file")
         }
     }
-
-    fn as_local(&self) -> Option<&LocalFs> {
-        Some(self)
-    }
 }
 
 fn storage_metadata_path(original_path: &Path) -> PathBuf {
@@ -332,7 +292,7 @@ fn storage_metadata_path(original_path: &Path) -> PathBuf {
 fn get_all_files<'a, P>(
     directory_path: P,
     recursive: bool,
-) -> Pin<Box<dyn Future<Output = anyhow::Result<Vec<RemoteObjectId>>> + Send + Sync + 'a>>
+) -> Pin<Box<dyn Future<Output = anyhow::Result<Vec<PathBuf>>> + Send + Sync + 'a>>
 where
     P: AsRef<Path> + Send + Sync + 'a,
 {
@@ -346,20 +306,20 @@ where
                     let file_type = dir_entry.file_type().await?;
                     let entry_path = dir_entry.path();
                     if file_type.is_symlink() {
-                        debug!("{:?} us a symlink, skipping", entry_path)
+                        debug!("{entry_path:?} us a symlink, skipping")
                     } else if file_type.is_dir() {
                         if recursive {
                             paths.extend(get_all_files(&entry_path, true).await?.into_iter())
                         } else {
-                            paths.push(remote_object_id_from_path(&dir_entry.path())?)
+                            paths.push(entry_path)
                         }
                     } else {
-                        paths.push(remote_object_id_from_path(&dir_entry.path())?);
+                        paths.push(entry_path);
                     }
                 }
                 Ok(paths)
             } else {
-                bail!("Path '{}' is not a directory", directory_path.display())
+                bail!("Path {directory_path:?} is not a directory")
             }
         } else {
             Ok(Vec::new())
@@ -394,173 +354,6 @@ fn file_exists(file_path: &Path) -> anyhow::Result<bool> {
     }
 }
 
-#[cfg(test)]
-mod pure_tests {
-    use tempfile::tempdir;
-
-    use super::*;
-
-    #[test]
-    fn storage_path_positive() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
-        let storage_root = PathBuf::from("somewhere").join("else");
-        let storage = LocalFs {
-            working_directory: workdir.clone(),
-            storage_root: storage_root.clone(),
-        };
-
-        let local_path = workdir
-            .join("timelines")
-            .join("some_timeline")
-            .join("file_name");
-        let expected_path = storage_root.join(local_path.strip_prefix(&workdir)?);
-
-        let actual_path = PathBuf::from(
-            storage
-                .remote_object_id(&local_path)
-                .expect("Matching path should map to storage path normally")
-                .0,
-        );
-        assert_eq!(
-            expected_path,
-            actual_path,
-            "File paths from workdir should be stored in local fs storage with the same path they have relative to the workdir"
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn storage_path_negatives() -> anyhow::Result<()> {
-        #[track_caller]
-        fn storage_path_error(storage: &LocalFs, mismatching_path: &Path) -> String {
-            match storage.remote_object_id(mismatching_path) {
-                Ok(wrong_path) => panic!(
-                    "Expected path '{}' to error, but got storage path: {:?}",
-                    mismatching_path.display(),
-                    wrong_path,
-                ),
-                Err(e) => format!("{:?}", e),
-            }
-        }
-
-        let workdir = tempdir()?.path().to_owned();
-        let storage_root = PathBuf::from("somewhere").join("else");
-        let storage = LocalFs {
-            working_directory: workdir.clone(),
-            storage_root,
-        };
-
-        let error_string = storage_path_error(&storage, &workdir);
-        assert!(error_string.contains("does not belong to this storage"));
-        assert!(error_string.contains(workdir.to_str().unwrap()));
-
-        let mismatching_path_str = "/something/else";
-        let error_message = storage_path_error(&storage, Path::new(mismatching_path_str));
-        assert!(
-            error_message.contains(mismatching_path_str),
-            "Error should mention wrong path"
-        );
-        assert!(
-            error_message.contains(workdir.to_str().unwrap()),
-            "Error should mention server workdir"
-        );
-        assert!(error_message.contains("does not belong to this storage"));
-
-        Ok(())
-    }
-
-    #[test]
-    fn local_path_positive() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-        let storage_root = PathBuf::from("somewhere").join("else");
-        let storage = LocalFs {
-            working_directory: workdir.clone(),
-            storage_root: storage_root.clone(),
-        };
-
-        let name = "not a metadata";
-        let local_path = workdir.join("timelines").join("some_timeline").join(name);
-        assert_eq!(
-            local_path,
-            storage
-                .local_path(&remote_object_id_from_path(
-                    &storage_root.join(local_path.strip_prefix(&workdir)?)
-                )?)
-                .expect("For a valid input, valid local path should be parsed"),
-            "Should be able to parse metadata out of the correctly named remote delta file"
-        );
-
-        let local_metadata_path = workdir
-            .join("timelines")
-            .join("some_timeline")
-            .join("metadata");
-        let remote_metadata_path = storage.remote_object_id(&local_metadata_path)?;
-        assert_eq!(
-            local_metadata_path,
-            storage
-                .local_path(&remote_metadata_path)
-                .expect("For a valid input, valid local path should be parsed"),
-            "Should be able to parse metadata out of the correctly named remote metadata file"
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn local_path_negatives() -> anyhow::Result<()> {
-        #[track_caller]
-        fn local_path_error(storage: &LocalFs, storage_path: &RemoteObjectId) -> String {
-            match storage.local_path(storage_path) {
-                Ok(wrong_path) => panic!(
-                    "Expected local path input {:?} to cause an error, but got file path: {:?}",
-                    storage_path, wrong_path,
-                ),
-                Err(e) => format!("{:?}", e),
-            }
-        }
-
-        let storage_root = PathBuf::from("somewhere").join("else");
-        let storage = LocalFs {
-            working_directory: tempdir()?.path().to_owned(),
-            storage_root,
-        };
-
-        let totally_wrong_path = "wrong_wrong_wrong";
-        let error_message =
-            local_path_error(&storage, &RemoteObjectId(totally_wrong_path.to_string()));
-        assert!(error_message.contains(totally_wrong_path));
-
-        Ok(())
-    }
-
-    #[test]
-    fn download_destination_matches_original_path() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-        let original_path = workdir
-            .join("timelines")
-            .join("some_timeline")
-            .join("some name");
-
-        let storage_root = PathBuf::from("somewhere").join("else");
-        let dummy_storage = LocalFs {
-            working_directory: workdir,
-            storage_root,
-        };
-
-        let storage_path = dummy_storage.remote_object_id(&original_path)?;
-        let download_destination = dummy_storage.local_path(&storage_path)?;
-
-        assert_eq!(
-            original_path, download_destination,
-            "'original path -> storage path -> matching fs path' transformation should produce the same path as the input one for the correct path"
-        );
-
-        Ok(())
-    }
-}
-
 #[cfg(test)]
 mod fs_tests {
     use super::*;
@@ -572,7 +365,7 @@ mod fs_tests {
         storage: &LocalFs,
         #[allow(clippy::ptr_arg)]
         // have to use &PathBuf due to `storage.local_path` parameter requirements
-        remote_storage_path: &RemoteObjectId,
+        remote_storage_path: &RemotePath,
         expected_metadata: Option<&StorageMetadata>,
     ) -> anyhow::Result<String> {
         let mut download = storage
@@ -595,41 +388,16 @@ mod fs_tests {
 
     #[tokio::test]
     async fn upload_file() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
         let storage = create_storage()?;
 
-        let (file, size) = create_file_for_upload(
-            &storage.working_directory.join("whatever"),
-            "whatever_contents",
-        )
-        .await?;
-        let target_path = "/somewhere/else";
-        match storage
-            .upload(
-                Box::new(file),
-                size,
-                &RemoteObjectId(target_path.to_string()),
-                None,
-            )
-            .await
-        {
-            Ok(()) => panic!("Should not allow storing files with wrong target path"),
-            Err(e) => {
-                let message = format!("{:?}", e);
-                assert!(message.contains(target_path));
-                assert!(message.contains("does not belong to the current storage"));
-            }
-        }
-        assert!(storage.list().await?.is_empty());
-
-        let target_path_1 = upload_dummy_file(&workdir, &storage, "upload_1", None).await?;
+        let target_path_1 = upload_dummy_file(&storage, "upload_1", None).await?;
         assert_eq!(
             storage.list().await?,
             vec![target_path_1.clone()],
             "Should list a single file after first upload"
         );
 
-        let target_path_2 = upload_dummy_file(&workdir, &storage, "upload_2", None).await?;
+        let target_path_2 = upload_dummy_file(&storage, "upload_2", None).await?;
         assert_eq!(
             list_files_sorted(&storage).await?,
             vec![target_path_1.clone(), target_path_2.clone()],
@@ -643,7 +411,7 @@ mod fs_tests {
     async fn upload_file_negatives() -> anyhow::Result<()> {
         let storage = create_storage()?;
 
-        let id = storage.remote_object_id(&storage.working_directory.join("dummy"))?;
+        let id = RemotePath::new(Path::new("dummy"))?;
         let content = std::io::Cursor::new(b"12345");
 
         // Check that you get an error if the size parameter doesn't match the actual
@@ -668,16 +436,14 @@ mod fs_tests {
     }
 
     fn create_storage() -> anyhow::Result<LocalFs> {
-        LocalFs::new(tempdir()?.path().to_owned(), tempdir()?.path().to_owned())
+        LocalFs::new(tempdir()?.path().to_owned())
     }
 
     #[tokio::test]
     async fn download_file() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
         let storage = create_storage()?;
         let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&workdir, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&storage, upload_name, None).await?;
 
         let contents = read_and_assert_remote_file_contents(&storage, &upload_target, None).await?;
         assert_eq!(
@@ -687,7 +453,7 @@ mod fs_tests {
         );
 
         let non_existing_path = "somewhere/else";
-        match storage.download(&RemoteObjectId(non_existing_path.to_string())).await {
+        match storage.download(&RemotePath::new(Path::new(non_existing_path))?).await {
             Err(DownloadError::NotFound) => {} // Should get NotFound for non existing keys
             other => panic!("Should get a NotFound error when downloading non-existing storage files, but got: {other:?}"),
         }
@@ -696,11 +462,9 @@ mod fs_tests {
 
     #[tokio::test]
     async fn download_file_range_positive() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
         let storage = create_storage()?;
         let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&workdir, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&storage, upload_name, None).await?;
 
         let full_range_download_contents =
             read_and_assert_remote_file_contents(&storage, &upload_target, None).await?;
@@ -766,11 +530,9 @@ mod fs_tests {
 
     #[tokio::test]
     async fn download_file_range_negative() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
         let storage = create_storage()?;
         let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&workdir, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&storage, upload_name, None).await?;
 
         let start = 1_000_000_000;
         let end = start + 1;
@@ -812,11 +574,9 @@ mod fs_tests {
 
     #[tokio::test]
     async fn delete_file() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
         let storage = create_storage()?;
         let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&workdir, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&storage, upload_name, None).await?;
 
         storage.delete(&upload_target).await?;
         assert!(storage.list().await?.is_empty());
@@ -826,7 +586,8 @@ mod fs_tests {
             Err(e) => {
                 let error_string = e.to_string();
                 assert!(error_string.contains("does not exist"));
-                assert!(error_string.contains(&upload_target.0));
+                let expected_path = upload_target.with_base(&storage.storage_root);
+                assert!(error_string.contains(expected_path.to_str().unwrap()));
             }
         }
         Ok(())
@@ -834,8 +595,6 @@ mod fs_tests {
 
     #[tokio::test]
     async fn file_with_metadata() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
         let storage = create_storage()?;
         let upload_name = "upload_1";
         let metadata = StorageMetadata(HashMap::from([
@@ -843,7 +602,7 @@ mod fs_tests {
             ("two".to_string(), "2".to_string()),
         ]));
         let upload_target =
-            upload_dummy_file(&workdir, &storage, upload_name, Some(metadata.clone())).await?;
+            upload_dummy_file(&storage, upload_name, Some(metadata.clone())).await?;
 
         let full_range_download_contents =
             read_and_assert_remote_file_contents(&storage, &upload_target, Some(&metadata)).await?;
@@ -883,23 +642,32 @@ mod fs_tests {
     }
 
     async fn upload_dummy_file(
-        workdir: &Path,
         storage: &LocalFs,
         name: &str,
         metadata: Option<StorageMetadata>,
-    ) -> anyhow::Result<RemoteObjectId> {
-        let timeline_path = workdir.join("timelines").join("some_timeline");
-        let relative_timeline_path = timeline_path.strip_prefix(&workdir)?;
-        let storage_path = storage.storage_root.join(relative_timeline_path).join(name);
-        let remote_object_id = RemoteObjectId(storage_path.to_str().unwrap().to_string());
-
-        let from_path = storage.working_directory.join(name);
+    ) -> anyhow::Result<RemotePath> {
+        let from_path = storage
+            .storage_root
+            .join("timelines")
+            .join("some_timeline")
+            .join(name);
         let (file, size) = create_file_for_upload(&from_path, &dummy_contents(name)).await?;
 
+        let relative_path = from_path
+            .strip_prefix(&storage.storage_root)
+            .context("Failed to strip storage root prefix")
+            .and_then(RemotePath::new)
+            .with_context(|| {
+                format!(
+                    "Failed to resolve remote part of path {:?} for base {:?}",
+                    from_path, storage.storage_root
+                )
+            })?;
+
         storage
-            .upload(Box::new(file), size, &remote_object_id, metadata)
+            .upload(Box::new(file), size, &relative_path, metadata)
             .await?;
-        remote_object_id_from_path(&storage_path)
+        Ok(relative_path)
     }
 
     async fn create_file_for_upload(
@@ -924,7 +692,7 @@ mod fs_tests {
         format!("contents for {name}")
     }
 
-    async fn list_files_sorted(storage: &LocalFs) -> anyhow::Result<Vec<RemoteObjectId>> {
+    async fn list_files_sorted(storage: &LocalFs) -> anyhow::Result<Vec<RemotePath>> {
         let mut files = storage.list().await?;
         files.sort_by(|a, b| a.0.cmp(&b.0));
         Ok(files)

libs/remote_storage/src/s3_bucket.rs

@@ -4,15 +4,13 @@
 //! allowing multiple api users to independently work with the same S3 bucket, if
 //! their bucket prefixes are both specified and different.
 
-use std::env::var;
-use std::path::{Path, PathBuf};
 use std::sync::Arc;
-use std::time::Duration;
 
 use anyhow::Context;
 use aws_config::{
-    environment::credentials::EnvironmentVariableCredentialsProvider, imds,
-    imds::credentials::ImdsCredentialsProvider, meta::credentials::provide_credentials_fn,
+    environment::credentials::EnvironmentVariableCredentialsProvider,
+    imds::credentials::ImdsCredentialsProvider,
+    meta::credentials::{CredentialsProviderChain, LazyCachingCredentialsProvider},
 };
 use aws_sdk_s3::{
     config::Config,
@@ -21,7 +19,6 @@ use aws_sdk_s3::{
     Client, Endpoint, Region,
 };
 use aws_smithy_http::body::SdkBody;
-use aws_types::credentials::{CredentialsError, ProvideCredentials};
 use hyper::Body;
 use tokio::{io, sync::Semaphore};
 use tokio_util::io::ReaderStream;
@@ -29,12 +26,9 @@ use tracing::debug;
 
 use super::StorageMetadata;
 use crate::{
-    strip_path_prefix, Download, DownloadError, RemoteObjectId, RemoteStorage, S3Config,
-    REMOTE_STORAGE_PREFIX_SEPARATOR,
+    Download, DownloadError, RemotePath, RemoteStorage, S3Config, REMOTE_STORAGE_PREFIX_SEPARATOR,
 };
 
-const DEFAULT_IMDS_TIMEOUT: Duration = Duration::from_secs(10);
-
 pub(super) mod metrics {
     use metrics::{register_int_counter_vec, IntCounterVec};
     use once_cell::sync::Lazy;
@@ -100,31 +94,8 @@ pub(super) mod metrics {
     }
 }
 
-fn download_destination(
-    id: &RemoteObjectId,
-    workdir: &Path,
-    prefix_to_strip: Option<&str>,
-) -> PathBuf {
-    let path_without_prefix = match prefix_to_strip {
-        Some(prefix) => id.0.strip_prefix(prefix).unwrap_or_else(|| {
-            panic!(
-                "Could not strip prefix '{}' from S3 object key '{}'",
-                prefix, id.0
-            )
-        }),
-        None => &id.0,
-    };
-
-    workdir.join(
-        path_without_prefix
-            .split(REMOTE_STORAGE_PREFIX_SEPARATOR)
-            .collect::<PathBuf>(),
-    )
-}
-
 /// AWS S3 storage.
 pub struct S3Bucket {
-    workdir: PathBuf,
     client: Client,
     bucket_name: String,
     prefix_in_bucket: Option<String>,
@@ -142,35 +113,28 @@ struct GetObjectRequest {
 }
 impl S3Bucket {
     /// Creates the S3 storage, errors if incorrect AWS S3 configuration provided.
-    pub fn new(aws_config: &S3Config, workdir: PathBuf) -> anyhow::Result<Self> {
+    pub fn new(aws_config: &S3Config) -> anyhow::Result<Self> {
         debug!(
             "Creating s3 remote storage for S3 bucket {}",
             aws_config.bucket_name
         );
+
+        let credentials_provider = {
+            // uses "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"
+            let env_creds = EnvironmentVariableCredentialsProvider::new();
+            // uses imds v2
+            let imds = ImdsCredentialsProvider::builder().build();
+
+            // finally add caching.
+            // this might change in future, see https://github.com/awslabs/aws-sdk-rust/issues/629
+            LazyCachingCredentialsProvider::builder()
+                .load(CredentialsProviderChain::first_try("env", env_creds).or_else("imds", imds))
+                .build()
+        };
+
         let mut config_builder = Config::builder()
             .region(Region::new(aws_config.bucket_region.clone()))
-            .credentials_provider(provide_credentials_fn(|| async {
-                match var("AWS_ACCESS_KEY_ID").is_ok() && var("AWS_SECRET_ACCESS_KEY").is_ok() {
-                    true => {
-                        EnvironmentVariableCredentialsProvider::new()
-                            .provide_credentials()
-                            .await
-                    }
-                    false => {
-                        let imds_client = imds::Client::builder()
-                            .connect_timeout(DEFAULT_IMDS_TIMEOUT)
-                            .read_timeout(DEFAULT_IMDS_TIMEOUT)
-                            .build()
-                            .await
-                            .map_err(CredentialsError::unhandled)?;
-                        ImdsCredentialsProvider::builder()
-                            .imds_client(imds_client)
-                            .build()
-                            .provide_credentials()
-                            .await
-                    }
-                }
-            }));
+            .credentials_provider(credentials_provider);
 
         if let Some(custom_endpoint) = aws_config.endpoint.clone() {
             let endpoint = Endpoint::immutable(
@@ -196,13 +160,39 @@ impl S3Bucket {
         });
         Ok(Self {
             client,
-            workdir,
             bucket_name: aws_config.bucket_name.clone(),
             prefix_in_bucket,
             concurrency_limiter: Semaphore::new(aws_config.concurrency_limit.get()),
         })
     }
 
+    fn s3_object_to_relative_path(&self, key: &str) -> RemotePath {
+        let relative_path =
+            match key.strip_prefix(self.prefix_in_bucket.as_deref().unwrap_or_default()) {
+                Some(stripped) => stripped,
+                // we rely on AWS to return properly prefixed paths
+                // for requests with a certain prefix
+                None => panic!(
+                    "Key {} does not start with bucket prefix {:?}",
+                    key, self.prefix_in_bucket
+                ),
+            };
+        RemotePath(
+            relative_path
+                .split(REMOTE_STORAGE_PREFIX_SEPARATOR)
+                .collect(),
+        )
+    }
+
+    fn relative_path_to_s3_object(&self, path: &RemotePath) -> String {
+        let mut full_path = self.prefix_in_bucket.clone().unwrap_or_default();
+        for segment in path.0.iter() {
+            full_path.push(REMOTE_STORAGE_PREFIX_SEPARATOR);
+            full_path.push_str(segment.to_str().unwrap_or_default());
+        }
+        full_path
+    }
+
     async fn download_object(&self, request: GetObjectRequest) -> Result<Download, DownloadError> {
         let _guard = self
             .concurrency_limiter
@@ -252,25 +242,7 @@ impl S3Bucket {
 
 #[async_trait::async_trait]
 impl RemoteStorage for S3Bucket {
-    fn remote_object_id(&self, local_path: &Path) -> anyhow::Result<RemoteObjectId> {
-        let relative_path = strip_path_prefix(&self.workdir, local_path)?;
-        let mut key = self.prefix_in_bucket.clone().unwrap_or_default();
-        for segment in relative_path {
-            key.push(REMOTE_STORAGE_PREFIX_SEPARATOR);
-            key.push_str(&segment.to_string_lossy());
-        }
-        Ok(RemoteObjectId(key))
-    }
-
-    fn local_path(&self, storage_path: &RemoteObjectId) -> anyhow::Result<PathBuf> {
-        Ok(download_destination(
-            storage_path,
-            &self.workdir,
-            self.prefix_in_bucket.as_deref(),
-        ))
-    }
-
-    async fn list(&self) -> anyhow::Result<Vec<RemoteObjectId>> {
+    async fn list(&self) -> anyhow::Result<Vec<RemotePath>> {
         let mut document_keys = Vec::new();
 
         let mut continuation_token = None;
@@ -300,7 +272,7 @@ impl RemoteStorage for S3Bucket {
                     .contents
                     .unwrap_or_default()
                     .into_iter()
-                    .filter_map(|o| Some(RemoteObjectId(o.key?))),
+                    .filter_map(|o| Some(self.s3_object_to_relative_path(o.key()?))),
             );
 
             match fetch_response.continuation_token {
@@ -314,13 +286,10 @@ impl RemoteStorage for S3Bucket {
 
     /// See the doc for `RemoteStorage::list_prefixes`
     /// Note: it wont include empty "directories"
-    async fn list_prefixes(
-        &self,
-        prefix: Option<&RemoteObjectId>,
-    ) -> anyhow::Result<Vec<RemoteObjectId>> {
+    async fn list_prefixes(&self, prefix: Option<&RemotePath>) -> anyhow::Result<Vec<RemotePath>> {
         // get the passed prefix or if it is not set use prefix_in_bucket value
         let list_prefix = prefix
-            .map(|p| p.0.clone())
+            .map(|p| self.relative_path_to_s3_object(p))
             .or_else(|| self.prefix_in_bucket.clone())
             .map(|mut p| {
                 // required to end with a separator
@@ -362,7 +331,7 @@ impl RemoteStorage for S3Bucket {
                     .common_prefixes
                     .unwrap_or_default()
                     .into_iter()
-                    .filter_map(|o| Some(RemoteObjectId(o.prefix?))),
+                    .filter_map(|o| Some(self.s3_object_to_relative_path(o.prefix()?))),
             );
 
             match fetch_response.continuation_token {
@@ -378,7 +347,7 @@ impl RemoteStorage for S3Bucket {
         &self,
         from: Box<(dyn io::AsyncRead + Unpin + Send + Sync + 'static)>,
         from_size_bytes: usize,
-        to: &RemoteObjectId,
+        to: &RemotePath,
         metadata: Option<StorageMetadata>,
     ) -> anyhow::Result<()> {
         let _guard = self
@@ -395,7 +364,7 @@ impl RemoteStorage for S3Bucket {
         self.client
             .put_object()
             .bucket(self.bucket_name.clone())
-            .key(to.0.to_owned())
+            .key(self.relative_path_to_s3_object(to))
             .set_metadata(metadata.map(|m| m.0))
             .content_length(from_size_bytes.try_into()?)
             .body(bytes_stream)
@@ -408,10 +377,10 @@ impl RemoteStorage for S3Bucket {
         Ok(())
     }
 
-    async fn download(&self, from: &RemoteObjectId) -> Result<Download, DownloadError> {
+    async fn download(&self, from: &RemotePath) -> Result<Download, DownloadError> {
         self.download_object(GetObjectRequest {
             bucket: self.bucket_name.clone(),
-            key: from.0.to_owned(),
+            key: self.relative_path_to_s3_object(from),
             ..GetObjectRequest::default()
         })
         .await
@@ -419,7 +388,7 @@ impl RemoteStorage for S3Bucket {
 
     async fn download_byte_range(
         &self,
-        from: &RemoteObjectId,
+        from: &RemotePath,
         start_inclusive: u64,
         end_exclusive: Option<u64>,
     ) -> Result<Download, DownloadError> {
@@ -427,19 +396,19 @@ impl RemoteStorage for S3Bucket {
         // and needs both ends to be exclusive
         let end_inclusive = end_exclusive.map(|end| end.saturating_sub(1));
         let range = Some(match end_inclusive {
-            Some(end_inclusive) => format!("bytes={}-{}", start_inclusive, end_inclusive),
-            None => format!("bytes={}-", start_inclusive),
+            Some(end_inclusive) => format!("bytes={start_inclusive}-{end_inclusive}"),
+            None => format!("bytes={start_inclusive}-"),
         });
 
         self.download_object(GetObjectRequest {
             bucket: self.bucket_name.clone(),
-            key: from.0.to_owned(),
+            key: self.relative_path_to_s3_object(from),
             range,
         })
         .await
     }
 
-    async fn delete(&self, remote_object_id: &RemoteObjectId) -> anyhow::Result<()> {
+    async fn delete(&self, path: &RemotePath) -> anyhow::Result<()> {
         let _guard = self
             .concurrency_limiter
             .acquire()
@@ -451,7 +420,7 @@ impl RemoteStorage for S3Bucket {
         self.client
             .delete_object()
             .bucket(self.bucket_name.clone())
-            .key(remote_object_id.0.to_owned())
+            .key(self.relative_path_to_s3_object(path))
             .send()
             .await
             .map_err(|e| {
@@ -461,181 +430,3 @@ impl RemoteStorage for S3Bucket {
         Ok(())
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use tempfile::tempdir;
-
-    use super::*;
-
-    #[test]
-    fn test_download_destination() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-        let local_path = workdir.join("one").join("two").join("test_name");
-        let relative_path = local_path.strip_prefix(&workdir)?;
-
-        let key = RemoteObjectId(format!(
-            "{}{}",
-            REMOTE_STORAGE_PREFIX_SEPARATOR,
-            relative_path
-                .iter()
-                .map(|segment| segment.to_str().unwrap())
-                .collect::<Vec<_>>()
-                .join(&REMOTE_STORAGE_PREFIX_SEPARATOR.to_string()),
-        ));
-
-        assert_eq!(
-            local_path,
-            download_destination(&key, &workdir, None),
-            "Download destination should consist of s3 path joined with the workdir prefix"
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn storage_path_positive() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-
-        let segment_1 = "matching";
-        let segment_2 = "file";
-        let local_path = &workdir.join(segment_1).join(segment_2);
-
-        let storage = dummy_storage(workdir);
-
-        let expected_key = RemoteObjectId(format!(
-            "{}{REMOTE_STORAGE_PREFIX_SEPARATOR}{segment_1}{REMOTE_STORAGE_PREFIX_SEPARATOR}{segment_2}",
-            storage.prefix_in_bucket.as_deref().unwrap_or_default(),
-        ));
-
-        let actual_key = storage
-            .remote_object_id(local_path)
-            .expect("Matching path should map to S3 path normally");
-        assert_eq!(
-            expected_key,
-            actual_key,
-            "S3 key from the matching path should contain all segments after the workspace prefix, separated with S3 separator"
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn storage_path_negatives() -> anyhow::Result<()> {
-        #[track_caller]
-        fn storage_path_error(storage: &S3Bucket, mismatching_path: &Path) -> String {
-            match storage.remote_object_id(mismatching_path) {
-                Ok(wrong_key) => panic!(
-                    "Expected path '{}' to error, but got S3 key: {:?}",
-                    mismatching_path.display(),
-                    wrong_key,
-                ),
-                Err(e) => e.to_string(),
-            }
-        }
-
-        let workdir = tempdir()?.path().to_owned();
-        let storage = dummy_storage(workdir.clone());
-
-        let error_message = storage_path_error(&storage, &workdir);
-        assert!(
-            error_message.contains("Prefix and the path are equal"),
-            "Message '{}' does not contain the required string",
-            error_message
-        );
-
-        let mismatching_path = PathBuf::from("somewhere").join("else");
-        let error_message = storage_path_error(&storage, &mismatching_path);
-        assert!(
-            error_message.contains(mismatching_path.to_str().unwrap()),
-            "Error should mention wrong path"
-        );
-        assert!(
-            error_message.contains(workdir.to_str().unwrap()),
-            "Error should mention server workdir"
-        );
-        assert!(
-            error_message.contains("is not prefixed with"),
-            "Message '{}' does not contain a required string",
-            error_message
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn local_path_positive() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-        let storage = dummy_storage(workdir.clone());
-        let timeline_dir = workdir.join("timelines").join("test_timeline");
-        let relative_timeline_path = timeline_dir.strip_prefix(&workdir)?;
-
-        let s3_key = create_s3_key(
-            &relative_timeline_path.join("not a metadata"),
-            storage.prefix_in_bucket.as_deref(),
-        );
-        assert_eq!(
-            download_destination(&s3_key, &workdir, storage.prefix_in_bucket.as_deref()),
-            storage
-                .local_path(&s3_key)
-                .expect("For a valid input, valid S3 info should be parsed"),
-            "Should be able to parse metadata out of the correctly named remote delta file"
-        );
-
-        let s3_key = create_s3_key(
-            &relative_timeline_path.join("metadata"),
-            storage.prefix_in_bucket.as_deref(),
-        );
-        assert_eq!(
-            download_destination(&s3_key, &workdir, storage.prefix_in_bucket.as_deref()),
-            storage
-                .local_path(&s3_key)
-                .expect("For a valid input, valid S3 info should be parsed"),
-            "Should be able to parse metadata out of the correctly named remote metadata file"
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn download_destination_matches_original_path() -> anyhow::Result<()> {
-        let workdir = tempdir()?.path().to_owned();
-        let original_path = workdir
-            .join("timelines")
-            .join("some_timeline")
-            .join("some name");
-
-        let dummy_storage = dummy_storage(workdir);
-
-        let key = dummy_storage.remote_object_id(&original_path)?;
-        let download_destination = dummy_storage.local_path(&key)?;
-
-        assert_eq!(
-            original_path, download_destination,
-            "'original path -> storage key -> matching fs path' transformation should produce the same path as the input one for the correct path"
-        );
-
-        Ok(())
-    }
-
-    fn dummy_storage(workdir: PathBuf) -> S3Bucket {
-        S3Bucket {
-            workdir,
-            client: Client::new(&aws_config::SdkConfig::builder().build()),
-            bucket_name: "dummy-bucket".to_string(),
-            prefix_in_bucket: Some("dummy_prefix/".to_string()),
-            concurrency_limiter: Semaphore::new(1),
-        }
-    }
-
-    fn create_s3_key(relative_file_path: &Path, prefix: Option<&str>) -> RemoteObjectId {
-        RemoteObjectId(relative_file_path.iter().fold(
-            prefix.unwrap_or_default().to_string(),
-            |mut path_string, segment| {
-                path_string.push(REMOTE_STORAGE_PREFIX_SEPARATOR);
-                path_string.push_str(segment.to_str().unwrap());
-                path_string
-            },
-        ))
-    }
-}

libs/safekeeper_api/src/models.rs

@@ -22,3 +22,40 @@ pub struct TimelineCreateRequest {
     // If not passed, it is assigned to the beginning of commit_lsn segment.
     pub local_start_lsn: Option<Lsn>,
 }
+
+fn lsn_invalid() -> Lsn {
+    Lsn::INVALID
+}
+
+/// Data about safekeeper's timeline, mirrors broker.proto.
+#[serde_as]
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct SkTimelineInfo {
+    /// Term of the last entry.
+    pub last_log_term: Option<u64>,
+    /// LSN of the last record.
+    #[serde_as(as = "DisplayFromStr")]
+    #[serde(default = "lsn_invalid")]
+    pub flush_lsn: Lsn,
+    /// Up to which LSN safekeeper regards its WAL as committed.
+    #[serde_as(as = "DisplayFromStr")]
+    #[serde(default = "lsn_invalid")]
+    pub commit_lsn: Lsn,
+    /// LSN up to which safekeeper has backed WAL.
+    #[serde_as(as = "DisplayFromStr")]
+    #[serde(default = "lsn_invalid")]
+    pub backup_lsn: Lsn,
+    /// LSN of last checkpoint uploaded by pageserver.
+    #[serde_as(as = "DisplayFromStr")]
+    #[serde(default = "lsn_invalid")]
+    pub remote_consistent_lsn: Lsn,
+    #[serde_as(as = "DisplayFromStr")]
+    #[serde(default = "lsn_invalid")]
+    pub peer_horizon_lsn: Lsn,
+    #[serde_as(as = "DisplayFromStr")]
+    #[serde(default = "lsn_invalid")]
+    pub local_start_lsn: Lsn,
+    /// A connection string to use for WAL receiving.
+    #[serde(default)]
+    pub safekeeper_connstr: Option<String>,
+}

libs/utils/Cargo.toml

@@ -4,6 +4,7 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
+sentry = "0.29.0"
 async-trait = "0.1"
 anyhow = "1.0"
 bincode = "1.3"

libs/utils/src/lib.rs

@@ -34,6 +34,7 @@ pub mod sock_split;
 pub mod logging;
 
 pub mod lock_file;
+pub mod pid_file;
 
 // Misc
 pub mod accum;
@@ -46,6 +47,7 @@ pub mod tcp_listener;
 pub mod nonblock;
 
 // Default signal handling
+pub mod sentry_init;
 pub mod signals;
 
 pub mod fs_ext;

libs/utils/src/lock_file.rs

@@ -1,81 +1,133 @@
-//! A module to create and read lock files. A lock file ensures that only one
-//! process is running at a time, in a particular directory.
+//! A module to create and read lock files.
 //!
-//! File locking is done using [`fcntl::flock`], which means that holding the
-//! lock on file only prevents acquiring another lock on it; all other
-//! operations are still possible on files. Other process can still open, read,
-//! write, or remove the file, for example.
-//! If the file is removed while a process is holding a lock on it,
-//! the process that holds the lock does not get any error or notification.
-//! Furthermore, you can create a new file with the same name and lock the new file,
-//! while the old process is still running.
-//! Deleting the lock file while the locking process is still running is a bad idea!
+//! File locking is done using [`fcntl::flock`] exclusive locks.
+//! The only consumer of this module is currently [`pid_file`].
+//! See the module-level comment there for potential pitfalls
+//! with lock files that are used to store PIDs (pidfiles).
 
-use std::{fs, os::unix::prelude::AsRawFd, path::Path};
+use std::{
+    fs,
+    io::{Read, Write},
+    ops::Deref,
+    os::unix::prelude::AsRawFd,
+    path::{Path, PathBuf},
+};
 
 use anyhow::Context;
-use nix::fcntl;
+use nix::{errno::Errno::EAGAIN, fcntl};
 
 use crate::crashsafe;
 
-pub enum LockCreationResult {
-    Created {
-        new_lock_contents: String,
-        file: fs::File,
-    },
-    AlreadyLocked {
-        existing_lock_contents: String,
-    },
-    CreationFailed(anyhow::Error),
+/// A handle to an open and unlocked, but not-yet-written lock file.
+/// Returned by [`create_exclusive`].
+#[must_use]
+pub struct UnwrittenLockFile {
+    path: PathBuf,
+    file: fs::File,
 }
 
-/// Creates a lock file in the path given and writes the given contents into the file.
-/// Note: The lock is automatically released when the file closed. You might want to use Box::leak to make sure it lives until the end of the program.
-pub fn create_lock_file(lock_file_path: &Path, contents: String) -> LockCreationResult {
-    let lock_file = match fs::OpenOptions::new()
+/// Returned by [`UnwrittenLockFile::write_content`].
+#[must_use]
+pub struct LockFileGuard(fs::File);
+
+impl Deref for LockFileGuard {
+    type Target = fs::File;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl UnwrittenLockFile {
+    /// Replace the content of this lock file with the byte representation of `contents`.
+    pub fn write_content(mut self, contents: String) -> anyhow::Result<LockFileGuard> {
+        self.file
+            .set_len(0)
+            .context("Failed to truncate lockfile")?;
+        self.file
+            .write_all(contents.as_bytes())
+            .with_context(|| format!("Failed to write '{contents}' contents into lockfile"))?;
+        crashsafe::fsync_file_and_parent(&self.path).context("fsync lockfile")?;
+        Ok(LockFileGuard(self.file))
+    }
+}
+
+/// Creates and opens a lock file in the path, grabs an exclusive flock on it, and returns
+/// a handle that allows overwriting the locked file's content.
+///
+/// The exclusive lock is released when dropping the returned handle.
+///
+/// It is not an error if the file already exists.
+/// It is an error if the file is already locked.
+pub fn create_exclusive(lock_file_path: &Path) -> anyhow::Result<UnwrittenLockFile> {
+    let lock_file = fs::OpenOptions::new()
         .create(true) // O_CREAT
         .write(true)
         .open(lock_file_path)
-        .context("Failed to open lock file")
-    {
-        Ok(file) => file,
-        Err(e) => return LockCreationResult::CreationFailed(e),
-    };
+        .context("open lock file")?;
 
-    match fcntl::flock(
+    let res = fcntl::flock(
         lock_file.as_raw_fd(),
         fcntl::FlockArg::LockExclusiveNonblock,
-    ) {
-        Ok(()) => {
-            match lock_file
-                .set_len(0)
-                .context("Failed to truncate lockfile")
-                .and_then(|()| {
-                    fs::write(lock_file_path, &contents).with_context(|| {
-                        format!("Failed to write '{contents}' contents into lockfile")
-                    })
-                })
-                .and_then(|()| {
-                    crashsafe::fsync_file_and_parent(lock_file_path)
-                        .context("Failed to fsync lockfile")
-                }) {
-                Ok(()) => LockCreationResult::Created {
-                    new_lock_contents: contents,
-                    file: lock_file,
-                },
-                Err(e) => LockCreationResult::CreationFailed(e),
-            }
-        }
-        Err(nix::errno::Errno::EAGAIN) => {
-            match fs::read_to_string(lock_file_path).context("Failed to read lockfile contents") {
-                Ok(existing_lock_contents) => LockCreationResult::AlreadyLocked {
-                    existing_lock_contents,
-                },
-                Err(e) => LockCreationResult::CreationFailed(e),
-            }
-        }
-        Err(e) => {
-            LockCreationResult::CreationFailed(anyhow::anyhow!("Failed to lock lockfile: {e}"))
-        }
+    );
+    match res {
+        Ok(()) => Ok(UnwrittenLockFile {
+            path: lock_file_path.to_owned(),
+            file: lock_file,
+        }),
+        Err(EAGAIN) => anyhow::bail!("file is already locked"),
+        Err(e) => Err(e).context("flock error"),
+    }
+}
+
+/// Returned by [`read_and_hold_lock_file`].
+/// Check out the [`pid_file`] module for what the variants mean
+/// and potential caveats if the lock files that are used to store PIDs.
+pub enum LockFileRead {
+    /// No file exists at the given path.
+    NotExist,
+    /// No other process held the lock file, so we grabbed an flock
+    /// on it and read its contents.
+    /// Release the flock by dropping the [`LockFileGuard`].
+    NotHeldByAnyProcess(LockFileGuard, String),
+    /// The file exists but another process was holding an flock on it.
+    LockedByOtherProcess {
+        not_locked_file: fs::File,
+        content: String,
+    },
+}
+
+/// Open & try to lock the lock file at the given `path`, returning a [handle][`LockFileRead`] to
+/// inspect its content. It is not an `Err(...)` if the file does not exist or is already locked.
+/// Check the [`LockFileRead`] variants for details.
+pub fn read_and_hold_lock_file(path: &Path) -> anyhow::Result<LockFileRead> {
+    let res = fs::OpenOptions::new().read(true).open(path);
+    let mut lock_file = match res {
+        Ok(f) => f,
+        Err(e) => match e.kind() {
+            std::io::ErrorKind::NotFound => return Ok(LockFileRead::NotExist),
+            _ => return Err(e).context("open lock file"),
+        },
+    };
+    let res = fcntl::flock(
+        lock_file.as_raw_fd(),
+        fcntl::FlockArg::LockExclusiveNonblock,
+    );
+    // We need the content regardless of lock success / failure.
+    // But, read it after flock so that, if it succeeded, the content is consistent.
+    let mut content = String::new();
+    lock_file
+        .read_to_string(&mut content)
+        .context("read lock file")?;
+    match res {
+        Ok(()) => Ok(LockFileRead::NotHeldByAnyProcess(
+            LockFileGuard(lock_file),
+            content,
+        )),
+        Err(EAGAIN) => Ok(LockFileRead::LockedByOtherProcess {
+            not_locked_file: lock_file,
+            content,
+        }),
+        Err(e) => Err(e).context("flock error"),
     }
 }

libs/utils/src/pid_file.rs

@@ -0,0 +1,165 @@
+//! Abstraction to create & read pidfiles.
+//!
+//! A pidfile is a file in the filesystem that stores a process's PID.
+//! Its purpose is to implement a singleton behavior where only
+//! one process of some "kind" is supposed to be running at a given time.
+//! The "kind" is identified by the pidfile.
+//!
+//! During process startup, the process that is supposed to be a singleton
+//! must [claim][`claim_for_current_process`] the pidfile first.
+//! If that is unsuccessful, the process must not act as the singleton, i.e.,
+//! it must not access any of the resources that only the singleton may access.
+//!
+//! A common need is to signal a running singleton process, e.g., to make
+//! it shut down and exit.
+//! For that, we have to [`read`] the pidfile. The result of the `read` operation
+//! tells us if there is any singleton process, and if so, what PID it has.
+//! We can then proceed to signal it, although some caveats still apply.
+//! Read the function-level documentation of [`read`] for that.
+//!
+//! ## Never Remove Pidfiles
+//!
+//! It would be natural to assume that the process who claimed the pidfile
+//! should remove it upon exit to avoid leaving a stale pidfile in place.
+//! However, we already have a reliable way to detect staleness of the pidfile,
+//! i.e., the `flock` that [claiming][`claim_for_current_process`] puts on it.
+//!
+//! And further, removing pidfiles would introduce a **catastrophic race condition**
+//! where two processes are running that are supposed to be singletons.
+//! Suppose we were to remove our pidfile during process shutdown.
+//! Here is how the race plays out:
+//! - Suppose we have a service called `myservice` with pidfile `myservice.pidfile`.
+//! - Process `A` starts to shut down.
+//! - Process `B` is just starting up
+//!     - It `open("myservice.pid", O_WRONLY|O_CREAT)` the file
+//!     - It blocks on `flock`
+//! - Process `A` removes the pidfile as the last step of its shutdown procedure
+//!     - `unlink("myservice.pid")
+//! - Process `A` exits
+//!     - This releases its `flock` and unblocks `B`
+//! - Process `B` still has the file descriptor for `myservice.pid` open
+//! - Process `B` writes its PID into `myservice.pid`.
+//! - But the `myservice.pid` file has been unlinked, so, there is `myservice.pid`
+//!   in the directory.
+//! - Process `C` starts
+//!     - It `open("myservice.pid", O_WRONLY|O_CREAT)` which creates a new file (new inode)
+//!     - It `flock`s the file, which, since it's a different file, does not block
+//!     - It writes its PID into the file
+//!
+//! At this point, `B` and `C` are running, which is hazardous.
+//! Morale of the story: don't unlink pidfiles, ever.
+
+use std::{ops::Deref, path::Path};
+
+use anyhow::Context;
+use nix::unistd::Pid;
+
+use crate::lock_file::{self, LockFileRead};
+
+/// Keeps a claim on a pidfile alive until it is dropped.
+/// Returned by [`claim_for_current_process`].
+#[must_use]
+pub struct PidFileGuard(lock_file::LockFileGuard);
+
+impl Deref for PidFileGuard {
+    type Target = lock_file::LockFileGuard;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+/// Try to claim `path` as a pidfile for the current process.
+///
+/// If another process has already claimed the pidfile, and it is still running,
+/// this function returns ane error.
+/// Otherwise, the function `flock`s the file and updates its contents to the
+/// current process's PID.
+/// If the update fails, the flock is released and an error returned.
+/// On success, the function returns a [`PidFileGuard`] to keep the flock alive.
+///
+/// ### Maintaining A Claim
+///
+/// It is the caller's responsibility to maintain the claim.
+/// The claim ends as soon as the returned guard object is dropped.
+/// To maintain the claim for the remaining lifetime of the current process,
+/// use [`std::mem::forget`] or similar.
+pub fn claim_for_current_process(path: &Path) -> anyhow::Result<PidFileGuard> {
+    let unwritten_lock_file = lock_file::create_exclusive(path).context("lock file")?;
+    // if any of the next steps fail, we drop the file descriptor and thereby release the lock
+    let guard = unwritten_lock_file
+        .write_content(Pid::this().to_string())
+        .context("write pid to lock file")?;
+    Ok(PidFileGuard(guard))
+}
+
+/// Returned by [`read`].
+pub enum PidFileRead {
+    /// No file exists at the given path.
+    NotExist,
+    /// The given pidfile is currently not claimed by any process.
+    /// To determine this, the [`read`] operation acquired
+    /// an exclusive flock on the file. The lock is still held and responsibility
+    /// to release it is returned through the guard object.
+    /// Before releasing it, other [`claim_for_current_process`] or [`read`] calls
+    /// will fail.
+    ///
+    /// ### Caveats
+    ///
+    /// Do not unlink the pidfile from the filesystem. See module-comment for why.
+    NotHeldByAnyProcess(PidFileGuard),
+    /// The given pidfile is still claimed by another process whose PID is given
+    /// as part of this variant.
+    ///
+    /// ### Caveats
+    ///
+    /// 1. The other process might exit at any time, turning the given PID stale.
+    /// 2. There is a small window in which `claim_for_current_process` has already
+    ///    locked the file but not yet updates its contents. [`read`] will return
+    ///    this variant here, but with the old file contents, i.e., a stale PID.
+    ///
+    /// The kernel is free to recycle PID once it has been `wait(2)`ed upon by
+    /// its creator. Thus, acting upon a stale PID, e.g., by issuing a `kill`
+    /// system call on it, bears the risk of killing an unrelated process.
+    /// This is an inherent limitation of using pidfiles.
+    /// The only race-free solution is to have a supervisor-process with a lifetime
+    /// that exceeds that of all of its child-processes (e.g., `runit`, `supervisord`).
+    LockedByOtherProcess(Pid),
+}
+
+/// Try to read the file at the given path as a pidfile that was previously created
+/// through [`claim_for_current_process`].
+///
+/// On success, this function returns a [`PidFileRead`].
+/// Check its docs for a description of the meaning of its different variants.
+pub fn read(pidfile: &Path) -> anyhow::Result<PidFileRead> {
+    let res = lock_file::read_and_hold_lock_file(pidfile).context("read and hold pid file")?;
+    let ret = match res {
+        LockFileRead::NotExist => PidFileRead::NotExist,
+        LockFileRead::NotHeldByAnyProcess(guard, _) => {
+            PidFileRead::NotHeldByAnyProcess(PidFileGuard(guard))
+        }
+        LockFileRead::LockedByOtherProcess {
+            not_locked_file: _not_locked_file,
+            content,
+        } => {
+            // XXX the read races with the write in claim_pid_file_for_pid().
+            // But pids are smaller than a page, so the kernel page cache will lock for us.
+            // The only problem is that we might get the old contents here.
+            // Can only fix that by implementing some scheme that downgrades the
+            // exclusive lock to shared lock in claim_pid_file_for_pid().
+            PidFileRead::LockedByOtherProcess(parse_pidfile_content(&content)?)
+        }
+    };
+    Ok(ret)
+}
+
+fn parse_pidfile_content(content: &str) -> anyhow::Result<Pid> {
+    let pid: i32 = content
+        .parse()
+        .map_err(|_| anyhow::anyhow!("parse pidfile content to PID"))?;
+    if pid < 1 {
+        anyhow::bail!("bad value in pidfile '{pid}'");
+    }
+    Ok(Pid::from_raw(pid))
+}

libs/utils/src/postgres_backend.rs

@@ -6,7 +6,7 @@
 use crate::sock_split::{BidiStream, ReadStream, WriteStream};
 use anyhow::{bail, ensure, Context, Result};
 use bytes::{Bytes, BytesMut};
-use pq_proto::{BeMessage, BeParameterStatusMessage, FeMessage, FeStartupPacket};
+use pq_proto::{BeMessage, FeMessage, FeStartupPacket};
 use rand::Rng;
 use serde::{Deserialize, Serialize};
 use std::fmt;
@@ -361,11 +361,9 @@ impl PostgresBackend {
                         match self.auth_type {
                             AuthType::Trust => {
                                 self.write_message_noflush(&BeMessage::AuthenticationOk)?
-                                    .write_message_noflush(&BeParameterStatusMessage::encoding())?
+                                    .write_message_noflush(&BeMessage::CLIENT_ENCODING)?
                                     // The async python driver requires a valid server_version
-                                    .write_message_noflush(&BeMessage::ParameterStatus(
-                                        BeParameterStatusMessage::ServerVersion("14.1"),
-                                    ))?
+                                    .write_message_noflush(&BeMessage::server_version("14.1"))?
                                     .write_message(&BeMessage::ReadyForQuery)?;
                                 self.state = ProtoState::Established;
                             }
@@ -413,7 +411,7 @@ impl PostgresBackend {
                     }
                 }
                 self.write_message_noflush(&BeMessage::AuthenticationOk)?
-                    .write_message_noflush(&BeParameterStatusMessage::encoding())?
+                    .write_message_noflush(&BeMessage::CLIENT_ENCODING)?
                     .write_message(&BeMessage::ReadyForQuery)?;
                 self.state = ProtoState::Established;
             }

libs/utils/src/postgres_backend_async.rs

@@ -6,7 +6,7 @@
 use crate::postgres_backend::AuthType;
 use anyhow::{bail, Context, Result};
 use bytes::{Bytes, BytesMut};
-use pq_proto::{BeMessage, BeParameterStatusMessage, FeMessage, FeStartupPacket};
+use pq_proto::{BeMessage, FeMessage, FeStartupPacket};
 use rand::Rng;
 use std::future::Future;
 use std::net::SocketAddr;
@@ -331,11 +331,9 @@ impl PostgresBackend {
                         match self.auth_type {
                             AuthType::Trust => {
                                 self.write_message(&BeMessage::AuthenticationOk)?
-                                    .write_message(&BeParameterStatusMessage::encoding())?
+                                    .write_message(&BeMessage::CLIENT_ENCODING)?
                                     // The async python driver requires a valid server_version
-                                    .write_message(&BeMessage::ParameterStatus(
-                                        BeParameterStatusMessage::ServerVersion("14.1"),
-                                    ))?
+                                    .write_message(&BeMessage::server_version("14.1"))?
                                     .write_message(&BeMessage::ReadyForQuery)?;
                                 self.state = ProtoState::Established;
                             }
@@ -384,7 +382,7 @@ impl PostgresBackend {
                     }
                 }
                 self.write_message(&BeMessage::AuthenticationOk)?
-                    .write_message(&BeParameterStatusMessage::encoding())?
+                    .write_message(&BeMessage::CLIENT_ENCODING)?
                     .write_message(&BeMessage::ReadyForQuery)?;
                 self.state = ProtoState::Established;
             }

libs/utils/src/sentry_init.rs

@@ -0,0 +1,29 @@
+use sentry::ClientInitGuard;
+use std::borrow::Cow;
+use std::env;
+
+pub use sentry::release_name;
+
+#[must_use]
+pub fn init_sentry(
+    release_name: Option<Cow<'static, str>>,
+    extra_options: &[(&str, &str)],
+) -> Option<ClientInitGuard> {
+    let dsn = env::var("SENTRY_DSN").ok()?;
+    let environment = env::var("SENTRY_ENVIRONMENT").unwrap_or_else(|_| "development".into());
+
+    let guard = sentry::init((
+        dsn,
+        sentry::ClientOptions {
+            release: release_name,
+            environment: Some(environment.into()),
+            ..Default::default()
+        },
+    ));
+    sentry::configure_scope(|scope| {
+        for &(key, value) in extra_options {
+            scope.set_extra(key, value.into());
+        }
+    });
+    Some(guard)
+}

libs/utils/src/seqwait.rs

@@ -11,11 +11,13 @@ use tokio::time::timeout;
 
 /// An error happened while waiting for a number
 #[derive(Debug, PartialEq, Eq, thiserror::Error)]
-#[error("SeqWaitError")]
 pub enum SeqWaitError {
     /// The wait timeout was reached
+    #[error("seqwait timeout was reached")]
     Timeout,
+
     /// [`SeqWait::shutdown`] was called
+    #[error("SeqWait::shutdown was called")]
     Shutdown,
 }
 

pageserver/Cargo.toml

@@ -18,7 +18,7 @@ async-stream = "0.3"
 async-trait = "0.1"
 byteorder = "1.4.3"
 bytes = "1.0.1"
-chrono = "0.4.19"
+chrono = { version = "0.4.23", default-features = false, features = ["clock"] }
 clap = { version = "4.0", features = ["string"] }
 close_fds = "0.3.2"
 const_format = "0.2.21"
@@ -36,9 +36,9 @@ nix = "0.25"
 num-traits = "0.2.15"
 once_cell = "1.13.0"
 pin-project-lite = "0.2.7"
-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
-postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
+postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
+postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
 pprof = { git = "https://github.com/neondatabase/pprof-rs.git", branch = "wallclock-profiling", features = ["flamegraph"], optional = true }
 rand = "0.8.3"
 regex = "1.4.5"
@@ -52,20 +52,20 @@ svg_fmt = "0.4.1"
 tar = "0.4.33"
 thiserror = "1.0"
 tokio = { version = "1.17", features = ["process", "sync", "macros", "fs", "rt", "io-util", "time"] }
-tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
 tokio-util = { version = "0.7.3", features = ["io", "io-util"] }
 toml_edit = { version = "0.14", features = ["easy"] }
 tracing = "0.1.36"
 url = "2"
 walkdir = "2.3.2"
 
-etcd_broker = { path = "../libs/etcd_broker" }
 metrics = { path = "../libs/metrics" }
 pageserver_api = { path = "../libs/pageserver_api" }
 postgres_connection = { path = "../libs/postgres_connection" }
 postgres_ffi = { path = "../libs/postgres_ffi" }
 pq_proto = { path = "../libs/pq_proto" }
 remote_storage = { path = "../libs/remote_storage" }
+storage_broker = { version = "0.1", path = "../storage_broker" }
 tenant_size_model = { path = "../libs/tenant_size_model" }
 utils = { path = "../libs/utils" }
 workspace_hack = { version = "0.1", path = "../workspace_hack" }

pageserver/benches/bench_layer_map.rs

@@ -1,10 +1,9 @@
 use anyhow::Result;
-use pageserver::repository::{Key, Value};
+use pageserver::repository::Key;
 use pageserver::tenant::filename::{DeltaFileName, ImageFileName};
 use pageserver::tenant::layer_map::LayerMap;
-use pageserver::tenant::storage_layer::Layer;
-use pageserver::tenant::storage_layer::ValueReconstructResult;
 use pageserver::tenant::storage_layer::ValueReconstructState;
+use pageserver::tenant::storage_layer::{Layer, ValueReconstructResult};
 use rand::prelude::{SeedableRng, SliceRandom, StdRng};
 use std::cmp::{max, min};
 use std::fs::File;
@@ -14,7 +13,7 @@ use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::Arc;
 use std::time::Instant;
-use utils::id::{TenantId, TimelineId};
+
 use utils::lsn::Lsn;
 
 use criterion::{criterion_group, criterion_main, Criterion};
@@ -25,14 +24,6 @@ struct DummyDelta {
 }
 
 impl Layer for DummyDelta {
-    fn get_tenant_id(&self) -> TenantId {
-        TenantId::from_str("00000000000000000000000000000000").unwrap()
-    }
-
-    fn get_timeline_id(&self) -> TimelineId {
-        TimelineId::from_str("00000000000000000000000000000000").unwrap()
-    }
-
     fn get_key_range(&self) -> Range<Key> {
         self.key_range.clone()
     }
@@ -40,15 +31,6 @@ impl Layer for DummyDelta {
     fn get_lsn_range(&self) -> Range<Lsn> {
         self.lsn_range.clone()
     }
-
-    fn filename(&self) -> PathBuf {
-        todo!()
-    }
-
-    fn local_path(&self) -> Option<PathBuf> {
-        todo!()
-    }
-
     fn get_value_reconstruct_data(
         &self,
         _key: Key,
@@ -62,24 +44,12 @@ impl Layer for DummyDelta {
         true
     }
 
-    fn is_in_memory(&self) -> bool {
-        false
-    }
-
-    fn iter(&self) -> Box<dyn Iterator<Item = Result<(Key, Lsn, Value)>> + '_> {
-        panic!()
-    }
-
-    fn key_iter(&self) -> Box<dyn Iterator<Item = (Key, Lsn, u64)> + '_> {
-        panic!("Not implemented")
-    }
-
-    fn delete(&self) -> Result<()> {
-        panic!()
-    }
-
     fn dump(&self, _verbose: bool) -> Result<()> {
-        todo!()
+        unimplemented!()
+    }
+
+    fn short_id(&self) -> String {
+        unimplemented!()
     }
 }
 
@@ -89,14 +59,6 @@ struct DummyImage {
 }
 
 impl Layer for DummyImage {
-    fn get_tenant_id(&self) -> TenantId {
-        TenantId::from_str("00000000000000000000000000000000").unwrap()
-    }
-
-    fn get_timeline_id(&self) -> TimelineId {
-        TimelineId::from_str("00000000000000000000000000000000").unwrap()
-    }
-
     fn get_key_range(&self) -> Range<Key> {
         self.key_range.clone()
     }
@@ -106,14 +68,6 @@ impl Layer for DummyImage {
         self.lsn..(self.lsn + 1)
     }
 
-    fn filename(&self) -> PathBuf {
-        todo!()
-    }
-
-    fn local_path(&self) -> Option<PathBuf> {
-        todo!()
-    }
-
     fn get_value_reconstruct_data(
         &self,
         _key: Key,
@@ -127,29 +81,17 @@ impl Layer for DummyImage {
         false
     }
 
-    fn is_in_memory(&self) -> bool {
-        false
-    }
-
-    fn iter(&self) -> Box<dyn Iterator<Item = Result<(Key, Lsn, Value)>> + '_> {
-        panic!()
-    }
-
-    fn key_iter(&self) -> Box<dyn Iterator<Item = (Key, Lsn, u64)> + '_> {
-        panic!("Not implemented")
-    }
-
-    fn delete(&self) -> Result<()> {
-        panic!()
-    }
-
     fn dump(&self, _verbose: bool) -> Result<()> {
-        todo!()
+        unimplemented!()
+    }
+
+    fn short_id(&self) -> String {
+        unimplemented!()
     }
 }
 
-fn build_layer_map(filename_dump: PathBuf) -> LayerMap {
-    let mut layer_map = LayerMap::default();
+fn build_layer_map(filename_dump: PathBuf) -> LayerMap<dyn Layer> {
+    let mut layer_map = LayerMap::<dyn Layer>::default();
 
     let mut min_lsn = Lsn(u64::MAX);
     let mut max_lsn = Lsn(0);
@@ -185,7 +127,7 @@ fn build_layer_map(filename_dump: PathBuf) -> LayerMap {
 }
 
 /// Construct a layer map query pattern for benchmarks
-fn uniform_query_pattern(layer_map: &LayerMap) -> Vec<(Key, Lsn)> {
+fn uniform_query_pattern(layer_map: &LayerMap<dyn Layer>) -> Vec<(Key, Lsn)> {
     // For each image layer we query one of the pages contained, at LSN right
     // before the image layer was created. This gives us a somewhat uniform
     // coverage of both the lsn and key space because image layers have
@@ -258,7 +200,7 @@ fn bench_from_real_project(c: &mut Criterion) {
 
 // Benchmark using synthetic data. Arrange image layers on stacked diagonal lines.
 fn bench_sequential(c: &mut Criterion) {
-    let mut layer_map = LayerMap::default();
+    let mut layer_map: LayerMap<dyn Layer> = LayerMap::default();
 
     // Init layer map. Create 100_000 layers arranged in 1000 diagonal lines.
     //

pageserver/benches/bench_walredo.rs

@@ -431,7 +431,7 @@ fn pg_record(will_init: bool, bytes: &'static [u8]) -> NeonWalRecord {
 struct Request {
     key: Key,
     lsn: Lsn,
-    base_img: Option<Bytes>,
+    base_img: Option<(Lsn, Bytes)>,
     records: Vec<(Lsn, NeonWalRecord)>,
     pg_version: u32,
 }

pageserver/src/bin/draw_timeline_dir.rs

@@ -11,8 +11,8 @@
 //!
 //! Example use:
 //! ```
-//! $ cd test_output/test_pgbench\[neon-45-684\]/repo/tenants/$TENANT/timelines/$TIMELINE
-//! $ ls | grep "__" | cargo run --release --bin draw_timeline_dir > out.svg
+//! $ ls test_output/test_pgbench\[neon-45-684\]/repo/tenants/$TENANT/timelines/$TIMELINE | \
+//! $   grep "__" | cargo run --release --bin draw_timeline_dir > out.svg
 //! $ firefox out.svg
 //! ```
 //!
@@ -25,6 +25,8 @@ use anyhow::Result;
 use pageserver::repository::Key;
 use std::cmp::Ordering;
 use std::io::{self, BufRead};
+use std::path::PathBuf;
+use std::str::FromStr;
 use std::{
     collections::{BTreeMap, BTreeSet},
     ops::Range,
@@ -65,7 +67,11 @@ fn main() -> Result<()> {
     let mut ranges: Vec<(Range<Key>, Range<Lsn>)> = vec![];
     let stdin = io::stdin();
     for line in stdin.lock().lines() {
-        let range = parse_filename(&line.unwrap());
+        let line = line.unwrap();
+        let line = PathBuf::from_str(&line).unwrap();
+        let filename = line.file_name().unwrap();
+        let filename = filename.to_str().unwrap();
+        let range = parse_filename(filename);
         ranges.push(range);
     }
 

pageserver/src/bin/pageserver.rs

@@ -7,7 +7,6 @@ use std::{env, ops::ControlFlow, path::Path, str::FromStr};
 use anyhow::{anyhow, Context};
 use clap::{Arg, ArgAction, Command};
 use fail::FailScenario;
-use nix::unistd::Pid;
 use tracing::*;
 
 use metrics::set_build_info_metric;
@@ -23,9 +22,10 @@ use pageserver::{
 use remote_storage::GenericRemoteStorage;
 use utils::{
     auth::JwtAuth,
-    lock_file, logging,
+    logging,
     postgres_backend::AuthType,
     project_git_version,
+    sentry_init::{init_sentry, release_name},
     signals::{self, Signal},
     tcp_listener,
 };
@@ -85,6 +85,9 @@ fn main() -> anyhow::Result<()> {
         }
     };
 
+    // initialize sentry if SENTRY_DSN is provided
+    let _sentry_guard = init_sentry(release_name!(), &[("node_id", &conf.id.to_string())]);
+
     let tenants_path = conf.tenants_path();
     if !tenants_path.exists() {
         utils::crashsafe::create_dir_all(conf.tenants_path()).with_context(|| {
@@ -198,8 +201,12 @@ fn initialize_config(
 }
 
 fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
+    // Initialize logging
     logging::init(conf.log_format)?;
+
+    // Print version to the log, and expose it as a prometheus metric too.
     info!("version: {}", version());
+    set_build_info_metric(GIT_VERSION);
 
     // If any failpoints were set from FAILPOINTS environment variable,
     // print them to the log for debugging purposes
@@ -215,53 +222,37 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
         )
     }
 
+    // Create and lock PID file. This ensures that there cannot be more than one
+    // pageserver process running at the same time.
     let lock_file_path = conf.workdir.join(PID_FILE_NAME);
-    let lock_file = match lock_file::create_lock_file(&lock_file_path, Pid::this().to_string()) {
-        lock_file::LockCreationResult::Created {
-            new_lock_contents,
-            file,
-        } => {
-            info!("Created lock file at {lock_file_path:?} with contenst {new_lock_contents}");
-            file
-        }
-        lock_file::LockCreationResult::AlreadyLocked {
-            existing_lock_contents,
-        } => anyhow::bail!(
-            "Could not lock pid file; pageserver is already running in {:?} with PID {}",
-            conf.workdir,
-            existing_lock_contents
-        ),
-        lock_file::LockCreationResult::CreationFailed(e) => {
-            return Err(e.context(format!("Failed to create lock file at {lock_file_path:?}")))
-        }
-    };
-    // ensure that the lock file is held even if the main thread of the process is panics
-    // we need to release the lock file only when the current process is gone
-    let _ = Box::leak(Box::new(lock_file));
+    let lock_file =
+        utils::pid_file::claim_for_current_process(&lock_file_path).context("claim pid file")?;
+    info!("Claimed pid file at {lock_file_path:?}");
 
-    // TODO: Check that it looks like a valid repository before going further
+    // Ensure that the lock file is held even if the main thread of the process panics.
+    // We need to release the lock file only when the process exits.
+    std::mem::forget(lock_file);
 
-    // bind sockets before daemonizing so we report errors early and do not return until we are listening
-    info!(
-        "Starting pageserver http handler on {}",
-        conf.listen_http_addr
-    );
-    let http_listener = tcp_listener::bind(conf.listen_http_addr.clone())?;
+    // Bind the HTTP and libpq ports early, so that if they are in use by some other
+    // process, we error out early.
+    let http_addr = &conf.listen_http_addr;
+    info!("Starting pageserver http handler on {http_addr}");
+    let http_listener = tcp_listener::bind(http_addr)?;
 
-    info!(
-        "Starting pageserver pg protocol handler on {}",
-        conf.listen_pg_addr
-    );
-    let pageserver_listener = tcp_listener::bind(conf.listen_pg_addr.clone())?;
+    let pg_addr = &conf.listen_pg_addr;
+    info!("Starting pageserver pg protocol handler on {pg_addr}");
+    let pageserver_listener = tcp_listener::bind(pg_addr)?;
 
+    // Install signal handlers
     let signals = signals::install_shutdown_handlers()?;
 
-    // start profiler (if enabled)
+    // Start profiler (if enabled)
     let profiler_guard = profiling::init_profiler(conf);
 
-    WALRECEIVER_RUNTIME.block_on(pageserver::walreceiver::init_etcd_client(conf))?;
+    // Launch broker client
+    WALRECEIVER_RUNTIME.block_on(pageserver::walreceiver::init_broker_client(conf))?;
 
-    // initialize authentication for incoming connections
+    // Initialize authentication for incoming connections
     let auth = match &conf.auth_type {
         AuthType::Trust | AuthType::MD5 => None,
         AuthType::NeonJWT => {
@@ -289,29 +280,26 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
         }
     };
 
+    // Set up remote storage client
     let remote_storage = conf
         .remote_storage_config
         .as_ref()
-        .map(|storage_config| {
-            GenericRemoteStorage::from_config(conf.workdir.clone(), storage_config)
-        })
+        .map(GenericRemoteStorage::from_config)
         .transpose()
         .context("Failed to init generic remote storage")?;
-    {
-        let _rt_guard = BACKGROUND_RUNTIME.enter();
-        tenant_mgr::init_tenant_mgr(conf, remote_storage.clone())?
-    };
 
-    // Spawn all HTTP related tasks in the MGMT_REQUEST_RUNTIME.
-    // bind before launching separate thread so the error reported before startup exits
+    // Scan the local 'tenants/' directory and start loading the tenants
+    BACKGROUND_RUNTIME.block_on(tenant_mgr::init_tenant_mgr(conf, remote_storage.clone()))?;
 
-    // Create a Service from the router above to handle incoming requests.
+    // Start up the service to handle HTTP mgmt API request. We created the
+    // listener earlier already.
     {
         let _rt_guard = MGMT_REQUEST_RUNTIME.enter();
 
-        let router = http::make_router(conf, auth.clone(), remote_storage)?;
-        let service =
-            utils::http::RouterService::new(router.build().map_err(|err| anyhow!(err))?).unwrap();
+        let router = http::make_router(conf, auth.clone(), remote_storage)?
+            .build()
+            .map_err(|err| anyhow!(err))?;
+        let service = utils::http::RouterService::new(router).unwrap();
         let server = hyper::Server::from_tcp(http_listener)?
             .serve(service)
             .with_graceful_shutdown(task_mgr::shutdown_watcher());
@@ -331,7 +319,7 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
     }
 
     // Spawn a task to listen for libpq connections. It will spawn further tasks
-    // for each connection.
+    // for each connection. We created the listener earlier already.
     task_mgr::spawn(
         COMPUTE_REQUEST_RUNTIME.handle(),
         TaskKind::LibpqEndpointListener,
@@ -344,8 +332,6 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
         },
     );
 
-    set_build_info_metric(GIT_VERSION);
-
     // All started up! Now just sit and wait for shutdown signal.
     signals.handle(|signal| match signal {
         Signal::Quit => {

pageserver/src/config.rs

@@ -5,8 +5,9 @@
 //! See also `settings.md` for better description on every parameter.
 
 use anyhow::{anyhow, bail, ensure, Context, Result};
-use remote_storage::RemoteStorageConfig;
+use remote_storage::{RemotePath, RemoteStorageConfig};
 use std::env;
+use storage_broker::Uri;
 use utils::crashsafe::path_with_suffix_extension;
 use utils::id::ConnectionId;
 
@@ -18,7 +19,7 @@ use std::sync::Arc;
 use std::time::Duration;
 use toml_edit;
 use toml_edit::{Document, Item};
-use url::Url;
+
 use utils::{
     id::{NodeId, TenantId, TimelineId},
     logging::LogFormat,
@@ -27,7 +28,9 @@ use utils::{
 
 use crate::tenant::{TENANT_ATTACHING_MARKER_FILENAME, TIMELINES_SEGMENT_NAME};
 use crate::tenant_config::{TenantConf, TenantConfOpt};
-use crate::{METADATA_FILE_NAME, TENANT_CONFIG_NAME, TIMELINE_UNINIT_MARK_SUFFIX};
+use crate::{
+    IGNORED_TENANT_FILE_NAME, METADATA_FILE_NAME, TENANT_CONFIG_NAME, TIMELINE_UNINIT_MARK_SUFFIX,
+};
 
 pub mod defaults {
     use crate::tenant_config::defaults::*;
@@ -37,6 +40,7 @@ pub mod defaults {
         DEFAULT_HTTP_LISTEN_ADDR, DEFAULT_HTTP_LISTEN_PORT, DEFAULT_PG_LISTEN_ADDR,
         DEFAULT_PG_LISTEN_PORT,
     };
+    pub use storage_broker::DEFAULT_ENDPOINT as BROKER_DEFAULT_ENDPOINT;
 
     pub const DEFAULT_WAIT_LSN_TIMEOUT: &str = "60 s";
     pub const DEFAULT_WAL_REDO_TIMEOUT: &str = "60 s";
@@ -57,7 +61,6 @@ pub mod defaults {
     pub const DEFAULT_CONFIG_FILE: &str = formatcp!(
         r###"
 # Initial configuration file created by 'pageserver --init'
-
 #listen_pg_addr = '{DEFAULT_PG_LISTEN_ADDR}'
 #listen_http_addr = '{DEFAULT_HTTP_LISTEN_ADDR}'
 
@@ -69,6 +72,8 @@ pub mod defaults {
 # initial superuser role name to use when creating a new tenant
 #initial_superuser_name = '{DEFAULT_SUPERUSER}'
 
+#broker_endpoint = '{BROKER_DEFAULT_ENDPOINT}'
+
 #log_format = '{DEFAULT_LOG_FORMAT}'
 
 #concurrent_tenant_size_logical_size_queries = '{DEFAULT_CONCURRENT_TENANT_SIZE_LOGICAL_SIZE_QUERIES}'
@@ -130,12 +135,9 @@ pub struct PageServerConf {
     pub profiling: ProfilingConfig,
     pub default_tenant_conf: TenantConf,
 
-    /// A prefix to add in etcd brokers before every key.
-    /// Can be used for isolating different pageserver groups within the same etcd cluster.
-    pub broker_etcd_prefix: String,
-
-    /// Etcd broker endpoints to connect to.
-    pub broker_endpoints: Vec<Url>,
+    /// Storage broker endpoints to connect to.
+    pub broker_endpoint: Uri,
+    pub broker_keepalive_interval: Duration,
 
     pub log_format: LogFormat,
 
@@ -146,8 +148,7 @@ pub struct PageServerConf {
 /// We do not want to store this in a PageServerConf because the latter may be logged
 /// and/or serialized at a whim, while the token is secret. Currently this token is the
 /// same for accessing all tenants/timelines, but may become per-tenant/per-timeline in
-/// the future, more tokens and auth may arrive for etcd and/or its rewrite (see
-/// https://github.com/neondatabase/neon/issues/2394), completely changing the logic.
+/// the future, more tokens and auth may arrive for storage broker, completely changing the logic.
 /// Hence, we resort to a global variable for now instead of passing the token from the
 /// startup code to the connection code through a dozen layers.
 pub static SAFEKEEPER_AUTH_TOKEN: OnceCell<Arc<String>> = OnceCell::new();
@@ -214,8 +215,8 @@ struct PageServerConfigBuilder {
     id: BuilderValue<NodeId>,
 
     profiling: BuilderValue<ProfilingConfig>,
-    broker_etcd_prefix: BuilderValue<String>,
-    broker_endpoints: BuilderValue<Vec<Url>>,
+    broker_endpoint: BuilderValue<Uri>,
+    broker_keepalive_interval: BuilderValue<Duration>,
 
     log_format: BuilderValue<LogFormat>,
 
@@ -245,8 +246,13 @@ impl Default for PageServerConfigBuilder {
             remote_storage_config: Set(None),
             id: NotSet,
             profiling: Set(ProfilingConfig::Disabled),
-            broker_etcd_prefix: Set(etcd_broker::DEFAULT_NEON_BROKER_ETCD_PREFIX.to_string()),
-            broker_endpoints: Set(Vec::new()),
+            broker_endpoint: Set(storage_broker::DEFAULT_ENDPOINT
+                .parse()
+                .expect("failed to parse default broker endpoint")),
+            broker_keepalive_interval: Set(humantime::parse_duration(
+                storage_broker::DEFAULT_KEEPALIVE_INTERVAL,
+            )
+            .expect("cannot parse default keepalive interval")),
             log_format: Set(LogFormat::from_str(DEFAULT_LOG_FORMAT).unwrap()),
 
             concurrent_tenant_size_logical_size_queries: Set(ConfigurableSemaphore::default()),
@@ -306,12 +312,12 @@ impl PageServerConfigBuilder {
         self.remote_storage_config = BuilderValue::Set(remote_storage_config)
     }
 
-    pub fn broker_endpoints(&mut self, broker_endpoints: Vec<Url>) {
-        self.broker_endpoints = BuilderValue::Set(broker_endpoints)
+    pub fn broker_endpoint(&mut self, broker_endpoint: Uri) {
+        self.broker_endpoint = BuilderValue::Set(broker_endpoint)
     }
 
-    pub fn broker_etcd_prefix(&mut self, broker_etcd_prefix: String) {
-        self.broker_etcd_prefix = BuilderValue::Set(broker_etcd_prefix)
+    pub fn broker_keepalive_interval(&mut self, broker_keepalive_interval: Duration) {
+        self.broker_keepalive_interval = BuilderValue::Set(broker_keepalive_interval)
     }
 
     pub fn id(&mut self, node_id: NodeId) {
@@ -331,10 +337,6 @@ impl PageServerConfigBuilder {
     }
 
     pub fn build(self) -> anyhow::Result<PageServerConf> {
-        let broker_endpoints = self
-            .broker_endpoints
-            .ok_or(anyhow!("No broker endpoints provided"))?;
-
         Ok(PageServerConf {
             listen_pg_addr: self
                 .listen_pg_addr
@@ -370,10 +372,12 @@ impl PageServerConfigBuilder {
             profiling: self.profiling.ok_or(anyhow!("missing profiling"))?,
             // TenantConf is handled separately
             default_tenant_conf: TenantConf::default(),
-            broker_endpoints,
-            broker_etcd_prefix: self
-                .broker_etcd_prefix
-                .ok_or(anyhow!("missing broker_etcd_prefix"))?,
+            broker_endpoint: self
+                .broker_endpoint
+                .ok_or(anyhow!("No broker endpoints provided"))?,
+            broker_keepalive_interval: self
+                .broker_keepalive_interval
+                .ok_or(anyhow!("No broker keepalive interval provided"))?,
             log_format: self.log_format.ok_or(anyhow!("missing log_format"))?,
             concurrent_tenant_size_logical_size_queries: self
                 .concurrent_tenant_size_logical_size_queries
@@ -402,6 +406,10 @@ impl PageServerConf {
             .join(TENANT_ATTACHING_MARKER_FILENAME)
     }
 
+    pub fn tenant_ignore_mark_file_path(&self, tenant_id: TenantId) -> PathBuf {
+        self.tenant_path(&tenant_id).join(IGNORED_TENANT_FILE_NAME)
+    }
+
     /// Points to a place in pageserver's local directory,
     /// where certain tenant's tenantconf file should be located.
     pub fn tenant_config_path(&self, tenant_id: TenantId) -> PathBuf {
@@ -450,6 +458,28 @@ impl PageServerConf {
             .join(METADATA_FILE_NAME)
     }
 
+    /// Files on the remote storage are stored with paths, relative to the workdir.
+    /// That path includes in itself both tenant and timeline ids, allowing to have a unique remote storage path.
+    ///
+    /// Errors if the path provided does not start from pageserver's workdir.
+    pub fn remote_path(&self, local_path: &Path) -> anyhow::Result<RemotePath> {
+        local_path
+            .strip_prefix(&self.workdir)
+            .context("Failed to strip workdir prefix")
+            .and_then(RemotePath::new)
+            .with_context(|| {
+                format!(
+                    "Failed to resolve remote part of path {:?} for base {:?}",
+                    local_path, self.workdir
+                )
+            })
+    }
+
+    /// Turns storage remote path of a file into its local path.
+    pub fn local_path(&self, remote_path: &RemotePath) -> PathBuf {
+        remote_path.with_base(&self.workdir)
+    }
+
     //
     // Postgres distribution paths
     //
@@ -486,7 +516,7 @@ impl PageServerConf {
         let mut builder = PageServerConfigBuilder::default();
         builder.workdir(workdir.to_owned());
 
-        let mut t_conf: TenantConfOpt = Default::default();
+        let mut t_conf = TenantConfOpt::default();
 
         for (key, item) in toml.iter() {
             match key {
@@ -507,24 +537,15 @@ impl PageServerConf {
                 )),
                 "auth_type" => builder.auth_type(parse_toml_from_str(key, item)?),
                 "remote_storage" => {
-                    builder.remote_storage_config(Some(RemoteStorageConfig::from_toml(item)?))
+                    builder.remote_storage_config(RemoteStorageConfig::from_toml(item)?)
                 }
                 "tenant_config" => {
                     t_conf = Self::parse_toml_tenant_conf(item)?;
                 }
                 "id" => builder.id(NodeId(parse_toml_u64(key, item)?)),
                 "profiling" => builder.profiling(parse_toml_from_str(key, item)?),
-                "broker_etcd_prefix" => builder.broker_etcd_prefix(parse_toml_string(key, item)?),
-                "broker_endpoints" => builder.broker_endpoints(
-                    parse_toml_array(key, item)?
-                        .into_iter()
-                        .map(|endpoint_str| {
-                            endpoint_str.parse::<Url>().with_context(|| {
-                                format!("Array item {endpoint_str} for key {key} is not a valid url endpoint")
-                            })
-                        })
-                        .collect::<anyhow::Result<_>>()?,
-                ),
+                "broker_endpoint" => builder.broker_endpoint(parse_toml_string(key, item)?.parse().context("failed to parse broker endpoint")?),
+                "broker_keepalive_interval" => builder.broker_keepalive_interval(parse_toml_duration(key, item)?),
                 "log_format" => builder.log_format(
                     LogFormat::from_config(&parse_toml_string(key, item)?)?
                 ),
@@ -617,6 +638,12 @@ impl PageServerConf {
         if let Some(max_lsn_wal_lag) = item.get("max_lsn_wal_lag") {
             t_conf.max_lsn_wal_lag = Some(parse_toml_from_str("max_lsn_wal_lag", max_lsn_wal_lag)?);
         }
+        if let Some(trace_read_requests) = item.get("trace_read_requests") {
+            t_conf.trace_read_requests =
+                Some(trace_read_requests.as_bool().with_context(|| {
+                    "configure option trace_read_requests is not a bool".to_string()
+                })?);
+        }
 
         Ok(t_conf)
     }
@@ -645,8 +672,8 @@ impl PageServerConf {
             remote_storage_config: None,
             profiling: ProfilingConfig::Disabled,
             default_tenant_conf: TenantConf::dummy_conf(),
-            broker_endpoints: Vec::new(),
-            broker_etcd_prefix: etcd_broker::DEFAULT_NEON_BROKER_ETCD_PREFIX.to_string(),
+            broker_endpoint: storage_broker::DEFAULT_ENDPOINT.parse().unwrap(),
+            broker_keepalive_interval: Duration::from_secs(5000),
             log_format: LogFormat::from_str(defaults::DEFAULT_LOG_FORMAT).unwrap(),
             concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::default(),
         }
@@ -698,22 +725,6 @@ where
     })
 }
 
-fn parse_toml_array(name: &str, item: &Item) -> anyhow::Result<Vec<String>> {
-    let array = item
-        .as_array()
-        .with_context(|| format!("configure option {name} is not an array"))?;
-
-    array
-        .iter()
-        .map(|value| {
-            value
-                .as_str()
-                .map(str::to_string)
-                .with_context(|| format!("Array item {value:?} for key {name} is not a string"))
-        })
-        .collect()
-}
-
 /// Configurable semaphore permits setting.
 ///
 /// Does not allow semaphore permits to be zero, because at runtime initially zero permits and empty
@@ -803,10 +814,10 @@ log_format = 'json'
     fn parse_defaults() -> anyhow::Result<()> {
         let tempdir = tempdir()?;
         let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?;
-        let broker_endpoint = "http://127.0.0.1:7777";
+        let broker_endpoint = storage_broker::DEFAULT_ENDPOINT;
         // we have to create dummy values to overcome the validation errors
         let config_string = format!(
-            "pg_distrib_dir='{}'\nid=10\nbroker_endpoints = ['{broker_endpoint}']",
+            "pg_distrib_dir='{}'\nid=10\nbroker_endpoint = '{broker_endpoint}'",
             pg_distrib_dir.display()
         );
         let toml = config_string.parse()?;
@@ -832,10 +843,10 @@ log_format = 'json'
                 remote_storage_config: None,
                 profiling: ProfilingConfig::Disabled,
                 default_tenant_conf: TenantConf::default(),
-                broker_endpoints: vec![broker_endpoint
-                    .parse()
-                    .expect("Failed to parse a valid broker endpoint URL")],
-                broker_etcd_prefix: etcd_broker::DEFAULT_NEON_BROKER_ETCD_PREFIX.to_string(),
+                broker_endpoint: storage_broker::DEFAULT_ENDPOINT.parse().unwrap(),
+                broker_keepalive_interval: humantime::parse_duration(
+                    storage_broker::DEFAULT_KEEPALIVE_INTERVAL
+                )?,
                 log_format: LogFormat::from_str(defaults::DEFAULT_LOG_FORMAT).unwrap(),
                 concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::default(),
             },
@@ -849,10 +860,10 @@ log_format = 'json'
     fn parse_basic_config() -> anyhow::Result<()> {
         let tempdir = tempdir()?;
         let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?;
-        let broker_endpoint = "http://127.0.0.1:7777";
+        let broker_endpoint = storage_broker::DEFAULT_ENDPOINT;
 
         let config_string = format!(
-            "{ALL_BASE_VALUES_TOML}pg_distrib_dir='{}'\nbroker_endpoints = ['{broker_endpoint}']",
+            "{ALL_BASE_VALUES_TOML}pg_distrib_dir='{}'\nbroker_endpoint = '{broker_endpoint}'",
             pg_distrib_dir.display()
         );
         let toml = config_string.parse()?;
@@ -878,10 +889,8 @@ log_format = 'json'
                 remote_storage_config: None,
                 profiling: ProfilingConfig::Disabled,
                 default_tenant_conf: TenantConf::default(),
-                broker_endpoints: vec![broker_endpoint
-                    .parse()
-                    .expect("Failed to parse a valid broker endpoint URL")],
-                broker_etcd_prefix: etcd_broker::DEFAULT_NEON_BROKER_ETCD_PREFIX.to_string(),
+                broker_endpoint: storage_broker::DEFAULT_ENDPOINT.parse().unwrap(),
+                broker_keepalive_interval: Duration::from_secs(5),
                 log_format: LogFormat::Json,
                 concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::default(),
             },
@@ -915,7 +924,7 @@ local_path = '{}'"#,
             let config_string = format!(
                 r#"{ALL_BASE_VALUES_TOML}
 pg_distrib_dir='{}'
-broker_endpoints = ['{broker_endpoint}']
+broker_endpoint = '{broker_endpoint}'
 
 {remote_storage_config_str}"#,
                 pg_distrib_dir.display(),
@@ -982,7 +991,7 @@ concurrency_limit = {s3_concurrency_limit}"#
             let config_string = format!(
                 r#"{ALL_BASE_VALUES_TOML}
 pg_distrib_dir='{}'
-broker_endpoints = ['{broker_endpoint}']
+broker_endpoint = '{broker_endpoint}'
 
 {remote_storage_config_str}"#,
                 pg_distrib_dir.display(),
@@ -1016,6 +1025,35 @@ broker_endpoints = ['{broker_endpoint}']
         Ok(())
     }
 
+    #[test]
+    fn parse_tenant_config() -> anyhow::Result<()> {
+        let tempdir = tempdir()?;
+        let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?;
+
+        let broker_endpoint = "http://127.0.0.1:7777";
+        let trace_read_requests = true;
+
+        let config_string = format!(
+            r#"{ALL_BASE_VALUES_TOML}
+pg_distrib_dir='{}'
+broker_endpoint = '{broker_endpoint}'
+
+[tenant_config]
+trace_read_requests = {trace_read_requests}"#,
+            pg_distrib_dir.display(),
+        );
+
+        let toml = config_string.parse()?;
+
+        let conf = PageServerConf::parse_and_validate(&toml, &workdir)?;
+        assert_eq!(
+            conf.default_tenant_conf.trace_read_requests, trace_read_requests,
+            "Tenant config from pageserver config file should be parsed and udpated values used as defaults for all tenants",
+        );
+
+        Ok(())
+    }
+
     fn prepare_fs(tempdir: &TempDir) -> anyhow::Result<(PathBuf, PathBuf)> {
         let tempdir_path = tempdir.path();
 

pageserver/src/http/openapi_spec.yml

@@ -274,6 +274,7 @@ paths:
         schema:
           type: string
           format: hex
+
     post:
       description: Schedules attach operation to happen in the background for given tenant
       responses:
@@ -325,7 +326,9 @@ paths:
           type: string
           format: hex
     post:
-      description: Detach local tenant
+      description: |
+        Remove tenant data (including all corresponding timelines) from pageserver's memory and file system.
+        Files on the remote storage are not affected.
       responses:
         "200":
           description: Tenant detached
@@ -354,6 +357,92 @@ paths:
               schema:
                 $ref: "#/components/schemas/Error"
 
+  /v1/tenant/{tenant_id}/ignore:
+    parameters:
+      - name: tenant_id
+        in: path
+        required: true
+        schema:
+          type: string
+          format: hex
+    post:
+      description: |
+        Remove tenant data (including all corresponding timelines) from pageserver's memory.
+        Files on local disk and remote storage are not affected.
+
+        Future pageserver restarts won't load the data back until `load` is called on such tenant.
+      responses:
+        "200":
+          description: Tenant ignored
+        "400":
+          description: Error when no tenant id found in path parameters
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+
+  /v1/tenant/{tenant_id}/load:
+    parameters:
+      - name: tenant_id
+        in: path
+        required: true
+        schema:
+          type: string
+          format: hex
+    post:
+      description: |
+        Schedules an operation that attempts to load a tenant from the local disk and
+        synchronise it with the remote storage (if enabled), repeating pageserver's restart logic for tenant load.
+        If the tenant was ignored before, removes the ignore mark and continues with load scheduling.
+
+        Errors if the tenant is absent on disk, already present in memory or fails to schedule its load.
+        Scheduling a load does not mean that the tenant would load successfully, check tenant status to ensure load correctness.
+      responses:
+        "202":
+          description: Tenant scheduled to load successfully
+        "400":
+          description: Error when no tenant id found in path parameters
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+        "401":
+          description: Unauthorized Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UnauthorizedError"
+        "403":
+          description: Forbidden Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ForbiddenError"
+        "500":
+          description: Generic operation error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Error"
+
   /v1/tenant/{tenant_id}/size:
     parameters:
       - name: tenant_id
@@ -659,7 +748,6 @@ components:
         - tenant_id
         - last_record_lsn
         - disk_consistent_lsn
-        - awaits_download
         - state
         - latest_gc_cutoff_lsn
       properties:
@@ -702,44 +790,11 @@ components:
           format: hex
         last_received_msg_ts:
           type: integer
-        awaits_download:
-          type: boolean
         state:
           type: string
         latest_gc_cutoff_lsn:
           type: string
           format: hex
-
-        # These 'local' and 'remote' fields just duplicate some of the fields
-        # above. They are kept for backwards-compatibility. They can be removed,
-        # when the control plane has been updated to look at the above fields
-        # directly.
-        local:
-          $ref: "#/components/schemas/LocalTimelineInfo"
-        remote:
-          $ref: "#/components/schemas/RemoteTimelineInfo"
-
-    LocalTimelineInfo:
-      type: object
-      properties:
-        ancestor_timeline_id:
-          type: string
-          format: hex
-        ancestor_lsn:
-          type: string
-          format: hex
-        current_logical_size:
-          type: integer
-        current_physical_size:
-          type: integer
-    RemoteTimelineInfo:
-      type: object
-      required:
-        - remote_consistent_lsn
-      properties:
-        remote_consistent_lsn:
-          type: string
-          format: hex
     Error:
       type: object
       required:

pageserver/src/http/routes.rs

@@ -3,14 +3,12 @@ use std::sync::Arc;
 use anyhow::{anyhow, Context, Result};
 use hyper::StatusCode;
 use hyper::{Body, Request, Response, Uri};
-use pageserver_api::models::TenantState;
 use remote_storage::GenericRemoteStorage;
-use tokio::task::JoinError;
 use tracing::*;
 
 use super::models::{
-    LocalTimelineInfo, RemoteTimelineInfo, StatusResponse, TenantConfigRequest,
-    TenantCreateRequest, TenantCreateResponse, TenantInfo, TimelineCreateRequest, TimelineInfo,
+    StatusResponse, TenantConfigRequest, TenantCreateRequest, TenantCreateResponse, TenantInfo,
+    TimelineCreateRequest, TimelineInfo,
 };
 use crate::pgdatadir_mapping::LsnForTimestamp;
 use crate::tenant::Timeline;
@@ -82,12 +80,11 @@ fn check_permission(request: &Request<Body>, tenant_id: Option<TenantId>) -> Res
 
 // Helper function to construct a TimelineInfo struct for a timeline
 fn build_timeline_info(
-    tenant_state: TenantState,
     timeline: &Arc<Timeline>,
     include_non_incremental_logical_size: bool,
     include_non_incremental_physical_size: bool,
 ) -> anyhow::Result<TimelineInfo> {
-    let mut info = build_timeline_info_common(tenant_state, timeline)?;
+    let mut info = build_timeline_info_common(timeline)?;
     if include_non_incremental_logical_size {
         info.current_logical_size_non_incremental =
             Some(timeline.get_current_logical_size_non_incremental(info.last_record_lsn)?);
@@ -99,10 +96,7 @@ fn build_timeline_info(
     Ok(info)
 }
 
-fn build_timeline_info_common(
-    tenant_state: TenantState,
-    timeline: &Arc<Timeline>,
-) -> anyhow::Result<TimelineInfo> {
+fn build_timeline_info_common(timeline: &Arc<Timeline>) -> anyhow::Result<TimelineInfo> {
     let last_record_lsn = timeline.get_last_record_lsn();
     let (wal_source_connstr, last_received_msg_lsn, last_received_msg_ts) = {
         let guard = timeline.last_received_wal.lock().unwrap();
@@ -153,22 +147,6 @@ fn build_timeline_info_common(
         pg_version: timeline.pg_version,
 
         state,
-
-        // XXX bring back tracking of downloads per timeline, or, introduce
-        // an 'Attaching' state for the timeline and get rid of this field.
-        awaits_download: tenant_state == TenantState::Attaching,
-
-        // Duplicate some fields in 'local' and 'remote' fields, for backwards-compatility
-        // with the control plane.
-        local: LocalTimelineInfo {
-            ancestor_timeline_id,
-            ancestor_lsn,
-            current_logical_size,
-            current_physical_size,
-        },
-        remote: RemoteTimelineInfo {
-            remote_consistent_lsn: Some(remote_consistent_lsn),
-        },
     };
     Ok(info)
 }
@@ -189,7 +167,9 @@ async fn timeline_create_handler(mut request: Request<Body>) -> Result<Response<
         .new_timeline_id
         .unwrap_or_else(TimelineId::generate);
 
-    let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
+    let tenant = tenant_mgr::get_tenant(tenant_id, true)
+        .await
+        .map_err(ApiError::NotFound)?;
     match tenant.create_timeline(
         new_timeline_id,
         request_data.ancestor_timeline_id.map(TimelineId::from),
@@ -200,7 +180,7 @@ async fn timeline_create_handler(mut request: Request<Body>) -> Result<Response<
     .await {
         Ok(Some(new_timeline)) => {
             // Created. Construct a TimelineInfo for it.
-            let timeline_info = build_timeline_info_common(tenant.current_state(), &new_timeline)
+            let timeline_info = build_timeline_info_common(&new_timeline)
                 .map_err(ApiError::InternalServerError)?;
             json_response(StatusCode::CREATED, timeline_info)
         }
@@ -217,26 +197,29 @@ async fn timeline_list_handler(request: Request<Body>) -> Result<Response<Body>,
         query_param_present(&request, "include-non-incremental-physical-size");
     check_permission(&request, Some(tenant_id))?;
 
-    let _entered = info_span!("timeline_list", tenant = %tenant_id).entered();
+    let response_data = async {
+        let tenant = tenant_mgr::get_tenant(tenant_id, true)
+            .await
+            .map_err(ApiError::NotFound)?;
+        let timelines = tenant.list_timelines();
 
-    let (tenant_state, timelines) = {
-        let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
-        (tenant.current_state(), tenant.list_timelines())
-    };
+        let mut response_data = Vec::with_capacity(timelines.len());
+        for timeline in timelines {
+            let timeline_info = build_timeline_info(
+                &timeline,
+                include_non_incremental_logical_size,
+                include_non_incremental_physical_size,
+            )
+            .context("Failed to convert tenant timeline {timeline_id} into the local one: {e:?}")
+            .map_err(ApiError::InternalServerError)?;
 
-    let mut response_data = Vec::with_capacity(timelines.len());
-    for timeline in timelines {
-        let timeline_info = build_timeline_info(
-            tenant_state,
-            &timeline,
-            include_non_incremental_logical_size,
-            include_non_incremental_physical_size,
-        )
-        .context("Failed to convert tenant timeline {timeline_id} into the local one: {e:?}")
-        .map_err(ApiError::InternalServerError)?;
+            response_data.push(timeline_info);
+        }
 
-        response_data.push(timeline_info);
+        Ok(response_data)
     }
+    .instrument(info_span!("timeline_list", tenant = %tenant_id))
+    .await?;
 
     json_response(StatusCode::OK, response_data)
 }
@@ -281,20 +264,15 @@ async fn timeline_detail_handler(request: Request<Body>) -> Result<Response<Body
     check_permission(&request, Some(tenant_id))?;
 
     let timeline_info = async {
-        let (tenant_state, timeline) = tokio::task::spawn_blocking(move || {
-            let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
-            Ok((
-                tenant.current_state(),
-                tenant.get_timeline(timeline_id, false),
-            ))
-        })
-        .await
-        .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??;
+        let tenant = tenant_mgr::get_tenant(tenant_id, true)
+            .await
+            .map_err(ApiError::NotFound)?;
 
-        let timeline = timeline.map_err(ApiError::NotFound)?;
+        let timeline = tenant
+            .get_timeline(timeline_id, false)
+            .map_err(ApiError::NotFound)?;
 
         let timeline_info = build_timeline_info(
-            tenant_state,
             &timeline,
             include_non_incremental_logical_size,
             include_non_incremental_physical_size,
@@ -322,6 +300,7 @@ async fn get_lsn_by_timestamp_handler(request: Request<Body>) -> Result<Response
     let timestamp_pg = postgres_ffi::to_pg_timestamp(timestamp);
 
     let timeline = tenant_mgr::get_tenant(tenant_id, true)
+        .await
         .and_then(|tenant| tenant.get_timeline(timeline_id, true))
         .map_err(ApiError::NotFound)?;
     let result = match timeline
@@ -347,13 +326,13 @@ async fn tenant_attach_handler(request: Request<Body>) -> Result<Response<Body>,
 
     if let Some(remote_storage) = &state.remote_storage {
         // FIXME: distinguish between "Tenant already exists" and other errors
-        tenant_mgr::attach_tenant(state.conf, tenant_id, remote_storage)
+        tenant_mgr::attach_tenant(state.conf, tenant_id, remote_storage.clone())
             .instrument(info_span!("tenant_attach", tenant = %tenant_id))
             .await
             .map_err(ApiError::InternalServerError)?;
     } else {
         return Err(ApiError::BadRequest(anyhow!(
-            "attach_tenant is possible because pageserver was configured without remote storage"
+            "attach_tenant is not possible because pageserver was configured without remote storage"
         )));
     }
 
@@ -392,23 +371,49 @@ async fn tenant_detach_handler(request: Request<Body>) -> Result<Response<Body>,
     json_response(StatusCode::OK, ())
 }
 
+async fn tenant_load_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
+    check_permission(&request, Some(tenant_id))?;
+
+    let state = get_state(&request);
+    tenant_mgr::load_tenant(state.conf, tenant_id, state.remote_storage.clone())
+        .instrument(info_span!("load", tenant = %tenant_id))
+        .await
+        .map_err(ApiError::InternalServerError)?;
+
+    json_response(StatusCode::ACCEPTED, ())
+}
+
+async fn tenant_ignore_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
+    check_permission(&request, Some(tenant_id))?;
+
+    let state = get_state(&request);
+    let conf = state.conf;
+    tenant_mgr::ignore_tenant(conf, tenant_id)
+        .instrument(info_span!("ignore_tenant", tenant = %tenant_id))
+        .await
+        // FIXME: Errors from `ignore_tenant` can be caused by both both user and internal errors.
+        // Replace this with better handling once the error type permits it.
+        .map_err(ApiError::InternalServerError)?;
+
+    json_response(StatusCode::OK, ())
+}
+
 async fn tenant_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
     check_permission(&request, None)?;
 
-    let response_data = tokio::task::spawn_blocking(move || {
-        let _enter = info_span!("tenant_list").entered();
-        tenant_mgr::list_tenants()
-            .iter()
-            .map(|(id, state)| TenantInfo {
-                id: *id,
-                state: *state,
-                current_physical_size: None,
-                has_in_progress_downloads: Some(state.has_in_progress_downloads()),
-            })
-            .collect::<Vec<TenantInfo>>()
-    })
-    .await
-    .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))?;
+    let response_data = tenant_mgr::list_tenants()
+        .instrument(info_span!("tenant_list"))
+        .await
+        .iter()
+        .map(|(id, state)| TenantInfo {
+            id: *id,
+            state: *state,
+            current_physical_size: None,
+            has_in_progress_downloads: Some(state.has_in_progress_downloads()),
+        })
+        .collect::<Vec<TenantInfo>>();
 
     json_response(StatusCode::OK, response_data)
 }
@@ -417,9 +422,8 @@ async fn tenant_status(request: Request<Body>) -> Result<Response<Body>, ApiErro
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
     check_permission(&request, Some(tenant_id))?;
 
-    let tenant_info = tokio::task::spawn_blocking(move || {
-        let _enter = info_span!("tenant_status_handler", tenant = %tenant_id).entered();
-        let tenant = tenant_mgr::get_tenant(tenant_id, false)?;
+    let tenant_info = async {
+        let tenant = tenant_mgr::get_tenant(tenant_id, false).await?;
 
         // Calculate total physical size of all timelines
         let mut current_physical_size = 0;
@@ -428,17 +432,15 @@ async fn tenant_status(request: Request<Body>) -> Result<Response<Body>, ApiErro
         }
 
         let state = tenant.current_state();
-        let tenant_info = TenantInfo {
+        Ok(TenantInfo {
             id: tenant_id,
             state,
             current_physical_size: Some(current_physical_size),
             has_in_progress_downloads: Some(state.has_in_progress_downloads()),
-        };
-
-        Ok::<_, anyhow::Error>(tenant_info)
-    })
+        })
+    }
+    .instrument(info_span!("tenant_status_handler", tenant = %tenant_id))
     .await
-    .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))?
     .map_err(ApiError::InternalServerError)?;
 
     json_response(StatusCode::OK, tenant_info)
@@ -448,7 +450,9 @@ async fn tenant_size_handler(request: Request<Body>) -> Result<Response<Body>, A
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
     check_permission(&request, Some(tenant_id))?;
 
-    let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::InternalServerError)?;
+    let tenant = tenant_mgr::get_tenant(tenant_id, true)
+        .await
+        .map_err(ApiError::InternalServerError)?;
 
     // this can be long operation, it currently is not backed by any request coalescing or similar
     let inputs = tenant
@@ -565,22 +569,19 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
         .map(TenantId::from)
         .unwrap_or_else(TenantId::generate);
 
-    let new_tenant = tokio::task::spawn_blocking(move || {
-        let _enter = info_span!("tenant_create", tenant = ?target_tenant_id).entered();
-        let state = get_state(&request);
+    let state = get_state(&request);
 
-        tenant_mgr::create_tenant(
-            state.conf,
-            tenant_conf,
-            target_tenant_id,
-            state.remote_storage.clone(),
-        )
-        // FIXME: `create_tenant` can fail from both user and internal errors. Replace this
-        // with better error handling once the type permits it
-        .map_err(ApiError::InternalServerError)
-    })
+    let new_tenant = tenant_mgr::create_tenant(
+        state.conf,
+        tenant_conf,
+        target_tenant_id,
+        state.remote_storage.clone(),
+    )
+    .instrument(info_span!("tenant_create", tenant = ?target_tenant_id))
     .await
-    .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??;
+    // FIXME: `create_tenant` can fail from both user and internal errors. Replace this
+    // with better error handling once the type permits it
+    .map_err(ApiError::InternalServerError)?;
 
     Ok(match new_tenant {
         Some(tenant) => {
@@ -671,17 +672,13 @@ async fn tenant_config_handler(mut request: Request<Body>) -> Result<Response<Bo
         );
     }
 
-    tokio::task::spawn_blocking(move || {
-        let _enter = info_span!("tenant_config", tenant = ?tenant_id).entered();
-
-        let state = get_state(&request);
-        tenant_mgr::update_tenant_config(state.conf, tenant_conf, tenant_id)
-            // FIXME: `update_tenant_config` can fail because of both user and internal errors.
-            // Replace this `map_err` with better error handling once the type permits it
-            .map_err(ApiError::InternalServerError)
-    })
-    .await
-    .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??;
+    let state = get_state(&request);
+    tenant_mgr::update_tenant_config(state.conf, tenant_conf, tenant_id)
+        .instrument(info_span!("tenant_config", tenant = ?tenant_id))
+        .await
+        // FIXME: `update_tenant_config` can fail because of both user and internal errors.
+        // Replace this `map_err` with better error handling once the type permits it
+        .map_err(ApiError::InternalServerError)?;
 
     json_response(StatusCode::OK, ())
 }
@@ -728,7 +725,7 @@ async fn timeline_gc_handler(mut request: Request<Body>) -> Result<Response<Body
 
     let gc_req: TimelineGcRequest = json_request(&mut request).await?;
 
-    let wait_task_done = tenant_mgr::immediate_gc(tenant_id, timeline_id, gc_req)?;
+    let wait_task_done = tenant_mgr::immediate_gc(tenant_id, timeline_id, gc_req).await?;
     let gc_result = wait_task_done
         .await
         .context("wait for gc task")
@@ -745,7 +742,9 @@ async fn timeline_compact_handler(request: Request<Body>) -> Result<Response<Bod
     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     check_permission(&request, Some(tenant_id))?;
 
-    let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
+    let tenant = tenant_mgr::get_tenant(tenant_id, true)
+        .await
+        .map_err(ApiError::NotFound)?;
     let timeline = tenant
         .get_timeline(timeline_id, true)
         .map_err(ApiError::NotFound)?;
@@ -764,7 +763,9 @@ async fn timeline_checkpoint_handler(request: Request<Body>) -> Result<Response<
     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     check_permission(&request, Some(tenant_id))?;
 
-    let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
+    let tenant = tenant_mgr::get_tenant(tenant_id, true)
+        .await
+        .map_err(ApiError::NotFound)?;
     let timeline = tenant
         .get_timeline(timeline_id, true)
         .map_err(ApiError::NotFound)?;
@@ -838,6 +839,8 @@ pub fn make_router(
         .post("/v1/tenant/:tenant_id/timeline", timeline_create_handler)
         .post("/v1/tenant/:tenant_id/attach", tenant_attach_handler)
         .post("/v1/tenant/:tenant_id/detach", tenant_detach_handler)
+        .post("/v1/tenant/:tenant_id/load", tenant_load_handler)
+        .post("/v1/tenant/:tenant_id/ignore", tenant_ignore_handler)
         .get(
             "/v1/tenant/:tenant_id/timeline/:timeline_id",
             timeline_detail_handler,

pageserver/src/lib.rs

@@ -125,6 +125,13 @@ pub const TEMP_FILE_SUFFIX: &str = "___temp";
 /// Full path: `tenants/<tenant_id>/timelines/<timeline_id>___uninit`.
 pub const TIMELINE_UNINIT_MARK_SUFFIX: &str = "___uninit";
 
+/// A marker file to prevent pageserver from loading a certain tenant on restart.
+/// Different from [`TIMELINE_UNINIT_MARK_SUFFIX`] due to semantics of the corresponding
+/// `ignore` management API command, that expects the ignored tenant to be properly loaded
+/// into pageserver's memory before being ignored.
+/// Full path: `tenants/<tenant_id>/___ignored_tenant`.
+pub const IGNORED_TENANT_FILE_NAME: &str = "___ignored_tenant";
+
 pub fn is_temporary(path: &Path) -> bool {
     match path.file_name() {
         Some(name) => name.to_string_lossy().ends_with(TEMP_FILE_SUFFIX),

pageserver/src/metrics.rs

@@ -96,6 +96,16 @@ static CURRENT_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
     .expect("failed to define a metric")
 });
 
+static REMOTE_PHYSICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
+    register_uint_gauge_vec!(
+        "pageserver_remote_physical_size",
+        "The size of the layer files present in the remote storage that are listed in the the remote index_part.json.",
+        // Corollary: If any files are missing from the index part, they won't be included here.
+        &["tenant_id", "timeline_id"]
+    )
+    .expect("failed to define a metric")
+});
+
 static CURRENT_LOGICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
     register_uint_gauge_vec!(
         "pageserver_current_logical_size",
@@ -201,7 +211,7 @@ pub static NUM_ONDISK_LAYERS: Lazy<IntGauge> = Lazy::new(|| {
 
 // remote storage metrics
 
-pub static REMOTE_UPLOAD_QUEUE_UNFINISHED_TASKS: Lazy<IntGaugeVec> = Lazy::new(|| {
+static REMOTE_UPLOAD_QUEUE_UNFINISHED_TASKS: Lazy<IntGaugeVec> = Lazy::new(|| {
     register_int_gauge_vec!(
         "pageserver_remote_upload_queue_unfinished_tasks",
         "Number of tasks in the upload queue that are not finished yet.",
@@ -210,14 +220,14 @@ pub static REMOTE_UPLOAD_QUEUE_UNFINISHED_TASKS: Lazy<IntGaugeVec> = Lazy::new(|
     .expect("failed to define a metric")
 });
 
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum RemoteOpKind {
     Upload,
     Download,
     Delete,
 }
 impl RemoteOpKind {
-    pub fn as_str(&self) -> &str {
+    pub fn as_str(&self) -> &'static str {
         match self {
             Self::Upload => "upload",
             Self::Download => "download",
@@ -226,13 +236,13 @@ impl RemoteOpKind {
     }
 }
 
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
 pub enum RemoteOpFileKind {
     Layer,
     Index,
 }
 impl RemoteOpFileKind {
-    pub fn as_str(&self) -> &str {
+    pub fn as_str(&self) -> &'static str {
         match self {
             Self::Layer => "layer",
             Self::Index => "index",
@@ -491,10 +501,114 @@ pub fn remove_tenant_metrics(tenant_id: &TenantId) {
 
 use futures::Future;
 use pin_project_lite::pin_project;
+use std::collections::HashMap;
 use std::pin::Pin;
+use std::sync::{Arc, Mutex};
 use std::task::{Context, Poll};
 use std::time::Instant;
 
+pub struct RemoteTimelineClientMetrics {
+    tenant_id: String,
+    timeline_id: String,
+    remote_physical_size_gauge: Mutex<Option<UIntGauge>>,
+    remote_operation_time: Mutex<HashMap<(&'static str, &'static str, &'static str), Histogram>>,
+    unfinished_tasks: Mutex<HashMap<(&'static str, &'static str), IntGauge>>,
+}
+
+impl RemoteTimelineClientMetrics {
+    pub fn new(tenant_id: &TenantId, timeline_id: &TimelineId) -> Self {
+        RemoteTimelineClientMetrics {
+            tenant_id: tenant_id.to_string(),
+            timeline_id: timeline_id.to_string(),
+            remote_operation_time: Mutex::new(HashMap::default()),
+            unfinished_tasks: Mutex::new(HashMap::default()),
+            remote_physical_size_gauge: Mutex::new(None),
+        }
+    }
+    pub fn remote_physical_size_gauge(&self) -> UIntGauge {
+        let mut guard = self.remote_physical_size_gauge.lock().unwrap();
+        guard
+            .get_or_insert_with(|| {
+                REMOTE_PHYSICAL_SIZE
+                    .get_metric_with_label_values(&[
+                        &self.tenant_id.to_string(),
+                        &self.timeline_id.to_string(),
+                    ])
+                    .unwrap()
+            })
+            .clone()
+    }
+    pub fn remote_operation_time(
+        &self,
+        file_kind: &RemoteOpFileKind,
+        op_kind: &RemoteOpKind,
+        status: &'static str,
+    ) -> Histogram {
+        // XXX would be nice to have an upgradable RwLock
+        let mut guard = self.remote_operation_time.lock().unwrap();
+        let key = (file_kind.as_str(), op_kind.as_str(), status);
+        let metric = guard.entry(key).or_insert_with(move || {
+            REMOTE_OPERATION_TIME
+                .get_metric_with_label_values(&[
+                    &self.tenant_id.to_string(),
+                    &self.timeline_id.to_string(),
+                    key.0,
+                    key.1,
+                    key.2,
+                ])
+                .unwrap()
+        });
+        metric.clone()
+    }
+    pub fn unfinished_tasks(
+        &self,
+        file_kind: &RemoteOpFileKind,
+        op_kind: &RemoteOpKind,
+    ) -> IntGauge {
+        // XXX would be nice to have an upgradable RwLock
+        let mut guard = self.unfinished_tasks.lock().unwrap();
+        let key = (file_kind.as_str(), op_kind.as_str());
+        let metric = guard.entry(key).or_insert_with(move || {
+            REMOTE_UPLOAD_QUEUE_UNFINISHED_TASKS
+                .get_metric_with_label_values(&[
+                    &self.tenant_id.to_string(),
+                    &self.timeline_id.to_string(),
+                    key.0,
+                    key.1,
+                ])
+                .unwrap()
+        });
+        metric.clone()
+    }
+}
+
+impl Drop for RemoteTimelineClientMetrics {
+    fn drop(&mut self) {
+        let RemoteTimelineClientMetrics {
+            tenant_id,
+            timeline_id,
+            remote_physical_size_gauge,
+            remote_operation_time,
+            unfinished_tasks,
+        } = self;
+        for ((a, b, c), _) in remote_operation_time.get_mut().unwrap().drain() {
+            let _ = REMOTE_OPERATION_TIME.remove_label_values(&[tenant_id, timeline_id, a, b, c]);
+        }
+        for ((a, b), _) in unfinished_tasks.get_mut().unwrap().drain() {
+            let _ = REMOTE_UPLOAD_QUEUE_UNFINISHED_TASKS.remove_label_values(&[
+                tenant_id,
+                timeline_id,
+                a,
+                b,
+            ]);
+        }
+        {
+            let _ = remote_physical_size_gauge; // use to avoid 'unused' warning in desctructuring above
+            let _ = REMOTE_PHYSICAL_SIZE.remove_label_values(&[tenant_id, timeline_id]);
+        }
+    }
+}
+
 /// Wrapper future that measures the time spent by a remote storage operation,
 /// and records the time and success/failure as a prometheus metric.
 pub trait MeasureRemoteOp: Sized {
@@ -504,6 +618,7 @@ pub trait MeasureRemoteOp: Sized {
         timeline_id: TimelineId,
         file_kind: RemoteOpFileKind,
         op: RemoteOpKind,
+        metrics: Arc<RemoteTimelineClientMetrics>,
     ) -> MeasuredRemoteOp<Self> {
         let start = Instant::now();
         MeasuredRemoteOp {
@@ -513,6 +628,7 @@ pub trait MeasureRemoteOp: Sized {
             file_kind,
             op,
             start,
+            metrics,
         }
     }
 }
@@ -529,6 +645,7 @@ pin_project! {
         file_kind: RemoteOpFileKind,
         op: RemoteOpKind,
         start: Instant,
+        metrics: Arc<RemoteTimelineClientMetrics>,
     }
 }
 
@@ -541,15 +658,8 @@ impl<F: Future<Output = Result<O, E>>, O, E> Future for MeasuredRemoteOp<F> {
         if let Poll::Ready(ref res) = poll_result {
             let duration = this.start.elapsed();
             let status = if res.is_ok() { &"success" } else { &"failure" };
-            REMOTE_OPERATION_TIME
-                .get_metric_with_label_values(&[
-                    &this.tenant_id.to_string(),
-                    &this.timeline_id.to_string(),
-                    this.file_kind.as_str(),
-                    this.op.as_str(),
-                    status,
-                ])
-                .unwrap()
+            this.metrics
+                .remote_operation_time(this.file_kind, this.op, status)
                 .observe(duration.as_secs_f64());
         }
         poll_result

pageserver/src/page_service.rs

@@ -941,7 +941,7 @@ impl postgres_backend_async::Handler for PageServerHandler {
 /// ensures that queries don't fail immediately after pageserver startup, because
 /// all tenants are still loading.
 async fn get_active_tenant_with_timeout(tenant_id: TenantId) -> Result<Arc<Tenant>> {
-    let tenant = tenant_mgr::get_tenant(tenant_id, false)?;
+    let tenant = tenant_mgr::get_tenant(tenant_id, false).await?;
     match tokio::time::timeout(Duration::from_secs(30), tenant.wait_to_become_active()).await {
         Ok(wait_result) => wait_result
             // no .context(), the error message is good enough and some tests depend on it