trim down merge_local_remote_metadata()

At the time of writing, the only logical difference between Attach and Load is
that Attach learns the list of timelines by querying the remote storage,
whereas Load learns it by listing the timelines directory of the tenant.

This patch restructures the code such that Attach

1. prepares the on-disk state and then
2. calls into the same `load()` routine that is used by Load.

Further, this patch provides the following fixes & improvements to Attach:

1. Make Attach durable before acknowledging it to the management API client.

   Before this change, we would acknowledge after just creating the in-memory
   tenant. In the event of a crash before creating the tenant directory and
   fsyncing the attaching marker, the pageserver would come up without the
   tenant present (404), even though we acknowledged success to the client.

2. Simplified resume logic if we crash during Attach.
   Before this patch, if we crashed during Attach with some timelines downloaded
   and others not downloaded, we would combine existing metadata files with
   remote ones one-by-one to figure out what's missing.
   That was necessary before on-demand download because we were downloading
   layer files as part of Attach. However, with on-demand download, Attach
   only downloads & writes the timeline metadata files.

   After this patch, when we crash during Attach, we blow away the tenant's
   directory while leaving its attach marker file in place. Then, we start over.

   IMO this is significantly easier to reason about compared to what we had before.
   Note that we were losing the work for the downloads even before this change,
   so that's not a regression (the old reconcile_with_remote would still need
   to download the `IndexPart`s when resuming Attach after a crash).

   If we want to improve on this in the future, I think the first order of
   business will be to avoiding re-downloading the `IndexPart`'s and the
   initial `list_remote_timelines()`.
   However, given that crashes should be rare, and attach events also, I don't
   think the number one priority with Attach code should be to make it as
   simple as possible.

For (2), I changed the location of the attach marker file to be outside the
tenant directory, so that we can use standard functions for removing the
tenant  directory.
I even wrote a migration function for it, although in retrospect, I think it's
quite unlikely that there are any tenants in attaching state deployed.
But oh well, now the code is there, and it even has unit tests.
We can delete the migration code once we've successfully rolled it out to all
regions.

The remaining wrinkle with this change is that Attach needs to hint the
downloaded `IndexPart`s to `load()` so that it doesn't download them twice
during Attach, which would be wasteful.
The mechanism for this is the new `TenantLoadReason` and `TimelineLoadReason`.

We could eliminate this particular case by on-demand downloading the metadata.
However, that might open up another can of worms which I'd like to avoid.
If we ever want to go that route, I suggest we start tracking the attachment
state of a timeline more formally, e.g., in a `timelines.json` file.

This is PR https://github.com/neondatabase/neon/pull/3466

.dockerignore

@@ -15,10 +15,8 @@
 !proxy/
 !safekeeper/
 !storage_broker/
-!trace/
 !vendor/postgres-v14/
 !vendor/postgres-v15/
 !workspace_hack/
 !neon_local/
 !scripts/ninstall.sh
-!vm-cgconfig.conf

.github/ansible/prod.ap-southeast-1.hosts.yaml

@@ -2,11 +2,11 @@ storage:
   vars:
     bucket_name: neon-prod-storage-ap-southeast-1
     bucket_region: ap-southeast-1
-    console_mgmt_base_url: http://neon-internal-api.aws.neon.tech
+    console_mgmt_base_url: http://console-release.local
     broker_endpoint: http://storage-broker-lb.epsilon.ap-southeast-1.internal.aws.neon.tech:50051
     pageserver_config_stub:
       pg_distrib_dir: /usr/local
-      metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events
+      metric_collection_endpoint: http://console-release.local/billing/api/v1/usage_events
       metric_collection_interval: 10min
       remote_storage:
         bucket_name: "{{ bucket_name }}"
@@ -32,7 +32,7 @@ storage:
       hosts:
         safekeeper-0.ap-southeast-1.aws.neon.tech:
           ansible_host:  i-0d6f1dc5161eef894
+        safekeeper-1.ap-southeast-1.aws.neon.tech:
+          ansible_host:  i-0e338adda8eb2d19f
         safekeeper-2.ap-southeast-1.aws.neon.tech:
           ansible_host:  i-04fb63634e4679eb9
-        safekeeper-3.ap-southeast-1.aws.neon.tech:
-          ansible_host:  i-05481f3bc88cfc2d4

.github/ansible/prod.eu-central-1.hosts.yaml

@@ -2,11 +2,11 @@ storage:
   vars:
     bucket_name: neon-prod-storage-eu-central-1
     bucket_region: eu-central-1
-    console_mgmt_base_url: http://neon-internal-api.aws.neon.tech
+    console_mgmt_base_url: http://console-release.local
     broker_endpoint: http://storage-broker-lb.gamma.eu-central-1.internal.aws.neon.tech:50051
     pageserver_config_stub:
       pg_distrib_dir: /usr/local
-      metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events
+      metric_collection_endpoint: http://console-release.local/billing/api/v1/usage_events
       metric_collection_interval: 10min
       remote_storage:
         bucket_name: "{{ bucket_name }}"

.github/ansible/prod.us-east-2.hosts.yaml

@@ -2,11 +2,11 @@ storage:
   vars:
     bucket_name: neon-prod-storage-us-east-2
     bucket_region: us-east-2
-    console_mgmt_base_url: http://neon-internal-api.aws.neon.tech
+    console_mgmt_base_url: http://console-release.local
     broker_endpoint: http://storage-broker-lb.delta.us-east-2.internal.aws.neon.tech:50051
     pageserver_config_stub:
       pg_distrib_dir: /usr/local
-      metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events
+      metric_collection_endpoint: http://console-release.local/billing/api/v1/usage_events
       metric_collection_interval: 10min
       remote_storage:
         bucket_name: "{{ bucket_name }}"

.github/ansible/prod.us-west-2.hosts.yaml

@@ -2,11 +2,11 @@ storage:
   vars:
     bucket_name: neon-prod-storage-us-west-2
     bucket_region: us-west-2
-    console_mgmt_base_url: http://neon-internal-api.aws.neon.tech
+    console_mgmt_base_url: http://console-release.local
     broker_endpoint: http://storage-broker-lb.eta.us-west-2.internal.aws.neon.tech:50051
     pageserver_config_stub:
       pg_distrib_dir: /usr/local
-      metric_collection_endpoint: http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events
+      metric_collection_endpoint: http://console-release.local/billing/api/v1/usage_events
       metric_collection_interval: 10min
       remote_storage:
         bucket_name: "{{ bucket_name }}"
@@ -29,8 +29,6 @@ storage:
           ansible_host: i-0c834be1dddba8b3f
         pageserver-2.us-west-2.aws.neon.tech:
           ansible_host: i-051642d372c0a4f32
-        pageserver-3.us-west-2.aws.neon.tech:
-          ansible_host: i-00c3844beb9ad1c6b
 
     safekeepers:
       hosts:

.github/ansible/production.hosts.yaml

@@ -0,0 +1,40 @@
+---
+storage:
+  vars:
+    console_mgmt_base_url: http://console-release.local
+    bucket_name: zenith-storage-oregon
+    bucket_region: us-west-2
+    broker_endpoint: http://storage-broker.prod.local:50051
+    pageserver_config_stub:
+      pg_distrib_dir: /usr/local
+      metric_collection_endpoint: http://console-release.local/billing/api/v1/usage_events
+      metric_collection_interval: 10min
+      remote_storage:
+        bucket_name: "{{ bucket_name }}"
+        bucket_region: "{{ bucket_region }}"
+        prefix_in_bucket: "{{ inventory_hostname }}"
+    safekeeper_s3_prefix: prod-1/wal
+    hostname_suffix: ".local"
+    remote_user: admin
+    sentry_environment: production
+
+  children:
+    pageservers:
+      hosts:
+        zenith-1-ps-2:
+          console_region_id: aws-us-west-2
+        zenith-1-ps-3:
+          console_region_id: aws-us-west-2
+        zenith-1-ps-4:
+          console_region_id: aws-us-west-2
+        zenith-1-ps-5:
+          console_region_id: aws-us-west-2
+
+    safekeepers:
+      hosts:
+        zenith-1-sk-1:
+          console_region_id: aws-us-west-2
+        zenith-1-sk-2:
+          console_region_id: aws-us-west-2
+        zenith-1-sk-4:
+          console_region_id: aws-us-west-2

.github/ansible/staging.eu-west-1.hosts.yaml

@@ -2,17 +2,12 @@ storage:
   vars:
     bucket_name: neon-dev-storage-eu-west-1
     bucket_region: eu-west-1
-    console_mgmt_base_url: http://neon-internal-api.aws.neon.build
+    console_mgmt_base_url: http://console-staging.local
     broker_endpoint: http://storage-broker-lb.zeta.eu-west-1.internal.aws.neon.build:50051
     pageserver_config_stub:
       pg_distrib_dir: /usr/local
-      metric_collection_endpoint: http://neon-internal-api.aws.neon.build/billing/api/v1/usage_events
+      metric_collection_endpoint: http://console-staging.local/billing/api/v1/usage_events
       metric_collection_interval: 10min
-      tenant_config:
-        eviction_policy:
-          kind: "LayerAccessThreshold"
-          period: "20m"
-          threshold: "20m"
       remote_storage:
         bucket_name: "{{ bucket_name }}"
         bucket_region: "{{ bucket_region }}"

.github/ansible/staging.us-east-2.hosts.yaml

@@ -2,17 +2,12 @@ storage:
   vars:
     bucket_name: neon-staging-storage-us-east-2
     bucket_region: us-east-2
-    console_mgmt_base_url: http://neon-internal-api.aws.neon.build
+    console_mgmt_base_url: http://console-staging.local
     broker_endpoint: http://storage-broker-lb.beta.us-east-2.internal.aws.neon.build:50051
     pageserver_config_stub:
       pg_distrib_dir: /usr/local
-      metric_collection_endpoint: http://neon-internal-api.aws.neon.build/billing/api/v1/usage_events
+      metric_collection_endpoint: http://console-staging.local/billing/api/v1/usage_events
       metric_collection_interval: 10min
-      tenant_config:
-        eviction_policy:
-          kind: "LayerAccessThreshold"
-          period: "20m"
-          threshold: "20m"
       remote_storage:
         bucket_name: "{{ bucket_name }}"
         bucket_region: "{{ bucket_region }}"
@@ -36,8 +31,6 @@ storage:
           ansible_host: i-01e31cdf7e970586a
         pageserver-3.us-east-2.aws.neon.build:
           ansible_host: i-0602a0291365ef7cc
-        pageserver-99.us-east-2.aws.neon.build:
-          ansible_host: i-0c39491109bb88824
 
     safekeepers:
       hosts:
@@ -47,5 +40,3 @@ storage:
           ansible_host: i-0171efc3604a7b907
         safekeeper-2.us-east-2.aws.neon.build:
           ansible_host: i-0de0b03a51676a6ce
-        safekeeper-99.us-east-2.aws.neon.build:
-          ansible_host: i-0d61b6a2ea32028d5

.github/helm-values/dev-eu-west-1-zeta.neon-proxy-scram.yaml

@@ -1,31 +1,16 @@
 # Helm chart values for neon-proxy-scram.
 # This is a YAML-formatted file.
 
-deploymentStrategy:
-  type: RollingUpdate
-  rollingUpdate:
-    maxSurge: 100%
-    maxUnavailable: 50%
-
-# Delay the kill signal by 7 days (7 * 24 * 60 * 60)
-# The pod(s) will stay in Terminating, keeps the existing connections
-# but doesn't receive new ones
-containerLifecycle:
-  preStop:
-    exec:
-      command: ["/bin/sh", "-c", "sleep 604800"]
-terminationGracePeriodSeconds: 604800
-
 image:
   repository: neondatabase/neon
 
 settings:
   authBackend: "console"
-  authEndpoint: "http://neon-internal-api.aws.neon.build/management/api/v2"
+  authEndpoint: "http://console-staging.local/management/api/v2"
   domain: "*.eu-west-1.aws.neon.build"
   sentryEnvironment: "staging"
   wssPort: 8443
-  metricCollectionEndpoint: "http://neon-internal-api.aws.neon.build/billing/api/v1/usage_events"
+  metricCollectionEndpoint: "http://console-staging.local/billing/api/v1/usage_events"
   metricCollectionInterval: "1min"
 
 # -- Additional labels for neon-proxy pods

.github/helm-values/dev-us-east-2-beta.neon-proxy-link.yaml

@@ -10,7 +10,7 @@ settings:
   uri: "https://console.stage.neon.tech/psql_session/"
   domain: "pg.neon.build"
   sentryEnvironment: "staging"
-  metricCollectionEndpoint: "http://neon-internal-api.aws.neon.build/billing/api/v1/usage_events"
+  metricCollectionEndpoint: "http://console-staging.local/billing/api/v1/usage_events"
   metricCollectionInterval: "1min"
 
 # -- Additional labels for neon-proxy-link pods

.github/helm-values/dev-us-east-2-beta.neon-proxy-scram-legacy.yaml

@@ -6,11 +6,11 @@ image:
 
 settings:
   authBackend: "console"
-  authEndpoint: "http://neon-internal-api.aws.neon.build/management/api/v2"
+  authEndpoint: "http://console-staging.local/management/api/v2"
   domain: "*.cloud.stage.neon.tech"
   sentryEnvironment: "staging"
   wssPort: 8443
-  metricCollectionEndpoint: "http://neon-internal-api.aws.neon.build/billing/api/v1/usage_events"
+  metricCollectionEndpoint: "http://console-staging.local/billing/api/v1/usage_events"
   metricCollectionInterval: "1min"
 
 # -- Additional labels for neon-proxy pods

.github/helm-values/dev-us-east-2-beta.neon-proxy-scram.yaml

@@ -1,31 +1,16 @@
 # Helm chart values for neon-proxy-scram.
 # This is a YAML-formatted file.
 
-deploymentStrategy:
-  type: RollingUpdate
-  rollingUpdate:
-    maxSurge: 100%
-    maxUnavailable: 50%
-
-# Delay the kill signal by 7 days (7 * 24 * 60 * 60)
-# The pod(s) will stay in Terminating, keeps the existing connections
-# but doesn't receive new ones
-containerLifecycle:
-  preStop:
-    exec:
-      command: ["/bin/sh", "-c", "sleep 604800"]
-terminationGracePeriodSeconds: 604800
-
 image:
   repository: neondatabase/neon
 
 settings:
   authBackend: "console"
-  authEndpoint: "http://neon-internal-api.aws.neon.build/management/api/v2"
+  authEndpoint: "http://console-staging.local/management/api/v2"
   domain: "*.us-east-2.aws.neon.build"
   sentryEnvironment: "staging"
   wssPort: 8443
-  metricCollectionEndpoint: "http://neon-internal-api.aws.neon.build/billing/api/v1/usage_events"
+  metricCollectionEndpoint: "http://console-staging.local/billing/api/v1/usage_events"
   metricCollectionInterval: "1min"
 
 # -- Additional labels for neon-proxy pods

.github/helm-values/prod-ap-southeast-1-epsilon.neon-proxy-scram.yaml

@@ -1,32 +1,16 @@
 # Helm chart values for neon-proxy-scram.
 # This is a YAML-formatted file.
 
-deploymentStrategy:
-  type: RollingUpdate
-  rollingUpdate:
-    maxSurge: 100%
-    maxUnavailable: 50%
-
-# Delay the kill signal by 7 days (7 * 24 * 60 * 60)
-# The pod(s) will stay in Terminating, keeps the existing connections
-# but doesn't receive new ones
-containerLifecycle:
-  preStop:
-    exec:
-      command: ["/bin/sh", "-c", "sleep 604800"]
-terminationGracePeriodSeconds: 604800
-
-
 image:
   repository: neondatabase/neon
 
 settings:
   authBackend: "console"
-  authEndpoint: "http://neon-internal-api.aws.neon.tech/management/api/v2"
+  authEndpoint: "http://console-release.local/management/api/v2"
   domain: "*.ap-southeast-1.aws.neon.tech"
   sentryEnvironment: "production"
   wssPort: 8443
-  metricCollectionEndpoint: "http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events"
+  metricCollectionEndpoint: "http://console-release.local/billing/api/v1/usage_events"
   metricCollectionInterval: "10min"
 
 # -- Additional labels for neon-proxy pods

.github/helm-values/prod-eu-central-1-gamma.neon-proxy-scram.yaml

@@ -1,32 +1,16 @@
 # Helm chart values for neon-proxy-scram.
 # This is a YAML-formatted file.
 
-deploymentStrategy:
-  type: RollingUpdate
-  rollingUpdate:
-    maxSurge: 100%
-    maxUnavailable: 50%
-
-# Delay the kill signal by 7 days (7 * 24 * 60 * 60)
-# The pod(s) will stay in Terminating, keeps the existing connections
-# but doesn't receive new ones
-containerLifecycle:
-  preStop:
-    exec:
-      command: ["/bin/sh", "-c", "sleep 604800"]
-terminationGracePeriodSeconds: 604800
-
-
 image:
   repository: neondatabase/neon
 
 settings:
   authBackend: "console"
-  authEndpoint: "http://neon-internal-api.aws.neon.tech/management/api/v2"
+  authEndpoint: "http://console-release.local/management/api/v2"
   domain: "*.eu-central-1.aws.neon.tech"
   sentryEnvironment: "production"
   wssPort: 8443
-  metricCollectionEndpoint: "http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events"
+  metricCollectionEndpoint: "http://console-release.local/billing/api/v1/usage_events"
   metricCollectionInterval: "10min"
 
 # -- Additional labels for neon-proxy pods

.github/helm-values/prod-us-east-2-delta.neon-proxy-scram.yaml

@@ -1,32 +1,16 @@
 # Helm chart values for neon-proxy-scram.
 # This is a YAML-formatted file.
 
-deploymentStrategy:
-  type: RollingUpdate
-  rollingUpdate:
-    maxSurge: 100%
-    maxUnavailable: 50%
-
-# Delay the kill signal by 7 days (7 * 24 * 60 * 60)
-# The pod(s) will stay in Terminating, keeps the existing connections
-# but doesn't receive new ones
-containerLifecycle:
-  preStop:
-    exec:
-      command: ["/bin/sh", "-c", "sleep 604800"]
-terminationGracePeriodSeconds: 604800
-
-
 image:
   repository: neondatabase/neon
 
 settings:
   authBackend: "console"
-  authEndpoint: "http://neon-internal-api.aws.neon.tech/management/api/v2"
+  authEndpoint: "http://console-release.local/management/api/v2"
   domain: "*.us-east-2.aws.neon.tech"
   sentryEnvironment: "production"
   wssPort: 8443
-  metricCollectionEndpoint: "http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events"
+  metricCollectionEndpoint: "http://console-release.local/billing/api/v1/usage_events"
   metricCollectionInterval: "10min"
 
 # -- Additional labels for neon-proxy pods

.github/helm-values/prod-us-west-2-eta.neon-proxy-scram-legacy.yaml

@@ -6,11 +6,11 @@ image:
 
 settings:
   authBackend: "console"
-  authEndpoint: "http://neon-internal-api.aws.neon.tech/management/api/v2"
+  authEndpoint: "http://console-release.local/management/api/v2"
   domain: "*.cloud.neon.tech"
   sentryEnvironment: "production"
   wssPort: 8443
-  metricCollectionEndpoint: "http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events"
+  metricCollectionEndpoint: "http://console-release.local/billing/api/v1/usage_events"
   metricCollectionInterval: "10min"
 
 # -- Additional labels for neon-proxy pods

.github/helm-values/prod-us-west-2-eta.neon-proxy-scram.yaml

@@ -1,32 +1,16 @@
 # Helm chart values for neon-proxy-scram.
 # This is a YAML-formatted file.
 
-deploymentStrategy:
-  type: RollingUpdate
-  rollingUpdate:
-    maxSurge: 100%
-    maxUnavailable: 50%
-
-# Delay the kill signal by 7 days (7 * 24 * 60 * 60)
-# The pod(s) will stay in Terminating, keeps the existing connections
-# but doesn't receive new ones
-containerLifecycle:
-  preStop:
-    exec:
-      command: ["/bin/sh", "-c", "sleep 604800"]
-terminationGracePeriodSeconds: 604800
-
-
 image:
   repository: neondatabase/neon
 
 settings:
   authBackend: "console"
-  authEndpoint: "http://neon-internal-api.aws.neon.tech/management/api/v2"
+  authEndpoint: "http://console-release.local/management/api/v2"
   domain: "*.us-west-2.aws.neon.tech"
   sentryEnvironment: "production"
   wssPort: 8443
-  metricCollectionEndpoint: "http://neon-internal-api.aws.neon.tech/billing/api/v1/usage_events"
+  metricCollectionEndpoint: "http://console-release.local/billing/api/v1/usage_events"
   metricCollectionInterval: "10min"
 
 # -- Additional labels for neon-proxy pods

.github/helm-values/production.neon-storage-broker.yaml

@@ -0,0 +1,56 @@
+# Helm chart values for neon-storage-broker
+podLabels:
+  neon_env: production
+  neon_service: storage-broker
+
+# Use L4 LB
+service:
+  # service.annotations -- Annotations to add to the service
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: external  # use newer AWS Load Balancer Controller
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+    service.beta.kubernetes.io/aws-load-balancer-scheme: internal  # deploy LB to private subnet
+    # assign service to this name at external-dns
+    external-dns.alpha.kubernetes.io/hostname: storage-broker.prod.local
+  # service.type -- Service type
+  type: LoadBalancer
+  # service.port -- broker listen port
+  port: 50051
+
+ingress:
+  enabled: false
+
+metrics:
+  enabled: true
+  serviceMonitor:
+    enabled: true
+    selector:
+      release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-storage-broker.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-storage-broker-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-storage-broker
+        app.kubernetes.io/instance: neon-storage-broker
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-storage-broker"
+      endpoints:
+        - port: broker
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"
+
+settings:
+  sentryEnvironment: "production"

.github/helm-values/production.proxy-scram.yaml

@@ -0,0 +1,54 @@
+settings:
+  authBackend: "console"
+  authEndpoint: "http://console-release.local/management/api/v2"
+  domain: "*.cloud.neon.tech"
+  sentryEnvironment: "production"
+  wssPort: 8443
+  metricCollectionEndpoint: "http://console-release.local/billing/api/v1/usage_events"
+  metricCollectionInterval: "10min"
+
+podLabels:
+  zenith_service: proxy-scram
+  zenith_env: production
+  zenith_region: us-west-2
+  zenith_region_slug: oregon
+
+exposedService:
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: external
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
+    external-dns.alpha.kubernetes.io/hostname: '*.cloud.neon.tech'
+  httpsPort: 443
+
+metrics:
+  enabled: true
+  serviceMonitor:
+    enabled: true
+    selector:
+      release: kube-prometheus-stack
+
+extraManifests:
+  - apiVersion: operator.victoriametrics.com/v1beta1
+    kind: VMServiceScrape
+    metadata:
+      name: "{{ include \"neon-proxy.fullname\" . }}"
+      labels:
+        helm.sh/chart: neon-proxy-{{ .Chart.Version }}
+        app.kubernetes.io/name: neon-proxy
+        app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}"
+        app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
+        app.kubernetes.io/managed-by: Helm
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: "neon-proxy"
+      endpoints:
+        - port: http
+          path: /metrics
+          interval: 10s
+          scrapeTimeout: 10s
+      namespaceSelector:
+        matchNames:
+          - "{{ .Release.Namespace }}"

.github/workflows/build_and_test.yml

@@ -1,4 +1,4 @@
-name: Build and Test
+name: Test and Deploy
 
 on:
   push:
@@ -54,7 +54,7 @@ jobs:
   check-codestyle-python:
     runs-on: [ self-hosted, gen3, small ]
     container:
-      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
+      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cloud:pinned
       options: --init
 
     steps:
@@ -337,7 +337,7 @@ jobs:
         uses: ./.github/actions/save-coverage-data
 
   benchmarks:
-    runs-on: [ self-hosted, gen3, small ]
+    runs-on: [ self-hosted, gen3, large ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
@@ -405,7 +405,7 @@ jobs:
           DATABASE_URL="$TEST_RESULT_CONNSTR" poetry run python3 scripts/ingest_regress_test_result.py --revision ${SHA} --reference ${GITHUB_REF} --build-type ${BUILD_TYPE} --ingest suites.json
 
   coverage-report:
-    runs-on: [ self-hosted, gen3, small ]
+    runs-on: [ self-hosted, gen3, large ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
@@ -481,7 +481,7 @@ jobs:
             }"
 
   trigger-e2e-tests:
-    runs-on: [ self-hosted, gen3, small ]
+    runs-on: [ self-hosted, gen3, large ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned
       options: --init
@@ -611,31 +611,34 @@ jobs:
       run:
         shell: sh -eu {0}
     env:
-      VM_BUILDER_VERSION: v0.4.6
+      VM_INFORMANT_VERSION: 0.1.1
 
     steps:
-      - name: Checkout
-        uses: actions/checkout@v1
-        with:
-          fetch-depth: 0
-
-      - name: Downloading vm-builder
+      - name: Downloading latest vm-builder
         run: |
-          curl -L https://github.com/neondatabase/neonvm/releases/download/$VM_BUILDER_VERSION/vm-builder -o vm-builder
+          curl -L https://github.com/neondatabase/neonvm/releases/latest/download/vm-builder -o vm-builder
           chmod +x vm-builder
 
       - name: Pulling compute-node image
         run: |
           docker pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}}
 
-      - name: Building VM compute-node rootfs
+      - name: Downloading VM informant version ${{ env.VM_INFORMANT_VERSION }}
         run: |
-          docker build -t temp-vm-compute-node --build-arg SRC_IMAGE=369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} -f Dockerfile.vm-compute-node .
+          curl -fL https://github.com/neondatabase/autoscaling/releases/download/${{ env.VM_INFORMANT_VERSION }}/vm-informant -o vm-informant
+          chmod +x vm-informant
+
+      - name: Adding VM informant to compute-node image
+        run: |
+          ID=$(docker create 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}})
+          docker cp vm-informant $ID:/bin/vm-informant
+          docker commit $ID temp-vm-compute-node
+          docker rm -f $ID
 
       - name: Build vm image
         run: |
           # note: as of 2023-01-12, vm-builder requires a trailing ":latest" for local images
-          ./vm-builder -use-inittab -src=temp-vm-compute-node:latest -dst=369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}}
+          ./vm-builder -src=temp-vm-compute-node:latest -dst=369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}}
 
       - name: Pushing vm-compute-node image
         run: |
@@ -810,6 +813,121 @@ jobs:
       - name: Cleanup ECR folder
         run: rm -rf ~/.ecr
 
+  calculate-deploy-targets:
+    runs-on: [ self-hosted, gen3, small ]
+    if: |
+      github.ref_name == 'release' &&
+      github.event_name != 'workflow_dispatch'
+    outputs:
+      matrix-include: ${{ steps.set-matrix.outputs.include }}
+    steps:
+      - id: set-matrix
+        run: |
+          if [[ "$GITHUB_REF_NAME" == "release" ]]; then
+            PRODUCTION='{"env_name": "production", "proxy_job": "neon-proxy", "proxy_config": "production.proxy", "storage_broker_ns": "neon-storage-broker", "storage_broker_config": "production.neon-storage-broker", "kubeconfig_secret": "PRODUCTION_KUBECONFIG_DATA", "console_api_key_secret": "NEON_PRODUCTION_API_KEY"}'
+            echo "include=[$PRODUCTION]" >> $GITHUB_OUTPUT
+          else
+            echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to 'release'"
+            exit 1
+          fi
+
+  deploy:
+    runs-on: [ self-hosted, gen3, small ]
+    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
+    # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version.
+    # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly
+    needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
+    if: |
+      github.ref_name == 'release' &&
+      github.event_name != 'workflow_dispatch'
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        include: ${{fromJSON(needs.calculate-deploy-targets.outputs.matrix-include)}}
+    environment:
+      name: prod-old
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Redeploy
+        run: |
+          export DOCKER_TAG=${{needs.tag.outputs.build-tag}}
+          cd "$(pwd)/.github/ansible"
+
+          if [[ "$GITHUB_REF_NAME" == "main" ]]; then
+            ./get_binaries.sh
+          elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
+            RELEASE=true ./get_binaries.sh
+          else
+            echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
+            exit 1
+          fi
+
+          eval $(ssh-agent)
+          echo "${{ secrets.TELEPORT_SSH_KEY }}"  | tr -d '\n'| base64 --decode >ssh-key
+          echo "${{ secrets.TELEPORT_SSH_CERT }}" | tr -d '\n'| base64 --decode >ssh-key-cert.pub
+          chmod 0600 ssh-key
+          ssh-add ssh-key
+          rm -f ssh-key ssh-key-cert.pub
+          ANSIBLE_CONFIG=./ansible.cfg ansible-galaxy collection install sivel.toiletwater
+          ANSIBLE_CONFIG=./ansible.cfg ansible-playbook deploy.yaml -i ${{ matrix.env_name }}.hosts.yaml -e CONSOLE_API_TOKEN=${{ secrets[matrix.console_api_key_secret] }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }}
+          rm -f neon_install.tar.gz .neon_current_version
+
+      # Cleanup script fails otherwise - rm: cannot remove '/nvme/actions-runner/_work/_temp/_github_home/.ansible/collections': Permission denied
+      - name: Cleanup ansible folder
+        run: rm -rf ~/.ansible
+
+  deploy-new:
+    runs-on: [ self-hosted, gen3, small ]
+    container:
+      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
+      options: --user root --privileged
+    # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version.
+    # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly
+    needs: [ push-docker-hub, tag, regress-tests ]
+    if: |
+      (github.ref_name == 'main') &&
+      github.event_name != 'workflow_dispatch'
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        target_region: [ eu-west-1, us-east-2 ]
+    environment:
+      name: dev-${{ matrix.target_region }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Redeploy
+        run: |
+          export DOCKER_TAG=${{needs.tag.outputs.build-tag}}
+          cd "$(pwd)/.github/ansible"
+          if [[ "$GITHUB_REF_NAME" == "main" ]]; then
+            ./get_binaries.sh
+          elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
+            RELEASE=true ./get_binaries.sh
+          else
+            echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
+            exit 1
+          fi
+          ansible-galaxy collection install sivel.toiletwater
+          ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_STAGING_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }}
+          rm -f neon_install.tar.gz .neon_current_version
+
+      - name: Cleanup ansible folder
+        run: rm -rf ~/.ansible
+
   deploy-pr-test-new:
     runs-on: [ self-hosted, gen3, small ]
     container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
@@ -846,37 +964,348 @@ jobs:
       - name: Cleanup ansible folder
         run: rm -rf ~/.ansible
 
-  deploy:
-    runs-on: [ self-hosted, gen3, small ]
-    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
+  deploy-prod-new:
+    runs-on: prod
+    container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
+    # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version.
+    # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly
     needs: [ push-docker-hub, tag, regress-tests ]
-    if: ( github.ref_name == 'main' || github.ref_name == 'release' ) && github.event_name != 'workflow_dispatch'
+    if: |
+      (github.ref_name == 'release') &&
+      github.event_name != 'workflow_dispatch'
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        target_region: [ us-east-2, us-west-2, eu-central-1, ap-southeast-1 ]
+    environment:
+      name: prod-${{ matrix.target_region }}
     steps:
       - name: Checkout
         uses: actions/checkout@v3
         with:
-          submodules: false
+          submodules: true
           fetch-depth: 0
 
-      - name: Trigger deploy workflow
-        env:
-          GH_TOKEN: ${{ github.token }}
+      - name: Redeploy
         run: |
+          export DOCKER_TAG=${{needs.tag.outputs.build-tag}}
+          cd "$(pwd)/.github/ansible"
+
           if [[ "$GITHUB_REF_NAME" == "main" ]]; then
-            gh workflow run deploy-dev.yml --ref main -f branch=${{ github.sha }} -f dockerTag=${{needs.tag.outputs.build-tag}}
+            ./get_binaries.sh
           elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
-            gh workflow run deploy-prod.yml --ref release -f branch=${{ github.sha }} -f dockerTag=${{needs.tag.outputs.build-tag}} -f disclamerAcknowledged=true
+            RELEASE=true ./get_binaries.sh
           else
             echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
             exit 1
           fi
 
+          ansible-galaxy collection install sivel.toiletwater
+          ansible-playbook deploy.yaml -i prod.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_PRODUCTION_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }}
+          rm -f neon_install.tar.gz .neon_current_version
+
+  deploy-proxy:
+    runs-on: [ self-hosted, gen3, small ]
+    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
+    # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
+    needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
+    if: |
+      github.ref_name == 'release' &&
+      github.event_name != 'workflow_dispatch'
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        include: ${{fromJSON(needs.calculate-deploy-targets.outputs.matrix-include)}}
+    environment:
+      name: prod-old
+    env:
+      KUBECONFIG: .kubeconfig
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Store kubeconfig file
+        run: |
+          echo "${{ secrets[matrix.kubeconfig_secret] }}" | base64 --decode > ${KUBECONFIG}
+          chmod 0600 ${KUBECONFIG}
+
+      - name: Add neon helm chart
+        run: helm repo add neondatabase https://neondatabase.github.io/helm-charts
+
+      - name: Re-deploy proxy
+        run: |
+          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
+          helm upgrade ${{ matrix.proxy_job }}-scram neondatabase/neon-proxy --namespace neon-proxy --install --atomic -f .github/helm-values/${{ matrix.proxy_config }}-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+
+      - name: Cleanup helm folder
+        run: rm -rf ~/.cache
+
+  deploy-storage-broker:
+    name: deploy storage broker on old staging and old prod
+    runs-on: [ self-hosted, gen3, small ]
+    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
+    # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
+    needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
+    if: |
+      github.ref_name == 'release' &&
+      github.event_name != 'workflow_dispatch'
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        include: ${{fromJSON(needs.calculate-deploy-targets.outputs.matrix-include)}}
+    environment:
+      name: prod-old
+    env:
+      KUBECONFIG: .kubeconfig
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Store kubeconfig file
+        run: |
+          echo "${{ secrets[matrix.kubeconfig_secret] }}" | base64 --decode > ${KUBECONFIG}
+          chmod 0600 ${KUBECONFIG}
+
+      - name: Add neon helm chart
+        run: helm repo add neondatabase https://neondatabase.github.io/helm-charts
+
+      - name: Deploy storage-broker
+        run:
+          helm upgrade neon-storage-broker neondatabase/neon-storage-broker --namespace ${{ matrix.storage_broker_ns }} --create-namespace --install --atomic -f .github/helm-values/${{ matrix.storage_broker_config }}.yaml --set image.tag=${{ needs.tag.outputs.build-tag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s
+
+      - name: Cleanup helm folder
+        run: rm -rf ~/.cache
+
+  deploy-proxy-new:
+    runs-on: [ self-hosted, gen3, small ]
+    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
+    # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
+    needs: [ push-docker-hub, tag, regress-tests ]
+    if: |
+      (github.ref_name == 'main') &&
+      github.event_name != 'workflow_dispatch'
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        include:
+          - target_region:  us-east-2
+            target_cluster: dev-us-east-2-beta
+            deploy_link_proxy: true
+            deploy_legacy_scram_proxy: true
+          - target_region:  eu-west-1
+            target_cluster: dev-eu-west-1-zeta
+            deploy_link_proxy: false
+            deploy_legacy_scram_proxy: false
+    environment:
+      name: dev-${{ matrix.target_region }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1-node16
+        with:
+          role-to-assume: arn:aws:iam::369495373322:role/github-runner
+          aws-region: eu-central-1
+          role-skip-session-tagging: true
+          role-duration-seconds: 1800
+
+      - name: Configure environment
+        run: |
+          helm repo add neondatabase https://neondatabase.github.io/helm-charts
+          aws --region ${{ matrix.target_region }} eks update-kubeconfig --name  ${{ matrix.target_cluster }}
+
+      - name: Re-deploy scram proxy
+        run: |
+          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
+          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+
+      - name: Re-deploy link proxy
+        if: matrix.deploy_link_proxy
+        run: |
+          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
+          helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+
+      - name: Re-deploy legacy scram proxy
+        if: matrix.deploy_legacy_scram_proxy
+        run: |
+          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
+          helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+
+      - name: Cleanup helm folder
+        run: rm -rf ~/.cache
+
+  deploy-storage-broker-dev-new:
+    runs-on: [ self-hosted, gen3, small ]
+    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
+    # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
+    needs: [ push-docker-hub, tag, regress-tests ]
+    if: |
+      (github.ref_name == 'main') &&
+      github.event_name != 'workflow_dispatch'
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        include:
+          - target_region:  us-east-2
+            target_cluster: dev-us-east-2-beta
+          - target_region:  eu-west-1
+            target_cluster: dev-eu-west-1-zeta
+    environment:
+      name: dev-${{ matrix.target_region }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1-node16
+        with:
+          role-to-assume: arn:aws:iam::369495373322:role/github-runner
+          aws-region: eu-central-1
+          role-skip-session-tagging: true
+          role-duration-seconds: 1800
+
+      - name: Configure environment
+        run: |
+          helm repo add neondatabase https://neondatabase.github.io/helm-charts
+          aws --region ${{ matrix.target_region }} eks update-kubeconfig --name  ${{ matrix.target_cluster }}
+
+      - name: Deploy storage-broker
+        run:
+          helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ needs.tag.outputs.build-tag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s
+
+      - name: Cleanup helm folder
+        run: rm -rf ~/.cache
+
+  deploy-proxy-prod-new:
+    runs-on: prod
+    container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
+    # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
+    needs: [ push-docker-hub, tag, regress-tests ]
+    if: |
+      (github.ref_name == 'release') &&
+      github.event_name != 'workflow_dispatch'
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        include:
+          - target_region:  us-east-2
+            target_cluster: prod-us-east-2-delta
+            deploy_link_proxy: true
+            deploy_legacy_scram_proxy: false
+          - target_region:  us-west-2
+            target_cluster: prod-us-west-2-eta
+            deploy_link_proxy: false
+            deploy_legacy_scram_proxy: true
+          - target_region: eu-central-1
+            target_cluster: prod-eu-central-1-gamma
+            deploy_link_proxy: false
+            deploy_legacy_scram_proxy: false
+          - target_region: ap-southeast-1
+            target_cluster: prod-ap-southeast-1-epsilon
+            deploy_link_proxy: false
+            deploy_legacy_scram_proxy: false
+    environment:
+      name: prod-${{ matrix.target_region }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Configure environment
+        run: |
+          helm repo add neondatabase https://neondatabase.github.io/helm-charts
+          aws --region ${{ matrix.target_region }} eks update-kubeconfig --name  ${{ matrix.target_cluster }}
+
+      - name: Re-deploy scram proxy
+        run: |
+          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
+          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+
+      - name: Re-deploy link proxy
+        if: matrix.deploy_link_proxy
+        run: |
+          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
+          helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+
+      - name: Re-deploy legacy scram proxy
+        if: matrix.deploy_legacy_scram_proxy
+        run: |
+          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
+          helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+
+  deploy-storage-broker-prod-new:
+    runs-on: prod
+    container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
+    # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
+    needs: [ push-docker-hub, tag, regress-tests ]
+    if: |
+      (github.ref_name == 'release') &&
+      github.event_name != 'workflow_dispatch'
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        include:
+          - target_region:  us-east-2
+            target_cluster: prod-us-east-2-delta
+          - target_region:  us-west-2
+            target_cluster: prod-us-west-2-eta
+          - target_region: eu-central-1
+            target_cluster: prod-eu-central-1-gamma
+          - target_region: ap-southeast-1
+            target_cluster: prod-ap-southeast-1-epsilon
+    environment:
+      name: prod-${{ matrix.target_region }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Configure environment
+        run: |
+          helm repo add neondatabase https://neondatabase.github.io/helm-charts
+          aws --region ${{ matrix.target_region }} eks update-kubeconfig --name  ${{ matrix.target_cluster }}
+
+      - name: Deploy storage-broker
+        run:
+          helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ needs.tag.outputs.build-tag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s
+
   promote-compatibility-data:
     runs-on: [ self-hosted, gen3, small ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
-    needs: [ push-docker-hub, tag, regress-tests ]
+    needs: [ deploy, deploy-proxy ]
     if: github.ref_name == 'release' && github.event_name != 'workflow_dispatch'
     steps:
       - name: Promote compatibility snapshot for the release

.github/workflows/deploy-dev.yml

@@ -1,179 +0,0 @@
-name: Neon Deploy dev
-
-on:
-  workflow_dispatch:
-    inputs:
-      dockerTag:
-        description: 'Docker tag to deploy'
-        required: true
-        type: string
-      branch:
-        description: 'Branch or commit used for deploy scripts and configs'
-        required: true
-        type: string
-        default: 'main'
-      deployStorage:
-        description: 'Deploy storage'
-        required: true
-        type: boolean
-        default: true
-      deployProxy:
-        description: 'Deploy proxy'
-        required: true
-        type: boolean
-        default: true
-      deployStorageBroker:
-        description: 'Deploy storage-broker'
-        required: true
-        type: boolean
-        default: true
-
-env:
-  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }}
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }}
-
-concurrency:
-  group: deploy-dev
-  cancel-in-progress: false
-
-jobs:
-  deploy-storage-new:
-    runs-on: [ self-hosted, gen3, small ]
-    container:
-      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
-      options: --user root --privileged
-    if: inputs.deployStorage
-    defaults:
-      run:
-        shell: bash
-    strategy:
-      matrix:
-        target_region: [ eu-west-1, us-east-2 ]
-    environment:
-      name: dev-${{ matrix.target_region }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          fetch-depth: 0
-          ref: ${{ inputs.branch }}
-
-      - name: Redeploy
-        run: |
-          export DOCKER_TAG=${{ inputs.dockerTag }}
-          cd "$(pwd)/.github/ansible"
-
-          ./get_binaries.sh
-
-          ansible-galaxy collection install sivel.toiletwater
-          ansible-playbook -v deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_STAGING_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }}
-          rm -f neon_install.tar.gz .neon_current_version
-
-      - name: Cleanup ansible folder
-        run: rm -rf ~/.ansible
-
-  deploy-proxy-new:
-    runs-on: [ self-hosted, gen3, small ]
-    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
-    if: inputs.deployProxy
-    defaults:
-      run:
-        shell: bash
-    strategy:
-      matrix:
-        include:
-          - target_region:  us-east-2
-            target_cluster: dev-us-east-2-beta
-            deploy_link_proxy: true
-            deploy_legacy_scram_proxy: true
-          - target_region:  eu-west-1
-            target_cluster: dev-eu-west-1-zeta
-            deploy_link_proxy: false
-            deploy_legacy_scram_proxy: false
-    environment:
-      name: dev-${{ matrix.target_region }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          fetch-depth: 0
-          ref: ${{ inputs.branch }}
-  
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1-node16
-        with:
-          role-to-assume: arn:aws:iam::369495373322:role/github-runner
-          aws-region: eu-central-1
-          role-skip-session-tagging: true
-          role-duration-seconds: 1800
-  
-      - name: Configure environment
-        run: |
-          helm repo add neondatabase https://neondatabase.github.io/helm-charts
-          aws --region ${{ matrix.target_region }} eks update-kubeconfig --name  ${{ matrix.target_cluster }}
-  
-      - name: Re-deploy scram proxy
-        run: |
-          DOCKER_TAG=${{ inputs.dockerTag }}
-          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
-  
-      - name: Re-deploy link proxy
-        if: matrix.deploy_link_proxy
-        run: |
-          DOCKER_TAG=${{ inputs.dockerTag }}
-          helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
-  
-      - name: Re-deploy legacy scram proxy
-        if: matrix.deploy_legacy_scram_proxy
-        run: |
-          DOCKER_TAG=${{ inputs.dockerTag }}
-          helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
-  
-      - name: Cleanup helm folder
-        run: rm -rf ~/.cache
-  
-  deploy-storage-broker-new:
-    runs-on: [ self-hosted, gen3, small ]
-    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
-    if: inputs.deployStorageBroker
-    defaults:
-      run:
-        shell: bash
-    strategy:
-      matrix:
-        include:
-          - target_region:  us-east-2
-            target_cluster: dev-us-east-2-beta
-          - target_region:  eu-west-1
-            target_cluster: dev-eu-west-1-zeta
-    environment:
-      name: dev-${{ matrix.target_region }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          fetch-depth: 0
-          ref: ${{ inputs.branch }}
-  
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1-node16
-        with:
-          role-to-assume: arn:aws:iam::369495373322:role/github-runner
-          aws-region: eu-central-1
-          role-skip-session-tagging: true
-          role-duration-seconds: 1800
-  
-      - name: Configure environment
-        run: |
-          helm repo add neondatabase https://neondatabase.github.io/helm-charts
-          aws --region ${{ matrix.target_region }} eks update-kubeconfig --name  ${{ matrix.target_cluster }}
-  
-      - name: Deploy storage-broker
-        run:
-          helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ inputs.dockerTag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s
-  
-      - name: Cleanup helm folder
-        run: rm -rf ~/.cache

.github/workflows/deploy-prod.yml

@@ -1,167 +0,0 @@
-name: Neon Deploy prod
-
-on:
-  workflow_dispatch:
-    inputs:
-      dockerTag:
-        description: 'Docker tag to deploy'
-        required: true
-        type: string
-      branch:
-        description: 'Branch or commit used for deploy scripts and configs'
-        required: true
-        type: string
-        default: 'release'
-      deployStorage:
-        description: 'Deploy storage'
-        required: true
-        type: boolean
-        default: true
-      deployProxy:
-        description: 'Deploy proxy'
-        required: true
-        type: boolean
-        default: true
-      deployStorageBroker:
-        description: 'Deploy storage-broker'
-        required: true
-        type: boolean
-        default: true
-      disclamerAcknowledged:
-        description: 'I confirm that there is an emergency and I can not use regular release workflow'
-        required: true
-        type: boolean
-        default: false
-
-concurrency:
-  group: deploy-prod
-  cancel-in-progress: false
-
-jobs:
-  deploy-prod-new:
-    runs-on: prod
-    container:
-      image: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
-      options: --user root --privileged
-    if: inputs.deployStorage && inputs.disclamerAcknowledged
-    defaults:
-      run:
-        shell: bash
-    strategy:
-      matrix:
-        target_region: [ us-east-2, us-west-2, eu-central-1, ap-southeast-1 ]
-    environment:
-      name: prod-${{ matrix.target_region }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          fetch-depth: 0
-          ref: ${{ inputs.branch }}
-
-      - name: Redeploy
-        run: |
-          export DOCKER_TAG=${{ inputs.dockerTag }}
-          cd "$(pwd)/.github/ansible"
-
-          ./get_binaries.sh
-
-          ansible-galaxy collection install sivel.toiletwater
-          ansible-playbook -v deploy.yaml -i prod.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_PRODUCTION_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }}
-          rm -f neon_install.tar.gz .neon_current_version
-
-  deploy-proxy-prod-new:
-    runs-on: prod
-    container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
-    if: inputs.deployProxy && inputs.disclamerAcknowledged
-    defaults:
-      run:
-        shell: bash
-    strategy:
-      matrix:
-        include:
-          - target_region:  us-east-2
-            target_cluster: prod-us-east-2-delta
-            deploy_link_proxy: true
-            deploy_legacy_scram_proxy: false
-          - target_region:  us-west-2
-            target_cluster: prod-us-west-2-eta
-            deploy_link_proxy: false
-            deploy_legacy_scram_proxy: true
-          - target_region: eu-central-1
-            target_cluster: prod-eu-central-1-gamma
-            deploy_link_proxy: false
-            deploy_legacy_scram_proxy: false
-          - target_region: ap-southeast-1
-            target_cluster: prod-ap-southeast-1-epsilon
-            deploy_link_proxy: false
-            deploy_legacy_scram_proxy: false
-    environment:
-      name: prod-${{ matrix.target_region }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          fetch-depth: 0
-          ref: ${{ inputs.branch }}
-
-      - name: Configure environment
-        run: |
-          helm repo add neondatabase https://neondatabase.github.io/helm-charts
-          aws --region ${{ matrix.target_region }} eks update-kubeconfig --name  ${{ matrix.target_cluster }}
-
-      - name: Re-deploy scram proxy
-        run: |
-          DOCKER_TAG=${{ inputs.dockerTag }}
-          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
-
-      - name: Re-deploy link proxy
-        if: matrix.deploy_link_proxy
-        run: |
-          DOCKER_TAG=${{ inputs.dockerTag }}
-          helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
-
-      - name: Re-deploy legacy scram proxy
-        if: matrix.deploy_legacy_scram_proxy
-        run: |
-          DOCKER_TAG=${{ inputs.dockerTag }}
-          helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
-
-  deploy-storage-broker-prod-new:
-    runs-on: prod
-    container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
-    if: inputs.deployStorageBroker && inputs.disclamerAcknowledged
-    defaults:
-      run:
-        shell: bash
-    strategy:
-      matrix:
-        include:
-          - target_region:  us-east-2
-            target_cluster: prod-us-east-2-delta
-          - target_region:  us-west-2
-            target_cluster: prod-us-west-2-eta
-          - target_region: eu-central-1
-            target_cluster: prod-eu-central-1-gamma
-          - target_region: ap-southeast-1
-            target_cluster: prod-ap-southeast-1-epsilon
-    environment:
-      name: prod-${{ matrix.target_region }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          fetch-depth: 0
-          ref: ${{ inputs.branch }}
-
-      - name: Configure environment
-        run: |
-          helm repo add neondatabase https://neondatabase.github.io/helm-charts
-          aws --region ${{ matrix.target_region }} eks update-kubeconfig --name  ${{ matrix.target_cluster }}
-
-      - name: Deploy storage-broker
-        run:
-          helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ inputs.dockerTag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s

.github/workflows/neon_extra_builds.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches:
     - main
-  pull_request:
 
 defaults:
   run:
@@ -21,7 +20,6 @@ env:
 
 jobs:
   check-macos-build:
-    if: github.ref_name == 'main' || contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos')
     timeout-minutes: 90
     runs-on: macos-latest
 
@@ -95,16 +93,11 @@ jobs:
         run: ./run_clippy.sh
 
   gather-rust-build-stats:
-    if: github.ref_name == 'main' || contains(github.event.pull_request.labels.*.name, 'run-extra-build-stats')
-    runs-on: [ self-hosted, gen3, large ]
-    container:
-      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
-      options: --init
+    timeout-minutes: 90
+    runs-on: ubuntu-latest
 
     env:
       BUILD_TYPE: release
-      # remove the cachepot wrapper and build without crate caches
-      RUSTC_WRAPPER: ""
       # build with incremental compilation produce partial results
       # so do not attempt to cache this build, also disable the incremental compilation
       CARGO_INCREMENTAL: 0
@@ -116,6 +109,11 @@ jobs:
           submodules: true
           fetch-depth: 1
 
+      - name: Install Ubuntu postgres dependencies
+        run: |
+          sudo apt update
+          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev libssl-dev protobuf-compiler
+
       # Some of our rust modules use FFI and need those to be checked
       - name: Get postgres headers
         run: make postgres-headers -j$(nproc)
@@ -124,31 +122,7 @@ jobs:
         run: cargo build --all --release --timings
 
       - name: Upload the build stats
-        id: upload-stats
-        env:
-          BUCKET: neon-github-public-dev
-          SHA: ${{ github.event.pull_request.head.sha || github.sha }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }}
-        run: |
-          REPORT_URL=https://${BUCKET}.s3.amazonaws.com/build-stats/${SHA}/${GITHUB_RUN_ID}/cargo-timing.html
-          aws s3 cp --only-show-errors ./target/cargo-timings/cargo-timing.html "s3://${BUCKET}/build-stats/${SHA}/${GITHUB_RUN_ID}/"
-          echo "report-url=${REPORT_URL}" >> $GITHUB_OUTPUT
-
-      - name: Publish build stats report
-        uses: actions/github-script@v6
-        env:
-          REPORT_URL: ${{ steps.upload-stats.outputs.report-url }}
-          SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+        uses: actions/upload-artifact@v3
         with:
-          script: |
-            const { REPORT_URL, SHA } = process.env
-
-            await github.rest.repos.createCommitStatus({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              sha: `${SHA}`,
-              state: 'success',
-              target_url: `${REPORT_URL}`,
-              context: `Build stats (release)`,
-            })
+          name: neon-${{ runner.os }}-release-build-stats
+          path: ./target/cargo-timings/

.github/workflows/release.yml

@@ -1,33 +0,0 @@
-name: Create Release Branch
-
-on:
-  schedule:
-    - cron: '0 10 * * 2'
-
-jobs:
-  create_release_branch:
-    runs-on: [ubuntu-latest]
-
-    steps:
-    - name: Check out code
-      uses: actions/checkout@v3
-      with:
-        ref: main
-
-    - name: Get current date
-      id: date
-      run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
-
-    - name: Create release branch
-      run: git checkout -b releases/${{ steps.date.outputs.date }}
-
-    - name: Push new branch
-      run: git push origin releases/${{ steps.date.outputs.date }}
-
-    - name: Create pull request into release
-      uses: thomaseizinger/create-pull-request@e3972219c86a56550fb70708d96800d8e24ba862 # 1.3.0
-      with:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        head: releases/${{ steps.date.outputs.date }}
-        base: release
-        title: Release ${{ steps.date.outputs.date }}

Cargo.toml

@@ -7,7 +7,6 @@ members = [
     "safekeeper",
     "storage_broker",
     "workspace_hack",
-    "trace",
     "libs/*",
 ]
 
@@ -32,15 +31,12 @@ bstr = "1.0"
 byteorder = "1.4"
 bytes = "1.0"
 chrono = { version = "0.4", default-features = false, features = ["clock"] }
-clap = { version = "4.0", features = ["derive"] }
+clap = "4.0"
 close_fds = "0.3.2"
 comfy-table = "6.1"
 const_format = "0.2"
 crc32c = "0.6"
 crossbeam-utils = "0.8.5"
-either = "1.8"
-enum-map = "2.4.2"
-enumset = "1.0.12"
 fail = "0.5.0"
 fs2 = "0.4.3"
 futures = "0.3"
@@ -48,7 +44,6 @@ futures-core = "0.3"
 futures-util = "0.3"
 git-version = "0.3"
 hashbrown = "0.13"
-hashlink = "0.8.1"
 hex = "0.4"
 hex-literal = "0.3"
 hmac = "0.12.1"
@@ -69,6 +64,7 @@ once_cell = "1.13"
 opentelemetry = "0.18.0"
 opentelemetry-otlp = { version = "0.11.0", default_features=false, features = ["http-proto", "trace", "http", "reqwest-client"] }
 opentelemetry-semantic-conventions = "0.10.0"
+tracing-opentelemetry = "0.18.0"
 parking_lot = "0.12"
 pin-project-lite = "0.2"
 prometheus = {version = "0.13", default_features=false, features = ["process"]} # removes protobuf dependency
@@ -76,8 +72,6 @@ prost = "0.11"
 rand = "0.8"
 regex = "1.4"
 reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] }
-reqwest-tracing = { version = "0.4.0", features = ["opentelemetry_0_18"] }
-reqwest-middleware = "0.2.0"
 routerify = "3"
 rpds = "0.12.0"
 rustls = "0.20"
@@ -94,7 +88,6 @@ socket2 = "0.4.4"
 strum = "0.24"
 strum_macros = "0.24"
 svg_fmt = "0.4.1"
-sync_wrapper = "0.1.2"
 tar = "0.4"
 thiserror = "1.0"
 tls-listener = { version = "0.6", features = ["rustls", "hyper-h1"] }
@@ -107,7 +100,6 @@ toml = "0.5"
 toml_edit = { version = "0.17", features = ["easy"] }
 tonic = {version = "0.8", features = ["tls", "tls-roots"]}
 tracing = "0.1"
-tracing-opentelemetry = "0.18.0"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 url = "2.2"
 uuid = { version = "1.2", features = ["v4", "serde"] }
@@ -126,9 +118,6 @@ postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", re
 tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
 tokio-tar = { git = "https://github.com/neondatabase/tokio-tar.git", rev="404df61437de0feef49ba2ccdbdd94eb8ad6e142" }
 
-## Other git libraries
-heapless = { default-features=false, features=[], git = "https://github.com/japaric/heapless.git", rev = "644653bf3b831c6bb4963be2de24804acf5e5001" } # upstream release pending
-
 ## Local libraries
 consumption_metrics = { version = "0.1", path = "./libs/consumption_metrics/" }
 metrics = { version = "0.1", path = "./libs/metrics/" }

Dockerfile.compute-node

@@ -1,4 +1,3 @@
-ARG PG_VERSION
 ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com
 ARG IMAGE=rust
 ARG TAG=pinned
@@ -11,8 +10,7 @@ ARG TAG=pinned
 FROM debian:bullseye-slim AS build-deps
 RUN apt update &&  \
     apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev \
-    zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libssl-dev \
-    libicu-dev libxslt1-dev
+    zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libssl-dev
 
 #########################################################################################
 #
@@ -24,8 +22,7 @@ FROM build-deps AS pg-build
 ARG PG_VERSION
 COPY vendor/postgres-${PG_VERSION} postgres
 RUN cd postgres && \
-    ./configure CFLAGS='-O2 -g3' --enable-debug --with-openssl --with-uuid=ossp --with-icu \
-    --with-libxml --with-libxslt && \
+    ./configure CFLAGS='-O2 -g3' --enable-debug --with-openssl --with-uuid=ossp && \
     make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s install && \
     make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C contrib/ install && \
     # Install headers
@@ -36,8 +33,7 @@ RUN cd postgres && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgrowlocks.control && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/intagg.control && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgstattuple.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/earthdistance.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/xml2.control
+    echo 'trusted = true' >> /usr/local/pgsql/share/extension/earthdistance.control
 
 #########################################################################################
 #
@@ -48,45 +44,25 @@ RUN cd postgres && \
 FROM build-deps AS postgis-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
 RUN apt update && \
-    apt install -y cmake gdal-bin libboost-dev libboost-thread-dev libboost-filesystem-dev \
-    libboost-system-dev libboost-iostreams-dev libboost-program-options-dev libboost-timer-dev \
-    libcgal-dev libgdal-dev libgmp-dev libmpfr-dev libopenscenegraph-dev libprotobuf-c-dev \
-    protobuf-c-compiler xsltproc
+    apt install -y gdal-bin libgdal-dev libprotobuf-c-dev protobuf-c-compiler xsltproc
 
-# SFCGAL > 1.3 requires CGAL > 5.2, Bullseye's libcgal-dev is 5.2
-RUN wget https://gitlab.com/Oslandia/SFCGAL/-/archive/v1.3.10/SFCGAL-v1.3.10.tar.gz -O SFCGAL.tar.gz && \
-    mkdir sfcgal-src && cd sfcgal-src && tar xvzf ../SFCGAL.tar.gz --strip-components=1 -C . && \
-    cmake . && make -j $(getconf _NPROCESSORS_ONLN) && \
-    DESTDIR=/sfcgal make install -j $(getconf _NPROCESSORS_ONLN) && \
-    make clean && cp -R /sfcgal/* /
-
-ENV PATH "/usr/local/pgsql/bin:$PATH"
-
-RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.2.tar.gz -O postgis.tar.gz && \
-    mkdir postgis-src && cd postgis-src && tar xvzf ../postgis.tar.gz --strip-components=1 -C . && \
+RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.1.tar.gz && \
+    tar xvzf postgis-3.3.1.tar.gz && \
+    cd postgis-3.3.1 && \
     ./autogen.sh && \
-    ./configure --with-sfcgal=/usr/local/bin/sfcgal-config && \
+    export PATH="/usr/local/pgsql/bin:$PATH" && \
+    ./configure && \
     make -j $(getconf _NPROCESSORS_ONLN) install && \
     cd extensions/postgis && \
     make clean && \
     make -j $(getconf _NPROCESSORS_ONLN) install && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis.control && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_raster.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_sfcgal.control && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_tiger_geocoder.control && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_topology.control && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/address_standardizer.control && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/address_standardizer_data_us.control
 
-RUN wget https://github.com/pgRouting/pgrouting/archive/v3.4.2.tar.gz -O pgrouting.tar.gz && \
-    mkdir pgrouting-src && cd pgrouting-src && tar xvzf ../pgrouting.tar.gz --strip-components=1 -C . && \
-    mkdir build && \
-    cd build && \
-    cmake .. && \
-    make -j $(getconf _NPROCESSORS_ONLN) && \
-    make -j $(getconf _NPROCESSORS_ONLN) install && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgrouting.control
-
 #########################################################################################
 #
 # Layer "plv8-build"
@@ -96,17 +72,30 @@ RUN wget https://github.com/pgRouting/pgrouting/archive/v3.4.2.tar.gz -O pgrouti
 FROM build-deps AS plv8-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
 RUN apt update && \
-    apt install -y ninja-build python3-dev libncurses5 binutils clang
+    apt install -y ninja-build python3-dev libc++-dev libc++abi-dev libncurses5 binutils
 
-RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.5.tar.gz -O plv8.tar.gz && \
-    mkdir plv8-src && cd plv8-src && tar xvzf ../plv8.tar.gz --strip-components=1 -C . && \
+# https://github.com/plv8/plv8/issues/475:
+#   v8 uses gold for linking and sets `--thread-count=4` which breaks
+#   gold version <= 1.35 (https://sourceware.org/bugzilla/show_bug.cgi?id=23607)
+# Install newer gold version manually as debian-testing binutils version updates
+# libc version, which in turn breaks other extension built against non-testing libc.
+RUN wget https://ftp.gnu.org/gnu/binutils/binutils-2.38.tar.gz && \
+    tar xvzf binutils-2.38.tar.gz && \
+    cd binutils-2.38 && \
+    cd libiberty && ./configure && make -j $(getconf _NPROCESSORS_ONLN) && \
+    cd ../bfd && ./configure && make bfdver.h && \
+    cd ../gold && ./configure && make -j $(getconf _NPROCESSORS_ONLN) && make install && \
+    cp /usr/local/bin/ld.gold /usr/bin/gold
+
+# Sed is used to patch for https://github.com/plv8/plv8/issues/503
+RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.4.tar.gz && \
+    tar xvzf v3.1.4.tar.gz && \
+    cd plv8-3.1.4 && \
     export PATH="/usr/local/pgsql/bin:$PATH" && \
+    sed -i 's/MemoryContextAlloc(/MemoryContextAllocZero(/' plv8.cc && \
     make DOCKER=1 -j $(getconf _NPROCESSORS_ONLN) install && \
     rm -rf /plv8-* && \
-    find /usr/local/pgsql/ -name "plv8-*.so" | xargs strip && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/plv8.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/plcoffee.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/plls.control
+    echo 'trusted = true' >> /usr/local/pgsql/share/extension/plv8.control
 
 #########################################################################################
 #
@@ -124,17 +113,20 @@ RUN wget https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2
       && /tmp/cmake-install.sh --skip-license --prefix=/usr/local/ \
       && rm /tmp/cmake-install.sh
 
-RUN wget https://github.com/uber/h3/archive/refs/tags/v4.1.0.tar.gz -O h3.tar.gz && \
-    mkdir h3-src && cd h3-src && tar xvzf ../h3.tar.gz --strip-components=1 -C . && \
-    mkdir build && cd build && \
+RUN wget https://github.com/uber/h3/archive/refs/tags/v4.0.1.tar.gz -O h3.tgz && \
+    tar xvzf h3.tgz  && \
+    cd h3-4.0.1 && \
+    mkdir build && \
+    cd build && \
     cmake .. -DCMAKE_BUILD_TYPE=Release && \
     make -j $(getconf _NPROCESSORS_ONLN) && \
     DESTDIR=/h3 make install && \
     cp -R /h3/usr / && \
     rm -rf build
 
-RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.1.2.tar.gz -O h3-pg.tar.gz && \
-    mkdir h3-pg-src && cd h3-pg-src && tar xvzf ../h3-pg.tar.gz --strip-components=1 -C . && \
+RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.0.1.tar.gz -O h3-pg.tgz && \
+    tar xvzf h3-pg.tgz && \
+    cd h3-pg-4.0.1 && \
     export PATH="/usr/local/pgsql/bin:$PATH" && \
     make -j $(getconf _NPROCESSORS_ONLN) && \
     make -j $(getconf _NPROCESSORS_ONLN) install && \
@@ -150,8 +142,9 @@ RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.1.2.tar.gz -O h3
 FROM build-deps AS unit-pg-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
 
-RUN wget https://github.com/df7cb/postgresql-unit/archive/refs/tags/7.7.tar.gz -O postgresql-unit.tar.gz && \
-    mkdir postgresql-unit-src && cd postgresql-unit-src && tar xvzf ../postgresql-unit.tar.gz --strip-components=1 -C . && \
+RUN wget https://github.com/df7cb/postgresql-unit/archive/refs/tags/7.7.tar.gz && \
+    tar xvzf 7.7.tar.gz && \
+    cd postgresql-unit-7.7 && \
     make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \
     make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \
     # unit extension's "create extension" script relies on absolute install path to fill some reference tables.
@@ -161,107 +154,6 @@ RUN wget https://github.com/df7cb/postgresql-unit/archive/refs/tags/7.7.tar.gz -
     find /usr/local/pgsql/share/extension/ -name "unit*.sql" -print0 | xargs -0 sed -i "s|pgsql/||g" && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/unit.control
 
-#########################################################################################
-#
-# Layer "vector-pg-build"
-# compile pgvector extension
-#
-#########################################################################################
-FROM build-deps AS vector-pg-build
-COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
-
-RUN wget https://github.com/pgvector/pgvector/archive/refs/tags/v0.4.0.tar.gz -O pgvector.tar.gz && \
-    mkdir pgvector-src && cd pgvector-src && tar xvzf ../pgvector.tar.gz --strip-components=1 -C . && \
-    make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \
-    make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/vector.control
-
-#########################################################################################
-#
-# Layer "pgjwt-pg-build"
-# compile pgjwt extension
-#
-#########################################################################################
-FROM build-deps AS pgjwt-pg-build
-COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
-
-# 9742dab1b2f297ad3811120db7b21451bca2d3c9 made on 13/11/2021
-RUN wget https://github.com/michelp/pgjwt/archive/9742dab1b2f297ad3811120db7b21451bca2d3c9.tar.gz -O pgjwt.tar.gz && \
-    mkdir pgjwt-src && cd pgjwt-src && tar xvzf ../pgjwt.tar.gz --strip-components=1 -C . && \
-    make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgjwt.control
-
-#########################################################################################
-#
-# Layer "hypopg-pg-build"
-# compile hypopg extension
-#
-#########################################################################################
-FROM build-deps AS hypopg-pg-build
-COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
-
-RUN wget https://github.com/HypoPG/hypopg/archive/refs/tags/1.3.1.tar.gz -O hypopg.tar.gz && \
-    mkdir hypopg-src && cd hypopg-src && tar xvzf ../hypopg.tar.gz --strip-components=1 -C . && \
-    make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \
-    make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/hypopg.control
-
-#########################################################################################
-# 
-# Layer "rust extensions"
-# This layer is used to build `pgx` deps
-#
-#########################################################################################
-FROM build-deps AS rust-extensions-build
-COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
-
-RUN apt-get update && \
-    apt-get install -y curl libclang-dev cmake && \
-    useradd -ms /bin/bash nonroot -b /home
-
-ENV HOME=/home/nonroot
-ENV PATH="/home/nonroot/.cargo/bin:/usr/local/pgsql/bin/:$PATH"
-USER nonroot
-WORKDIR /home/nonroot
-ARG PG_VERSION
-
-RUN curl -sSO https://static.rust-lang.org/rustup/dist/$(uname -m)-unknown-linux-gnu/rustup-init && \
-    chmod +x rustup-init && \
-    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain stable && \
-    rm rustup-init && \
-    cargo install --git https://github.com/vadim2404/pgx --branch neon_abi_v0.6.1 --locked cargo-pgx && \
-    /bin/bash -c 'cargo pgx init --pg${PG_VERSION:1}=/usr/local/pgsql/bin/pg_config'
-
-USER root
-
-#########################################################################################
-# 
-# Layer "pg-jsonschema-pg-build"
-# Compile "pg_jsonschema" extension
-#
-#########################################################################################
-
-FROM rust-extensions-build AS pg-jsonschema-pg-build
-
-RUN git clone --depth=1 --single-branch --branch neon_abi_v0.1.4 https://github.com/vadim2404/pg_jsonschema/ && \
-    cd pg_jsonschema && \
-    cargo pgx install --release && \
-    echo "trusted = true" >> /usr/local/pgsql/share/extension/pg_jsonschema.control
-
-#########################################################################################
-# 
-# Layer "pg-graphql-pg-build"
-# Compile "pg_graphql" extension
-#
-#########################################################################################
-
-FROM rust-extensions-build AS pg-graphql-pg-build
-
-RUN git clone --depth=1 --single-branch --branch neon_abi_v1.1.0 https://github.com/vadim2404/pg_graphql && \
-    cd pg_graphql && \  
-    cargo pgx install --release && \
-    echo "trusted = true" >> /usr/local/pgsql/share/extension/pg_graphql.control
-
 #########################################################################################
 #
 # Layer "neon-pg-ext-build"
@@ -270,16 +162,10 @@ RUN git clone --depth=1 --single-branch --branch neon_abi_v1.1.0 https://github.
 #########################################################################################
 FROM build-deps AS neon-pg-ext-build
 COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/
-COPY --from=postgis-build /sfcgal/* /
 COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=h3-pg-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=h3-pg-build /h3/usr /
 COPY --from=unit-pg-build /usr/local/pgsql/ /usr/local/pgsql/
-COPY --from=vector-pg-build /usr/local/pgsql/ /usr/local/pgsql/
-COPY --from=pgjwt-pg-build /usr/local/pgsql/ /usr/local/pgsql/
-COPY --from=pg-jsonschema-pg-build /usr/local/pgsql/ /usr/local/pgsql/
-COPY --from=pg-graphql-pg-build /usr/local/pgsql/ /usr/local/pgsql/
-COPY --from=hypopg-pg-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY pgxn/ pgxn/
 
 RUN make -j $(getconf _NPROCESSORS_ONLN) \
@@ -330,36 +216,25 @@ RUN mkdir /var/db && useradd -m -d /var/db/postgres postgres && \
     mkdir /var/db/postgres/compute && mkdir /var/db/postgres/specs && \
     chown -R postgres:postgres /var/db/postgres && \
     chmod 0750 /var/db/postgres/compute && \
-    echo '/usr/local/lib' >> /etc/ld.so.conf && /sbin/ldconfig && \
-    # create folder for file cache
-    mkdir -p -m 777 /neon/cache
+    echo '/usr/local/lib' >> /etc/ld.so.conf && /sbin/ldconfig
 
 COPY --from=postgres-cleanup-layer --chown=postgres /usr/local/pgsql /usr/local
 COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-debug-size-lto/compute_ctl /usr/local/bin/compute_ctl
 
 # Install:
 # libreadline8 for psql
-# libicu67, locales for collations (including ICU)
 # libossp-uuid16 for extension ossp-uuid
-# libgeos, libgdal, libsfcgal1, libproj and libprotobuf-c1 for PostGIS
-# libxml2, libxslt1.1 for xml2
+# libgeos, libgdal, libproj and libprotobuf-c1 for PostGIS
 RUN apt update &&  \
     apt install --no-install-recommends -y \
-        locales \
-        libicu67 \
         libreadline8 \
         libossp-uuid16 \
         libgeos-c1v5 \
         libgdal28 \
         libproj19 \
         libprotobuf-c1 \
-        libsfcgal1 \
-        libxml2 \
-        libxslt1.1 \
         gdb && \
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
-    localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-ENV LANG en_US.utf8
 USER postgres
 ENTRYPOINT ["/usr/local/bin/compute_ctl"]

Dockerfile.vm-compute-node

@@ -1,32 +0,0 @@
-# Note: this file *mostly* just builds on Dockerfile.compute-node
-
-ARG SRC_IMAGE
-ARG VM_INFORMANT_VERSION=v0.1.6
-
-# Pull VM informant and set up inittab
-FROM neondatabase/vm-informant:$VM_INFORMANT_VERSION as informant
-
-RUN set -e \
-	&& rm -f /etc/inittab \
-	&& touch /etc/inittab
-
-ADD vm-cgconfig.conf /etc/cgconfig.conf
-RUN set -e \
-	&& echo "::sysinit:cgconfigparser -l /etc/cgconfig.conf -s 1664" >> /etc/inittab \
-	&& echo "::respawn:su vm-informant -c '/usr/local/bin/vm-informant --auto-restart --cgroup=neon-postgres'" >> /etc/inittab
-
-# Combine, starting from non-VM compute node image.
-FROM $SRC_IMAGE as base
-
-# Temporarily set user back to root so we can run apt update and adduser
-USER root
-RUN apt update && \
-	apt install --no-install-recommends -y \
-        cgroup-tools
-RUN adduser vm-informant --disabled-password --no-create-home
-USER postgres
-
-COPY --from=informant /etc/inittab /etc/inittab
-COPY --from=informant /usr/bin/vm-informant /usr/local/bin/vm-informant
-
-ENTRYPOINT ["/usr/sbin/cgexec", "-g", "*:neon-postgres", "/usr/local/bin/compute_ctl"]

Makefile

@@ -136,15 +136,9 @@ neon-pg-ext-%: postgres-%
 
 .PHONY: neon-pg-ext-clean-%
 neon-pg-ext-clean-%:
-	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/$*/bin/pg_config \
-	-C $(POSTGRES_INSTALL_DIR)/build/neon-$* \
-	-f $(ROOT_PROJECT_DIR)/pgxn/neon/Makefile clean
-	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/$*/bin/pg_config \
-	-C $(POSTGRES_INSTALL_DIR)/build/neon-walredo-$* \
-	-f $(ROOT_PROJECT_DIR)/pgxn/neon_walredo/Makefile clean
-	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/$*/bin/pg_config \
-	-C $(POSTGRES_INSTALL_DIR)/build/neon-test-utils-$* \
-	-f $(ROOT_PROJECT_DIR)/pgxn/neon_test_utils/Makefile clean
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/pgxn/neon-$* -f $(ROOT_PROJECT_DIR)/pgxn/neon/Makefile clean
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/pgxn/neon_walredo-$* -f $(ROOT_PROJECT_DIR)/pgxn/neon_walredo/Makefile clean
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/pgxn/neon_test_utils-$* -f $(ROOT_PROJECT_DIR)/pgxn/neon_test_utils/Makefile clean
 
 .PHONY: neon-pg-ext
 neon-pg-ext: \

README.md

@@ -34,11 +34,6 @@ dnf install flex bison readline-devel zlib-devel openssl-devel \
   libseccomp-devel perl clang cmake postgresql postgresql-contrib protobuf-compiler \
   protobuf-devel
 ```
-* On Arch based systems, these packages are needed:
-```bash
-pacman -S base-devel readline zlib libseccomp openssl clang \
-postgresql-libs cmake postgresql protobuf
-```
 
 2. [Install Rust](https://www.rust-lang.org/tools/install)
 ```
@@ -88,10 +83,9 @@ cd neon
 
 # The preferred and default is to make a debug build. This will create a
 # demonstrably slower build than a release build. For a release build,
-# use "BUILD_TYPE=release make -j`nproc` -s"
-# Remove -s for the verbose build log
+# use "BUILD_TYPE=release make -j`nproc`"
 
-make -j`nproc` -s
+make -j`nproc`
 ```
 
 #### Building on OSX
@@ -105,17 +99,16 @@ cd neon
 
 # The preferred and default is to make a debug build. This will create a
 # demonstrably slower build than a release build. For a release build,
-# use "BUILD_TYPE=release make -j`sysctl -n hw.logicalcpu` -s"
-# Remove -s for the verbose build log
+# use "BUILD_TYPE=release make -j`sysctl -n hw.logicalcpu`"
 
-make -j`sysctl -n hw.logicalcpu` -s
+make -j`sysctl -n hw.logicalcpu`
 ```
 
 #### Dependency installation notes
 To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `pg_install/bin` and `pg_install/lib`, respectively.
 
 To run the integration tests or Python scripts (not required to use the code), install
-Python (3.9 or higher), and install python3 packages using `./scripts/pysync` (requires [poetry>=1.3](https://python-poetry.org/)) in the project directory.
+Python (3.9 or higher), and install python3 packages using `./scripts/pysync` (requires [poetry](https://python-poetry.org/)) in the project directory.
 
 
 #### Running neon database

compute_tools/src/bin/compute_ctl.rs

@@ -44,6 +44,7 @@ use tracing::{error, info};
 
 use compute_tools::compute::{ComputeMetrics, ComputeNode, ComputeState, ComputeStatus};
 use compute_tools::http::api::launch_http_server;
+use compute_tools::informant::spawn_vm_informant_if_present;
 use compute_tools::logger::*;
 use compute_tools::monitor::launch_monitor;
 use compute_tools::params::*;
@@ -140,6 +141,8 @@ fn main() -> Result<()> {
     // requests, while configuration is still in progress.
     let _http_handle = launch_http_server(&compute).expect("cannot launch http endpoint thread");
     let _monitor_handle = launch_monitor(&compute).expect("cannot launch compute monitor thread");
+    // Also spawn the thread responsible for handling the VM informant -- if it's present
+    let _vm_informant_handle = spawn_vm_informant_if_present().expect("cannot launch VM informant");
 
     // Start Postgres
     let mut delay_exit = false;

compute_tools/src/http/api.rs

@@ -3,7 +3,6 @@ use std::net::SocketAddr;
 use std::sync::Arc;
 use std::thread;
 
-use crate::compute::ComputeNode;
 use anyhow::Result;
 use hyper::service::{make_service_fn, service_fn};
 use hyper::{Body, Method, Request, Response, Server, StatusCode};
@@ -11,6 +10,8 @@ use serde_json;
 use tracing::{error, info};
 use tracing_utils::http::OtelName;
 
+use crate::compute::ComputeNode;
+
 // Service function to handle all available routes.
 async fn routes(req: Request<Body>, compute: &Arc<ComputeNode>) -> Response<Body> {
     //

compute_tools/src/informant.rs

@@ -0,0 +1,50 @@
+use std::path::Path;
+use std::process;
+use std::thread;
+use std::time::Duration;
+use tracing::{info, warn};
+
+use anyhow::{Context, Result};
+
+const VM_INFORMANT_PATH: &str = "/bin/vm-informant";
+const RESTART_INFORMANT_AFTER_MILLIS: u64 = 5000;
+
+/// Launch a thread to start the VM informant if it's present (and restart, on failure)
+pub fn spawn_vm_informant_if_present() -> Result<Option<thread::JoinHandle<()>>> {
+    let exists = Path::new(VM_INFORMANT_PATH)
+        .try_exists()
+        .context("could not check if path exists")?;
+
+    if !exists {
+        return Ok(None);
+    }
+
+    Ok(Some(
+        thread::Builder::new()
+            .name("run-vm-informant".into())
+            .spawn(move || run_informant())?,
+    ))
+}
+
+fn run_informant() -> ! {
+    let restart_wait = Duration::from_millis(RESTART_INFORMANT_AFTER_MILLIS);
+
+    info!("starting VM informant");
+
+    loop {
+        let mut cmd = process::Command::new(VM_INFORMANT_PATH);
+        // Block on subprocess:
+        let result = cmd.status();
+
+        match result {
+            Err(e) => warn!("failed to run VM informant at {VM_INFORMANT_PATH:?}: {e}"),
+            Ok(status) if !status.success() => {
+                warn!("{VM_INFORMANT_PATH} exited with code {status:?}, retrying")
+            }
+            Ok(_) => info!("{VM_INFORMANT_PATH} ended gracefully (unexpectedly). Retrying"),
+        }
+
+        // Wait before retrying
+        thread::sleep(restart_wait);
+    }
+}

compute_tools/src/lib.rs

@@ -8,6 +8,7 @@ pub mod http;
 #[macro_use]
 pub mod logger;
 pub mod compute;
+pub mod informant;
 pub mod monitor;
 pub mod params;
 pub mod pg_helpers;

compute_tools/src/spec.rs

@@ -387,13 +387,13 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
                     name.pg_quote(),
                     db.owner.pg_quote()
                 );
-                let _guard = info_span!("executing", query).entered();
+                let _ = info_span!("executing", query).entered();
                 client.execute(query.as_str(), &[])?;
             }
             DatabaseAction::Create => {
                 let mut query: String = format!("CREATE DATABASE {} ", name.pg_quote());
                 query.push_str(&db.to_pg_options());
-                let _guard = info_span!("executing", query).entered();
+                let _ = info_span!("executing", query).entered();
                 client.execute(query.as_str(), &[])?;
             }
         };

control_plane/Cargo.toml

@@ -15,7 +15,6 @@ postgres.workspace = true
 regex.workspace = true
 reqwest = { workspace = true, features = ["blocking", "json"] }
 serde.workspace = true
-serde_json.workspace = true
 serde_with.workspace = true
 tar.workspace = true
 thiserror.workspace = true

control_plane/src/pageserver.rs

@@ -419,11 +419,6 @@ impl PageServerNode {
                     .map(|x| x.parse::<bool>())
                     .transpose()
                     .context("Failed to parse 'trace_read_requests' as bool")?,
-                eviction_policy: settings
-                    .get("eviction_policy")
-                    .map(|x| serde_json::from_str(x))
-                    .transpose()
-                    .context("Failed to parse 'eviction_policy' json")?,
             })
             .send()?
             .error_from_body()?;

docs/settings.md

@@ -16,7 +16,7 @@ listen_http_addr = '127.0.0.1:9898'
 checkpoint_distance = '268435456' # in bytes
 checkpoint_timeout = '10m'
 
-gc_period = '1 hour'
+gc_period = '100 s'
 gc_horizon = '67108864'
 
 max_file_descriptors = '100'
@@ -101,7 +101,7 @@ away.
 
 #### gc_period
 
-Interval at which garbage collection is triggered. Default is 1 hour.
+Interval at which garbage collection is triggered. Default is 100 s.
 
 #### image_creation_threshold
 
@@ -109,7 +109,7 @@ L0 delta layer threshold for L1 image layer creation. Default is 3.
 
 #### pitr_interval
 
-WAL retention duration for PITR branching. Default is 7 days.
+WAL retention duration for PITR branching. Default is 30 days.
 
 #### walreceiver_connect_timeout
 

docs/synthetic-size.md

@@ -1,335 +0,0 @@
-# Synthetic size
-
-Neon storage has copy-on-write branching, which makes it difficult to
-answer the question "how large is my database"? To give one reasonable
-answer, we calculate _synthetic size_ for a project.
-
-The calculation is called "synthetic", because it is based purely on
-the user-visible logical size, which is the size that you would see on
-a standalone PostgreSQL installation, and the amount of WAL, which is
-also the same as what you'd see on a standalone PostgreSQL, for the
-same set of updates.
-
-The synthetic size does *not* depend on the actual physical size
-consumed in the storage, or implementation details of the Neon storage
-like garbage collection, compaction and compression.  There is a
-strong *correlation* between the physical size and the synthetic size,
-but the synthetic size is designed to be independent of the
-implementation details, so that any improvements we make in the
-storage system simply reduce our COGS. And vice versa: any bugs or bad
-implementation where we keep more data than we would need to, do not
-change the synthetic size or incur any costs to the user.
-
-The synthetic size is calculated for the whole project. It is not
-straighforward to attribute size to individual branches. See "What is
-the size of an individual branch?" for discussion on those
-difficulties.
-
-The synthetic size is designed to:
-
-- Take into account the copy-on-write nature of the storage. For
-  example, if you create a branch, it doesn't immediately add anything
-  to the synthetic size. It starts to affect the synthetic size only
-  as it diverges from the parent branch.
-
-- Be independent of any implementation details of the storage, like
-  garbage collection, remote storage, or compression.
-
-## Terms & assumptions
-
-- logical size is the size of a branch *at a given point in
-  time*. It's the total size of all tables in all databases, as you
-  see with "\l+" in psql for example, plus the Postgres SLRUs and some
-  small amount of metadata. NOTE that currently, Neon does not include
-  the SLRUs and metadata in the logical size. See comment to `get_current_logical_size_non_incremental()`.
-
-- a "point in time" is defined as an LSN value. You can convert a
-  timestamp to an LSN, but the storage internally works with LSNs.
-
-- PITR horizon can be set per-branch.
-
-- PITR horizon can be set as a time interval, e.g. 5 days or hours, or
-  as amount of WAL, in bytes.  If it's given as a time interval, it's
-  converted to an LSN for the calculation.
-
-- PITR horizon can be set to 0, if you don't want to retain any history.
-
-## Calculation
-
-Inputs to the calculation are:
-- logical size of the database at different points in time,
-- amount of WAL generated, and
-- the PITR horizon settings
-
-The synthetic size is based on an idealistic model of the storage
-system, where we pretend that the storage consists of two things:
-- snapshots, containing a full snapshot of the database, at a given
-  point in time, and
-- WAL.
-
-In the simple case that the project contains just one branch (main),
-and a fixed PITR horizon, the synthetic size is the sum of:
-
-- the logical size of the branch *at the beginning of the PITR
-  horizon*, i.e. at the oldest point that you can still recover to, and
-- the size of the WAL covering the PITR horizon.
-
-The snapshot allows you to recover to the beginning of the PITR
-horizon, and the WAL allows you to recover from that point to any
-point within the horizon.
-
-```
-                             WAL
-   -----------------------#########>
-                          ^
-                       snapshot
-
-Legend:
-  ##### PITR horizon. This is the region that you can still access
-        with Point-in-time query and you can still create branches
-        from.
-  ----- history that has fallen out of the PITR horizon, and can no
-        longer be accessed
-```
-
-NOTE: This is not how the storage system actually works! The actual
-implementation is also based on snapshots and WAL, but the snapshots
-are taken for individual database pages and ranges of pages rather
-than the whole database, and it is much more complicated. This model
-is a reasonable approximation, however, to make the synthetic size a
-useful proxy for the actual storage consumption.
-
-
-## Example: Data is INSERTed
-
-For example, let's assume that your database contained 10 GB of data
-at the beginning of the PITR horizon, and you have since then inserted
-5 GB of additional data into it. The additional insertions of 5 GB of
-data consume roughly 5 GB of WAL. In that case, the synthetic size is:
-
-> 10 GB (snapshot) +  5 GB (WAL) = 15 GB
-
-If you now set the PITR horizon on the project to 0, so that no
-historical data is retained, then the beginning PITR horizon would be
-at the end of the branch, so the size of the snapshot would be
-calculated at the end of the branch, after the insertions. Then the
-synthetic size is:
-
-> 15 GB (snapshot) + 0 GB (WAL) = 15 GB.
-
-In this case, the synthetic size is the same, regardless of the PITR horizon,
-because all the history consists of inserts. The newly inserted data takes
-up the same amount of space, whether it's stored as part of the logical
-snapshot, or as WAL. (*)
-
-(*) This is a rough approximation. In reality, the WAL contains
-headers and other overhead, and on the other hand, the logical
-snapshot includes empty space on pages, so the size of insertions in
-WAL can be smaller or greater than the size of the final table after
-the insertions. But in most cases, it's in the same ballpark.
-
-## Example: Data is DELETEd
-
-Let's look at another example:
-
-Let's start again with a database that contains 10 GB of data. Then,
-you DELETE 5 GB of the data, and run VACUUM to free up the space, so
-that the logical size of the database is now only 5 GB.
-
-Let's assume that the WAL for the deletions and the vacuum take up
-100 MB of space. In that case, the synthetic size of the project is:
-
-> 10 GB (snapshot) + 100 MB (WAL) = 10.1 GB
-
-This is much larger than the logical size of the database after the
-deletions (5 GB). That's because the system still needs to retain the
-deleted data, because it's still accessible to queries and branching
-in the PITR window.
-
-If you now set the PITR horizon to 0 or just wait for time to pass so
-that the data falls out of the PITR horizon, making the deleted data
-inaccessible, the synthetic size shrinks:
-
-> 5 GB (snapshot) + 0 GB (WAL) = 5 GB
-
-
-# Branching
-
-Things get more complicated with branching. Branches in Neon are
-copy-on-write, which is also reflected in the synthetic size.
-
-When you create a branch, it doesn't immediately change the synthetic
-size at all. The branch point is within the PITR horizon, and all the
-data needed to recover to that point in time needs to be retained
-anyway.
-
-However, if you make modifications on the branch, the system needs to
-keep the WAL of those modifications. The WAL is included in the
-synthetic size.
-
-## Example: branch and INSERT
-
-Let's assume that you again start with a 10 GB database.
-On the main branch, you insert 2 GB of data. Then you create
-a branch at that point, and insert another 3 GB of data on the
-main branch, and 1 GB of data on the child branch
-
-```
-  child                 +#####>
-                        |
-                        |    WAL
-  main    ---------###############>
-                   ^
-                snapshot
-```
-
-In this case, the synthetic size consists of:
-- the snapshot at the beginning of the PITR horizon (10 GB)
-- the WAL on the main branch (2 GB + 3 GB = 5 GB)
-- the WAL on the child branch (1 GB)
-
-Total: 16 GB
-
-# Diverging branches
-
-If there is only a small amount of changes in the database on the
-different branches, as in the previous example, the synthetic size
-consists of a snapshot before the branch point, containing all the
-shared data, and the WAL on both branches. However, if the branches
-diverge a lot, it is more efficient to store a separate snapshot of
-branches.
-
-## Example: diverging branches
-
-You start with a 10 GB database. You insert 5 GB of data on the main
-branch. Then you create a branch, and immediately delete all the data
-on the child branch and insert 5 GB of new data to it. Then you do the
-same on the main branch. Let's assume
-that the PITR horizon requires keeping the last 1 GB of WAL on the
-both branches.
-
-```
-                              snapshot
-                                  v     WAL
-  child                 +---------##############>
-                        |
-                        |
-  main     -------------+---------##############>
-                                  ^     WAL
-                              snapshot
-```
-
-In this case, the synthetic size consists of:
-- snapshot at the beginning of the PITR horizon on the main branch (4 GB)
-- WAL on the main branch (1 GB)
-- snapshot at the beginning of the PITR horizon on the child branch (4 GB)
-- last 1 GB of WAL on the child branch (1 GB)
-
-Total: 10 GB
-
-The alternative way to store this would be to take only one snapshot
-at the beginning of branch point, and keep all the WAL on both
-branches.  However, the size with that method would be larger, as it
-would require one 10 GB snapshot, and 5 GB + 5 GB of WAL. It depends
-on the amount of changes (WAL) on both branches, and the logical size
-at the branch point, which method would result in a smaller synthetic
-size. On each branch point, the system performs the calculation with
-both methods, and uses the method that is cheaper, i.e. the one that
-results in a smaller synthetic size.
-
-One way to think about this is that when you create a branch, it
-starts out as a thin branch that only stores the WAL since the branch
-point.  As you modify it, and the amount of WAL grows, at some point
-it becomes cheaper to store a completely new snapshot of the branch
-and truncate the WAL.
-
-
-# What is the size of an individual branch?
-
-Synthetic size is calculated for the whole project, and includes all
-branches. There is no such thing as the size of a branch, because it
-is not straighforward to attribute the parts of size to individual
-branches.
-
-## Example: attributing size to branches
-
-(copied from https://github.com/neondatabase/neon/pull/2884#discussion_r1029365278)
-
-Imagine that you create two branches, A and B, at the same point from
-main branch, and do a couple of small updates on both branches. Then
-six months pass, and during those six months the data on the main
-branch churns over completely multiple times. The retention period is,
-say 1 month.
-
-```
-                      +------> A
-                     /
---------------------*-------------------------------> main
-                     \
-                      +--------> B
-```
-
-In that situation, the synthetic tenant size would be calculated based
-on a "logical snapshot" at the branch point, that is, the logical size
-of the database at that point. Plus the WAL on branches A and B. Let's
-say that the snapshot size is 10 GB, and the WAL is 1 MB on both
-branches A and B. So the total synthetic storage size is 10002
-MB. (Let's ignore the main branch for now, that would be just added to
-the sum)
-
-How would you break that down per branch? I can think of three
-different ways to do it, and all of them have their own problems:
-
-### Subtraction method
-
-For each branch, calculate how much smaller the total synthetic size
-would be, if that branch didn't exist. In other words, how much would
-you save if you dropped the branch. With this method, the size of
-branches A and B is 1 MB.
-
-With this method, the 10 GB shared logical snapshot is not included
-for A nor B. So the size of all branches is not equal to the total
-synthetic size of the tenant. If you drop branch A, you save 1 MB as
-you'd expect, but also the size of B suddenly jumps from 1 MB to 10001
-MB, which might feel surprising.
-
-### Division method
-
-Divide the common parts evenly across all branches that need
-them. With this method, the size of branches A and B would be 5001 MB.
-
-With this method, the sum of all branches adds up to the total
-synthetic size. But it's surprising in other ways: if you drop branch
-A, you might think that you save 5001 MB, but in reality you only save
-1 MB, and the size of branch B suddenly grows from 5001 to 10001 MB.
-
-### Addition method
-
-For each branch, include all the snapshots and WAL that it depends on,
-even if some of them are shared by other branches. With this method,
-the size of branches A and B would be 10001 MB.
-
-The surprise with this method is that the sum of all the branches is
-larger than the total synthetic size. And if you drop branch A, the
-total synthetic size doesn't fall by 10001 MB as you might think.
-
-# Alternatives
-
-A sort of cop-out method would be to show the whole tree of branches
-graphically, and for each section of WAL or logical snapshot, display
-the size of that section. You can then see which branches depend on
-which sections, which sections are shared etc. That would be good to
-have in the UI anyway.
-
-Or perhaps calculate per-branch numbers using the subtraction method,
-and in addition to that, one more number for "shared size" that
-includes all the data that is needed by more than one branch.
-
-## Which is the right method?
-
-The bottom line is that it's not straightforward to attribute the
-synthetic size to individual branches. There are things we can do, and
-all of those methods are pretty straightforward to implement, but they
-all have their own problems. What makes sense depends a lot on what
-you want to do with the number, what question you are trying to
-answer.

libs/metrics/Cargo.toml

@@ -8,6 +8,5 @@ license.workspace = true
 prometheus.workspace = true
 libc.workspace = true
 once_cell.workspace = true
-chrono.workspace = true
 
 workspace_hack.workspace = true

libs/metrics/src/launch_timestamp.rs

@@ -1,34 +0,0 @@
-//! A timestamp captured at process startup to identify restarts of the process, e.g., in logs and metrics.
-
-use chrono::Utc;
-
-use super::register_uint_gauge;
-use std::fmt::Display;
-
-pub struct LaunchTimestamp(chrono::DateTime<Utc>);
-
-impl LaunchTimestamp {
-    pub fn generate() -> Self {
-        LaunchTimestamp(Utc::now())
-    }
-}
-
-impl Display for LaunchTimestamp {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.0)
-    }
-}
-
-pub fn set_launch_timestamp_metric(launch_ts: &'static LaunchTimestamp) {
-    let millis_since_epoch: u64 = launch_ts
-        .0
-        .timestamp_millis()
-        .try_into()
-        .expect("we're after the epoch, this should be positive");
-    let metric = register_uint_gauge!(
-        "libmetrics_launch_timestamp",
-        "Timestamp (millis since epoch) at wich the process launched."
-    )
-    .unwrap();
-    metric.set(millis_since_epoch);
-}

libs/metrics/src/lib.rs

@@ -8,7 +8,6 @@ pub use prometheus::opts;
 pub use prometheus::register;
 pub use prometheus::{core, default_registry, proto};
 pub use prometheus::{exponential_buckets, linear_buckets};
-pub use prometheus::{register_counter_vec, Counter, CounterVec};
 pub use prometheus::{register_gauge, Gauge};
 pub use prometheus::{register_gauge_vec, GaugeVec};
 pub use prometheus::{register_histogram, Histogram};
@@ -20,7 +19,6 @@ pub use prometheus::{register_int_gauge_vec, IntGaugeVec};
 pub use prometheus::{Encoder, TextEncoder};
 use prometheus::{Registry, Result};
 
-pub mod launch_timestamp;
 mod wrappers;
 pub use wrappers::{CountedReader, CountedWriter};
 
@@ -35,14 +33,6 @@ macro_rules! register_uint_gauge_vec {
     }};
 }
 
-#[macro_export]
-macro_rules! register_uint_gauge {
-    ($NAME:expr, $HELP:expr $(,)?) => {{
-        let gauge = $crate::UIntGauge::new($NAME, $HELP).unwrap();
-        $crate::register(Box::new(gauge.clone())).map(|_| gauge)
-    }};
-}
-
 /// Special internal registry, to collect metrics independently from the default registry.
 /// Was introduced to fix deadlock with lazy registration of metrics in the default registry.
 static INTERNAL_REGISTRY: Lazy<Registry> = Lazy::new(Registry::new);

libs/pageserver_api/Cargo.toml

@@ -13,7 +13,5 @@ bytes.workspace = true
 byteorder.workspace = true
 utils.workspace = true
 postgres_ffi.workspace = true
-enum-map.workspace = true
-serde_json.workspace = true
 
 workspace_hack.workspace = true

libs/pageserver_api/src/models.rs

@@ -1,14 +1,9 @@
-use std::{
-    collections::HashMap,
-    num::{NonZeroU64, NonZeroUsize},
-    time::SystemTime,
-};
+use std::num::{NonZeroU64, NonZeroUsize};
 
 use byteorder::{BigEndian, ReadBytesExt};
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
 use utils::{
-    history_buffer::HistoryBufferWithDropCounter,
     id::{NodeId, TenantId, TimelineId},
     lsn::Lsn,
 };
@@ -34,14 +29,6 @@ pub enum TenantState {
     Broken,
 }
 
-pub mod state {
-    pub const LOADING: &str = "loading";
-    pub const ATTACHING: &str = "attaching";
-    pub const ACTIVE: &str = "active";
-    pub const STOPPING: &str = "stopping";
-    pub const BROKEN: &str = "broken";
-}
-
 impl TenantState {
     pub fn has_in_progress_downloads(&self) -> bool {
         match self {
@@ -52,16 +39,6 @@ impl TenantState {
             Self::Broken => false,
         }
     }
-
-    pub fn as_str(&self) -> &'static str {
-        match self {
-            TenantState::Loading => state::LOADING,
-            TenantState::Attaching => state::ATTACHING,
-            TenantState::Active => state::ACTIVE,
-            TenantState::Stopping => state::STOPPING,
-            TenantState::Broken => state::BROKEN,
-        }
-    }
 }
 
 /// A state of a timeline in pageserver's memory.
@@ -142,6 +119,7 @@ pub struct TenantConfigRequest {
     #[serde_as(as = "DisplayFromStr")]
     pub tenant_id: TenantId,
     #[serde(default)]
+    #[serde_as(as = "Option<DisplayFromStr>")]
     pub checkpoint_distance: Option<u64>,
     pub checkpoint_timeout: Option<String>,
     pub compaction_target_size: Option<u64>,
@@ -155,11 +133,6 @@ pub struct TenantConfigRequest {
     pub lagging_wal_timeout: Option<String>,
     pub max_lsn_wal_lag: Option<NonZeroU64>,
     pub trace_read_requests: Option<bool>,
-    // We defer the parsing of the eviction_policy field to the request handler.
-    // Otherwise we'd have to move the types for eviction policy into this package.
-    // We might do that once the eviction feature has stabilizied.
-    // For now, this field is not even documented in the openapi_spec.yml.
-    pub eviction_policy: Option<serde_json::Value>,
 }
 
 impl TenantConfigRequest {
@@ -179,7 +152,6 @@ impl TenantConfigRequest {
             lagging_wal_timeout: None,
             max_lsn_wal_lag: None,
             trace_read_requests: None,
-            eviction_policy: None,
         }
     }
 }
@@ -237,130 +209,6 @@ pub struct TimelineInfo {
     pub state: TimelineState,
 }
 
-#[derive(Debug, Clone, Serialize)]
-pub struct LayerMapInfo {
-    pub in_memory_layers: Vec<InMemoryLayerInfo>,
-    pub historic_layers: Vec<HistoricLayerInfo>,
-}
-
-#[derive(Debug, Hash, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, enum_map::Enum)]
-#[repr(usize)]
-pub enum LayerAccessKind {
-    GetValueReconstructData,
-    Iter,
-    KeyIter,
-    Dump,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct LayerAccessStatFullDetails {
-    pub when_millis_since_epoch: u64,
-    pub task_kind: &'static str,
-    pub access_kind: LayerAccessKind,
-}
-
-/// An event that impacts the layer's residence status.
-#[serde_as]
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct LayerResidenceEvent {
-    /// The time when the event occurred.
-    /// NB: this timestamp is captured while the residence status changes.
-    /// So, it might be behind/ahead of the actual residence change by a short amount of time.
-    ///
-    #[serde(rename = "timestamp_millis_since_epoch")]
-    #[serde_as(as = "serde_with::TimestampMilliSeconds")]
-    pub timestamp: SystemTime,
-    /// The new residence status of the layer.
-    pub status: LayerResidenceStatus,
-    /// The reason why we had to record this event.
-    pub reason: LayerResidenceEventReason,
-}
-
-/// The reason for recording a given [`ResidenceEvent`].
-#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
-pub enum LayerResidenceEventReason {
-    /// The layer map is being populated, e.g. during timeline load or attach.
-    /// This includes [`RemoteLayer`] objects created in [`reconcile_with_remote`].
-    /// We need to record such events because there is no persistent storage for the events.
-    LayerLoad,
-    /// We just created the layer (e.g., freeze_and_flush or compaction).
-    /// Such layers are always [`LayerResidenceStatus::Resident`].
-    LayerCreate,
-    /// We on-demand downloaded or evicted the given layer.
-    ResidenceChange,
-}
-
-/// The residence status of the layer, after the given [`LayerResidenceEvent`].
-#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
-pub enum LayerResidenceStatus {
-    /// Residence status for a layer file that exists locally.
-    /// It may also exist on the remote, we don't care here.
-    Resident,
-    /// Residence status for a layer file that only exists on the remote.
-    Evicted,
-}
-
-impl LayerResidenceEvent {
-    pub fn new(status: LayerResidenceStatus, reason: LayerResidenceEventReason) -> Self {
-        Self {
-            status,
-            reason,
-            timestamp: SystemTime::now(),
-        }
-    }
-}
-
-#[derive(Debug, Clone, Serialize)]
-pub struct LayerAccessStats {
-    pub access_count_by_access_kind: HashMap<LayerAccessKind, u64>,
-    pub task_kind_access_flag: Vec<&'static str>,
-    pub first: Option<LayerAccessStatFullDetails>,
-    pub accesses_history: HistoryBufferWithDropCounter<LayerAccessStatFullDetails, 16>,
-    pub residence_events_history: HistoryBufferWithDropCounter<LayerResidenceEvent, 16>,
-}
-
-#[serde_as]
-#[derive(Debug, Clone, Serialize)]
-#[serde(tag = "kind")]
-pub enum InMemoryLayerInfo {
-    Open {
-        #[serde_as(as = "DisplayFromStr")]
-        lsn_start: Lsn,
-    },
-    Frozen {
-        #[serde_as(as = "DisplayFromStr")]
-        lsn_start: Lsn,
-        #[serde_as(as = "DisplayFromStr")]
-        lsn_end: Lsn,
-    },
-}
-
-#[serde_as]
-#[derive(Debug, Clone, Serialize)]
-#[serde(tag = "kind")]
-pub enum HistoricLayerInfo {
-    Delta {
-        layer_file_name: String,
-        layer_file_size: Option<u64>,
-
-        #[serde_as(as = "DisplayFromStr")]
-        lsn_start: Lsn,
-        #[serde_as(as = "DisplayFromStr")]
-        lsn_end: Lsn,
-        remote: bool,
-        access_stats: LayerAccessStats,
-    },
-    Image {
-        layer_file_name: String,
-        layer_file_size: Option<u64>,
-
-        #[serde_as(as = "DisplayFromStr")]
-        lsn_start: Lsn,
-        remote: bool,
-        access_stats: LayerAccessStats,
-    },
-}
-
 #[derive(Debug, Serialize, Deserialize)]
 pub struct DownloadRemoteLayersTaskSpawnRequest {
     pub max_concurrent_downloads: NonZeroUsize,
@@ -401,7 +249,7 @@ pub struct TimelineGcRequest {
 }
 
 // Wrapped in libpq CopyData
-#[derive(PartialEq, Eq, Debug)]
+#[derive(PartialEq, Eq)]
 pub enum PagestreamFeMessage {
     Exists(PagestreamExistsRequest),
     Nblocks(PagestreamNblocksRequest),

libs/pq_proto/src/lib.rs

@@ -75,36 +75,27 @@ impl StartupMessageParams {
     /// taking into account all escape sequences but leaving them as-is.
     /// [`None`] means that there's no `options` in [`Self`].
     pub fn options_raw(&self) -> Option<impl Iterator<Item = &str>> {
-        self.get("options").map(Self::parse_options_raw)
-    }
-
-    /// Split command-line options according to PostgreSQL's logic,
-    /// applying all escape sequences (using owned strings as needed).
-    /// [`None`] means that there's no `options` in [`Self`].
-    pub fn options_escaped(&self) -> Option<impl Iterator<Item = Cow<'_, str>>> {
-        self.get("options").map(Self::parse_options_escaped)
-    }
-
-    /// Split command-line options according to PostgreSQL's logic,
-    /// taking into account all escape sequences but leaving them as-is.
-    pub fn parse_options_raw(input: &str) -> impl Iterator<Item = &str> {
         // See `postgres: pg_split_opts`.
         let mut last_was_escape = false;
-        input
+        let iter = self
+            .get("options")?
             .split(move |c: char| {
                 // We split by non-escaped whitespace symbols.
                 let should_split = c.is_ascii_whitespace() && !last_was_escape;
                 last_was_escape = c == '\\' && !last_was_escape;
                 should_split
             })
-            .filter(|s| !s.is_empty())
+            .filter(|s| !s.is_empty());
+
+        Some(iter)
     }
 
     /// Split command-line options according to PostgreSQL's logic,
     /// applying all escape sequences (using owned strings as needed).
-    pub fn parse_options_escaped(input: &str) -> impl Iterator<Item = Cow<'_, str>> {
+    /// [`None`] means that there's no `options` in [`Self`].
+    pub fn options_escaped(&self) -> Option<impl Iterator<Item = Cow<'_, str>>> {
         // See `postgres: pg_split_opts`.
-        Self::parse_options_raw(input).map(|s| {
+        let iter = self.options_raw()?.map(|s| {
             let mut preserve_next_escape = false;
             let escape = |c| {
                 // We should remove '\\' unless it's preceded by '\\'.
@@ -117,12 +108,9 @@ impl StartupMessageParams {
                 true => Cow::Owned(s.replace(escape, "")),
                 false => Cow::Borrowed(s),
             }
-        })
-    }
+        });
 
-    /// Iterate through key-value pairs in an arbitrary order.
-    pub fn iter(&self) -> impl Iterator<Item = (&str, &str)> {
-        self.params.iter().map(|(k, v)| (k.as_str(), v.as_str()))
+        Some(iter)
     }
 
     // This function is mostly useful in tests.

libs/remote_storage/Cargo.toml

@@ -21,7 +21,7 @@ toml_edit.workspace = true
 tracing.workspace = true
 metrics.workspace = true
 utils.workspace = true
-pin-project-lite.workspace = true
+
 workspace_hack.workspace = true
 
 [dev-dependencies]

libs/remote_storage/src/s3_bucket.rs

@@ -20,10 +20,7 @@ use aws_sdk_s3::{
 };
 use aws_smithy_http::body::SdkBody;
 use hyper::Body;
-use tokio::{
-    io::{self, AsyncRead},
-    sync::Semaphore,
-};
+use tokio::{io, sync::Semaphore};
 use tokio_util::io::ReaderStream;
 use tracing::debug;
 
@@ -105,7 +102,7 @@ pub struct S3Bucket {
     // Every request to S3 can be throttled or cancelled, if a certain number of requests per second is exceeded.
     // Same goes to IAM, which is queried before every S3 request, if enabled. IAM has even lower RPS threshold.
     // The helps to ensure we don't exceed the thresholds.
-    concurrency_limiter: Arc<Semaphore>,
+    concurrency_limiter: Semaphore,
 }
 
 #[derive(Default)]
@@ -165,7 +162,7 @@ impl S3Bucket {
             client,
             bucket_name: aws_config.bucket_name.clone(),
             prefix_in_bucket,
-            concurrency_limiter: Arc::new(Semaphore::new(aws_config.concurrency_limit.get())),
+            concurrency_limiter: Semaphore::new(aws_config.concurrency_limit.get()),
         })
     }
 
@@ -197,10 +194,9 @@ impl S3Bucket {
     }
 
     async fn download_object(&self, request: GetObjectRequest) -> Result<Download, DownloadError> {
-        let permit = self
+        let _guard = self
             .concurrency_limiter
-            .clone()
-            .acquire_owned()
+            .acquire()
             .await
             .context("Concurrency limiter semaphore got closed during S3 download")
             .map_err(DownloadError::Other)?;
@@ -221,10 +217,9 @@ impl S3Bucket {
                 let metadata = object_output.metadata().cloned().map(StorageMetadata);
                 Ok(Download {
                     metadata,
-                    download_stream: Box::pin(io::BufReader::new(RatelimitedAsyncRead::new(
-                        permit,
+                    download_stream: Box::pin(io::BufReader::new(
                         object_output.body.into_async_read(),
-                    ))),
+                    )),
                 })
             }
             Err(SdkError::ServiceError {
@@ -245,32 +240,6 @@ impl S3Bucket {
     }
 }
 
-pin_project_lite::pin_project! {
-    /// An `AsyncRead` adapter which carries a permit for the lifetime of the value.
-    struct RatelimitedAsyncRead<S> {
-        permit: tokio::sync::OwnedSemaphorePermit,
-        #[pin]
-        inner: S,
-    }
-}
-
-impl<S: AsyncRead> RatelimitedAsyncRead<S> {
-    fn new(permit: tokio::sync::OwnedSemaphorePermit, inner: S) -> Self {
-        RatelimitedAsyncRead { permit, inner }
-    }
-}
-
-impl<S: AsyncRead> AsyncRead for RatelimitedAsyncRead<S> {
-    fn poll_read(
-        self: std::pin::Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-        buf: &mut io::ReadBuf<'_>,
-    ) -> std::task::Poll<std::io::Result<()>> {
-        let this = self.project();
-        this.inner.poll_read(cx, buf)
-    }
-}
-
 #[async_trait::async_trait]
 impl RemoteStorage for S3Bucket {
     async fn list(&self) -> anyhow::Result<Vec<RemotePath>> {

libs/tenant_size_model/Cargo.toml

@@ -7,7 +7,5 @@ license.workspace = true
 
 [dependencies]
 anyhow.workspace = true
-serde.workspace = true
-serde_json.workspace = true
 
 workspace_hack.workspace = true

libs/tenant_size_model/src/calculation.rs

@@ -1,219 +0,0 @@
-use crate::{SegmentMethod, SegmentSizeResult, SizeResult, StorageModel};
-
-//
-//                 *-g--*---D--->
-//                /
-//               /
-//              /                 *---b----*-B--->
-//             /                 /
-//            /                 /
-//      -----*--e---*-----f----* C
-//           E                  \
-//                               \
-//                                *--a---*---A-->
-//
-// If A and B need to be retained, is it cheaper to store
-// snapshot at C+a+b, or snapshots at A and B ?
-//
-// If D also needs to be retained, which is cheaper:
-//
-// 1. E+g+e+f+a+b
-// 2. D+C+a+b
-// 3. D+A+B
-
-/// [`Segment`] which has had it's size calculated.
-#[derive(Clone, Debug)]
-struct SegmentSize {
-    method: SegmentMethod,
-
-    // calculated size of this subtree, using this method
-    accum_size: u64,
-
-    seg_id: usize,
-    children: Vec<SegmentSize>,
-}
-
-struct SizeAlternatives {
-    // cheapest alternative if parent is available.
-    incremental: SegmentSize,
-
-    // cheapest alternative if parent node is not available
-    non_incremental: Option<SegmentSize>,
-}
-
-impl StorageModel {
-    pub fn calculate(&self) -> SizeResult {
-        // Build adjacency list. 'child_list' is indexed by segment id. Each entry
-        // contains a list of all child segments of the segment.
-        let mut roots: Vec<usize> = Vec::new();
-        let mut child_list: Vec<Vec<usize>> = Vec::new();
-        child_list.resize(self.segments.len(), Vec::new());
-
-        for (seg_id, seg) in self.segments.iter().enumerate() {
-            if let Some(parent_id) = seg.parent {
-                child_list[parent_id].push(seg_id);
-            } else {
-                roots.push(seg_id);
-            }
-        }
-
-        let mut segment_results = Vec::new();
-        segment_results.resize(
-            self.segments.len(),
-            SegmentSizeResult {
-                method: SegmentMethod::Skipped,
-                accum_size: 0,
-            },
-        );
-
-        let mut total_size = 0;
-        for root in roots {
-            if let Some(selected) = self.size_here(root, &child_list).non_incremental {
-                StorageModel::fill_selected_sizes(&selected, &mut segment_results);
-                total_size += selected.accum_size;
-            } else {
-                // Couldn't find any way to get this root. Error?
-            }
-        }
-
-        SizeResult {
-            total_size,
-            segments: segment_results,
-        }
-    }
-
-    fn fill_selected_sizes(selected: &SegmentSize, result: &mut Vec<SegmentSizeResult>) {
-        result[selected.seg_id] = SegmentSizeResult {
-            method: selected.method,
-            accum_size: selected.accum_size,
-        };
-        // recurse to children
-        for child in selected.children.iter() {
-            StorageModel::fill_selected_sizes(child, result);
-        }
-    }
-
-    //
-    // This is the core of the sizing calculation.
-    //
-    // This is a recursive function, that for each Segment calculates the best way
-    // to reach all the Segments that are marked as needed in this subtree, under two
-    // different conditions:
-    // a) when the parent of this segment is available (as a snaphot or through WAL), and
-    // b) when the parent of this segment is not available.
-    //
-    fn size_here(&self, seg_id: usize, child_list: &Vec<Vec<usize>>) -> SizeAlternatives {
-        let seg = &self.segments[seg_id];
-        // First figure out the best way to get each child
-        let mut children = Vec::new();
-        for child_id in &child_list[seg_id] {
-            children.push(self.size_here(*child_id, child_list))
-        }
-
-        // Method 1. If this node is not needed, we can skip it as long as we
-        // take snapshots later in each sub-tree
-        let snapshot_later = if !seg.needed {
-            let mut snapshot_later = SegmentSize {
-                seg_id,
-                method: SegmentMethod::Skipped,
-                accum_size: 0,
-                children: Vec::new(),
-            };
-
-            let mut possible = true;
-            for child in children.iter() {
-                if let Some(non_incremental) = &child.non_incremental {
-                    snapshot_later.accum_size += non_incremental.accum_size;
-                    snapshot_later.children.push(non_incremental.clone())
-                } else {
-                    possible = false;
-                    break;
-                }
-            }
-            if possible {
-                Some(snapshot_later)
-            } else {
-                None
-            }
-        } else {
-            None
-        };
-
-        // Method 2. Get a snapshot here. This assumed to be possible, if the 'size' of
-        // this Segment was given.
-        let snapshot_here = if !seg.needed || seg.parent.is_none() {
-            if let Some(snapshot_size) = seg.size {
-                let mut snapshot_here = SegmentSize {
-                    seg_id,
-                    method: SegmentMethod::SnapshotHere,
-                    accum_size: snapshot_size,
-                    children: Vec::new(),
-                };
-                for child in children.iter() {
-                    snapshot_here.accum_size += child.incremental.accum_size;
-                    snapshot_here.children.push(child.incremental.clone())
-                }
-                Some(snapshot_here)
-            } else {
-                None
-            }
-        } else {
-            None
-        };
-
-        // Method 3. Use WAL to get here from parent
-        let wal_here = {
-            let mut wal_here = SegmentSize {
-                seg_id,
-                method: SegmentMethod::Wal,
-                accum_size: if let Some(parent_id) = seg.parent {
-                    seg.lsn - self.segments[parent_id].lsn
-                } else {
-                    0
-                },
-                children: Vec::new(),
-            };
-            for child in children {
-                wal_here.accum_size += child.incremental.accum_size;
-                wal_here.children.push(child.incremental)
-            }
-            wal_here
-        };
-
-        // If the parent is not available, what's the cheapest method involving
-        // a snapshot here or later?
-        let mut cheapest_non_incremental: Option<SegmentSize> = None;
-        if let Some(snapshot_here) = snapshot_here {
-            cheapest_non_incremental = Some(snapshot_here);
-        }
-        if let Some(snapshot_later) = snapshot_later {
-            // Use <=, to prefer skipping if the size is equal
-            if let Some(parent) = &cheapest_non_incremental {
-                if snapshot_later.accum_size <= parent.accum_size {
-                    cheapest_non_incremental = Some(snapshot_later);
-                }
-            } else {
-                cheapest_non_incremental = Some(snapshot_later);
-            }
-        }
-
-        // And what's the cheapest method, if the parent is available?
-        let cheapest_incremental = if let Some(cheapest_non_incremental) = &cheapest_non_incremental
-        {
-            // Is it cheaper to use a snapshot here or later, anyway?
-            // Use <, to prefer Wal over snapshot if the cost is the same
-            if wal_here.accum_size < cheapest_non_incremental.accum_size {
-                wal_here
-            } else {
-                cheapest_non_incremental.clone()
-            }
-        } else {
-            wal_here
-        };
-
-        SizeAlternatives {
-            incremental: cheapest_incremental,
-            non_incremental: cheapest_non_incremental,
-        }
-    }
-}

libs/tenant_size_model/src/lib.rs

@@ -1,70 +1,401 @@
-//! Synthetic size calculation
+use std::borrow::Cow;
+use std::collections::HashMap;
 
-mod calculation;
-pub mod svg;
+use anyhow::Context;
 
-/// StorageModel is the input to the synthetic size calculation. It represents
-/// a tree of timelines, with just the information that's needed for the
-/// calculation. This doesn't track timeline names or where each timeline
-/// begins and ends, for example. Instead, it consists of "points of interest"
-/// on the timelines. A point of interest could be the timeline start or end point,
-/// the oldest point on a timeline that needs to be retained because of PITR
-/// cutoff, or snapshot points named by the user. For each such point, and the
-/// edge connecting the points (implicit in Segment), we store information about
-/// whether we need to be able to recover to the point, and if known, the logical
-/// size at the point.
+/// Pricing model or history size builder.
 ///
-/// The segments must form a well-formed tree, with no loops.
-#[derive(serde::Serialize)]
-pub struct StorageModel {
-    pub segments: Vec<Segment>,
+/// Maintains knowledge of the branches and their modifications. Generic over the branch name key
+/// type.
+pub struct Storage<K: 'static> {
+    segments: Vec<Segment>,
+
+    /// Mapping from the branch name to the index of a segment describing it's latest state.
+    branches: HashMap<K, usize>,
 }
 
-/// Segment represents one point in the tree of branches, *and* the edge that leads
-/// to it (if any). We don't need separate structs for points and edges, because each
-/// point can have only one parent.
-///
-/// When 'needed' is true, it means that we need to be able to reconstruct
-/// any version between 'parent.lsn' and 'lsn'. If you want to represent that only
-/// a single point is needed, create two Segments with the same lsn, and mark only
-/// the child as needed.
-///
-#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
+/// Snapshot of a branch.
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct Segment {
     /// Previous segment index into ['Storage::segments`], if any.
-    pub parent: Option<usize>,
+    parent: Option<usize>,
 
-    /// LSN at this point
-    pub lsn: u64,
+    /// Description of how did we get to this state.
+    ///
+    /// Mainly used in the original scenarios 1..=4 with insert, delete and update. Not used when
+    /// modifying a branch directly.
+    pub op: Cow<'static, str>,
 
-    /// Logical size at this node, if known.
-    pub size: Option<u64>,
+    /// LSN before this state
+    start_lsn: u64,
 
-    /// If true, the segment from parent to this node is needed by `retention_period`
+    /// LSN at this state
+    pub end_lsn: u64,
+
+    /// Logical size before this state
+    start_size: u64,
+
+    /// Logical size at this state. Can be None in the last Segment of a branch.
+    pub end_size: Option<u64>,
+
+    /// Indices to [`Storage::segments`]
+    ///
+    /// FIXME: this could be an Option<usize>
+    children_after: Vec<usize>,
+
+    /// Determined by `retention_period` given to [`Storage::calculate`]
     pub needed: bool,
 }
 
-/// Result of synthetic size calculation. Returned by StorageModel::calculate()
-pub struct SizeResult {
-    pub total_size: u64,
+//
+//
+//
+//
+//                 *-g--*---D--->
+//                /
+//               /
+//              /                 *---b----*-B--->
+//             /                 /
+//            /                 /
+//      -----*--e---*-----f----* C
+//           E                  \
+//                               \
+//                                *--a---*---A-->
+//
+// If A and B need to be retained, is it cheaper to store
+// snapshot at C+a+b, or snapshots at A and B ?
+//
+// If D also needs to be retained, which is cheaper:
+//
+// 1. E+g+e+f+a+b
+// 2. D+C+a+b
+// 3. D+A+B
 
-    // This has same length as the StorageModel::segments vector in the input.
-    // Each entry in this array corresponds to the entry with same index in
-    // StorageModel::segments.
-    pub segments: Vec<SegmentSizeResult>,
+/// [`Segment`] which has had it's size calculated.
+pub struct SegmentSize {
+    pub seg_id: usize,
+
+    pub method: SegmentMethod,
+
+    this_size: u64,
+
+    pub children: Vec<SegmentSize>,
 }
 
-#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
-pub struct SegmentSizeResult {
-    pub method: SegmentMethod,
-    // calculated size of this subtree, using this method
-    pub accum_size: u64,
+impl SegmentSize {
+    fn total(&self) -> u64 {
+        self.this_size + self.children.iter().fold(0, |acc, x| acc + x.total())
+    }
+
+    pub fn total_children(&self) -> u64 {
+        if self.method == SnapshotAfter {
+            self.this_size + self.children.iter().fold(0, |acc, x| acc + x.total())
+        } else {
+            self.children.iter().fold(0, |acc, x| acc + x.total())
+        }
+    }
 }
 
 /// Different methods to retain history from a particular state
-#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub enum SegmentMethod {
-    SnapshotHere, // A logical snapshot is needed after this segment
-    Wal,          // Keep WAL leading up to this node
+    SnapshotAfter,
+    Wal,
+    WalNeeded,
     Skipped,
 }
+
+use SegmentMethod::*;
+
+impl<K: std::hash::Hash + Eq + 'static> Storage<K> {
+    /// Creates a new storage with the given default branch name.
+    pub fn new(initial_branch: K) -> Storage<K> {
+        let init_segment = Segment {
+            op: "".into(),
+            needed: false,
+            parent: None,
+            start_lsn: 0,
+            end_lsn: 0,
+            start_size: 0,
+            end_size: Some(0),
+            children_after: Vec::new(),
+        };
+
+        Storage {
+            segments: vec![init_segment],
+            branches: HashMap::from([(initial_branch, 0)]),
+        }
+    }
+
+    /// Advances the branch with a new point, at given LSN.
+    pub fn insert_point<Q: ?Sized>(
+        &mut self,
+        branch: &Q,
+        op: Cow<'static, str>,
+        lsn: u64,
+        size: Option<u64>,
+    ) -> anyhow::Result<()>
+    where
+        K: std::borrow::Borrow<Q>,
+        Q: std::hash::Hash + Eq + std::fmt::Debug,
+    {
+        let Some(lastseg_id) = self.branches.get(branch).copied() else { anyhow::bail!("branch not found: {branch:?}") };
+        let newseg_id = self.segments.len();
+        let lastseg = &mut self.segments[lastseg_id];
+
+        assert!(lsn > lastseg.end_lsn);
+
+        let Some(start_size) = lastseg.end_size else { anyhow::bail!("no end_size on latest segment for {branch:?}") };
+
+        let newseg = Segment {
+            op,
+            parent: Some(lastseg_id),
+            start_lsn: lastseg.end_lsn,
+            end_lsn: lsn,
+            start_size,
+            end_size: size,
+            children_after: Vec::new(),
+            needed: false,
+        };
+        lastseg.children_after.push(newseg_id);
+
+        self.segments.push(newseg);
+        *self.branches.get_mut(branch).expect("read already") = newseg_id;
+
+        Ok(())
+    }
+
+    /// Advances the branch with the named operation, by the relative LSN and logical size bytes.
+    pub fn modify_branch<Q: ?Sized>(
+        &mut self,
+        branch: &Q,
+        op: Cow<'static, str>,
+        lsn_bytes: u64,
+        size_bytes: i64,
+    ) -> anyhow::Result<()>
+    where
+        K: std::borrow::Borrow<Q>,
+        Q: std::hash::Hash + Eq + std::fmt::Debug,
+    {
+        let Some(lastseg_id) = self.branches.get(branch).copied() else { anyhow::bail!("branch not found: {branch:?}") };
+        let newseg_id = self.segments.len();
+        let lastseg = &mut self.segments[lastseg_id];
+
+        let Some(last_end_size) = lastseg.end_size else { anyhow::bail!("no end_size on latest segment for {branch:?}") };
+
+        let newseg = Segment {
+            op,
+            parent: Some(lastseg_id),
+            start_lsn: lastseg.end_lsn,
+            end_lsn: lastseg.end_lsn + lsn_bytes,
+            start_size: last_end_size,
+            end_size: Some((last_end_size as i64 + size_bytes) as u64),
+            children_after: Vec::new(),
+            needed: false,
+        };
+        lastseg.children_after.push(newseg_id);
+
+        self.segments.push(newseg);
+        *self.branches.get_mut(branch).expect("read already") = newseg_id;
+        Ok(())
+    }
+
+    pub fn insert<Q: ?Sized>(&mut self, branch: &Q, bytes: u64) -> anyhow::Result<()>
+    where
+        K: std::borrow::Borrow<Q>,
+        Q: std::hash::Hash + Eq + std::fmt::Debug,
+    {
+        self.modify_branch(branch, "insert".into(), bytes, bytes as i64)
+    }
+
+    pub fn update<Q: ?Sized>(&mut self, branch: &Q, bytes: u64) -> anyhow::Result<()>
+    where
+        K: std::borrow::Borrow<Q>,
+        Q: std::hash::Hash + Eq + std::fmt::Debug,
+    {
+        self.modify_branch(branch, "update".into(), bytes, 0i64)
+    }
+
+    pub fn delete<Q: ?Sized>(&mut self, branch: &Q, bytes: u64) -> anyhow::Result<()>
+    where
+        K: std::borrow::Borrow<Q>,
+        Q: std::hash::Hash + Eq + std::fmt::Debug,
+    {
+        self.modify_branch(branch, "delete".into(), bytes, -(bytes as i64))
+    }
+
+    pub fn branch<Q: ?Sized>(&mut self, parent: &Q, name: K) -> anyhow::Result<()>
+    where
+        K: std::borrow::Borrow<Q> + std::fmt::Debug,
+        Q: std::hash::Hash + Eq + std::fmt::Debug,
+    {
+        // Find the right segment
+        let branchseg_id = *self.branches.get(parent).with_context(|| {
+            format!(
+                "should had found the parent {:?} by key. in branches {:?}",
+                parent, self.branches
+            )
+        })?;
+
+        let _branchseg = &mut self.segments[branchseg_id];
+
+        // Create branch name for it
+        self.branches.insert(name, branchseg_id);
+        Ok(())
+    }
+
+    pub fn calculate(&mut self, retention_period: u64) -> anyhow::Result<SegmentSize> {
+        // Phase 1: Mark all the segments that need to be retained
+        for (_branch, &last_seg_id) in self.branches.iter() {
+            let last_seg = &self.segments[last_seg_id];
+            let cutoff_lsn = last_seg.start_lsn.saturating_sub(retention_period);
+            let mut seg_id = last_seg_id;
+            loop {
+                let seg = &mut self.segments[seg_id];
+                if seg.end_lsn < cutoff_lsn {
+                    break;
+                }
+                seg.needed = true;
+                if let Some(prev_seg_id) = seg.parent {
+                    seg_id = prev_seg_id;
+                } else {
+                    break;
+                }
+            }
+        }
+
+        // Phase 2: For each oldest segment in a chain that needs to be retained,
+        // calculate if we should store snapshot or WAL
+        self.size_from_snapshot_later(0)
+    }
+
+    fn size_from_wal(&self, seg_id: usize) -> anyhow::Result<SegmentSize> {
+        let seg = &self.segments[seg_id];
+
+        let this_size = seg.end_lsn - seg.start_lsn;
+
+        let mut children = Vec::new();
+
+        // try both ways
+        for &child_id in seg.children_after.iter() {
+            // try each child both ways
+            let child = &self.segments[child_id];
+            let p1 = self.size_from_wal(child_id)?;
+
+            let p = if !child.needed {
+                let p2 = self.size_from_snapshot_later(child_id)?;
+                if p1.total() < p2.total() {
+                    p1
+                } else {
+                    p2
+                }
+            } else {
+                p1
+            };
+            children.push(p);
+        }
+        Ok(SegmentSize {
+            seg_id,
+            method: if seg.needed { WalNeeded } else { Wal },
+            this_size,
+            children,
+        })
+    }
+
+    fn size_from_snapshot_later(&self, seg_id: usize) -> anyhow::Result<SegmentSize> {
+        // If this is needed, then it's time to do the snapshot and continue
+        // with wal method.
+        let seg = &self.segments[seg_id];
+        //eprintln!("snap: seg{}: {} needed: {}", seg_id, seg.children_after.len(), seg.needed);
+        if seg.needed {
+            let mut children = Vec::new();
+
+            for &child_id in seg.children_after.iter() {
+                // try each child both ways
+                let child = &self.segments[child_id];
+                let p1 = self.size_from_wal(child_id)?;
+
+                let p = if !child.needed {
+                    let p2 = self.size_from_snapshot_later(child_id)?;
+                    if p1.total() < p2.total() {
+                        p1
+                    } else {
+                        p2
+                    }
+                } else {
+                    p1
+                };
+                children.push(p);
+            }
+            Ok(SegmentSize {
+                seg_id,
+                method: WalNeeded,
+                this_size: seg.start_size,
+                children,
+            })
+        } else {
+            // If any of the direct children are "needed", need to be able to reconstruct here
+            let mut children_needed = false;
+            for &child in seg.children_after.iter() {
+                let seg = &self.segments[child];
+                if seg.needed {
+                    children_needed = true;
+                    break;
+                }
+            }
+
+            let method1 = if !children_needed {
+                let mut children = Vec::new();
+                for child in seg.children_after.iter() {
+                    children.push(self.size_from_snapshot_later(*child)?);
+                }
+                Some(SegmentSize {
+                    seg_id,
+                    method: Skipped,
+                    this_size: 0,
+                    children,
+                })
+            } else {
+                None
+            };
+
+            // If this a junction, consider snapshotting here
+            let method2 = if children_needed || seg.children_after.len() >= 2 {
+                let mut children = Vec::new();
+                for child in seg.children_after.iter() {
+                    children.push(self.size_from_wal(*child)?);
+                }
+                let Some(this_size) = seg.end_size else { anyhow::bail!("no end_size at junction {seg_id}") };
+                Some(SegmentSize {
+                    seg_id,
+                    method: SnapshotAfter,
+                    this_size,
+                    children,
+                })
+            } else {
+                None
+            };
+
+            Ok(match (method1, method2) {
+                (None, None) => anyhow::bail!(
+                    "neither method was applicable: children_after={}, children_needed={}",
+                    seg.children_after.len(),
+                    children_needed
+                ),
+                (Some(method), None) => method,
+                (None, Some(method)) => method,
+                (Some(method1), Some(method2)) => {
+                    if method1.total() < method2.total() {
+                        method1
+                    } else {
+                        method2
+                    }
+                }
+            })
+        }
+    }
+
+    pub fn into_segments(self) -> Vec<Segment> {
+        self.segments
+    }
+}

libs/tenant_size_model/src/main.rs

@@ -0,0 +1,269 @@
+//! Tenant size model testing ground.
+//!
+//! Has a number of scenarios and a `main` for invoking these by number, calculating the history
+//! size, outputs graphviz graph. Makefile in directory shows how to use graphviz to turn scenarios
+//! into pngs.
+
+use tenant_size_model::{Segment, SegmentSize, Storage};
+
+// Main branch only. Some updates on it.
+fn scenario_1() -> anyhow::Result<(Vec<Segment>, SegmentSize)> {
+    // Create main branch
+    let mut storage = Storage::new("main");
+
+    // Bulk load 5 GB of data to it
+    storage.insert("main", 5_000)?;
+
+    // Stream of updates
+    for _ in 0..5 {
+        storage.update("main", 1_000)?;
+    }
+
+    let size = storage.calculate(1000)?;
+
+    Ok((storage.into_segments(), size))
+}
+
+// Main branch only. Some updates on it.
+fn scenario_2() -> anyhow::Result<(Vec<Segment>, SegmentSize)> {
+    // Create main branch
+    let mut storage = Storage::new("main");
+
+    // Bulk load 5 GB of data to it
+    storage.insert("main", 5_000)?;
+
+    // Stream of updates
+    for _ in 0..5 {
+        storage.update("main", 1_000)?;
+    }
+
+    // Branch
+    storage.branch("main", "child")?;
+    storage.update("child", 1_000)?;
+
+    // More updates on parent
+    storage.update("main", 1_000)?;
+
+    let size = storage.calculate(1000)?;
+
+    Ok((storage.into_segments(), size))
+}
+
+// Like 2, but more updates on main
+fn scenario_3() -> anyhow::Result<(Vec<Segment>, SegmentSize)> {
+    // Create main branch
+    let mut storage = Storage::new("main");
+
+    // Bulk load 5 GB of data to it
+    storage.insert("main", 5_000)?;
+
+    // Stream of updates
+    for _ in 0..5 {
+        storage.update("main", 1_000)?;
+    }
+
+    // Branch
+    storage.branch("main", "child")?;
+    storage.update("child", 1_000)?;
+
+    // More updates on parent
+    for _ in 0..5 {
+        storage.update("main", 1_000)?;
+    }
+
+    let size = storage.calculate(1000)?;
+
+    Ok((storage.into_segments(), size))
+}
+
+// Diverged branches
+fn scenario_4() -> anyhow::Result<(Vec<Segment>, SegmentSize)> {
+    // Create main branch
+    let mut storage = Storage::new("main");
+
+    // Bulk load 5 GB of data to it
+    storage.insert("main", 5_000)?;
+
+    // Stream of updates
+    for _ in 0..5 {
+        storage.update("main", 1_000)?;
+    }
+
+    // Branch
+    storage.branch("main", "child")?;
+    storage.update("child", 1_000)?;
+
+    // More updates on parent
+    for _ in 0..8 {
+        storage.update("main", 1_000)?;
+    }
+
+    let size = storage.calculate(1000)?;
+
+    Ok((storage.into_segments(), size))
+}
+
+fn scenario_5() -> anyhow::Result<(Vec<Segment>, SegmentSize)> {
+    let mut storage = Storage::new("a");
+    storage.insert("a", 5000)?;
+    storage.branch("a", "b")?;
+    storage.update("b", 4000)?;
+    storage.update("a", 2000)?;
+    storage.branch("a", "c")?;
+    storage.insert("c", 4000)?;
+    storage.insert("a", 2000)?;
+
+    let size = storage.calculate(5000)?;
+
+    Ok((storage.into_segments(), size))
+}
+
+fn scenario_6() -> anyhow::Result<(Vec<Segment>, SegmentSize)> {
+    use std::borrow::Cow;
+
+    const NO_OP: Cow<'static, str> = Cow::Borrowed("");
+
+    let branches = [
+        Some(0x7ff1edab8182025f15ae33482edb590a_u128),
+        Some(0xb1719e044db05401a05a2ed588a3ad3f),
+        Some(0xb68d6691c895ad0a70809470020929ef),
+    ];
+
+    // compared to other scenarios, this one uses bytes instead of kB
+
+    let mut storage = Storage::new(None);
+
+    storage.branch(&None, branches[0])?; // at 0
+    storage.modify_branch(&branches[0], NO_OP, 108951064, 43696128)?; // at 108951064
+    storage.branch(&branches[0], branches[1])?; // at 108951064
+    storage.modify_branch(&branches[1], NO_OP, 15560408, -1851392)?; // at 124511472
+    storage.modify_branch(&branches[0], NO_OP, 174464360, -1531904)?; // at 283415424
+    storage.branch(&branches[0], branches[2])?; // at 283415424
+    storage.modify_branch(&branches[2], NO_OP, 15906192, 8192)?; // at 299321616
+    storage.modify_branch(&branches[0], NO_OP, 18909976, 32768)?; // at 302325400
+
+    let size = storage.calculate(100_000)?;
+
+    Ok((storage.into_segments(), size))
+}
+
+fn main() {
+    let args: Vec<String> = std::env::args().collect();
+
+    let scenario = if args.len() < 2 { "1" } else { &args[1] };
+
+    let (segments, size) = match scenario {
+        "1" => scenario_1(),
+        "2" => scenario_2(),
+        "3" => scenario_3(),
+        "4" => scenario_4(),
+        "5" => scenario_5(),
+        "6" => scenario_6(),
+        other => {
+            eprintln!("invalid scenario {}", other);
+            std::process::exit(1);
+        }
+    }
+    .unwrap();
+
+    graphviz_tree(&segments, &size);
+}
+
+fn graphviz_recurse(segments: &[Segment], node: &SegmentSize) {
+    use tenant_size_model::SegmentMethod::*;
+
+    let seg_id = node.seg_id;
+    let seg = segments.get(seg_id).unwrap();
+    let lsn = seg.end_lsn;
+    let size = seg.end_size.unwrap_or(0);
+    let method = node.method;
+
+    println!("  {{");
+    println!("    node [width=0.1 height=0.1 shape=oval]");
+
+    let tenant_size = node.total_children();
+
+    let penwidth = if seg.needed { 6 } else { 3 };
+    let x = match method {
+        SnapshotAfter =>
+            format!("label=\"lsn: {lsn}\\nsize: {size}\\ntenant_size: {tenant_size}\" style=filled penwidth={penwidth}"),
+        Wal =>
+            format!("label=\"lsn: {lsn}\\nsize: {size}\\ntenant_size: {tenant_size}\" color=\"black\" penwidth={penwidth}"),
+        WalNeeded =>
+            format!("label=\"lsn: {lsn}\\nsize: {size}\\ntenant_size: {tenant_size}\" color=\"black\" penwidth={penwidth}"),
+        Skipped =>
+            format!("label=\"lsn: {lsn}\\nsize: {size}\\ntenant_size: {tenant_size}\" color=\"gray\" penwidth={penwidth}"),
+    };
+
+    println!("    \"seg{seg_id}\" [{x}]");
+    println!("  }}");
+
+    // Recurse. Much of the data is actually on the edge
+    for child in node.children.iter() {
+        let child_id = child.seg_id;
+        graphviz_recurse(segments, child);
+
+        let edge_color = match child.method {
+            SnapshotAfter => "gray",
+            Wal => "black",
+            WalNeeded => "black",
+            Skipped => "gray",
+        };
+
+        println!("  {{");
+        println!("    edge [] ");
+        print!("    \"seg{seg_id}\" -> \"seg{child_id}\" [");
+        print!("color={edge_color}");
+        if child.method == WalNeeded {
+            print!(" penwidth=6");
+        }
+        if child.method == Wal {
+            print!(" penwidth=3");
+        }
+
+        let next = segments.get(child_id).unwrap();
+
+        if next.op.is_empty() {
+            print!(
+                " label=\"{} / {}\"",
+                next.end_lsn - seg.end_lsn,
+                (next.end_size.unwrap_or(0) as i128 - seg.end_size.unwrap_or(0) as i128)
+            );
+        } else {
+            print!(" label=\"{}: {}\"", next.op, next.end_lsn - seg.end_lsn);
+        }
+        println!("]");
+        println!("  }}");
+    }
+}
+
+fn graphviz_tree(segments: &[Segment], tree: &SegmentSize) {
+    println!("digraph G {{");
+    println!("  fontname=\"Helvetica,Arial,sans-serif\"");
+    println!("  node [fontname=\"Helvetica,Arial,sans-serif\"]");
+    println!("  edge [fontname=\"Helvetica,Arial,sans-serif\"]");
+    println!("  graph [center=1 rankdir=LR]");
+    println!("  edge [dir=none]");
+
+    graphviz_recurse(segments, tree);
+
+    println!("}}");
+}
+
+#[test]
+fn scenarios_return_same_size() {
+    type ScenarioFn = fn() -> anyhow::Result<(Vec<Segment>, SegmentSize)>;
+    let truths: &[(u32, ScenarioFn, _)] = &[
+        (line!(), scenario_1, 8000),
+        (line!(), scenario_2, 9000),
+        (line!(), scenario_3, 13000),
+        (line!(), scenario_4, 16000),
+        (line!(), scenario_5, 17000),
+        (line!(), scenario_6, 333_792_000),
+    ];
+
+    for (line, scenario, expected) in truths {
+        let (_, size) = scenario().unwrap();
+        assert_eq!(*expected, size.total_children(), "scenario on line {line}");
+    }
+}

libs/tenant_size_model/src/svg.rs

@@ -1,193 +0,0 @@
-use crate::{SegmentMethod, SegmentSizeResult, SizeResult, StorageModel};
-use std::fmt::Write;
-
-const SVG_WIDTH: f32 = 500.0;
-
-struct SvgDraw<'a> {
-    storage: &'a StorageModel,
-    branches: &'a [String],
-    seg_to_branch: &'a [usize],
-    sizes: &'a [SegmentSizeResult],
-
-    // layout
-    xscale: f32,
-    min_lsn: u64,
-    seg_coordinates: Vec<(f32, f32)>,
-}
-
-fn draw_legend(result: &mut String) -> anyhow::Result<()> {
-    writeln!(
-        result,
-        "<circle cx=\"10\" cy=\"10\" r=\"5\" stroke=\"red\"/>"
-    )?;
-    writeln!(result, "<text x=\"20\" y=\"15\">logical snapshot</text>")?;
-    writeln!(
-        result,
-        "<line x1=\"5\" y1=\"30\" x2=\"15\" y2=\"30\" stroke-width=\"6\" stroke=\"black\" />"
-    )?;
-    writeln!(
-        result,
-        "<text x=\"20\" y=\"35\">WAL within retention period</text>"
-    )?;
-    writeln!(
-        result,
-        "<line x1=\"5\" y1=\"50\" x2=\"15\" y2=\"50\" stroke-width=\"3\" stroke=\"black\" />"
-    )?;
-    writeln!(
-        result,
-        "<text x=\"20\" y=\"55\">WAL retained to avoid copy</text>"
-    )?;
-    writeln!(
-        result,
-        "<line x1=\"5\" y1=\"70\" x2=\"15\" y2=\"70\" stroke-width=\"1\" stroke=\"gray\" />"
-    )?;
-    writeln!(result, "<text x=\"20\" y=\"75\">WAL not retained</text>")?;
-    Ok(())
-}
-
-pub fn draw_svg(
-    storage: &StorageModel,
-    branches: &[String],
-    seg_to_branch: &[usize],
-    sizes: &SizeResult,
-) -> anyhow::Result<String> {
-    let mut draw = SvgDraw {
-        storage,
-        branches,
-        seg_to_branch,
-        sizes: &sizes.segments,
-
-        xscale: 0.0,
-        min_lsn: 0,
-        seg_coordinates: Vec::new(),
-    };
-
-    let mut result = String::new();
-
-    writeln!(result, "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" height=\"300\" width=\"500\">")?;
-
-    draw.calculate_svg_layout();
-
-    // Draw the tree
-    for (seg_id, _seg) in storage.segments.iter().enumerate() {
-        draw.draw_seg_phase1(seg_id, &mut result)?;
-    }
-
-    // Draw snapshots
-    for (seg_id, _seg) in storage.segments.iter().enumerate() {
-        draw.draw_seg_phase2(seg_id, &mut result)?;
-    }
-
-    draw_legend(&mut result)?;
-
-    write!(result, "</svg>")?;
-
-    Ok(result)
-}
-
-impl<'a> SvgDraw<'a> {
-    fn calculate_svg_layout(&mut self) {
-        // Find x scale
-        let segments = &self.storage.segments;
-        let min_lsn = segments.iter().map(|s| s.lsn).fold(u64::MAX, std::cmp::min);
-        let max_lsn = segments.iter().map(|s| s.lsn).fold(0, std::cmp::max);
-
-        // Start with 1 pixel = 1 byte. Double the scale until it fits into the image
-        let mut xscale = 1.0;
-        while (max_lsn - min_lsn) as f32 / xscale > SVG_WIDTH {
-            xscale *= 2.0;
-        }
-
-        // Layout the timelines on Y dimension.
-        // TODO
-        let mut y = 100.0;
-        let mut branch_y_coordinates = Vec::new();
-        for _branch in self.branches {
-            branch_y_coordinates.push(y);
-            y += 40.0;
-        }
-
-        // Calculate coordinates for each point
-        let seg_coordinates = std::iter::zip(segments, self.seg_to_branch)
-            .map(|(seg, branch_id)| {
-                let x = (seg.lsn - min_lsn) as f32 / xscale;
-                let y = branch_y_coordinates[*branch_id];
-                (x, y)
-            })
-            .collect();
-
-        self.xscale = xscale;
-        self.min_lsn = min_lsn;
-        self.seg_coordinates = seg_coordinates;
-    }
-
-    /// Draws lines between points
-    fn draw_seg_phase1(&self, seg_id: usize, result: &mut String) -> anyhow::Result<()> {
-        let seg = &self.storage.segments[seg_id];
-
-        let wal_bytes = if let Some(parent_id) = seg.parent {
-            seg.lsn - self.storage.segments[parent_id].lsn
-        } else {
-            0
-        };
-
-        let style = match self.sizes[seg_id].method {
-            SegmentMethod::SnapshotHere => "stroke-width=\"1\" stroke=\"gray\"",
-            SegmentMethod::Wal if seg.needed && wal_bytes > 0 => {
-                "stroke-width=\"6\" stroke=\"black\""
-            }
-            SegmentMethod::Wal => "stroke-width=\"3\" stroke=\"black\"",
-            SegmentMethod::Skipped => "stroke-width=\"1\" stroke=\"gray\"",
-        };
-        if let Some(parent_id) = seg.parent {
-            let (x1, y1) = self.seg_coordinates[parent_id];
-            let (x2, y2) = self.seg_coordinates[seg_id];
-
-            writeln!(
-                result,
-                "<line x1=\"{x1}\" y1=\"{y1}\" x2=\"{x2}\" y2=\"{y2}\" {style}>",
-            )?;
-            writeln!(
-                result,
-                "  <title>{wal_bytes} bytes of WAL (seg {seg_id})</title>"
-            )?;
-            writeln!(result, "</line>")?;
-        } else {
-            // draw a little dash to mark the starting point of this branch
-            let (x, y) = self.seg_coordinates[seg_id];
-            let (x1, y1) = (x, y - 5.0);
-            let (x2, y2) = (x, y + 5.0);
-
-            writeln!(
-                result,
-                "<line x1=\"{x1}\" y1=\"{y1}\" x2=\"{x2}\" y2=\"{y2}\" {style}>",
-            )?;
-            writeln!(result, "  <title>(seg {seg_id})</title>")?;
-            writeln!(result, "</line>")?;
-        }
-
-        Ok(())
-    }
-
-    /// Draw circles where snapshots are taken
-    fn draw_seg_phase2(&self, seg_id: usize, result: &mut String) -> anyhow::Result<()> {
-        let seg = &self.storage.segments[seg_id];
-
-        // draw a snapshot point if it's needed
-        let (coord_x, coord_y) = self.seg_coordinates[seg_id];
-        if self.sizes[seg_id].method == SegmentMethod::SnapshotHere {
-            writeln!(
-                result,
-                "<circle cx=\"{coord_x}\" cy=\"{coord_y}\" r=\"5\" stroke=\"red\">",
-            )?;
-            writeln!(
-                result,
-                "  <title>logical size {}</title>",
-                seg.size.unwrap()
-            )?;
-            write!(result, "</circle>")?;
-        }
-
-        Ok(())
-    }
-}

libs/tenant_size_model/tests/tests.rs

@@ -1,313 +0,0 @@
-//! Tenant size model tests.
-
-use tenant_size_model::{Segment, SizeResult, StorageModel};
-
-use std::collections::HashMap;
-
-struct ScenarioBuilder {
-    segments: Vec<Segment>,
-
-    /// Mapping from the branch name to the index of a segment describing its latest state.
-    branches: HashMap<String, usize>,
-}
-
-impl ScenarioBuilder {
-    /// Creates a new storage with the given default branch name.
-    pub fn new(initial_branch: &str) -> ScenarioBuilder {
-        let init_segment = Segment {
-            parent: None,
-            lsn: 0,
-            size: Some(0),
-            needed: false, // determined later
-        };
-
-        ScenarioBuilder {
-            segments: vec![init_segment],
-            branches: HashMap::from([(initial_branch.into(), 0)]),
-        }
-    }
-
-    /// Advances the branch with the named operation, by the relative LSN and logical size bytes.
-    pub fn modify_branch(&mut self, branch: &str, lsn_bytes: u64, size_bytes: i64) {
-        let lastseg_id = *self.branches.get(branch).unwrap();
-        let newseg_id = self.segments.len();
-        let lastseg = &mut self.segments[lastseg_id];
-
-        let newseg = Segment {
-            parent: Some(lastseg_id),
-            lsn: lastseg.lsn + lsn_bytes,
-            size: Some((lastseg.size.unwrap() as i64 + size_bytes) as u64),
-            needed: false,
-        };
-
-        self.segments.push(newseg);
-        *self.branches.get_mut(branch).expect("read already") = newseg_id;
-    }
-
-    pub fn insert(&mut self, branch: &str, bytes: u64) {
-        self.modify_branch(branch, bytes, bytes as i64);
-    }
-
-    pub fn update(&mut self, branch: &str, bytes: u64) {
-        self.modify_branch(branch, bytes, 0i64);
-    }
-
-    pub fn _delete(&mut self, branch: &str, bytes: u64) {
-        self.modify_branch(branch, bytes, -(bytes as i64));
-    }
-
-    /// Panics if the parent branch cannot be found.
-    pub fn branch(&mut self, parent: &str, name: &str) {
-        // Find the right segment
-        let branchseg_id = *self
-            .branches
-            .get(parent)
-            .expect("should had found the parent by key");
-        let _branchseg = &mut self.segments[branchseg_id];
-
-        // Create branch name for it
-        self.branches.insert(name.to_string(), branchseg_id);
-    }
-
-    pub fn calculate(&mut self, retention_period: u64) -> (StorageModel, SizeResult) {
-        // Phase 1: Mark all the segments that need to be retained
-        for (_branch, &last_seg_id) in self.branches.iter() {
-            let last_seg = &self.segments[last_seg_id];
-            let cutoff_lsn = last_seg.lsn.saturating_sub(retention_period);
-            let mut seg_id = last_seg_id;
-            loop {
-                let seg = &mut self.segments[seg_id];
-                if seg.lsn <= cutoff_lsn {
-                    break;
-                }
-                seg.needed = true;
-                if let Some(prev_seg_id) = seg.parent {
-                    seg_id = prev_seg_id;
-                } else {
-                    break;
-                }
-            }
-        }
-
-        // Perform the calculation
-        let storage_model = StorageModel {
-            segments: self.segments.clone(),
-        };
-        let size_result = storage_model.calculate();
-        (storage_model, size_result)
-    }
-}
-
-// Main branch only. Some updates on it.
-#[test]
-fn scenario_1() {
-    // Create main branch
-    let mut scenario = ScenarioBuilder::new("main");
-
-    // Bulk load 5 GB of data to it
-    scenario.insert("main", 5_000);
-
-    // Stream of updates
-    for _ in 0..5 {
-        scenario.update("main", 1_000);
-    }
-
-    // Calculate the synthetic size with retention horizon 1000
-    let (_model, result) = scenario.calculate(1000);
-
-    // The end of the branch is at LSN 10000. Need to retain
-    // a logical snapshot at LSN 9000, plus the WAL between 9000-10000.
-    // The logical snapshot has size 5000.
-    assert_eq!(result.total_size, 5000 + 1000);
-}
-
-// Main branch only. Some updates on it.
-#[test]
-fn scenario_2() {
-    // Create main branch
-    let mut scenario = ScenarioBuilder::new("main");
-
-    // Bulk load 5 GB of data to it
-    scenario.insert("main", 5_000);
-
-    // Stream of updates
-    for _ in 0..5 {
-        scenario.update("main", 1_000);
-    }
-
-    // Branch
-    scenario.branch("main", "child");
-    scenario.update("child", 1_000);
-
-    // More updates on parent
-    scenario.update("main", 1_000);
-
-    //
-    // The history looks like this now:
-    //
-    //         10000          11000
-    // *----*----*--------------*    main
-    //           |
-    //           |            11000
-    //           +--------------     child
-    //
-    //
-    // With retention horizon 1000, we need to retain logical snapshot
-    // at the branch point, size 5000, and the WAL from 10000-11000 on
-    // both branches.
-    let (_model, result) = scenario.calculate(1000);
-
-    assert_eq!(result.total_size, 5000 + 1000 + 1000);
-}
-
-// Like 2, but more updates on main
-#[test]
-fn scenario_3() {
-    // Create main branch
-    let mut scenario = ScenarioBuilder::new("main");
-
-    // Bulk load 5 GB of data to it
-    scenario.insert("main", 5_000);
-
-    // Stream of updates
-    for _ in 0..5 {
-        scenario.update("main", 1_000);
-    }
-
-    // Branch
-    scenario.branch("main", "child");
-    scenario.update("child", 1_000);
-
-    // More updates on parent
-    for _ in 0..5 {
-        scenario.update("main", 1_000);
-    }
-
-    //
-    // The history looks like this now:
-    //
-    //         10000                                 15000
-    // *----*----*------------------------------------*    main
-    //           |
-    //           |            11000
-    //           +--------------     child
-    //
-    //
-    // With retention horizon 1000, it's still cheapest to retain
-    // - snapshot at branch point (size 5000)
-    // - WAL on child between 10000-11000
-    // - WAL on main between 10000-15000
-    //
-    // This is in total 5000 + 1000 + 5000
-    //
-    let (_model, result) = scenario.calculate(1000);
-
-    assert_eq!(result.total_size, 5000 + 1000 + 5000);
-}
-
-// Diverged branches
-#[test]
-fn scenario_4() {
-    // Create main branch
-    let mut scenario = ScenarioBuilder::new("main");
-
-    // Bulk load 5 GB of data to it
-    scenario.insert("main", 5_000);
-
-    // Stream of updates
-    for _ in 0..5 {
-        scenario.update("main", 1_000);
-    }
-
-    // Branch
-    scenario.branch("main", "child");
-    scenario.update("child", 1_000);
-
-    // More updates on parent
-    for _ in 0..8 {
-        scenario.update("main", 1_000);
-    }
-
-    //
-    // The history looks like this now:
-    //
-    //         10000                                 18000
-    // *----*----*------------------------------------*    main
-    //           |
-    //           |            11000
-    //           +--------------     child
-    //
-    //
-    // With retention horizon 1000, it's now cheapest to retain
-    // separate snapshots on both branches:
-    // - snapshot on main branch at LSN 17000 (size 5000)
-    // - WAL on main between 17000-18000
-    // - snapshot on child branch at LSN 10000 (size 5000)
-    // - WAL on child between 10000-11000
-    //
-    // This is in total 5000 + 1000 + 5000 + 1000 = 12000
-    //
-    // (If we used the the method from the previous scenario, and
-    // kept only snapshot at the branch point, we'd need to keep
-    // all the WAL between 10000-18000 on the main branch, so
-    // the total size would be 5000 + 1000 + 8000 = 14000. The
-    // calculation always picks the cheapest alternative)
-
-    let (_model, result) = scenario.calculate(1000);
-
-    assert_eq!(result.total_size, 5000 + 1000 + 5000 + 1000);
-}
-
-#[test]
-fn scenario_5() {
-    let mut scenario = ScenarioBuilder::new("a");
-    scenario.insert("a", 5000);
-    scenario.branch("a", "b");
-    scenario.update("b", 4000);
-    scenario.update("a", 2000);
-    scenario.branch("a", "c");
-    scenario.insert("c", 4000);
-    scenario.insert("a", 2000);
-
-    let (_model, result) = scenario.calculate(1000);
-
-    assert_eq!(result.total_size, 17000);
-}
-
-#[test]
-fn scenario_6() {
-    let branches = [
-        "7ff1edab8182025f15ae33482edb590a",
-        "b1719e044db05401a05a2ed588a3ad3f",
-        "0xb68d6691c895ad0a70809470020929ef",
-    ];
-
-    // compared to other scenarios, this one uses bytes instead of kB
-
-    let mut scenario = ScenarioBuilder::new("");
-
-    scenario.branch("", branches[0]); // at 0
-    scenario.modify_branch(branches[0], 108951064, 43696128); // at 108951064
-    scenario.branch(branches[0], branches[1]); // at 108951064
-    scenario.modify_branch(branches[1], 15560408, -1851392); // at 124511472
-    scenario.modify_branch(branches[0], 174464360, -1531904); // at 283415424
-    scenario.branch(branches[0], branches[2]); // at 283415424
-    scenario.modify_branch(branches[2], 15906192, 8192); // at 299321616
-    scenario.modify_branch(branches[0], 18909976, 32768); // at 302325400
-
-    let (model, result) = scenario.calculate(100_000);
-
-    // FIXME: We previously calculated 333_792_000. But with this PR, we get
-    // a much lower number. At a quick look at the model output and the
-    // calculations here, the new result seems correct to me.
-    eprintln!(
-        " MODEL: {}",
-        serde_json::to_string(&model.segments).unwrap()
-    );
-    eprintln!(
-        "RESULT: {}",
-        serde_json::to_string(&result.segments).unwrap()
-    );
-
-    assert_eq!(result.total_size, 136_236_928);
-}

libs/utils/Cargo.toml

@@ -11,7 +11,6 @@ async-trait.workspace = true
 anyhow.workspace = true
 bincode.workspace = true
 bytes.workspace = true
-heapless.workspace = true
 hyper = { workspace = true, features = ["full"] }
 routerify.workspace = true
 serde.workspace = true
@@ -38,7 +37,6 @@ metrics.workspace = true
 pq_proto.workspace = true
 
 workspace_hack.workspace = true
-url.workspace = true
 
 [dev-dependencies]
 byteorder.workspace = true

libs/utils/src/history_buffer.rs

@@ -1,161 +0,0 @@
-//! A heapless buffer for events of sorts.
-
-use std::ops;
-
-use heapless::HistoryBuffer;
-
-#[derive(Debug, Clone)]
-pub struct HistoryBufferWithDropCounter<T, const L: usize> {
-    buffer: HistoryBuffer<T, L>,
-    drop_count: u64,
-}
-
-impl<T, const L: usize> HistoryBufferWithDropCounter<T, L> {
-    pub fn write(&mut self, data: T) {
-        let len_before = self.buffer.len();
-        self.buffer.write(data);
-        let len_after = self.buffer.len();
-        self.drop_count += u64::from(len_before == len_after);
-    }
-    pub fn drop_count(&self) -> u64 {
-        self.drop_count
-    }
-    pub fn map<U, F: Fn(&T) -> U>(&self, f: F) -> HistoryBufferWithDropCounter<U, L> {
-        let mut buffer = HistoryBuffer::new();
-        buffer.extend(self.buffer.oldest_ordered().map(f));
-        HistoryBufferWithDropCounter::<U, L> {
-            buffer,
-            drop_count: self.drop_count,
-        }
-    }
-}
-
-impl<T, const L: usize> Default for HistoryBufferWithDropCounter<T, L> {
-    fn default() -> Self {
-        Self {
-            buffer: HistoryBuffer::default(),
-            drop_count: 0,
-        }
-    }
-}
-
-impl<T, const L: usize> ops::Deref for HistoryBufferWithDropCounter<T, L> {
-    type Target = HistoryBuffer<T, L>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.buffer
-    }
-}
-
-#[derive(serde::Serialize)]
-struct SerdeRepr<T> {
-    buffer: Vec<T>,
-    drop_count: u64,
-}
-
-impl<'a, T, const L: usize> From<&'a HistoryBufferWithDropCounter<T, L>> for SerdeRepr<T>
-where
-    T: Clone + serde::Serialize,
-{
-    fn from(value: &'a HistoryBufferWithDropCounter<T, L>) -> Self {
-        let HistoryBufferWithDropCounter { buffer, drop_count } = value;
-        SerdeRepr {
-            buffer: buffer.iter().cloned().collect(),
-            drop_count: *drop_count,
-        }
-    }
-}
-
-impl<T, const L: usize> serde::Serialize for HistoryBufferWithDropCounter<T, L>
-where
-    T: Clone + serde::Serialize,
-{
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        SerdeRepr::from(self).serialize(serializer)
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::HistoryBufferWithDropCounter;
-
-    #[test]
-    fn test_basics() {
-        let mut b = HistoryBufferWithDropCounter::<_, 2>::default();
-        b.write(1);
-        b.write(2);
-        b.write(3);
-        assert!(b.iter().any(|e| *e == 2));
-        assert!(b.iter().any(|e| *e == 3));
-        assert!(!b.iter().any(|e| *e == 1));
-    }
-
-    #[test]
-    fn test_drop_count_works() {
-        let mut b = HistoryBufferWithDropCounter::<_, 2>::default();
-        b.write(1);
-        assert_eq!(b.drop_count(), 0);
-        b.write(2);
-        assert_eq!(b.drop_count(), 0);
-        b.write(3);
-        assert_eq!(b.drop_count(), 1);
-        b.write(4);
-        assert_eq!(b.drop_count(), 2);
-    }
-
-    #[test]
-    fn test_clone_works() {
-        let mut b = HistoryBufferWithDropCounter::<_, 2>::default();
-        b.write(1);
-        b.write(2);
-        b.write(3);
-        assert_eq!(b.drop_count(), 1);
-        let mut c = b.clone();
-        assert_eq!(c.drop_count(), 1);
-        assert!(c.iter().any(|e| *e == 2));
-        assert!(c.iter().any(|e| *e == 3));
-        assert!(!c.iter().any(|e| *e == 1));
-
-        c.write(4);
-        assert!(c.iter().any(|e| *e == 4));
-        assert!(!b.iter().any(|e| *e == 4));
-    }
-
-    #[test]
-    fn test_map() {
-        let mut b = HistoryBufferWithDropCounter::<_, 2>::default();
-
-        b.write(1);
-        assert_eq!(b.drop_count(), 0);
-        {
-            let c = b.map(|i| i + 10);
-            assert_eq!(c.oldest_ordered().cloned().collect::<Vec<_>>(), vec![11]);
-            assert_eq!(c.drop_count(), 0);
-        }
-
-        b.write(2);
-        assert_eq!(b.drop_count(), 0);
-        {
-            let c = b.map(|i| i + 10);
-            assert_eq!(
-                c.oldest_ordered().cloned().collect::<Vec<_>>(),
-                vec![11, 12]
-            );
-            assert_eq!(c.drop_count(), 0);
-        }
-
-        b.write(3);
-        assert_eq!(b.drop_count(), 1);
-        {
-            let c = b.map(|i| i + 10);
-            assert_eq!(
-                c.oldest_ordered().cloned().collect::<Vec<_>>(),
-                vec![12, 13]
-            );
-            assert_eq!(c.drop_count(), 1);
-        }
-    }
-}

libs/utils/src/http/endpoint.rs

@@ -1,21 +1,18 @@
 use crate::auth::{Claims, JwtAuth};
 use crate::http::error;
-use anyhow::{anyhow, Context};
-use hyper::header::{HeaderName, AUTHORIZATION};
-use hyper::http::HeaderValue;
+use anyhow::anyhow;
+use hyper::header::AUTHORIZATION;
 use hyper::{header::CONTENT_TYPE, Body, Request, Response, Server};
-use hyper::{Method, StatusCode};
 use metrics::{register_int_counter, Encoder, IntCounter, TextEncoder};
 use once_cell::sync::Lazy;
 use routerify::ext::RequestExt;
 use routerify::RequestInfo;
 use routerify::{Middleware, Router, RouterBuilder, RouterService};
 use tokio::task::JoinError;
-use tracing;
+use tracing::info;
 
 use std::future::Future;
 use std::net::TcpListener;
-use std::str::FromStr;
 
 use super::error::ApiError;
 
@@ -28,14 +25,7 @@ static SERVE_METRICS_COUNT: Lazy<IntCounter> = Lazy::new(|| {
 });
 
 async fn logger(res: Response<Body>, info: RequestInfo) -> Result<Response<Body>, ApiError> {
-    // cannot factor out the Level to avoid the repetition
-    // because tracing can only work with const Level
-    // which is not the case here
-    if info.method() == Method::GET && res.status() == StatusCode::OK {
-        tracing::debug!("{} {} {}", info.method(), info.uri().path(), res.status());
-    } else {
-        tracing::info!("{} {} {}", info.method(), info.uri().path(), res.status());
-    }
+    info!("{} {} {}", info.method(), info.uri().path(), res.status(),);
     Ok(res)
 }
 
@@ -153,38 +143,6 @@ pub fn auth_middleware<B: hyper::body::HttpBody + Send + Sync + 'static>(
     })
 }
 
-pub fn add_response_header_middleware<B>(
-    header: &str,
-    value: &str,
-) -> anyhow::Result<Middleware<B, ApiError>>
-where
-    B: hyper::body::HttpBody + Send + Sync + 'static,
-{
-    let name =
-        HeaderName::from_str(header).with_context(|| format!("invalid header name: {header}"))?;
-    let value =
-        HeaderValue::from_str(value).with_context(|| format!("invalid header value: {value}"))?;
-    Ok(Middleware::post_with_info(
-        move |mut response, request_info| {
-            let name = name.clone();
-            let value = value.clone();
-            async move {
-                let headers = response.headers_mut();
-                if headers.contains_key(&name) {
-                    tracing::warn!(
-                        "{} response already contains header {:?}",
-                        request_info.uri(),
-                        &name,
-                    );
-                } else {
-                    headers.insert(name, value);
-                }
-                Ok(response)
-            }
-        },
-    ))
-}
-
 pub fn check_permission_with(
     req: &Request<Body>,
     check_permission: impl Fn(&Claims) -> Result<(), anyhow::Error>,
@@ -211,7 +169,7 @@ pub fn serve_thread_main<S>(
 where
     S: Future<Output = ()> + Send + Sync,
 {
-    tracing::info!("Starting an HTTP endpoint at {}", listener.local_addr()?);
+    info!("Starting an HTTP endpoint at {}", listener.local_addr()?);
 
     // Create a Service from the router above to handle incoming requests.
     let service = RouterService::new(router_builder.build().map_err(|err| anyhow!(err))?).unwrap();

libs/utils/src/http/request.rs

@@ -1,5 +1,4 @@
-use core::fmt;
-use std::{borrow::Cow, str::FromStr};
+use std::str::FromStr;
 
 use super::error::ApiError;
 use anyhow::anyhow;
@@ -30,50 +29,6 @@ pub fn parse_request_param<T: FromStr>(
     }
 }
 
-fn get_query_param<'a>(
-    request: &'a Request<Body>,
-    param_name: &str,
-) -> Result<Option<Cow<'a, str>>, ApiError> {
-    let query = match request.uri().query() {
-        Some(q) => q,
-        None => return Ok(None),
-    };
-    let mut values = url::form_urlencoded::parse(query.as_bytes())
-        .filter_map(|(k, v)| if k == param_name { Some(v) } else { None })
-        // we call .next() twice below. If it's None the first time, .fuse() ensures it's None afterwards
-        .fuse();
-
-    let value1 = values.next();
-    if values.next().is_some() {
-        return Err(ApiError::BadRequest(anyhow!(
-            "param {param_name} specified more than once"
-        )));
-    }
-    Ok(value1)
-}
-
-pub fn must_get_query_param<'a>(
-    request: &'a Request<Body>,
-    param_name: &str,
-) -> Result<Cow<'a, str>, ApiError> {
-    get_query_param(request, param_name)?.ok_or_else(|| {
-        ApiError::BadRequest(anyhow!("no {param_name} specified in query parameters"))
-    })
-}
-
-pub fn parse_query_param<E: fmt::Display, T: FromStr<Err = E>>(
-    request: &Request<Body>,
-    param_name: &str,
-) -> Result<Option<T>, ApiError> {
-    get_query_param(request, param_name)?
-        .map(|v| {
-            v.parse().map_err(|e| {
-                ApiError::BadRequest(anyhow!("cannot parse query param {param_name}: {e}"))
-            })
-        })
-        .transpose()
-}
-
 pub async fn ensure_no_body(request: &mut Request<Body>) -> Result<(), ApiError> {
     match request.body_mut().data().await {
         Some(_) => Err(ApiError::BadRequest(anyhow!("Unexpected request body"))),

libs/utils/src/lib.rs

@@ -52,8 +52,6 @@ pub mod signals;
 
 pub mod fs_ext;
 
-pub mod history_buffer;
-
 /// use with fail::cfg("$name", "return(2000)")
 #[macro_export]
 macro_rules! failpoint_sleep_millis_async {

libs/utils/src/logging.rs

@@ -45,115 +45,3 @@ pub fn init(log_format: LogFormat) -> anyhow::Result<()> {
 
     Ok(())
 }
-
-/// Disable the default rust panic hook by using `set_hook`.
-///
-/// For neon binaries, the assumption is that tracing is configured before with [`init`], after
-/// that sentry is configured (if needed). sentry will install it's own on top of this, always
-/// processing the panic before we log it.
-///
-/// When the return value is dropped, the hook is reverted to std default hook (prints to stderr).
-/// If the assumptions about the initialization order are not held, use
-/// [`TracingPanicHookGuard::disarm`] but keep in mind, if tracing is stopped, then panics will be
-/// lost.
-#[must_use]
-pub fn replace_panic_hook_with_tracing_panic_hook() -> TracingPanicHookGuard {
-    std::panic::set_hook(Box::new(tracing_panic_hook));
-    TracingPanicHookGuard::new()
-}
-
-/// Drop guard which restores the std panic hook on drop.
-///
-/// Tracing should not be used when it's not configured, but we cannot really latch on to any
-/// imaginary lifetime of tracing.
-pub struct TracingPanicHookGuard {
-    act: bool,
-}
-
-impl TracingPanicHookGuard {
-    fn new() -> Self {
-        TracingPanicHookGuard { act: true }
-    }
-
-    /// Make this hook guard not do anything when dropped.
-    pub fn forget(&mut self) {
-        self.act = false;
-    }
-}
-
-impl Drop for TracingPanicHookGuard {
-    fn drop(&mut self) {
-        if self.act {
-            let _ = std::panic::take_hook();
-        }
-    }
-}
-
-/// Named symbol for our panic hook, which logs the panic.
-fn tracing_panic_hook(info: &std::panic::PanicInfo) {
-    // following rust 1.66.1 std implementation:
-    // https://github.com/rust-lang/rust/blob/90743e7298aca107ddaa0c202a4d3604e29bfeb6/library/std/src/panicking.rs#L235-L288
-    let location = info.location();
-
-    let msg = match info.payload().downcast_ref::<&'static str>() {
-        Some(s) => *s,
-        None => match info.payload().downcast_ref::<String>() {
-            Some(s) => &s[..],
-            None => "Box<dyn Any>",
-        },
-    };
-
-    let thread = std::thread::current();
-    let thread = thread.name().unwrap_or("<unnamed>");
-    let backtrace = std::backtrace::Backtrace::capture();
-
-    let _entered = if let Some(location) = location {
-        tracing::error_span!("panic", %thread, location = %PrettyLocation(location))
-    } else {
-        // very unlikely to hit here, but the guarantees of std could change
-        tracing::error_span!("panic", %thread)
-    }
-    .entered();
-
-    if backtrace.status() == std::backtrace::BacktraceStatus::Captured {
-        // this has an annoying extra '\n' in the end which anyhow doesn't do, but we cannot really
-        // get rid of it as we cannot get in between of std::fmt::Formatter<'_>; we could format to
-        // string, maybe even to a TLS one but tracing already does that.
-        tracing::error!("{msg}\n\nStack backtrace:\n{backtrace}");
-    } else {
-        tracing::error!("{msg}");
-    }
-
-    // ensure that we log something on the panic if this hook is left after tracing has been
-    // unconfigured. worst case when teardown is racing the panic is to log the panic twice.
-    tracing::dispatcher::get_default(|d| {
-        if let Some(_none) = d.downcast_ref::<tracing::subscriber::NoSubscriber>() {
-            let location = location.map(PrettyLocation);
-            log_panic_to_stderr(thread, msg, location, &backtrace);
-        }
-    });
-}
-
-#[cold]
-fn log_panic_to_stderr(
-    thread: &str,
-    msg: &str,
-    location: Option<PrettyLocation<'_, '_>>,
-    backtrace: &std::backtrace::Backtrace,
-) {
-    eprintln!("panic while tracing is unconfigured: thread '{thread}' panicked at '{msg}', {location:?}\nStack backtrace:\n{backtrace}");
-}
-
-struct PrettyLocation<'a, 'b>(&'a std::panic::Location<'b>);
-
-impl std::fmt::Display for PrettyLocation<'_, '_> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}:{}:{}", self.0.file(), self.0.line(), self.0.column())
-    }
-}
-
-impl std::fmt::Debug for PrettyLocation<'_, '_> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        <Self as std::fmt::Display>::fmt(self, f)
-    }
-}

pageserver/Cargo.toml

@@ -23,7 +23,6 @@ const_format.workspace = true
 consumption_metrics.workspace = true
 crc32c.workspace = true
 crossbeam-utils.workspace = true
-either.workspace = true
 fail.workspace = true
 futures.workspace = true
 git-version.workspace = true
@@ -52,7 +51,7 @@ thiserror.workspace = true
 tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time"] }
 tokio-postgres.workspace = true
 tokio-util.workspace = true
-toml_edit = { workspace = true, features = [ "serde" ] }
+toml_edit.workspace = true
 tracing.workspace = true
 url.workspace = true
 walkdir.workspace = true
@@ -68,10 +67,6 @@ utils.workspace = true
 workspace_hack.workspace = true
 reqwest.workspace = true
 rpds.workspace = true
-enum-map.workspace = true
-enumset.workspace = true
-strum.workspace = true
-strum_macros.workspace = true
 
 [dev-dependencies]
 criterion.workspace = true

pageserver/benches/bench_layer_map.rs

@@ -1,7 +1,8 @@
 use pageserver::keyspace::{KeyPartitioning, KeySpace};
 use pageserver::repository::Key;
 use pageserver::tenant::layer_map::LayerMap;
-use pageserver::tenant::storage_layer::{Layer, LayerDescriptor, LayerFileName};
+use pageserver::tenant::storage_layer::Layer;
+use pageserver::tenant::storage_layer::{DeltaFileName, ImageFileName, LayerDescriptor};
 use rand::prelude::{SeedableRng, SliceRandom, StdRng};
 use std::cmp::{max, min};
 use std::fs::File;
@@ -25,15 +26,30 @@ fn build_layer_map(filename_dump: PathBuf) -> LayerMap<LayerDescriptor> {
 
     let mut updates = layer_map.batch_update();
     for fname in filenames {
-        let fname = fname.unwrap();
-        let fname = LayerFileName::from_str(&fname).unwrap();
-        let layer = LayerDescriptor::from(fname);
-
-        let lsn_range = layer.get_lsn_range();
-        min_lsn = min(min_lsn, lsn_range.start);
-        max_lsn = max(max_lsn, Lsn(lsn_range.end.0 - 1));
-
-        updates.insert_historic(Arc::new(layer));
+        let fname = &fname.unwrap();
+        if let Some(imgfilename) = ImageFileName::parse_str(fname) {
+            let layer = LayerDescriptor {
+                key: imgfilename.key_range,
+                lsn: imgfilename.lsn..(imgfilename.lsn + 1),
+                is_incremental: false,
+                short_id: fname.to_string(),
+            };
+            updates.insert_historic(Arc::new(layer));
+            min_lsn = min(min_lsn, imgfilename.lsn);
+            max_lsn = max(max_lsn, imgfilename.lsn);
+        } else if let Some(deltafilename) = DeltaFileName::parse_str(fname) {
+            let layer = LayerDescriptor {
+                key: deltafilename.key_range.clone(),
+                lsn: deltafilename.lsn_range.clone(),
+                is_incremental: true,
+                short_id: fname.to_string(),
+            };
+            updates.insert_historic(Arc::new(layer));
+            min_lsn = min(min_lsn, deltafilename.lsn_range.start);
+            max_lsn = max(max_lsn, deltafilename.lsn_range.end);
+        } else {
+            panic!("unexpected filename {fname}");
+        }
     }
 
     println!("min: {min_lsn}, max: {max_lsn}");

pageserver/src/bin/layer_map_analyzer.rs

@@ -1,230 +0,0 @@
-//! Tool for extracting content-dependent metadata about layers. Useful for scanning real project layer files and evaluating the effectiveness of different heuristics on them.
-//!
-//! Currently it only analyzes holes, which are regions within the layer range that the layer contains no updates for. In the future it might do more analysis (maybe key quantiles?) but it should never return sensitive data.
-
-use anyhow::Result;
-use std::cmp::Ordering;
-use std::collections::BinaryHeap;
-use std::ops::Range;
-use std::{env, fs, path::Path, path::PathBuf, str, str::FromStr};
-
-use pageserver::page_cache::PAGE_SZ;
-use pageserver::repository::{Key, KEY_SIZE};
-use pageserver::tenant::block_io::{BlockReader, FileBlockReader};
-use pageserver::tenant::disk_btree::{DiskBtreeReader, VisitDirection};
-use pageserver::tenant::storage_layer::delta_layer::{Summary, DELTA_KEY_SIZE};
-use pageserver::tenant::storage_layer::range_overlaps;
-use pageserver::virtual_file::VirtualFile;
-
-use utils::{bin_ser::BeSer, lsn::Lsn};
-
-const MIN_HOLE_LENGTH: i128 = (128 * 1024 * 1024 / PAGE_SZ) as i128;
-const DEFAULT_MAX_HOLES: usize = 10;
-
-/// Wrapper for key range to provide reverse ordering by range length for BinaryHeap
-#[derive(PartialEq, Eq)]
-struct Hole(Range<Key>);
-
-impl Ord for Hole {
-    fn cmp(&self, other: &Self) -> Ordering {
-        let other_len = other.0.end.to_i128() - other.0.start.to_i128();
-        let self_len = self.0.end.to_i128() - self.0.start.to_i128();
-        other_len.cmp(&self_len)
-    }
-}
-
-impl PartialOrd for Hole {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-struct LayerFile {
-    key_range: Range<Key>,
-    lsn_range: Range<Lsn>,
-    is_delta: bool,
-    holes: Vec<Hole>,
-}
-
-impl LayerFile {
-    fn skips(&self, key_range: &Range<Key>) -> bool {
-        if !range_overlaps(&self.key_range, key_range) {
-            return false;
-        }
-        let start = match self
-            .holes
-            .binary_search_by_key(&key_range.start, |hole| hole.0.start)
-        {
-            Ok(index) => index,
-            Err(index) => {
-                if index == 0 {
-                    return false;
-                }
-                index - 1
-            }
-        };
-        self.holes[start].0.end >= key_range.end
-    }
-}
-
-fn parse_filename(name: &str) -> Option<LayerFile> {
-    let split: Vec<&str> = name.split("__").collect();
-    if split.len() != 2 {
-        return None;
-    }
-    let keys: Vec<&str> = split[0].split('-').collect();
-    let mut lsns: Vec<&str> = split[1].split('-').collect();
-    let is_delta = if lsns.len() == 1 {
-        lsns.push(lsns[0]);
-        false
-    } else {
-        true
-    };
-
-    let key_range = Key::from_hex(keys[0]).unwrap()..Key::from_hex(keys[1]).unwrap();
-    let lsn_range = Lsn::from_hex(lsns[0]).unwrap()..Lsn::from_hex(lsns[1]).unwrap();
-    let holes = Vec::new();
-    Some(LayerFile {
-        key_range,
-        lsn_range,
-        is_delta,
-        holes,
-    })
-}
-
-// Finds the max_holes largest holes, ignoring any that are smaller than MIN_HOLE_LENGTH"
-fn get_holes(path: &Path, max_holes: usize) -> Result<Vec<Hole>> {
-    let file = FileBlockReader::new(VirtualFile::open(path)?);
-    let summary_blk = file.read_blk(0)?;
-    let actual_summary = Summary::des_prefix(summary_blk.as_ref())?;
-    let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new(
-        actual_summary.index_start_blk,
-        actual_summary.index_root_blk,
-        file,
-    );
-    // min-heap (reserve space for one more element added before eviction)
-    let mut heap: BinaryHeap<Hole> = BinaryHeap::with_capacity(max_holes + 1);
-    let mut prev_key: Option<Key> = None;
-    tree_reader.visit(
-        &[0u8; DELTA_KEY_SIZE],
-        VisitDirection::Forwards,
-        |key, _value| {
-            let curr = Key::from_slice(&key[..KEY_SIZE]);
-            if let Some(prev) = prev_key {
-                if curr.to_i128() - prev.to_i128() >= MIN_HOLE_LENGTH {
-                    heap.push(Hole(prev..curr));
-                    if heap.len() > max_holes {
-                        heap.pop(); // remove smallest hole
-                    }
-                }
-            }
-            prev_key = Some(curr.next());
-            true
-        },
-    )?;
-    let mut holes = heap.into_vec();
-    holes.sort_by_key(|hole| hole.0.start);
-    Ok(holes)
-}
-
-fn main() -> Result<()> {
-    let args: Vec<String> = env::args().collect();
-    if args.len() < 2 {
-        println!("Usage: layer_map_analyzer PAGESERVER_DATA_DIR [MAX_HOLES]");
-        return Ok(());
-    }
-    let storage_path = PathBuf::from_str(&args[1])?;
-    let max_holes = if args.len() > 2 {
-        args[2].parse::<usize>().unwrap()
-    } else {
-        DEFAULT_MAX_HOLES
-    };
-
-    // Initialize virtual_file (file desriptor cache) and page cache which are needed to access layer persistent B-Tree.
-    pageserver::virtual_file::init(10);
-    pageserver::page_cache::init(100);
-
-    let mut total_delta_layers = 0usize;
-    let mut total_image_layers = 0usize;
-    let mut total_excess_layers = 0usize;
-    for tenant in fs::read_dir(storage_path.join("tenants"))? {
-        let tenant = tenant?;
-        if !tenant.file_type()?.is_dir() {
-            continue;
-        }
-        for timeline in fs::read_dir(tenant.path().join("timelines"))? {
-            let timeline = timeline?;
-            if !timeline.file_type()?.is_dir() {
-                continue;
-            }
-            // Collect sorted vec of layers and count deltas
-            let mut layers = Vec::new();
-            let mut n_deltas = 0usize;
-
-            for layer in fs::read_dir(timeline.path())? {
-                let layer = layer?;
-                if let Some(mut layer_file) =
-                    parse_filename(&layer.file_name().into_string().unwrap())
-                {
-                    if layer_file.is_delta {
-                        layer_file.holes = get_holes(&layer.path(), max_holes)?;
-                        n_deltas += 1;
-                    }
-                    layers.push(layer_file);
-                }
-            }
-            layers.sort_by_key(|layer| layer.lsn_range.end);
-
-            // Count the number of holes and number of excess layers.
-            // Excess layer is image layer generated when holes in delta layers are not considered.
-            let mut n_excess_layers = 0usize;
-            let mut n_holes = 0usize;
-
-            for i in 0..layers.len() {
-                if !layers[i].is_delta {
-                    let mut n_deltas_since_last_image = 0usize;
-                    let mut n_skipped = 0usize;
-                    let img_key_range = &layers[i].key_range;
-                    for j in (0..i).rev() {
-                        if range_overlaps(img_key_range, &layers[j].key_range) {
-                            if layers[j].is_delta {
-                                n_deltas_since_last_image += 1;
-                                if layers[j].skips(img_key_range) {
-                                    n_skipped += 1;
-                                }
-                            } else {
-                                // Image layer is always dense, despite to the fact that it doesn't contain all possible
-                                // key values in the specified range: there are may be no keys in the storage belonging
-                                // to the image layer range but not present in the image layer.
-                                break;
-                            }
-                        }
-                    }
-                    if n_deltas_since_last_image >= 3 && n_deltas_since_last_image - n_skipped < 3 {
-                        // It is just approximation: it doesn't take in account all image coverage.
-                        // Moreover the new layer map doesn't count total deltas, but the max stack of overlapping deltas.
-                        n_excess_layers += 1;
-                    }
-                    n_holes += n_skipped;
-                }
-            }
-            println!(
-                "Tenant {} timeline {} delta layers {} image layers {} excess layers {} holes {}",
-                tenant.file_name().into_string().unwrap(),
-                timeline.file_name().into_string().unwrap(),
-                n_deltas,
-                layers.len() - n_deltas,
-                n_excess_layers,
-                n_holes
-            );
-            total_delta_layers += n_deltas;
-            total_image_layers += layers.len() - n_deltas;
-            total_excess_layers += n_excess_layers;
-        }
-    }
-    println!(
-        "Total delta layers {} image layers {} excess layers {}",
-        total_delta_layers, total_image_layers, total_excess_layers
-    );
-    Ok(())
-}

pageserver/src/bin/pageserver.rs

@@ -7,7 +7,6 @@ use std::{env, ops::ControlFlow, path::Path, str::FromStr};
 use anyhow::{anyhow, Context};
 use clap::{Arg, ArgAction, Command};
 use fail::FailScenario;
-use metrics::launch_timestamp::{set_launch_timestamp_metric, LaunchTimestamp};
 use remote_storage::GenericRemoteStorage;
 use tracing::*;
 
@@ -53,8 +52,6 @@ fn version() -> String {
 }
 
 fn main() -> anyhow::Result<()> {
-    let launch_ts = Box::leak(Box::new(LaunchTimestamp::generate()));
-
     let arg_matches = cli().get_matches();
 
     if arg_matches.get_flag("enabled-features") {
@@ -88,13 +85,6 @@ fn main() -> anyhow::Result<()> {
         }
     };
 
-    // Initialize logging, which must be initialized before the custom panic hook is installed.
-    logging::init(conf.log_format)?;
-
-    // mind the order required here: 1. logging, 2. panic_hook, 3. sentry.
-    // disarming this hook on pageserver, because we never tear down tracing.
-    logging::replace_panic_hook_with_tracing_panic_hook().forget();
-
     // initialize sentry if SENTRY_DSN is provided
     let _sentry_guard = init_sentry(
         Some(GIT_VERSION.into()),
@@ -118,7 +108,7 @@ fn main() -> anyhow::Result<()> {
     virtual_file::init(conf.max_file_descriptors);
     page_cache::init(conf.page_cache_size);
 
-    start_pageserver(launch_ts, conf).context("Failed to start pageserver")?;
+    start_pageserver(conf).context("Failed to start pageserver")?;
 
     scenario.teardown();
     Ok(())
@@ -213,21 +203,13 @@ fn initialize_config(
     })
 }
 
-fn start_pageserver(
-    launch_ts: &'static LaunchTimestamp,
-    conf: &'static PageServerConf,
-) -> anyhow::Result<()> {
-    // Print version and launch timestamp to the log,
-    // and expose them as prometheus metrics.
-    // A changed version string indicates changed software.
-    // A changed launch timestamp indicates a pageserver restart.
-    info!(
-        "version: {} launch_timestamp: {}",
-        version(),
-        launch_ts.to_string()
-    );
+fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
+    // Initialize logging
+    logging::init(conf.log_format)?;
+
+    // Print version to the log, and expose it as a prometheus metric too.
+    info!("version: {}", version());
     set_build_info_metric(GIT_VERSION);
-    set_launch_timestamp_metric(launch_ts);
 
     // If any failpoints were set from FAILPOINTS environment variable,
     // print them to the log for debugging purposes
@@ -325,7 +307,7 @@ fn start_pageserver(
     {
         let _rt_guard = MGMT_REQUEST_RUNTIME.enter();
 
-        let router = http::make_router(conf, launch_ts, auth.clone(), remote_storage)?
+        let router = http::make_router(conf, auth.clone(), remote_storage)?
             .build()
             .map_err(|err| anyhow!(err))?;
         let service = utils::http::RouterService::new(router).unwrap();
@@ -365,7 +347,6 @@ fn start_pageserver(
                     pageserver::consumption_metrics::collect_metrics(
                         metric_collection_endpoint,
                         conf.metric_collection_interval,
-                        conf.cached_metric_collection_interval,
                         conf.synthetic_size_calculation_interval,
                         conf.id,
                         metrics_ctx,

pageserver/src/bin/pageserver_binutils.rs

@@ -12,9 +12,7 @@ use anyhow::Context;
 use clap::{value_parser, Arg, Command};
 
 use pageserver::{
-    context::{DownloadBehavior, RequestContext},
     page_cache,
-    task_mgr::TaskKind,
     tenant::{dump_layerfile_from_path, metadata::TimelineMetadata},
     virtual_file,
 };
@@ -77,8 +75,7 @@ fn print_layerfile(path: &Path) -> anyhow::Result<()> {
     // Basic initialization of things that don't change after startup
     virtual_file::init(10);
     page_cache::init(100);
-    let ctx = RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error);
-    dump_layerfile_from_path(path, true, &ctx)
+    dump_layerfile_from_path(path, true)
 }
 
 fn handle_metadata(path: &Path, arg_matches: &clap::ArgMatches) -> Result<(), anyhow::Error> {

pageserver/src/config.rs

@@ -29,7 +29,7 @@ use utils::{
 
 use crate::tenant::config::TenantConf;
 use crate::tenant::config::TenantConfOpt;
-use crate::tenant::{TENANT_ATTACHING_MARKER_FILENAME, TIMELINES_SEGMENT_NAME};
+use crate::tenant::{TENANT_ATTACHING_MARKER_SUFFIX, TIMELINES_SEGMENT_NAME};
 use crate::{
     IGNORED_TENANT_FILE_NAME, METADATA_FILE_NAME, TENANT_CONFIG_NAME, TIMELINE_UNINIT_MARK_SUFFIX,
 };
@@ -58,7 +58,6 @@ pub mod defaults {
         super::ConfigurableSemaphore::DEFAULT_INITIAL.get();
 
     pub const DEFAULT_METRIC_COLLECTION_INTERVAL: &str = "10 min";
-    pub const DEFAULT_CACHED_METRIC_COLLECTION_INTERVAL: &str = "1 hour";
     pub const DEFAULT_METRIC_COLLECTION_ENDPOINT: Option<reqwest::Url> = None;
     pub const DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL: &str = "10 min";
 
@@ -86,7 +85,6 @@ pub mod defaults {
 #concurrent_tenant_size_logical_size_queries = '{DEFAULT_CONCURRENT_TENANT_SIZE_LOGICAL_SIZE_QUERIES}'
 
 #metric_collection_interval = '{DEFAULT_METRIC_COLLECTION_INTERVAL}'
-#cached_metric_collection_interval = '{DEFAULT_CACHED_METRIC_COLLECTION_INTERVAL}'
 #synthetic_size_calculation_interval = '{DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL}'
 
 # [tenant_config]
@@ -156,8 +154,6 @@ pub struct PageServerConf {
 
     // How often to collect metrics and send them to the metrics endpoint.
     pub metric_collection_interval: Duration,
-    // How often to send unchanged cached metrics to the metrics endpoint.
-    pub cached_metric_collection_interval: Duration,
     pub metric_collection_endpoint: Option<Url>,
     pub synthetic_size_calculation_interval: Duration,
 
@@ -224,7 +220,6 @@ struct PageServerConfigBuilder {
     concurrent_tenant_size_logical_size_queries: BuilderValue<ConfigurableSemaphore>,
 
     metric_collection_interval: BuilderValue<Duration>,
-    cached_metric_collection_interval: BuilderValue<Duration>,
     metric_collection_endpoint: BuilderValue<Option<Url>>,
     synthetic_size_calculation_interval: BuilderValue<Duration>,
 
@@ -269,10 +264,6 @@ impl Default for PageServerConfigBuilder {
                 DEFAULT_METRIC_COLLECTION_INTERVAL,
             )
             .expect("cannot parse default metric collection interval")),
-            cached_metric_collection_interval: Set(humantime::parse_duration(
-                DEFAULT_CACHED_METRIC_COLLECTION_INTERVAL,
-            )
-            .expect("cannot parse default cached_metric_collection_interval")),
             synthetic_size_calculation_interval: Set(humantime::parse_duration(
                 DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL,
             )
@@ -362,14 +353,6 @@ impl PageServerConfigBuilder {
         self.metric_collection_interval = BuilderValue::Set(metric_collection_interval)
     }
 
-    pub fn cached_metric_collection_interval(
-        &mut self,
-        cached_metric_collection_interval: Duration,
-    ) {
-        self.cached_metric_collection_interval =
-            BuilderValue::Set(cached_metric_collection_interval)
-    }
-
     pub fn metric_collection_endpoint(&mut self, metric_collection_endpoint: Option<Url>) {
         self.metric_collection_endpoint = BuilderValue::Set(metric_collection_endpoint)
     }
@@ -444,9 +427,6 @@ impl PageServerConfigBuilder {
             metric_collection_interval: self
                 .metric_collection_interval
                 .ok_or(anyhow!("missing metric_collection_interval"))?,
-            cached_metric_collection_interval: self
-                .cached_metric_collection_interval
-                .ok_or(anyhow!("missing cached_metric_collection_interval"))?,
             metric_collection_endpoint: self
                 .metric_collection_endpoint
                 .ok_or(anyhow!("missing metric_collection_endpoint"))?,
@@ -479,8 +459,7 @@ impl PageServerConf {
     }
 
     pub fn tenant_attaching_mark_file_path(&self, tenant_id: &TenantId) -> PathBuf {
-        self.tenant_path(tenant_id)
-            .join(TENANT_ATTACHING_MARKER_FILENAME)
+        path_with_suffix_extension(self.tenant_path(tenant_id), TENANT_ATTACHING_MARKER_SUFFIX)
     }
 
     pub fn tenant_ignore_mark_file_path(&self, tenant_id: TenantId) -> PathBuf {
@@ -632,7 +611,6 @@ impl PageServerConf {
                     ConfigurableSemaphore::new(permits)
                 }),
                 "metric_collection_interval" => builder.metric_collection_interval(parse_toml_duration(key, item)?),
-                "cached_metric_collection_interval" => builder.cached_metric_collection_interval(parse_toml_duration(key, item)?),
                 "metric_collection_endpoint" => {
                     let endpoint = parse_toml_string(key, item)?.parse().context("failed to parse metric_collection_endpoint")?;
                     builder.metric_collection_endpoint(Some(endpoint));
@@ -731,13 +709,6 @@ impl PageServerConf {
                 })?);
         }
 
-        if let Some(eviction_policy) = item.get("eviction_policy") {
-            t_conf.eviction_policy = Some(
-                toml_edit::de::from_item(eviction_policy.clone())
-                    .context("parse eviction_policy")?,
-            );
-        }
-
         Ok(t_conf)
     }
 
@@ -769,7 +740,6 @@ impl PageServerConf {
             log_format: LogFormat::from_str(defaults::DEFAULT_LOG_FORMAT).unwrap(),
             concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::default(),
             metric_collection_interval: Duration::from_secs(60),
-            cached_metric_collection_interval: Duration::from_secs(60 * 60),
             metric_collection_endpoint: defaults::DEFAULT_METRIC_COLLECTION_ENDPOINT,
             synthetic_size_calculation_interval: Duration::from_secs(60),
             test_remote_failures: 0,
@@ -910,7 +880,6 @@ initial_superuser_name = 'zzzz'
 id = 10
 
 metric_collection_interval = '222 s'
-cached_metric_collection_interval = '22200 s'
 metric_collection_endpoint = 'http://localhost:80/metrics'
 synthetic_size_calculation_interval = '333 s'
 log_format = 'json'
@@ -958,9 +927,6 @@ log_format = 'json'
                 metric_collection_interval: humantime::parse_duration(
                     defaults::DEFAULT_METRIC_COLLECTION_INTERVAL
                 )?,
-                cached_metric_collection_interval: humantime::parse_duration(
-                    defaults::DEFAULT_CACHED_METRIC_COLLECTION_INTERVAL
-                )?,
                 metric_collection_endpoint: defaults::DEFAULT_METRIC_COLLECTION_ENDPOINT,
                 synthetic_size_calculation_interval: humantime::parse_duration(
                     defaults::DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL
@@ -1011,7 +977,6 @@ log_format = 'json'
                 log_format: LogFormat::Json,
                 concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::default(),
                 metric_collection_interval: Duration::from_secs(222),
-                cached_metric_collection_interval: Duration::from_secs(22200),
                 metric_collection_endpoint: Some(Url::parse("http://localhost:80/metrics")?),
                 synthetic_size_calculation_interval: Duration::from_secs(333),
                 test_remote_failures: 0,

pageserver/src/consumption_metrics.rs

@@ -25,7 +25,7 @@ const REMOTE_STORAGE_SIZE: &str = "remote_storage_size";
 const TIMELINE_LOGICAL_SIZE: &str = "timeline_logical_size";
 
 #[serde_as]
-#[derive(Serialize, Debug)]
+#[derive(Serialize)]
 struct Ids {
     #[serde_as(as = "DisplayFromStr")]
     tenant_id: TenantId,
@@ -46,12 +46,12 @@ pub struct PageserverConsumptionMetricsKey {
 pub async fn collect_metrics(
     metric_collection_endpoint: &Url,
     metric_collection_interval: Duration,
-    cached_metric_collection_interval: Duration,
     synthetic_size_calculation_interval: Duration,
     node_id: NodeId,
     ctx: RequestContext,
 ) -> anyhow::Result<()> {
     let mut ticker = tokio::time::interval(metric_collection_interval);
+
     info!("starting collect_metrics");
 
     // spin up background worker that caclulates tenant sizes
@@ -75,7 +75,6 @@ pub async fn collect_metrics(
     // define client here to reuse it for all requests
     let client = reqwest::Client::new();
     let mut cached_metrics: HashMap<PageserverConsumptionMetricsKey, u64> = HashMap::new();
-    let mut prev_iteration_time: std::time::Instant = std::time::Instant::now();
 
     loop {
         tokio::select! {
@@ -84,15 +83,10 @@ pub async fn collect_metrics(
                 return Ok(());
             },
             _ = ticker.tick() => {
-
-                // send cached metrics every cached_metric_collection_interval
-                let send_cached = prev_iteration_time.elapsed() >= cached_metric_collection_interval;
-
-                if send_cached {
-                    prev_iteration_time = std::time::Instant::now();
+                if let Err(err) = collect_metrics_iteration(&client, &mut cached_metrics, metric_collection_endpoint, node_id, &ctx).await
+                {
+                    error!("metrics collection failed: {err:?}");
                 }
-
-                collect_metrics_iteration(&client, &mut cached_metrics, metric_collection_endpoint, node_id, &ctx, send_cached).await;
             }
         }
     }
@@ -103,19 +97,17 @@ pub async fn collect_metrics(
 /// Gather per-tenant and per-timeline metrics and send them to the `metric_collection_endpoint`.
 /// Cache metrics to avoid sending the same metrics multiple times.
 ///
-/// This function handles all errors internally
-/// and doesn't break iteration if just one tenant fails.
-///
 /// TODO
 /// - refactor this function (chunking+sending part) to reuse it in proxy module;
+/// - improve error handling. Now if one tenant fails to collect metrics,
+/// the whole iteration fails and metrics for other tenants are not collected.
 pub async fn collect_metrics_iteration(
     client: &reqwest::Client,
     cached_metrics: &mut HashMap<PageserverConsumptionMetricsKey, u64>,
     metric_collection_endpoint: &reqwest::Url,
     node_id: NodeId,
     ctx: &RequestContext,
-    send_cached: bool,
-) {
+) -> anyhow::Result<()> {
     let mut current_metrics: Vec<(PageserverConsumptionMetricsKey, u64)> = Vec::new();
     trace!(
         "starting collect_metrics_iteration. metric_collection_endpoint: {}",
@@ -123,13 +115,7 @@ pub async fn collect_metrics_iteration(
     );
 
     // get list of tenants
-    let tenants = match mgr::list_tenants().await {
-        Ok(tenants) => tenants,
-        Err(err) => {
-            error!("failed to list tenants: {:?}", err);
-            return;
-        }
-    };
+    let tenants = mgr::list_tenants().await?;
 
     // iterate through list of Active tenants and collect metrics
     for (tenant_id, tenant_state) in tenants {
@@ -137,15 +123,7 @@ pub async fn collect_metrics_iteration(
             continue;
         }
 
-        let tenant = match mgr::get_tenant(tenant_id, true).await {
-            Ok(tenant) => tenant,
-            Err(err) => {
-                // It is possible that tenant was deleted between
-                // `list_tenants` and `get_tenant`, so just warn about it.
-                warn!("failed to get tenant {tenant_id:?}: {err:?}");
-                continue;
-            }
-        };
+        let tenant = mgr::get_tenant(tenant_id, true).await?;
 
         let mut tenant_resident_size = 0;
 
@@ -164,51 +142,29 @@ pub async fn collect_metrics_iteration(
                     timeline_written_size,
                 ));
 
-                match timeline.get_current_logical_size(ctx) {
-                    // Only send timeline logical size when it is fully calculated.
-                    Ok((size, is_exact)) if is_exact => {
-                        current_metrics.push((
-                            PageserverConsumptionMetricsKey {
-                                tenant_id,
-                                timeline_id: Some(timeline.timeline_id),
-                                metric: TIMELINE_LOGICAL_SIZE,
-                            },
-                            size,
-                        ));
-                    }
-                    Ok((_, _)) => {}
-                    Err(err) => {
-                        error!(
-                            "failed to get current logical size for timeline {}: {err:?}",
-                            timeline.timeline_id
-                        );
-                        continue;
-                    }
-                };
+                let (timeline_logical_size, is_exact) = timeline.get_current_logical_size(ctx)?;
+                // Only send timeline logical size when it is fully calculated.
+                if is_exact {
+                    current_metrics.push((
+                        PageserverConsumptionMetricsKey {
+                            tenant_id,
+                            timeline_id: Some(timeline.timeline_id),
+                            metric: TIMELINE_LOGICAL_SIZE,
+                        },
+                        timeline_logical_size,
+                    ));
+                }
             }
 
             let timeline_resident_size = timeline.get_resident_physical_size();
             tenant_resident_size += timeline_resident_size;
         }
 
-        match tenant.get_remote_size().await {
-            Ok(tenant_remote_size) => {
-                current_metrics.push((
-                    PageserverConsumptionMetricsKey {
-                        tenant_id,
-                        timeline_id: None,
-                        metric: REMOTE_STORAGE_SIZE,
-                    },
-                    tenant_remote_size,
-                ));
-            }
-            Err(err) => {
-                error!(
-                    "failed to get remote size for tenant {}: {err:?}",
-                    tenant_id
-                );
-            }
-        }
+        let tenant_remote_size = tenant.get_remote_size().await?;
+        debug!(
+            "collected current metrics for tenant: {}: state={:?} resident_size={} remote_size={}",
+            tenant_id, tenant_state, tenant_resident_size, tenant_remote_size
+        );
 
         current_metrics.push((
             PageserverConsumptionMetricsKey {
@@ -219,6 +175,15 @@ pub async fn collect_metrics_iteration(
             tenant_resident_size,
         ));
 
+        current_metrics.push((
+            PageserverConsumptionMetricsKey {
+                tenant_id,
+                timeline_id: None,
+                metric: REMOTE_STORAGE_SIZE,
+            },
+            tenant_remote_size,
+        ));
+
         // Note that this metric is calculated in a separate bgworker
         // Here we only use cached value, which may lag behind the real latest one
         let tenant_synthetic_size = tenant.get_cached_synthetic_size();
@@ -232,18 +197,15 @@ pub async fn collect_metrics_iteration(
         ));
     }
 
-    // Filter metrics, unless we want to send all metrics, including cached ones.
-    // See: https://github.com/neondatabase/neon/issues/3485
-    if !send_cached {
-        current_metrics.retain(|(curr_key, curr_val)| match cached_metrics.get(curr_key) {
-            Some(val) => val != curr_val,
-            None => true,
-        });
-    }
+    // Filter metrics
+    current_metrics.retain(|(curr_key, curr_val)| match cached_metrics.get(curr_key) {
+        Some(val) => val != curr_val,
+        None => true,
+    });
 
     if current_metrics.is_empty() {
         trace!("no new metrics to send");
-        return;
+        return Ok(());
     }
 
     // Send metrics.
@@ -287,12 +249,6 @@ pub async fn collect_metrics_iteration(
                     }
                 } else {
                     error!("metrics endpoint refused the sent metrics: {:?}", res);
-                    for metric in chunk_to_send.iter() {
-                        // Report if the metric value is suspiciously large
-                        if metric.value > (1u64 << 40) {
-                            error!("potentially abnormal metric value: {:?}", metric);
-                        }
-                    }
                 }
             }
             Err(err) => {
@@ -300,6 +256,8 @@ pub async fn collect_metrics_iteration(
             }
         }
     }
+
+    Ok(())
 }
 
 /// Caclculate synthetic size for each active tenant

pageserver/src/http/openapi_spec.yml

@@ -437,13 +437,6 @@ paths:
           type: boolean
         description: |
           When true, skip calculation and only provide the model inputs (for debugging). Defaults to false.
-      - name: retention_period
-        in: query
-        required: false
-        schema:
-          type: integer
-        description: |
-          Override the default retention period (in bytes) used for size calculation.
     get:
       description: |
         Calculate tenant's size, which is a mixture of WAL (bytes) and logical_size (bytes).
@@ -671,55 +664,6 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/Error"
-  /v1/tenant/{tenant_id}/config/:
-    parameters:
-      - name: tenant_id
-        in: path
-        required: true
-        schema:
-          type: string
-          format: hex
-    get:
-      description: |
-        Returns tenant's config description: specific config overrides a tenant has
-        and the effective config.
-      responses:
-        "200":
-          description: Tenant config, specific and effective
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TenantConfig"
-        "400":
-          description: Malformed get tenanant config request
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-        "401":
-          description: Unauthorized Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/UnauthorizedError"
-        "403":
-          description: Forbidden Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ForbiddenError"
-        "404":
-          description: Tenand or timeline were not found
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/NotFoundError"
-        "500":
-          description: Generic operation error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
 components:
   securitySchemes:
     JWT:
@@ -780,33 +724,10 @@ components:
           type: integer
         checkpoint_timeout:
           type: string
-        compaction_target_size:
-          type: integer
         compaction_period:
           type: string
         compaction_threshold:
           type: string
-        image_creation_threshold:
-          type: integer
-        walreceiver_connect_timeout:
-          type: string
-        lagging_wal_timeout:
-          type: string
-        max_lsn_wal_lag:
-          type: integer
-        trace_read_requests:
-          type: boolean
-    TenantConfig:
-      type: object
-      properties:
-        tenant_specific_overrides:
-          type: object
-          schema:
-            $ref: "#/components/schemas/TenantConfigInfo"
-        effective_config:
-          type: object
-          schema:
-            $ref: "#/components/schemas/TenantConfigInfo"
     TimelineInfo:
       type: object
       required:

pageserver/src/http/routes.rs

@@ -1,28 +1,22 @@
-use std::collections::HashMap;
 use std::sync::Arc;
 
 use anyhow::{anyhow, Context, Result};
 use hyper::StatusCode;
 use hyper::{Body, Request, Response, Uri};
-use metrics::launch_timestamp::LaunchTimestamp;
 use pageserver_api::models::DownloadRemoteLayersTaskSpawnRequest;
 use remote_storage::GenericRemoteStorage;
-use tenant_size_model::{SizeResult, StorageModel};
 use tokio_util::sync::CancellationToken;
 use tracing::*;
-use utils::http::request::{get_request_param, must_get_query_param, parse_query_param};
 
 use super::models::{
     StatusResponse, TenantConfigRequest, TenantCreateRequest, TenantCreateResponse, TenantInfo,
-    TimelineCreateRequest, TimelineGcRequest, TimelineInfo,
+    TimelineCreateRequest, TimelineInfo,
 };
 use crate::context::{DownloadBehavior, RequestContext};
 use crate::pgdatadir_mapping::LsnForTimestamp;
 use crate::task_mgr::TaskKind;
 use crate::tenant::config::TenantConfOpt;
 use crate::tenant::mgr::TenantMapInsertError;
-use crate::tenant::size::ModelInputs;
-use crate::tenant::storage_layer::LayerAccessStatsReset;
 use crate::tenant::{PageReconstructError, Timeline};
 use crate::{config::PageServerConf, tenant::mgr};
 use utils::{
@@ -40,7 +34,7 @@ use utils::{
 
 // Imports only used for testing APIs
 #[cfg(feature = "testing")]
-use super::models::ConfigureFailpointsRequest;
+use super::models::{ConfigureFailpointsRequest, TimelineGcRequest};
 
 struct State {
     conf: &'static PageServerConf,
@@ -93,7 +87,9 @@ fn apierror_from_prerror(err: PageReconstructError) -> ApiError {
         PageReconstructError::NeedsDownload(_, _) => {
             // This shouldn't happen, because we use a RequestContext that requests to
             // download any missing layer files on-demand.
-            ApiError::InternalServerError(anyhow::anyhow!("need to download remote layer file"))
+            ApiError::InternalServerError(anyhow::anyhow!(
+                "would need to download remote layer file"
+            ))
         }
         PageReconstructError::Cancelled => {
             ApiError::InternalServerError(anyhow::anyhow!("request was cancelled"))
@@ -241,8 +237,8 @@ async fn timeline_create_handler(mut request: Request<Body>) -> Result<Response<
 
 async fn timeline_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
-    let include_non_incremental_logical_size: Option<bool> =
-        parse_query_param(&request, "include-non-incremental-logical-size")?;
+    let include_non_incremental_logical_size =
+        query_param_present(&request, "include-non-incremental-logical-size");
     check_permission(&request, Some(tenant_id))?;
 
     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
@@ -255,14 +251,13 @@ async fn timeline_list_handler(request: Request<Body>) -> Result<Response<Body>,
 
         let mut response_data = Vec::with_capacity(timelines.len());
         for timeline in timelines {
-            let timeline_info = build_timeline_info(
-                &timeline,
-                include_non_incremental_logical_size.unwrap_or(false),
-                &ctx,
-            )
-            .await
-            .context("Failed to convert tenant timeline {timeline_id} into the local one: {e:?}")
-            .map_err(ApiError::InternalServerError)?;
+            let timeline_info =
+                build_timeline_info(&timeline, include_non_incremental_logical_size, &ctx)
+                    .await
+                    .context(
+                        "Failed to convert tenant timeline {timeline_id} into the local one: {e:?}",
+                    )
+                    .map_err(ApiError::InternalServerError)?;
 
             response_data.push(timeline_info);
         }
@@ -274,11 +269,36 @@ async fn timeline_list_handler(request: Request<Body>) -> Result<Response<Body>,
     json_response(StatusCode::OK, response_data)
 }
 
+/// Checks if a query param is present in the request's URL
+fn query_param_present(request: &Request<Body>, param: &str) -> bool {
+    request
+        .uri()
+        .query()
+        .map(|v| url::form_urlencoded::parse(v.as_bytes()).any(|(p, _)| p == param))
+        .unwrap_or(false)
+}
+
+fn get_query_param(request: &Request<Body>, param_name: &str) -> Result<String, ApiError> {
+    request.uri().query().map_or(
+        Err(ApiError::BadRequest(anyhow!("empty query in request"))),
+        |v| {
+            url::form_urlencoded::parse(v.as_bytes())
+                .find(|(k, _)| k == param_name)
+                .map_or(
+                    Err(ApiError::BadRequest(anyhow!(
+                        "no {param_name} specified in query parameters"
+                    ))),
+                    |(_, v)| Ok(v.into_owned()),
+                )
+        },
+    )
+}
+
 async fn timeline_detail_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
-    let include_non_incremental_logical_size: Option<bool> =
-        parse_query_param(&request, "include-non-incremental-logical-size")?;
+    let include_non_incremental_logical_size =
+        query_param_present(&request, "include-non-incremental-logical-size");
     check_permission(&request, Some(tenant_id))?;
 
     // Logical size calculation needs downloading.
@@ -293,14 +313,11 @@ async fn timeline_detail_handler(request: Request<Body>) -> Result<Response<Body
             .get_timeline(timeline_id, false)
             .map_err(ApiError::NotFound)?;
 
-        let timeline_info = build_timeline_info(
-            &timeline,
-            include_non_incremental_logical_size.unwrap_or(false),
-            &ctx,
-        )
-        .await
-        .context("get local timeline info")
-        .map_err(ApiError::InternalServerError)?;
+        let timeline_info =
+            build_timeline_info(&timeline, include_non_incremental_logical_size, &ctx)
+                .await
+                .context("get local timeline info")
+                .map_err(ApiError::InternalServerError)?;
 
         Ok::<_, ApiError>(timeline_info)
     }
@@ -315,14 +332,17 @@ async fn get_lsn_by_timestamp_handler(request: Request<Body>) -> Result<Response
     check_permission(&request, Some(tenant_id))?;
 
     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
-    let timestamp_raw = must_get_query_param(&request, "timestamp")?;
-    let timestamp = humantime::parse_rfc3339(&timestamp_raw)
+    let timestamp_raw = get_query_param(&request, "timestamp")?;
+    let timestamp = humantime::parse_rfc3339(timestamp_raw.as_str())
         .with_context(|| format!("Invalid time: {:?}", timestamp_raw))
         .map_err(ApiError::BadRequest)?;
     let timestamp_pg = postgres_ffi::to_pg_timestamp(timestamp);
 
     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
-    let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?;
+    let timeline = mgr::get_tenant(tenant_id, true)
+        .await
+        .and_then(|tenant| tenant.get_timeline(timeline_id, true))
+        .map_err(ApiError::NotFound)?;
     let result = timeline
         .find_lsn_for_timestamp(timestamp_pg, &ctx)
         .await
@@ -330,7 +350,7 @@ async fn get_lsn_by_timestamp_handler(request: Request<Body>) -> Result<Response
 
     let result = match result {
         LsnForTimestamp::Present(lsn) => format!("{lsn}"),
-        LsnForTimestamp::Future(lsn) => format!("{lsn}"),
+        LsnForTimestamp::Future(_lsn) => "future".into(),
         LsnForTimestamp::Past(_lsn) => "past".into(),
         LsnForTimestamp::NoData(_lsn) => "nodata".into(),
     };
@@ -481,19 +501,17 @@ async fn tenant_status(request: Request<Body>) -> Result<Response<Body>, ApiErro
 /// to debug any of the calculations. Requires `tenant_id` request parameter, supports
 /// `inputs_only=true|false` (default false) which supports debugging failure to calculate model
 /// values.
-///
-/// 'retention_period' query parameter overrides the cutoff that is used to calculate the size
-/// (only if it is shorter than the real cutoff).
-///
-/// Note: we don't update the cached size and prometheus metric here.
-/// The retention period might be different, and it's nice to have a method to just calculate it
-/// without modifying anything anyway.
 async fn tenant_size_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
     check_permission(&request, Some(tenant_id))?;
-    let inputs_only: Option<bool> = parse_query_param(&request, "inputs_only")?;
-    let retention_period: Option<u64> = parse_query_param(&request, "retention_period")?;
-    let headers = request.headers();
+
+    let inputs_only = if query_param_present(&request, "inputs_only") {
+        get_query_param(&request, "inputs_only")?
+            .parse()
+            .map_err(|_| ApiError::BadRequest(anyhow!("failed to parse inputs_only")))?
+    } else {
+        false
+    };
 
     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
     let tenant = mgr::get_tenant(tenant_id, true)
@@ -502,29 +520,20 @@ async fn tenant_size_handler(request: Request<Body>) -> Result<Response<Body>, A
 
     // this can be long operation
     let inputs = tenant
-        .gather_size_inputs(retention_period, &ctx)
+        .gather_size_inputs(&ctx)
         .await
         .map_err(ApiError::InternalServerError)?;
 
-    let mut sizes = None;
-    if !inputs_only.unwrap_or(false) {
-        let storage_model = inputs
-            .calculate_model()
-            .map_err(ApiError::InternalServerError)?;
-        let size = storage_model.calculate();
+    let size = if !inputs_only {
+        Some(inputs.calculate().map_err(ApiError::InternalServerError)?)
+    } else {
+        None
+    };
 
-        // If request header expects html, return html
-        if headers["Accept"] == "text/html" {
-            return synthetic_size_html_response(inputs, storage_model, size);
-        }
-        sizes = Some(size);
-    } else if headers["Accept"] == "text/html" {
-        return Err(ApiError::BadRequest(anyhow!(
-            "inputs_only parameter is incompatible with html output request"
-        )));
-    }
-
-    /// The type resides in the pageserver not to expose `ModelInputs`.
+    /// Private response type with the additional "unstable" `inputs` field.
+    ///
+    /// The type is described with `id` and `size` in the openapi_spec file, but the `inputs` is
+    /// intentionally left out. The type resides in the pageserver not to expose `ModelInputs`.
     #[serde_with::serde_as]
     #[derive(serde::Serialize)]
     struct TenantHistorySize {
@@ -534,9 +543,6 @@ async fn tenant_size_handler(request: Request<Body>) -> Result<Response<Body>, A
         ///
         /// Will be none if `?inputs_only=true` was given.
         size: Option<u64>,
-        /// Size of each segment used in the model.
-        /// Will be null if `?inputs_only=true` was given.
-        segment_sizes: Option<Vec<tenant_size_model::SegmentSizeResult>>,
         inputs: crate::tenant::size::ModelInputs,
     }
 
@@ -544,128 +550,12 @@ async fn tenant_size_handler(request: Request<Body>) -> Result<Response<Body>, A
         StatusCode::OK,
         TenantHistorySize {
             id: tenant_id,
-            size: sizes.as_ref().map(|x| x.total_size),
-            segment_sizes: sizes.map(|x| x.segments),
+            size,
             inputs,
         },
     )
 }
 
-async fn layer_map_info_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
-    let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
-    let reset: LayerAccessStatsReset =
-        parse_query_param(&request, "reset")?.unwrap_or(LayerAccessStatsReset::NoReset);
-
-    check_permission(&request, Some(tenant_id))?;
-
-    let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?;
-    let layer_map_info = timeline.layer_map_info(reset);
-
-    json_response(StatusCode::OK, layer_map_info)
-}
-
-async fn layer_download_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
-    check_permission(&request, Some(tenant_id))?;
-    let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
-    let layer_file_name = get_request_param(&request, "layer_file_name")?;
-    check_permission(&request, Some(tenant_id))?;
-
-    let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?;
-    let downloaded = timeline
-        .download_layer(layer_file_name)
-        .await
-        .map_err(ApiError::InternalServerError)?;
-
-    match downloaded {
-        Some(true) => json_response(StatusCode::OK, ()),
-        Some(false) => json_response(StatusCode::NOT_MODIFIED, ()),
-        None => json_response(
-            StatusCode::BAD_REQUEST,
-            format!("Layer {tenant_id}/{timeline_id}/{layer_file_name} not found"),
-        ),
-    }
-}
-
-async fn evict_timeline_layer_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
-    check_permission(&request, Some(tenant_id))?;
-    let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
-    let layer_file_name = get_request_param(&request, "layer_file_name")?;
-
-    let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?;
-    let evicted = timeline
-        .evict_layer(layer_file_name)
-        .await
-        .map_err(ApiError::InternalServerError)?;
-
-    match evicted {
-        Some(true) => json_response(StatusCode::OK, ()),
-        Some(false) => json_response(StatusCode::NOT_MODIFIED, ()),
-        None => json_response(
-            StatusCode::BAD_REQUEST,
-            format!("Layer {tenant_id}/{timeline_id}/{layer_file_name} not found"),
-        ),
-    }
-}
-
-/// Get tenant_size SVG graph along with the JSON data.
-fn synthetic_size_html_response(
-    inputs: ModelInputs,
-    storage_model: StorageModel,
-    sizes: SizeResult,
-) -> Result<Response<Body>, ApiError> {
-    let mut timeline_ids: Vec<String> = Vec::new();
-    let mut timeline_map: HashMap<TimelineId, usize> = HashMap::new();
-    for (index, ti) in inputs.timeline_inputs.iter().enumerate() {
-        timeline_map.insert(ti.timeline_id, index);
-        timeline_ids.push(ti.timeline_id.to_string());
-    }
-    let seg_to_branch: Vec<usize> = inputs
-        .segments
-        .iter()
-        .map(|seg| *timeline_map.get(&seg.timeline_id).unwrap())
-        .collect();
-
-    let svg =
-        tenant_size_model::svg::draw_svg(&storage_model, &timeline_ids, &seg_to_branch, &sizes)
-            .map_err(ApiError::InternalServerError)?;
-
-    let mut response = String::new();
-
-    use std::fmt::Write;
-    write!(response, "<html>\n<body>\n").unwrap();
-    write!(response, "<div>\n{svg}\n</div>").unwrap();
-    writeln!(response, "Project size: {}", sizes.total_size).unwrap();
-    writeln!(response, "<pre>").unwrap();
-    writeln!(
-        response,
-        "{}",
-        serde_json::to_string_pretty(&inputs).unwrap()
-    )
-    .unwrap();
-    writeln!(
-        response,
-        "{}",
-        serde_json::to_string_pretty(&sizes.segments).unwrap()
-    )
-    .unwrap();
-    writeln!(response, "</pre>").unwrap();
-    write!(response, "</body>\n</html>\n").unwrap();
-
-    html_response(StatusCode::OK, response)
-}
-
-pub fn html_response(status: StatusCode, data: String) -> Result<Response<Body>, ApiError> {
-    let response = Response::builder()
-        .status(status)
-        .header(hyper::header::CONTENT_TYPE, "text/html")
-        .body(Body::from(data.as_bytes().to_vec()))
-        .map_err(|e| ApiError::InternalServerError(e.into()))?;
-    Ok(response)
-}
-
 // Helper function to standardize the error messages we produce on bad durations
 //
 // Intended to be used with anyhow's `with_context`, e.g.:
@@ -782,40 +672,12 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
     )
 }
 
-async fn get_tenant_config_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
-    let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
-    check_permission(&request, Some(tenant_id))?;
-
-    let tenant = mgr::get_tenant(tenant_id, false)
-        .await
-        .map_err(ApiError::NotFound)?;
-
-    let response = HashMap::from([
-        (
-            "tenant_specific_overrides",
-            serde_json::to_value(tenant.tenant_specific_overrides())
-                .context("serializing tenant specific overrides")
-                .map_err(ApiError::InternalServerError)?,
-        ),
-        (
-            "effective_config",
-            serde_json::to_value(tenant.effective_config())
-                .context("serializing effective config")
-                .map_err(ApiError::InternalServerError)?,
-        ),
-    ]);
-
-    json_response(StatusCode::OK, response)
-}
-
-async fn update_tenant_config_handler(
-    mut request: Request<Body>,
-) -> Result<Response<Body>, ApiError> {
+async fn tenant_config_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
     let request_data: TenantConfigRequest = json_request(&mut request).await?;
     let tenant_id = request_data.tenant_id;
     check_permission(&request, Some(tenant_id))?;
 
-    let mut tenant_conf = TenantConfOpt::default();
+    let mut tenant_conf: TenantConfOpt = Default::default();
     if let Some(gc_period) = request_data.gc_period {
         tenant_conf.gc_period = Some(
             humantime::parse_duration(&gc_period)
@@ -850,8 +712,12 @@ async fn update_tenant_config_handler(
                 .map_err(ApiError::BadRequest)?,
         );
     }
-    tenant_conf.max_lsn_wal_lag = request_data.max_lsn_wal_lag;
-    tenant_conf.trace_read_requests = request_data.trace_read_requests;
+    if let Some(max_lsn_wal_lag) = request_data.max_lsn_wal_lag {
+        tenant_conf.max_lsn_wal_lag = Some(max_lsn_wal_lag);
+    }
+    if let Some(trace_read_requests) = request_data.trace_read_requests {
+        tenant_conf.trace_read_requests = Some(trace_read_requests);
+    }
 
     tenant_conf.checkpoint_distance = request_data.checkpoint_distance;
     if let Some(checkpoint_timeout) = request_data.checkpoint_timeout {
@@ -872,16 +738,8 @@ async fn update_tenant_config_handler(
         );
     }
 
-    if let Some(eviction_policy) = request_data.eviction_policy {
-        tenant_conf.eviction_policy = Some(
-            serde_json::from_value(eviction_policy)
-                .context("parse field `eviction_policy`")
-                .map_err(ApiError::BadRequest)?,
-        );
-    }
-
     let state = get_state(&request);
-    mgr::set_new_tenant_config(state.conf, tenant_conf, tenant_id)
+    mgr::update_tenant_config(state.conf, tenant_conf, tenant_id)
         .instrument(info_span!("tenant_config", tenant = ?tenant_id))
         .await
         // FIXME: `update_tenant_config` can fail because of both user and internal errors.
@@ -925,6 +783,7 @@ async fn failpoints_handler(mut request: Request<Body>) -> Result<Response<Body>
 }
 
 // Run GC immediately on given timeline.
+#[cfg(feature = "testing")]
 async fn timeline_gc_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
@@ -973,7 +832,12 @@ async fn timeline_checkpoint_handler(request: Request<Body>) -> Result<Response<
     check_permission(&request, Some(tenant_id))?;
 
     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
-    let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?;
+    let tenant = mgr::get_tenant(tenant_id, true)
+        .await
+        .map_err(ApiError::NotFound)?;
+    let timeline = tenant
+        .get_timeline(timeline_id, true)
+        .map_err(ApiError::NotFound)?;
     timeline
         .freeze_and_flush()
         .await
@@ -994,7 +858,12 @@ async fn timeline_download_remote_layers_handler_post(
     let body: DownloadRemoteLayersTaskSpawnRequest = json_request(&mut request).await?;
     check_permission(&request, Some(tenant_id))?;
 
-    let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?;
+    let tenant = mgr::get_tenant(tenant_id, true)
+        .await
+        .map_err(ApiError::NotFound)?;
+    let timeline = tenant
+        .get_timeline(timeline_id, true)
+        .map_err(ApiError::NotFound)?;
     match timeline.spawn_download_all_remote_layers(body).await {
         Ok(st) => json_response(StatusCode::ACCEPTED, st),
         Err(st) => json_response(StatusCode::CONFLICT, st),
@@ -1005,10 +874,15 @@ async fn timeline_download_remote_layers_handler_get(
     request: Request<Body>,
 ) -> Result<Response<Body>, ApiError> {
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
-    check_permission(&request, Some(tenant_id))?;
     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
+    check_permission(&request, Some(tenant_id))?;
 
-    let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?;
+    let tenant = mgr::get_tenant(tenant_id, true)
+        .await
+        .map_err(ApiError::NotFound)?;
+    let timeline = tenant
+        .get_timeline(timeline_id, true)
+        .map_err(ApiError::NotFound)?;
     let info = timeline
         .get_download_all_remote_layers_task_info()
         .context("task never started since last pageserver process start")
@@ -1016,29 +890,6 @@ async fn timeline_download_remote_layers_handler_get(
     json_response(StatusCode::OK, info)
 }
 
-async fn active_timeline_of_active_tenant(
-    tenant_id: TenantId,
-    timeline_id: TimelineId,
-) -> Result<Arc<Timeline>, ApiError> {
-    let tenant = mgr::get_tenant(tenant_id, true)
-        .await
-        .map_err(ApiError::NotFound)?;
-    tenant
-        .get_timeline(timeline_id, true)
-        .map_err(ApiError::NotFound)
-}
-
-async fn always_panic_handler(req: Request<Body>) -> Result<Response<Body>, ApiError> {
-    // Deliberately cause a panic to exercise the panic hook registered via std::panic::set_hook().
-    // For pageserver, the relevant panic hook is `tracing_panic_hook` , and the `sentry` crate's wrapper around it.
-    // Use catch_unwind to ensure that tokio nor hyper are distracted by our panic.
-    let query = req.uri().query();
-    let _ = std::panic::catch_unwind(|| {
-        panic!("unconditional panic for testing panic hook integration; request query: {query:?}")
-    });
-    json_response(StatusCode::NO_CONTENT, ())
-}
-
 async fn handler_404(_: Request<Body>) -> Result<Response<Body>, ApiError> {
     json_response(
         StatusCode::NOT_FOUND,
@@ -1048,7 +899,6 @@ async fn handler_404(_: Request<Body>) -> Result<Response<Body>, ApiError> {
 
 pub fn make_router(
     conf: &'static PageServerConf,
-    launch_ts: &'static LaunchTimestamp,
     auth: Option<Arc<JwtAuth>>,
     remote_storage: Option<GenericRemoteStorage>,
 ) -> anyhow::Result<RouterBuilder<hyper::Body, ApiError>> {
@@ -1065,14 +915,6 @@ pub fn make_router(
         }))
     }
 
-    router = router.middleware(
-        endpoint::add_response_header_middleware(
-            "PAGESERVER_LAUNCH_TIMESTAMP",
-            &launch_ts.to_string(),
-        )
-        .expect("construct launch timestamp header middleware"),
-    );
-
     macro_rules! testing_api {
         ($handler_desc:literal, $handler:path $(,)?) => {{
             #[cfg(not(feature = "testing"))]
@@ -1104,9 +946,8 @@ pub fn make_router(
         .get("/v1/tenant", tenant_list_handler)
         .post("/v1/tenant", tenant_create_handler)
         .get("/v1/tenant/:tenant_id", tenant_status)
-        .get("/v1/tenant/:tenant_id/synthetic_size", tenant_size_handler)
-        .put("/v1/tenant/config", update_tenant_config_handler)
-        .get("/v1/tenant/:tenant_id/config", get_tenant_config_handler)
+        .get("/v1/tenant/:tenant_id/size", tenant_size_handler)
+        .put("/v1/tenant/config", tenant_config_handler)
         .get("/v1/tenant/:tenant_id/timeline", timeline_list_handler)
         .post("/v1/tenant/:tenant_id/timeline", timeline_create_handler)
         .post("/v1/tenant/:tenant_id/attach", tenant_attach_handler)
@@ -1123,7 +964,7 @@ pub fn make_router(
         )
         .put(
             "/v1/tenant/:tenant_id/timeline/:timeline_id/do_gc",
-            timeline_gc_handler,
+            testing_api!("run timeline GC", timeline_gc_handler),
         )
         .put(
             "/v1/tenant/:tenant_id/timeline/:timeline_id/compact",
@@ -1145,18 +986,5 @@ pub fn make_router(
             "/v1/tenant/:tenant_id/timeline/:timeline_id",
             timeline_delete_handler,
         )
-        .get(
-            "/v1/tenant/:tenant_id/timeline/:timeline_id/layer",
-            layer_map_info_handler,
-        )
-        .get(
-            "/v1/tenant/:tenant_id/timeline/:timeline_id/layer/:layer_file_name",
-            layer_download_handler,
-        )
-        .delete(
-            "/v1/tenant/:tenant_id/timeline/:timeline_id/layer/:layer_file_name",
-            evict_timeline_layer_handler,
-        )
-        .get("/v1/panic", always_panic_handler)
         .any(handler_404))
 }

pageserver/src/metrics.rs

@@ -1,12 +1,10 @@
 use metrics::core::{AtomicU64, GenericCounter};
 use metrics::{
-    register_counter_vec, register_histogram, register_histogram_vec, register_int_counter,
-    register_int_counter_vec, register_int_gauge, register_int_gauge_vec, register_uint_gauge_vec,
-    Counter, CounterVec, Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec,
-    UIntGauge, UIntGaugeVec,
+    register_histogram, register_histogram_vec, register_int_counter, register_int_counter_vec,
+    register_int_gauge, register_int_gauge_vec, register_uint_gauge_vec, Histogram, HistogramVec,
+    IntCounter, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
 };
 use once_cell::sync::Lazy;
-use pageserver_api::models::state;
 use utils::id::{TenantId, TimelineId};
 
 /// Prometheus histogram buckets (in seconds) that capture the majority of
@@ -37,29 +35,11 @@ const STORAGE_TIME_OPERATIONS: &[&str] = &[
     "gc",
 ];
 
-pub static STORAGE_TIME_SUM_PER_TIMELINE: Lazy<CounterVec> = Lazy::new(|| {
-    register_counter_vec!(
-        "pageserver_storage_operations_seconds_sum",
-        "Total time spent on storage operations with operation, tenant and timeline dimensions",
-        &["operation", "tenant_id", "timeline_id"],
-    )
-    .expect("failed to define a metric")
-});
-
-pub static STORAGE_TIME_COUNT_PER_TIMELINE: Lazy<IntCounterVec> = Lazy::new(|| {
-    register_int_counter_vec!(
-        "pageserver_storage_operations_seconds_count",
-        "Count of storage operations with operation, tenant and timeline dimensions",
-        &["operation", "tenant_id", "timeline_id"],
-    )
-    .expect("failed to define a metric")
-});
-
-pub static STORAGE_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
+pub static STORAGE_TIME: Lazy<HistogramVec> = Lazy::new(|| {
     register_histogram_vec!(
-        "pageserver_storage_operations_seconds_global",
+        "pageserver_storage_operations_seconds",
         "Time spent on storage operations",
-        &["operation"],
+        &["operation", "tenant_id", "timeline_id"],
         get_buckets_for_critical_operations(),
     )
     .expect("failed to define a metric")
@@ -132,33 +112,6 @@ static CURRENT_LOGICAL_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
     .expect("failed to define current logical size metric")
 });
 
-// Metrics collected on tenant states.
-const TENANT_STATE_OPTIONS: &[&str] = &[
-    state::LOADING,
-    state::ATTACHING,
-    state::ACTIVE,
-    state::STOPPING,
-    state::BROKEN,
-];
-
-pub static TENANT_STATE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
-    register_uint_gauge_vec!(
-        "pageserver_tenant_states_count",
-        "Count of tenants per state",
-        &["tenant_id", "state"]
-    )
-    .expect("Failed to register pageserver_tenant_states_count metric")
-});
-
-pub static TENANT_SYNTHETIC_SIZE_METRIC: Lazy<UIntGaugeVec> = Lazy::new(|| {
-    register_uint_gauge_vec!(
-        "pageserver_tenant_synthetic_size",
-        "Synthetic size of each tenant",
-        &["tenant_id"]
-    )
-    .expect("Failed to register pageserver_tenant_synthetic_size metric")
-});
-
 // Metrics for cloud upload. These metrics reflect data uploaded to cloud storage,
 // or in testing they estimate how much we would upload if we did.
 static NUM_PERSISTENT_FILES_CREATED: Lazy<IntCounterVec> = Lazy::new(|| {
@@ -422,81 +375,18 @@ pub static WAL_REDO_RECORD_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
     .unwrap()
 });
 
-/// Similar to [`prometheus::HistogramTimer`] but does not record on drop.
-pub struct StorageTimeMetricsTimer {
-    metrics: StorageTimeMetrics,
-    start: Instant,
-}
-
-impl StorageTimeMetricsTimer {
-    fn new(metrics: StorageTimeMetrics) -> Self {
-        Self {
-            metrics,
-            start: Instant::now(),
-        }
-    }
-
-    /// Record the time from creation to now.
-    pub fn stop_and_record(self) {
-        let duration = self.start.elapsed().as_secs_f64();
-        self.metrics.timeline_sum.inc_by(duration);
-        self.metrics.timeline_count.inc();
-        self.metrics.global_histogram.observe(duration);
-    }
-}
-
-/// Timing facilities for an globally histogrammed metric, which is supported by per tenant and
-/// timeline total sum and count.
-#[derive(Clone, Debug)]
-pub struct StorageTimeMetrics {
-    /// Sum of f64 seconds, per operation, tenant_id and timeline_id
-    timeline_sum: Counter,
-    /// Number of oeprations, per operation, tenant_id and timeline_id
-    timeline_count: IntCounter,
-    /// Global histogram having only the "operation" label.
-    global_histogram: Histogram,
-}
-
-impl StorageTimeMetrics {
-    pub fn new(operation: &str, tenant_id: &str, timeline_id: &str) -> Self {
-        let timeline_sum = STORAGE_TIME_SUM_PER_TIMELINE
-            .get_metric_with_label_values(&[operation, tenant_id, timeline_id])
-            .unwrap();
-        let timeline_count = STORAGE_TIME_COUNT_PER_TIMELINE
-            .get_metric_with_label_values(&[operation, tenant_id, timeline_id])
-            .unwrap();
-        let global_histogram = STORAGE_TIME_GLOBAL
-            .get_metric_with_label_values(&[operation])
-            .unwrap();
-
-        StorageTimeMetrics {
-            timeline_sum,
-            timeline_count,
-            global_histogram,
-        }
-    }
-
-    /// Starts timing a new operation.
-    ///
-    /// Note: unlike [`prometheus::HistogramTimer`] the returned timer does not record on drop.
-    pub fn start_timer(&self) -> StorageTimeMetricsTimer {
-        StorageTimeMetricsTimer::new(self.clone())
-    }
-}
-
 #[derive(Debug)]
 pub struct TimelineMetrics {
     tenant_id: String,
     timeline_id: String,
     pub reconstruct_time_histo: Histogram,
     pub materialized_page_cache_hit_counter: GenericCounter<AtomicU64>,
-    pub flush_time_histo: StorageTimeMetrics,
-    pub compact_time_histo: StorageTimeMetrics,
-    pub create_images_time_histo: StorageTimeMetrics,
-    pub init_logical_size_histo: StorageTimeMetrics,
-    pub logical_size_histo: StorageTimeMetrics,
-    pub load_layer_map_histo: StorageTimeMetrics,
-    pub garbage_collect_histo: StorageTimeMetrics,
+    pub flush_time_histo: Histogram,
+    pub compact_time_histo: Histogram,
+    pub create_images_time_histo: Histogram,
+    pub init_logical_size_histo: Histogram,
+    pub logical_size_histo: Histogram,
+    pub load_layer_map_histo: Histogram,
     pub last_record_gauge: IntGauge,
     pub wait_lsn_time_histo: Histogram,
     pub resident_physical_size_gauge: UIntGauge,
@@ -516,16 +406,24 @@ impl TimelineMetrics {
         let materialized_page_cache_hit_counter = MATERIALIZED_PAGE_CACHE_HIT
             .get_metric_with_label_values(&[&tenant_id, &timeline_id])
             .unwrap();
-        let flush_time_histo = StorageTimeMetrics::new("layer flush", &tenant_id, &timeline_id);
-        let compact_time_histo = StorageTimeMetrics::new("compact", &tenant_id, &timeline_id);
-        let create_images_time_histo =
-            StorageTimeMetrics::new("create images", &tenant_id, &timeline_id);
-        let init_logical_size_histo =
-            StorageTimeMetrics::new("init logical size", &tenant_id, &timeline_id);
-        let logical_size_histo = StorageTimeMetrics::new("logical size", &tenant_id, &timeline_id);
-        let load_layer_map_histo =
-            StorageTimeMetrics::new("load layer map", &tenant_id, &timeline_id);
-        let garbage_collect_histo = StorageTimeMetrics::new("gc", &tenant_id, &timeline_id);
+        let flush_time_histo = STORAGE_TIME
+            .get_metric_with_label_values(&["layer flush", &tenant_id, &timeline_id])
+            .unwrap();
+        let compact_time_histo = STORAGE_TIME
+            .get_metric_with_label_values(&["compact", &tenant_id, &timeline_id])
+            .unwrap();
+        let create_images_time_histo = STORAGE_TIME
+            .get_metric_with_label_values(&["create images", &tenant_id, &timeline_id])
+            .unwrap();
+        let init_logical_size_histo = STORAGE_TIME
+            .get_metric_with_label_values(&["init logical size", &tenant_id, &timeline_id])
+            .unwrap();
+        let logical_size_histo = STORAGE_TIME
+            .get_metric_with_label_values(&["logical size", &tenant_id, &timeline_id])
+            .unwrap();
+        let load_layer_map_histo = STORAGE_TIME
+            .get_metric_with_label_values(&["load layer map", &tenant_id, &timeline_id])
+            .unwrap();
         let last_record_gauge = LAST_RECORD_LSN
             .get_metric_with_label_values(&[&tenant_id, &timeline_id])
             .unwrap();
@@ -555,7 +453,6 @@ impl TimelineMetrics {
             create_images_time_histo,
             init_logical_size_histo,
             logical_size_histo,
-            garbage_collect_histo,
             load_layer_map_histo,
             last_record_gauge,
             wait_lsn_time_histo,
@@ -581,10 +478,7 @@ impl Drop for TimelineMetrics {
         let _ = PERSISTENT_BYTES_WRITTEN.remove_label_values(&[tenant_id, timeline_id]);
 
         for op in STORAGE_TIME_OPERATIONS {
-            let _ =
-                STORAGE_TIME_SUM_PER_TIMELINE.remove_label_values(&[op, tenant_id, timeline_id]);
-            let _ =
-                STORAGE_TIME_COUNT_PER_TIMELINE.remove_label_values(&[op, tenant_id, timeline_id]);
+            let _ = STORAGE_TIME.remove_label_values(&[op, tenant_id, timeline_id]);
         }
         for op in STORAGE_IO_TIME_OPERATIONS {
             let _ = STORAGE_IO_TIME.remove_label_values(&[op, tenant_id, timeline_id]);
@@ -601,11 +495,7 @@ impl Drop for TimelineMetrics {
 }
 
 pub fn remove_tenant_metrics(tenant_id: &TenantId) {
-    let tid = tenant_id.to_string();
-    let _ = TENANT_SYNTHETIC_SIZE_METRIC.remove_label_values(&[&tid]);
-    for state in TENANT_STATE_OPTIONS {
-        let _ = TENANT_STATE_METRIC.remove_label_values(&[&tid, state]);
-    }
+    let _ = STORAGE_TIME.remove_label_values(&["gc", &tenant_id.to_string(), "-"]);
 }
 
 use futures::Future;

pageserver/src/pgdatadir_mapping.rs

@@ -290,7 +290,6 @@ impl Timeline {
         }
     }
 
-    ///
     /// Locate LSN, such that all transactions that committed before
     /// 'search_timestamp' are visible, but nothing newer is.
     ///
@@ -304,11 +303,7 @@ impl Timeline {
         ctx: &RequestContext,
     ) -> Result<LsnForTimestamp, PageReconstructError> {
         let gc_cutoff_lsn_guard = self.get_latest_gc_cutoff_lsn();
-        // We use this method to figure out the branching LSN for new branch, but
-        // GC cutoff could be before branching point and we cannot create new branch
-        // with LSN < `ancestor_lsn`. Thus, pick the maximum of these two just to be
-        // on the safe side.
-        let min_lsn = std::cmp::max(*gc_cutoff_lsn_guard, self.get_ancestor_lsn());
+        let min_lsn = *gc_cutoff_lsn_guard;
         let max_lsn = self.get_last_record_lsn();
 
         // LSNs are always 8-byte aligned. low/mid/high represent the
@@ -332,22 +327,12 @@ impl Timeline {
                 )
                 .await?;
 
-            if !cmp {
-                // We either 1) found commit with timestamp **before** `search_timestamp`;
-                // or 2) we haven't found any commit records at all.
-                // Search with a larger LSN, in case 1) to try to find a more recent commit
-                // (but still **before** target timestamp); and in case 2) to fetch more
-                // SLRU segments for `clog`.
-                low = mid + 1;
-            } else {
-                // We found only more recent commits, search in the older range.
+            if cmp {
                 high = mid;
+            } else {
+                low = mid + 1;
             }
         }
-        // If `found_smaller == true`, `low` is the LSN of the first commit record
-        // **before** the `search_timestamp` + 1 (to hit the while loop exit condition).
-        // Substitute 1 to get exactly the commit LSN.
-        let commit_lsn = Lsn((low - 1) * 8);
         match (found_smaller, found_larger) {
             (false, false) => {
                 // This can happen if no commit records have been processed yet, e.g.
@@ -355,26 +340,32 @@ impl Timeline {
                 Ok(LsnForTimestamp::NoData(max_lsn))
             }
             (true, false) => {
-                // Only found a commit with timestamp smaller than the request.
-                // It's still a valid case for branch creation, return it.
-                // And `update_gc_info()` ignores LSN for a `LsnForTimestamp::Future`
-                // case, anyway.
-                Ok(LsnForTimestamp::Future(commit_lsn))
+                // Didn't find any commit timestamps larger than the request
+                Ok(LsnForTimestamp::Future(max_lsn))
             }
             (false, true) => {
                 // Didn't find any commit timestamps smaller than the request
                 Ok(LsnForTimestamp::Past(max_lsn))
             }
-            (true, true) => Ok(LsnForTimestamp::Present(commit_lsn)),
+            (true, true) => {
+                // low is the LSN of the first commit record *after* the search_timestamp,
+                // Back off by one to get to the point just before the commit.
+                //
+                // FIXME: it would be better to get the LSN of the previous commit.
+                // Otherwise, if you restore to the returned LSN, the database will
+                // include physical changes from later commits that will be marked
+                // as aborted, and will need to be vacuumed away.
+                Ok(LsnForTimestamp::Present(Lsn((low - 1) * 8)))
+            }
         }
     }
 
     ///
-    /// Subroutine of `find_lsn_for_timestamp()`. Returns `true`, if there are any
-    /// commits that committed after `search_timestamp`, at LSN `probe_lsn`.
+    /// Subroutine of find_lsn_for_timestamp(). Returns true, if there are any
+    /// commits that committed after 'search_timestamp', at LSN 'probe_lsn'.
     ///
-    /// Additionally, sets `found_smaller` / `found_larger`, if encounters any commits
-    /// with a smaller / larger timestamp.
+    /// Additionally, sets 'found_smaller'/'found_Larger, if encounters any commits
+    /// with a smaller/larger timestamp.
     ///
     pub async fn is_latest_commit_timestamp_ge_than(
         &self,

pageserver/src/repository.rs

@@ -7,11 +7,11 @@ use std::fmt;
 use std::ops::{AddAssign, Range};
 use std::time::Duration;
 
+#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize)]
 /// Key used in the Repository kv-store.
 ///
 /// The Repository treats this as an opaque struct, but see the code in pgdatadir_mapping.rs
 /// for what we actually store in these fields.
-#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize)]
 pub struct Key {
     pub field1: u8,
     pub field2: u32,

pageserver/src/task_mgr.rs

@@ -169,14 +169,7 @@ task_local! {
 /// Note that we don't try to limit how many task of a certain kind can be running
 /// at the same time.
 ///
-#[derive(
-    Debug,
-    // NB: enumset::EnumSetType derives PartialEq, Eq, Clone, Copy
-    enumset::EnumSetType,
-    serde::Serialize,
-    serde::Deserialize,
-    strum_macros::IntoStaticStr,
-)]
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
 pub enum TaskKind {
     // Pageserver startup, i.e., `main`
     Startup,
@@ -231,9 +224,6 @@ pub enum TaskKind {
     // Compaction. One per tenant.
     Compaction,
 
-    // Eviction. One per timeline.
-    Eviction,
-
     // Initial logical size calculation
     InitialLogicalSizeCalculation,
 
@@ -248,11 +238,17 @@ pub enum TaskKind {
     // Task that downloads a file from remote storage
     RemoteDownloadTask,
 
-    // task that handles the initial downloading of all tenants
-    InitialLoad,
+    // task that handles loading of a tenant during pageserver startup
+    TenantLoadStartup,
+
+    // task that handles loading of a tenant in response to a /load HTTP API request
+    TenantLoadApi,
+
+    // task that handles loading of a tenant as part of the tenant creation procedure
+    TenantLoadCreate,
 
     // task that handles attaching a tenant
-    Attach,
+    TenantAttach,
 
     // task that handhes metrics collection
     MetricsCollection,
@@ -265,8 +261,6 @@ pub enum TaskKind {
     // A request that comes in via the pageserver HTTP API.
     MgmtRequest,
 
-    DebugTool,
-
     #[cfg(test)]
     UnitTest,
 }

pageserver/src/tenant/config.rs

@@ -51,7 +51,6 @@ pub struct TenantConf {
     pub checkpoint_distance: u64,
     // Inmemory layer is also flushed at least once in checkpoint_timeout to
     // eventually upload WAL after activity is stopped.
-    #[serde(with = "humantime_serde")]
     pub checkpoint_timeout: Duration,
     // Target file size, when creating image and delta layers.
     // This parameter determines L1 layer file size.
@@ -91,97 +90,30 @@ pub struct TenantConf {
     /// to avoid eager reconnects.
     pub max_lsn_wal_lag: NonZeroU64,
     pub trace_read_requests: bool,
-    pub eviction_policy: EvictionPolicy,
 }
 
 /// Same as TenantConf, but this struct preserves the information about
 /// which parameters are set and which are not.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
 pub struct TenantConfOpt {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(default)]
     pub checkpoint_distance: Option<u64>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(default)]
     pub checkpoint_timeout: Option<Duration>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(default)]
     pub compaction_target_size: Option<u64>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(with = "humantime_serde")]
-    #[serde(default)]
     pub compaction_period: Option<Duration>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(default)]
     pub compaction_threshold: Option<usize>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(default)]
     pub gc_horizon: Option<u64>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(with = "humantime_serde")]
-    #[serde(default)]
     pub gc_period: Option<Duration>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(default)]
     pub image_creation_threshold: Option<usize>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(with = "humantime_serde")]
-    #[serde(default)]
     pub pitr_interval: Option<Duration>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(with = "humantime_serde")]
-    #[serde(default)]
     pub walreceiver_connect_timeout: Option<Duration>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(with = "humantime_serde")]
-    #[serde(default)]
     pub lagging_wal_timeout: Option<Duration>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(default)]
     pub max_lsn_wal_lag: Option<NonZeroU64>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(default)]
     pub trace_read_requests: Option<bool>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(default)]
-    pub eviction_policy: Option<EvictionPolicy>,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(tag = "kind")]
-pub enum EvictionPolicy {
-    NoEviction,
-    LayerAccessThreshold(EvictionPolicyLayerAccessThreshold),
-}
-
-impl EvictionPolicy {
-    pub fn discriminant_str(&self) -> &'static str {
-        match self {
-            EvictionPolicy::NoEviction => "NoEviction",
-            EvictionPolicy::LayerAccessThreshold(_) => "LayerAccessThreshold",
-        }
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
-pub struct EvictionPolicyLayerAccessThreshold {
-    #[serde(with = "humantime_serde")]
-    pub period: Duration,
-    #[serde(with = "humantime_serde")]
-    pub threshold: Duration,
 }
 
 impl TenantConfOpt {
@@ -218,7 +150,6 @@ impl TenantConfOpt {
             trace_read_requests: self
                 .trace_read_requests
                 .unwrap_or(global_conf.trace_read_requests),
-            eviction_policy: self.eviction_policy.unwrap_or(global_conf.eviction_policy),
         }
     }
 
@@ -291,28 +222,6 @@ impl Default for TenantConf {
             max_lsn_wal_lag: NonZeroU64::new(DEFAULT_MAX_WALRECEIVER_LSN_WAL_LAG)
                 .expect("cannot parse default max walreceiver Lsn wal lag"),
             trace_read_requests: false,
-            eviction_policy: EvictionPolicy::NoEviction,
         }
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn de_serializing_pageserver_config_omits_empty_values() {
-        let small_conf = TenantConfOpt {
-            gc_horizon: Some(42),
-            ..TenantConfOpt::default()
-        };
-
-        let toml_form = toml_edit::easy::to_string(&small_conf).unwrap();
-        assert_eq!(toml_form, "gc_horizon = 42\n");
-        assert_eq!(small_conf, toml_edit::easy::from_str(&toml_form).unwrap());
-
-        let json_form = serde_json::to_string(&small_conf).unwrap();
-        assert_eq!(json_form, "{\"gc_horizon\":42}");
-        assert_eq!(small_conf, serde_json::from_str(&json_form).unwrap());
-    }
-}

pageserver/src/tenant/layer_map.rs

@@ -46,7 +46,6 @@
 mod historic_layer_coverage;
 mod layer_coverage;
 
-use crate::context::RequestContext;
 use crate::keyspace::KeyPartitioning;
 use crate::metrics::NUM_ONDISK_LAYERS;
 use crate::repository::Key;
@@ -59,7 +58,6 @@ use std::sync::Arc;
 use utils::lsn::Lsn;
 
 use historic_layer_coverage::BufferedHistoricLayerCoverage;
-pub use historic_layer_coverage::Replacement;
 
 use super::storage_layer::range_eq;
 
@@ -139,30 +137,6 @@ where
         self.layer_map.remove_historic_noflush(layer)
     }
 
-    /// Replaces existing layer iff it is the `expected`.
-    ///
-    /// If the expected layer has been removed it will not be inserted by this function.
-    ///
-    /// Returned `Replacement` describes succeeding in replacement or the reason why it could not
-    /// be done.
-    ///
-    /// TODO replacement can be done without buffering and rebuilding layer map updates.
-    ///      One way to do that is to add a layer of indirection for returned values, so
-    ///      that we can replace values only by updating a hashmap.
-    pub fn replace_historic(
-        &mut self,
-        expected: &Arc<L>,
-        new: Arc<L>,
-    ) -> anyhow::Result<Replacement<Arc<L>>> {
-        fail::fail_point!("layermap-replace-notfound", |_| Ok(
-            // this is not what happens if an L0 layer was not found a anyhow error but perhaps
-            // that should be changed. this is good enough to show a replacement failure.
-            Replacement::NotFound
-        ));
-
-        self.layer_map.replace_historic_noflush(expected, new)
-    }
-
     // We will flush on drop anyway, but this method makes it
     // more explicit that there is some work being done.
     /// Apply all updates
@@ -279,8 +253,14 @@ where
     /// Helper function for BatchedUpdates::insert_historic
     ///
     pub(self) fn insert_historic_noflush(&mut self, layer: Arc<L>) {
+        let kr = layer.get_key_range();
+        let lr = layer.get_lsn_range();
         self.historic.insert(
-            historic_layer_coverage::LayerKey::from(&*layer),
+            historic_layer_coverage::LayerKey {
+                key: kr.start.to_i128()..kr.end.to_i128(),
+                lsn: lr.start.0..lr.end.0,
+                is_image: !layer.is_incremental(),
+            },
             Arc::clone(&layer),
         );
 
@@ -297,72 +277,30 @@ where
     /// Helper function for BatchedUpdates::remove_historic
     ///
     pub fn remove_historic_noflush(&mut self, layer: Arc<L>) {
-        self.historic
-            .remove(historic_layer_coverage::LayerKey::from(&*layer));
+        let kr = layer.get_key_range();
+        let lr = layer.get_lsn_range();
+        self.historic.remove(historic_layer_coverage::LayerKey {
+            key: kr.start.to_i128()..kr.end.to_i128(),
+            lsn: lr.start.0..lr.end.0,
+            is_image: !layer.is_incremental(),
+        });
 
         if Self::is_l0(&layer) {
             let len_before = self.l0_delta_layers.len();
+
+            // FIXME: ptr_eq might fail to return true for 'dyn'
+            // references.  Clippy complains about this. In practice it
+            // seems to work, the assertion below would be triggered
+            // otherwise but this ought to be fixed.
+            #[allow(clippy::vtable_address_comparisons)]
             self.l0_delta_layers
-                .retain(|other| !Self::compare_arced_layers(other, &layer));
-            // this assertion is related to use of Arc::ptr_eq in Self::compare_arced_layers,
-            // there's a chance that the comparison fails at runtime due to it comparing (pointer,
-            // vtable) pairs.
-            assert_eq!(
-                self.l0_delta_layers.len(),
-                len_before - 1,
-                "failed to locate removed historic layer from l0_delta_layers"
-            );
+                .retain(|other| !Arc::ptr_eq(other, &layer));
+            assert_eq!(self.l0_delta_layers.len(), len_before - 1);
         }
 
         NUM_ONDISK_LAYERS.dec();
     }
 
-    pub(self) fn replace_historic_noflush(
-        &mut self,
-        expected: &Arc<L>,
-        new: Arc<L>,
-    ) -> anyhow::Result<Replacement<Arc<L>>> {
-        let key = historic_layer_coverage::LayerKey::from(&**expected);
-        let other = historic_layer_coverage::LayerKey::from(&*new);
-
-        let expected_l0 = Self::is_l0(expected);
-        let new_l0 = Self::is_l0(&new);
-
-        anyhow::ensure!(
-            key == other,
-            "expected and new must have equal LayerKeys: {key:?} != {other:?}"
-        );
-
-        anyhow::ensure!(
-            expected_l0 == new_l0,
-            "expected and new must both be l0 deltas or neither should be: {expected_l0} != {new_l0}"
-        );
-
-        let l0_index = if expected_l0 {
-            // find the index in case replace worked, we need to replace that as well
-            Some(
-                self.l0_delta_layers
-                    .iter()
-                    .position(|slot| Self::compare_arced_layers(slot, expected))
-                    .ok_or_else(|| anyhow::anyhow!("existing l0 delta layer was not found"))?,
-            )
-        } else {
-            None
-        };
-
-        let replaced = self.historic.replace(&key, new.clone(), |existing| {
-            Self::compare_arced_layers(existing, expected)
-        });
-
-        if let Replacement::Replaced { .. } = &replaced {
-            if let Some(index) = l0_index {
-                self.l0_delta_layers[index] = new;
-            }
-        }
-
-        Ok(replaced)
-    }
-
     /// Helper function for BatchedUpdates::drop.
     pub(self) fn flush_updates(&mut self) {
         self.historic.rebuild();
@@ -716,139 +654,24 @@ where
 
     /// debugging function to print out the contents of the layer map
     #[allow(unused)]
-    pub fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()> {
+    pub fn dump(&self, verbose: bool) -> Result<()> {
         println!("Begin dump LayerMap");
 
         println!("open_layer:");
         if let Some(open_layer) = &self.open_layer {
-            open_layer.dump(verbose, ctx)?;
+            open_layer.dump(verbose)?;
         }
 
         println!("frozen_layers:");
         for frozen_layer in self.frozen_layers.iter() {
-            frozen_layer.dump(verbose, ctx)?;
+            frozen_layer.dump(verbose)?;
         }
 
         println!("historic_layers:");
         for layer in self.iter_historic_layers() {
-            layer.dump(verbose, ctx)?;
+            layer.dump(verbose)?;
         }
         println!("End dump LayerMap");
         Ok(())
     }
-
-    /// Similar to `Arc::ptr_eq`, but only compares the object pointers, not vtables.
-    ///
-    /// Returns `true` if the two `Arc` point to the same layer, false otherwise.
-    #[inline(always)]
-    pub fn compare_arced_layers(left: &Arc<L>, right: &Arc<L>) -> bool {
-        // "dyn Trait" objects are "fat pointers" in that they have two components:
-        // - pointer to the object
-        // - pointer to the vtable
-        //
-        // rust does not provide a guarantee that these vtables are unique, but however
-        // `Arc::ptr_eq` as of writing (at least up to 1.67) uses a comparison where both the
-        // pointer and the vtable need to be equal.
-        //
-        // See: https://github.com/rust-lang/rust/issues/103763
-        //
-        // A future version of rust will most likely use this form below, where we cast each
-        // pointer into a pointer to unit, which drops the inaccessible vtable pointer, making it
-        // not affect the comparison.
-        //
-        // See: https://github.com/rust-lang/rust/pull/106450
-        let left = Arc::as_ptr(left) as *const ();
-        let right = Arc::as_ptr(right) as *const ();
-
-        left == right
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::{LayerMap, Replacement};
-    use crate::tenant::storage_layer::{Layer, LayerDescriptor, LayerFileName};
-    use std::str::FromStr;
-    use std::sync::Arc;
-
-    mod l0_delta_layers_updated {
-
-        use super::*;
-
-        #[test]
-        fn for_full_range_delta() {
-            // l0_delta_layers are used by compaction, and should observe all buffered updates
-            l0_delta_layers_updated_scenario(
-                "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000053423C21-0000000053424D69",
-                true
-            )
-        }
-
-        #[test]
-        fn for_non_full_range_delta() {
-            // has minimal uncovered areas compared to l0_delta_layers_updated_on_insert_replace_remove_for_full_range_delta
-            l0_delta_layers_updated_scenario(
-                "000000000000000000000000000000000001-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE__0000000053423C21-0000000053424D69",
-                // because not full range
-                false
-            )
-        }
-
-        #[test]
-        fn for_image() {
-            l0_delta_layers_updated_scenario(
-                "000000000000000000000000000000000000-000000000000000000000000000000010000__0000000053424D69",
-                // code only checks if it is a full range layer, doesn't care about images, which must
-                // mean we should in practice never have full range images
-                false
-            )
-        }
-
-        fn l0_delta_layers_updated_scenario(layer_name: &str, expected_l0: bool) {
-            let name = LayerFileName::from_str(layer_name).unwrap();
-            let skeleton = LayerDescriptor::from(name);
-
-            let remote: Arc<dyn Layer> = Arc::new(skeleton.clone());
-            let downloaded: Arc<dyn Layer> = Arc::new(skeleton);
-
-            let mut map = LayerMap::default();
-
-            // two disjoint Arcs in different lifecycle phases.
-            assert!(!LayerMap::compare_arced_layers(&remote, &downloaded));
-
-            let expected_in_counts = (1, usize::from(expected_l0));
-
-            map.batch_update().insert_historic(remote.clone());
-            assert_eq!(count_layer_in(&map, &remote), expected_in_counts);
-
-            let replaced = map
-                .batch_update()
-                .replace_historic(&remote, downloaded.clone())
-                .expect("name derived attributes are the same");
-            assert!(
-                matches!(replaced, Replacement::Replaced { .. }),
-                "{replaced:?}"
-            );
-            assert_eq!(count_layer_in(&map, &downloaded), expected_in_counts);
-
-            map.batch_update().remove_historic(downloaded.clone());
-            assert_eq!(count_layer_in(&map, &downloaded), (0, 0));
-        }
-
-        fn count_layer_in(map: &LayerMap<dyn Layer>, layer: &Arc<dyn Layer>) -> (usize, usize) {
-            let historic = map
-                .iter_historic_layers()
-                .filter(|x| LayerMap::compare_arced_layers(x, layer))
-                .count();
-            let l0s = map
-                .get_level0_deltas()
-                .expect("why does this return a result");
-            let l0 = l0s
-                .iter()
-                .filter(|x| LayerMap::compare_arced_layers(x, layer))
-                .count();
-
-            (historic, l0)
-        }
-    }
 }

pageserver/src/tenant/layer_map/historic_layer_coverage.rs

@@ -41,18 +41,6 @@ impl Ord for LayerKey {
     }
 }
 
-impl<'a, L: crate::tenant::storage_layer::Layer + ?Sized> From<&'a L> for LayerKey {
-    fn from(layer: &'a L) -> Self {
-        let kr = layer.get_key_range();
-        let lr = layer.get_lsn_range();
-        LayerKey {
-            key: kr.start.to_i128()..kr.end.to_i128(),
-            lsn: lr.start.0..lr.end.0,
-            is_image: !layer.is_incremental(),
-        }
-    }
-}
-
 /// Efficiently queryable layer coverage for each LSN.
 ///
 /// Allows answering layer map queries very efficiently,
@@ -94,13 +82,15 @@ impl<Value: Clone> HistoricLayerCoverage<Value> {
         }
 
         // Insert into data structure
-        let target = if layer_key.is_image {
-            &mut self.head.image_coverage
+        if layer_key.is_image {
+            self.head
+                .image_coverage
+                .insert(layer_key.key, layer_key.lsn.clone(), value);
         } else {
-            &mut self.head.delta_coverage
-        };
-
-        target.insert(layer_key.key, layer_key.lsn.clone(), value);
+            self.head
+                .delta_coverage
+                .insert(layer_key.key, layer_key.lsn.clone(), value);
+        }
 
         // Remember history. Clone is O(1)
         self.historic.insert(layer_key.lsn.start, self.head.clone());
@@ -425,59 +415,6 @@ impl<Value: Clone> BufferedHistoricLayerCoverage<Value> {
         self.buffer.insert(layer_key, None);
     }
 
-    /// Replaces a previous layer with a new layer value.
-    ///
-    /// The replacement is conditional on:
-    /// - there is an existing `LayerKey` record
-    /// - there is no buffered removal for the given `LayerKey`
-    /// - the given closure returns true for the current `Value`
-    ///
-    /// The closure is used to compare the latest value (buffered insert, or existing layer)
-    /// against some expectation. This allows to use `Arc::ptr_eq` or similar which would be
-    /// inaccessible via `PartialEq` trait.
-    ///
-    /// Returns a `Replacement` value describing the outcome; only the case of
-    /// `Replacement::Replaced` modifies the map and requires a rebuild.
-    pub fn replace<F>(
-        &mut self,
-        layer_key: &LayerKey,
-        new: Value,
-        check_expected: F,
-    ) -> Replacement<Value>
-    where
-        F: FnOnce(&Value) -> bool,
-    {
-        let (slot, in_buffered) = match self.buffer.get(layer_key) {
-            Some(inner @ Some(_)) => {
-                // we compare against the buffered version, because there will be a later
-                // rebuild before querying
-                (inner.as_ref(), true)
-            }
-            Some(None) => {
-                // buffer has removal for this key; it will not be equivalent by any check_expected.
-                return Replacement::RemovalBuffered;
-            }
-            None => {
-                // no pending modification for the key, check layers
-                (self.layers.get(layer_key), false)
-            }
-        };
-
-        match slot {
-            Some(existing) if !check_expected(existing) => {
-                // unfortunate clone here, but otherwise the nll borrowck grows the region of
-                // 'a to cover the whole function, and we could not mutate in the other
-                // Some(existing) branch
-                Replacement::Unexpected(existing.clone())
-            }
-            None => Replacement::NotFound,
-            Some(_existing) => {
-                self.insert(layer_key.to_owned(), new);
-                Replacement::Replaced { in_buffered }
-            }
-        }
-    }
-
     pub fn rebuild(&mut self) {
         // Find the first LSN that needs to be rebuilt
         let rebuild_since: u64 = match self.buffer.iter().next() {
@@ -546,22 +483,6 @@ impl<Value: Clone> BufferedHistoricLayerCoverage<Value> {
     }
 }
 
-/// Outcome of the replace operation.
-#[derive(Debug)]
-pub enum Replacement<Value> {
-    /// Previous value was replaced with the new value.
-    Replaced {
-        /// Replacement happened for a scheduled insert.
-        in_buffered: bool,
-    },
-    /// Key was not found buffered updates or existing layers.
-    NotFound,
-    /// Key has been scheduled for removal, it was not replaced.
-    RemovalBuffered,
-    /// Previous value was rejected by the closure.
-    Unexpected(Value),
-}
-
 #[test]
 fn test_retroactive_regression_1() {
     let mut map = BufferedHistoricLayerCoverage::new();
@@ -627,7 +548,7 @@ fn test_retroactive_simple() {
         LayerKey {
             key: 2..5,
             lsn: 105..106,
-            is_image: false,
+            is_image: true,
         },
         "Delta 1".to_string(),
     );
@@ -635,24 +556,17 @@ fn test_retroactive_simple() {
     // Rebuild so we can start querying
     map.rebuild();
 
-    {
-        let map = map.get().expect("rebuilt");
-
-        let version = map.get_version(90);
-        assert!(version.is_none());
-        let version = map.get_version(102).unwrap();
-        assert_eq!(version.image_coverage.query(4), Some("Image 1".to_string()));
-
-        let version = map.get_version(107).unwrap();
-        assert_eq!(version.image_coverage.query(4), Some("Image 1".to_string()));
-        assert_eq!(version.delta_coverage.query(4), Some("Delta 1".to_string()));
-
-        let version = map.get_version(115).unwrap();
-        assert_eq!(version.image_coverage.query(4), Some("Image 2".to_string()));
-
-        let version = map.get_version(125).unwrap();
-        assert_eq!(version.image_coverage.query(4), Some("Image 3".to_string()));
-    }
+    // Query key 4
+    let version = map.get().unwrap().get_version(90);
+    assert!(version.is_none());
+    let version = map.get().unwrap().get_version(102).unwrap();
+    assert_eq!(version.image_coverage.query(4), Some("Image 1".to_string()));
+    let version = map.get().unwrap().get_version(107).unwrap();
+    assert_eq!(version.image_coverage.query(4), Some("Delta 1".to_string()));
+    let version = map.get().unwrap().get_version(115).unwrap();
+    assert_eq!(version.image_coverage.query(4), Some("Image 2".to_string()));
+    let version = map.get().unwrap().get_version(125).unwrap();
+    assert_eq!(version.image_coverage.query(4), Some("Image 3".to_string()));
 
     // Remove Image 3
     map.remove(LayerKey {
@@ -662,147 +576,8 @@ fn test_retroactive_simple() {
     });
     map.rebuild();
 
-    {
-        // Check deletion worked
-        let map = map.get().expect("rebuilt");
-        let version = map.get_version(125).unwrap();
-        assert_eq!(version.image_coverage.query(4), Some("Image 2".to_string()));
-        assert_eq!(version.image_coverage.query(8), Some("Image 4".to_string()));
-    }
-}
-
-#[test]
-fn test_retroactive_replacement() {
-    let mut map = BufferedHistoricLayerCoverage::new();
-
-    let keys = [
-        LayerKey {
-            key: 0..5,
-            lsn: 100..101,
-            is_image: true,
-        },
-        LayerKey {
-            key: 3..9,
-            lsn: 110..111,
-            is_image: true,
-        },
-        LayerKey {
-            key: 4..6,
-            lsn: 120..121,
-            is_image: true,
-        },
-    ];
-
-    let layers = [
-        "Image 1".to_string(),
-        "Image 2".to_string(),
-        "Image 3".to_string(),
-    ];
-
-    for (key, layer) in keys.iter().zip(layers.iter()) {
-        map.insert(key.to_owned(), layer.to_owned());
-    }
-
-    // rebuild is not necessary here, because replace works for both buffered updates and existing
-    // layers.
-
-    for (key, orig_layer) in keys.iter().zip(layers.iter()) {
-        let replacement = format!("Remote {orig_layer}");
-
-        // evict
-        let ret = map.replace(key, replacement.clone(), |l| l == orig_layer);
-        assert!(
-            matches!(ret, Replacement::Replaced { .. }),
-            "replace {orig_layer}: {ret:?}"
-        );
-        map.rebuild();
-
-        let at = key.lsn.end + 1;
-
-        let version = map.get().expect("rebuilt").get_version(at).unwrap();
-        assert_eq!(
-            version.image_coverage.query(4).as_deref(),
-            Some(replacement.as_str()),
-            "query for 4 at version {at} after eviction",
-        );
-
-        // download
-        let ret = map.replace(key, orig_layer.clone(), |l| l == &replacement);
-        assert!(
-            matches!(ret, Replacement::Replaced { .. }),
-            "replace {orig_layer} back: {ret:?}"
-        );
-        map.rebuild();
-        let version = map.get().expect("rebuilt").get_version(at).unwrap();
-        assert_eq!(
-            version.image_coverage.query(4).as_deref(),
-            Some(orig_layer.as_str()),
-            "query for 4 at version {at} after download",
-        );
-    }
-}
-
-#[test]
-fn missing_key_is_not_inserted_with_replace() {
-    let mut map = BufferedHistoricLayerCoverage::new();
-    let key = LayerKey {
-        key: 0..5,
-        lsn: 100..101,
-        is_image: true,
-    };
-
-    let ret = map.replace(&key, "should not replace", |_| true);
-    assert!(matches!(ret, Replacement::NotFound), "{ret:?}");
-    map.rebuild();
-    assert!(map
-        .get()
-        .expect("no changes to rebuild")
-        .get_version(102)
-        .is_none());
-}
-
-#[test]
-fn replacing_buffered_insert_and_remove() {
-    let mut map = BufferedHistoricLayerCoverage::new();
-    let key = LayerKey {
-        key: 0..5,
-        lsn: 100..101,
-        is_image: true,
-    };
-
-    map.insert(key.clone(), "Image 1");
-    let ret = map.replace(&key, "Remote Image 1", |&l| l == "Image 1");
-    assert!(
-        matches!(ret, Replacement::Replaced { in_buffered: true }),
-        "{ret:?}"
-    );
-    map.rebuild();
-
-    assert_eq!(
-        map.get()
-            .expect("rebuilt")
-            .get_version(102)
-            .unwrap()
-            .image_coverage
-            .query(4),
-        Some("Remote Image 1")
-    );
-
-    map.remove(key.clone());
-    let ret = map.replace(&key, "should not replace", |_| true);
-    assert!(
-        matches!(ret, Replacement::RemovalBuffered),
-        "cannot replace after scheduled remove: {ret:?}"
-    );
-
-    map.rebuild();
-
-    let ret = map.replace(&key, "should not replace", |_| true);
-    assert!(
-        matches!(ret, Replacement::NotFound),
-        "cannot replace after remove + rebuild: {ret:?}"
-    );
-
-    let at_version = map.get().expect("rebuilt").get_version(102);
-    assert!(at_version.is_none());
+    // Check deletion worked
+    let version = map.get().unwrap().get_version(125).unwrap();
+    assert_eq!(version.image_coverage.query(4), Some("Image 2".to_string()));
+    assert_eq!(version.image_coverage.query(8), Some("Image 4".to_string()));
 }

pageserver/src/tenant/mgr.rs

@@ -19,12 +19,13 @@ use crate::config::PageServerConf;
 use crate::context::{DownloadBehavior, RequestContext};
 use crate::task_mgr::{self, TaskKind};
 use crate::tenant::config::TenantConfOpt;
-use crate::tenant::{Tenant, TenantState};
+use crate::tenant::{Tenant, TenantState, TENANT_ATTACHING_LEGACY_MARKER_FILENAME};
 use crate::IGNORED_TENANT_FILE_NAME;
 
-use utils::fs_ext::PathExt;
 use utils::id::{TenantId, TimelineId};
 
+use super::{TenantLoadReasonNotAttach, TENANT_ATTACHING_MARKER_SUFFIX};
+
 /// The tenants known to the pageserver.
 /// The enum variants are used to distinguish the different states that the pageserver can be in.
 enum TenantsMap {
@@ -66,6 +67,11 @@ pub async fn init_tenant_mgr(
     // Scan local filesystem for attached tenants
     let tenants_dir = conf.tenants_path();
 
+    // Other code in pageserver assumes new attaching markers.
+    // Do the migration here, abort startup if it fails.
+    Tenant::migrate_attaching_marker_files(&conf.tenants_path())
+        .context("attaching marker migration failed")?;
+
     let mut tenants = HashMap::new();
 
     let mut dir_entries = fs::read_dir(&tenants_dir)
@@ -92,18 +98,12 @@ pub async fn init_tenant_mgr(
                         );
                     }
                 } else {
-                    // This case happens if we crash during attach before creating the attach marker file
-                    let is_empty = tenant_dir_path.is_empty_dir().with_context(|| {
-                        format!("Failed to check whether {tenant_dir_path:?} is an empty dir")
-                    })?;
-                    if is_empty {
-                        info!("removing empty tenant directory {tenant_dir_path:?}");
-                        if let Err(e) = fs::remove_dir(&tenant_dir_path).await {
-                            error!(
-                                "Failed to remove empty tenant directory '{}': {e:#}",
-                                tenant_dir_path.display()
-                            )
-                        }
+                    if tenant_dir_path
+                        .to_string_lossy()
+                        .ends_with(TENANT_ATTACHING_MARKER_SUFFIX)
+                    {
+                        // schedule_local_tenant_processing checks for marker when it encounters a tenant dir
+                        info!("found a tenant attaching marker {tenant_dir_path:?}, skipping");
                         continue;
                     }
 
@@ -117,6 +117,7 @@ pub async fn init_tenant_mgr(
                         conf,
                         &tenant_dir_path,
                         remote_storage.clone(),
+                        TenantLoadReasonNotAttach::PageserverStartup,
                         &ctx,
                     ) {
                         Ok(tenant) => {
@@ -147,10 +148,11 @@ pub async fn init_tenant_mgr(
     Ok(())
 }
 
-pub fn schedule_local_tenant_processing(
+pub(crate) fn schedule_local_tenant_processing(
     conf: &'static PageServerConf,
     tenant_path: &Path,
     remote_storage: Option<GenericRemoteStorage>,
+    load_reason: TenantLoadReasonNotAttach,
     ctx: &RequestContext,
 ) -> anyhow::Result<Arc<Tenant>> {
     anyhow::ensure!(
@@ -162,10 +164,10 @@ pub fn schedule_local_tenant_processing(
         "Cannot load tenant from temporary path {tenant_path:?}"
     );
     anyhow::ensure!(
-        !tenant_path.is_empty_dir().with_context(|| {
-            format!("Failed to check whether {tenant_path:?} is an empty dir")
-        })?,
-        "Cannot load tenant from empty directory {tenant_path:?}"
+        !tenant_path
+            .to_string_lossy()
+            .ends_with(TENANT_ATTACHING_MARKER_SUFFIX),
+        "Caller must filter these out: {tenant_path:?}"
     );
 
     let tenant_id = tenant_path
@@ -183,10 +185,22 @@ pub fn schedule_local_tenant_processing(
         "Cannot load tenant, ignore mark found at {tenant_ignore_mark:?}"
     );
 
+    let legacy_attaching_marker = tenant_path.join(TENANT_ATTACHING_LEGACY_MARKER_FILENAME);
+    anyhow::ensure!(
+        !legacy_attaching_marker.exists(),
+        "legacy attaching marker still present, migration code must have been not called or has a bug: {legacy_attaching_marker:?}"
+    );
+
     let tenant = if conf.tenant_attaching_mark_file_path(&tenant_id).exists() {
         info!("tenant {tenant_id} has attaching mark file, resuming its attach operation");
         if let Some(remote_storage) = remote_storage {
-            Tenant::spawn_attach(conf, tenant_id, remote_storage, ctx)
+            match Tenant::spawn_resume_attach(conf, tenant_id, remote_storage, ctx) {
+                Ok(tenant) => tenant,
+                Err(e) => {
+                    warn!("tenant {tenant_id} failed to resume attach operation: {e:#}");
+                    Tenant::create_broken_tenant(conf, tenant_id)
+                }
+            }
         } else {
             warn!("tenant {tenant_id} has attaching mark file, but pageserver has no remote storage configured");
             Tenant::create_broken_tenant(conf, tenant_id)
@@ -194,7 +208,7 @@ pub fn schedule_local_tenant_processing(
     } else {
         info!("tenant {tenant_id} is assumed to be loadable, starting load operation");
         // Start loading the tenant into memory. It will initially be in Loading state.
-        Tenant::spawn_load(conf, tenant_id, remote_storage, ctx)
+        Tenant::spawn_load(conf, tenant_id, remote_storage, load_reason, ctx)
     };
     Ok(tenant)
 }
@@ -274,7 +288,7 @@ pub async fn create_tenant(
         // and do the work in that state.
         let tenant_directory = super::create_tenant_files(conf, tenant_conf, tenant_id)?;
         let created_tenant =
-            schedule_local_tenant_processing(conf, &tenant_directory, remote_storage, ctx)?;
+            schedule_local_tenant_processing(conf, &tenant_directory, remote_storage, TenantLoadReasonNotAttach::Create, ctx)?;
         let crated_tenant_id = created_tenant.tenant_id();
         anyhow::ensure!(
                 tenant_id == crated_tenant_id,
@@ -285,22 +299,17 @@ pub async fn create_tenant(
     }).await
 }
 
-pub async fn set_new_tenant_config(
+pub async fn update_tenant_config(
     conf: &'static PageServerConf,
-    new_tenant_conf: TenantConfOpt,
+    tenant_conf: TenantConfOpt,
     tenant_id: TenantId,
 ) -> anyhow::Result<()> {
     info!("configuring tenant {tenant_id}");
     let tenant = get_tenant(tenant_id, true).await?;
 
+    tenant.update_tenant_config(tenant_conf);
     let tenant_config_path = conf.tenant_config_path(tenant_id);
-    Tenant::persist_tenant_config(
-        &tenant.tenant_id(),
-        &tenant_config_path,
-        new_tenant_conf,
-        false,
-    )?;
-    tenant.set_new_tenant_config(new_tenant_conf);
+    Tenant::persist_tenant_config(&tenant.tenant_id(), &tenant_config_path, tenant_conf, false)?;
     Ok(())
 }
 
@@ -366,7 +375,7 @@ pub async fn load_tenant(
                 .with_context(|| format!("Failed to remove tenant ignore mark {tenant_ignore_mark:?} during tenant loading"))?;
         }
 
-        let new_tenant = schedule_local_tenant_processing(conf, &tenant_path, remote_storage, ctx)
+        let new_tenant = schedule_local_tenant_processing(conf, &tenant_path, remote_storage, TenantLoadReasonNotAttach::LoadApi, ctx)
             .with_context(|| {
                 format!("Failed to schedule tenant processing in path {tenant_path:?}")
             })?;
@@ -426,13 +435,8 @@ pub async fn attach_tenant(
     ctx: &RequestContext,
 ) -> Result<(), TenantMapInsertError> {
     tenant_map_insert(tenant_id, |vacant_entry| {
-        let tenant_path = conf.tenant_path(&tenant_id);
-        anyhow::ensure!(
-            !tenant_path.exists(),
-            "Cannot attach tenant {tenant_id}, local tenant directory already exists"
-        );
-
-        let tenant = Tenant::spawn_attach(conf, tenant_id, remote_storage, ctx);
+        let tenant = Tenant::spawn_start_attach(conf, tenant_id, remote_storage, ctx)
+            .map_err(|source| anyhow::anyhow!("attach tenant {tenant_id}: {source:#}"))?;
         vacant_entry.insert(tenant);
         Ok(())
     })
@@ -540,11 +544,13 @@ where
     }
 }
 
+#[cfg(feature = "testing")]
 use {
     crate::repository::GcResult, pageserver_api::models::TimelineGcRequest,
     utils::http::error::ApiError,
 };
 
+#[cfg(feature = "testing")]
 pub async fn immediate_gc(
     tenant_id: TenantId,
     timeline_id: TimelineId,

pageserver/src/tenant/remote_timeline_client.rs

@@ -157,17 +157,26 @@
 //! downloading files from the remote storage. Downloads are performed immediately
 //! against the `RemoteStorage`, independently of the upload queue.
 //!
-//! When we attach a tenant, we perform the following steps:
+//! When we attach a tenant, we prepare the on-disk state based on the remote state,
+//! then use the same code that's used to set up the tenant during pageserver startup:
+//!
 //! - create `Tenant` object in `TenantState::Attaching` state
+//! - Create an attaching marker file for this tenant on disk.
 //! - List timelines that are present in remote storage, and for each:
 //!   - download their remote [`IndexPart`]s
+//!   - create the local `metadata` file from the [`IndexPart`] contents
+//! - Remove the attaching marker file.
+//! - tell the `Tenant` object to load the prepared on-disk state.
+//!
+//! Loading the on-disk state performs the following steps:
+//!
 //!   - create `Timeline` struct and a `RemoteTimelineClient`
-//!   - initialize the client's upload queue with its `IndexPart`
+//!   - initialize the client's upload queue with the `IndexPart`
+//!     - for attach, we carry this over in memory
+//!     - during pageserver startup, we refresh the IndexParts from the remote
 //!   - create [`RemoteLayer`] instances for layers that are referenced by `IndexPart`
 //!     but not present locally
 //!   - schedule uploads for layers that are only present locally.
-//!   - if the remote `IndexPart`'s metadata was newer than the metadata in
-//!     the local filesystem, write the remote metadata to the local filesystem
 //! - After the above is done for each timeline, open the tenant for business by
 //!   transitioning it from `TenantState::Attaching` to `TenantState::Active` state.
 //!   This starts the timelines' WAL-receivers and the tenant's GC & Compaction loops.
@@ -571,15 +580,14 @@ impl RemoteTimelineClient {
         Ok(())
     }
 
-    /// Launch a delete operation in the background.
     ///
-    /// The operation does not modify local state but assumes the local files have already been
-    /// deleted, and is used to mirror those changes to remote.
+    /// Launch a delete operation in the background.
     ///
     /// Note: This schedules an index file upload before the deletions.  The
     /// deletion won't actually be performed, until any previously scheduled
     /// upload operations, and the index file upload, have completed
     /// succesfully.
+    ///
     pub fn schedule_layer_file_deletion(
         self: &Arc<Self>,
         names: &[LayerFileName],
@@ -1136,29 +1144,18 @@ mod tests {
         client.init_upload_queue_for_empty_remote(&metadata)?;
 
         // Create a couple of dummy files,  schedule upload for them
-        let layer_file_name_1: LayerFileName = "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap();
-        let layer_file_name_2: LayerFileName = "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D9-00000000016B5A52".parse().unwrap();
-        let layer_file_name_3: LayerFileName = "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59DA-00000000016B5A53".parse().unwrap();
-        let content_1 = dummy_contents("foo");
-        let content_2 = dummy_contents("bar");
-        let content_3 = dummy_contents("baz");
-        std::fs::write(
-            timeline_path.join(layer_file_name_1.file_name()),
-            &content_1,
-        )?;
-        std::fs::write(
-            timeline_path.join(layer_file_name_2.file_name()),
-            &content_2,
-        )?;
-        std::fs::write(timeline_path.join(layer_file_name_3.file_name()), content_3)?;
+        let content_foo = dummy_contents("foo");
+        let content_bar = dummy_contents("bar");
+        std::fs::write(timeline_path.join("foo"), &content_foo)?;
+        std::fs::write(timeline_path.join("bar"), &content_bar)?;
 
         client.schedule_layer_file_upload(
-            &layer_file_name_1,
-            &LayerFileMetadata::new(content_1.len() as u64),
+            &LayerFileName::Test("foo".to_owned()),
+            &LayerFileMetadata::new(content_foo.len() as u64),
         )?;
         client.schedule_layer_file_upload(
-            &layer_file_name_2,
-            &LayerFileMetadata::new(content_2.len() as u64),
+            &LayerFileName::Test("bar".to_owned()),
+            &LayerFileMetadata::new(content_bar.len() as u64),
         )?;
 
         // Check that they are started immediately, not queued
@@ -1195,13 +1192,7 @@ mod tests {
 
         // Download back the index.json, and check that the list of files is correct
         let index_part = runtime.block_on(client.download_index_file())?;
-        assert_file_list(
-            &index_part.timeline_layers,
-            &[
-                &layer_file_name_1.file_name(),
-                &layer_file_name_2.file_name(),
-            ],
-        );
+        assert_file_list(&index_part.timeline_layers, &["foo", "bar"]);
         let downloaded_metadata = index_part.parse_metadata()?;
         assert_eq!(downloaded_metadata, metadata);
 
@@ -1209,10 +1200,10 @@ mod tests {
         let content_baz = dummy_contents("baz");
         std::fs::write(timeline_path.join("baz"), &content_baz)?;
         client.schedule_layer_file_upload(
-            &layer_file_name_3,
+            &LayerFileName::Test("baz".to_owned()),
             &LayerFileMetadata::new(content_baz.len() as u64),
         )?;
-        client.schedule_layer_file_deletion(&[layer_file_name_1.clone()])?;
+        client.schedule_layer_file_deletion(&[LayerFileName::Test("foo".to_owned())])?;
         {
             let mut guard = client.upload_queue.lock().unwrap();
             let upload_queue = guard.initialized_mut().unwrap();
@@ -1224,26 +1215,12 @@ mod tests {
             assert!(upload_queue.num_inprogress_deletions == 0);
             assert!(upload_queue.latest_files_changes_since_metadata_upload_scheduled == 0);
         }
-        assert_remote_files(
-            &[
-                &layer_file_name_1.file_name(),
-                &layer_file_name_2.file_name(),
-                "index_part.json",
-            ],
-            &remote_timeline_dir,
-        );
+        assert_remote_files(&["foo", "bar", "index_part.json"], &remote_timeline_dir);
 
         // Finish them
         runtime.block_on(client.wait_completion())?;
 
-        assert_remote_files(
-            &[
-                &layer_file_name_2.file_name(),
-                &layer_file_name_3.file_name(),
-                "index_part.json",
-            ],
-            &remote_timeline_dir,
-        );
+        assert_remote_files(&["bar", "baz", "index_part.json"], &remote_timeline_dir);
 
         Ok(())
     }

pageserver/src/tenant/remote_timeline_client/index.rs

@@ -8,8 +8,7 @@ use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
 use tracing::warn;
 
-use crate::tenant::metadata::TimelineMetadata;
-use crate::tenant::storage_layer::LayerFileName;
+use crate::tenant::{metadata::TimelineMetadata, storage_layer::LayerFileName};
 
 use utils::lsn::Lsn;
 
@@ -275,7 +274,7 @@ mod tests {
             "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
             "layer_metadata":{
                 "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
-                "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
+                "LAYER_FILE_NAME::test/not_a_real_layer_but_adding_coverage": { "file_size": 9007199254741001 }
             },
             "disk_consistent_lsn":"0/16960E8",
             "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
@@ -289,7 +288,7 @@ mod tests {
                 ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata {
                     file_size: Some(25600000),
                 }),
-                ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata {
+                (LayerFileName::new_test("not_a_real_layer_but_adding_coverage"), IndexLayerMetadata {
                     // serde_json should always parse this but this might be a double with jq for
                     // example.
                     file_size: Some(9007199254741001),
@@ -313,7 +312,7 @@ mod tests {
             "missing_layers":["This shouldn't fail deserialization"],
             "layer_metadata":{
                 "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
-                "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
+                "LAYER_FILE_NAME::test/not_a_real_layer_but_adding_coverage": { "file_size": 9007199254741001 }
             },
             "disk_consistent_lsn":"0/16960E8",
             "metadata_bytes":[112,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
@@ -327,7 +326,7 @@ mod tests {
                 ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata {
                     file_size: Some(25600000),
                 }),
-                ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata {
+                (LayerFileName::new_test("not_a_real_layer_but_adding_coverage"), IndexLayerMetadata {
                     // serde_json should always parse this but this might be a double with jq for
                     // example.
                     file_size: Some(9007199254741001),

pageserver/src/tenant/storage_layer.rs

@@ -1,30 +1,18 @@
 //! Common traits and structs for layers
 
-pub mod delta_layer;
+mod delta_layer;
 mod filename;
 mod image_layer;
 mod inmemory_layer;
 mod remote_layer;
 
-use crate::config::PageServerConf;
-use crate::context::RequestContext;
 use crate::repository::{Key, Value};
-use crate::task_mgr::TaskKind;
 use crate::walrecord::NeonWalRecord;
 use anyhow::Result;
 use bytes::Bytes;
-use either::Either;
-use enum_map::EnumMap;
-use enumset::EnumSet;
-use pageserver_api::models::LayerAccessKind;
-use pageserver_api::models::{
-    HistoricLayerInfo, LayerResidenceEvent, LayerResidenceEventReason, LayerResidenceStatus,
-};
 use std::ops::Range;
 use std::path::PathBuf;
-use std::sync::{Arc, Mutex};
-use std::time::{SystemTime, UNIX_EPOCH};
-use utils::history_buffer::HistoryBufferWithDropCounter;
+use std::sync::Arc;
 
 use utils::{
     id::{TenantId, TimelineId},
@@ -32,7 +20,7 @@ use utils::{
 };
 
 pub use delta_layer::{DeltaLayer, DeltaLayerWriter};
-pub use filename::{DeltaFileName, ImageFileName, LayerFileName};
+pub use filename::{DeltaFileName, ImageFileName, LayerFileName, PathOrConf};
 pub use image_layer::{ImageLayer, ImageLayerWriter};
 pub use inmemory_layer::InMemoryLayer;
 pub use remote_layer::RemoteLayer;
@@ -92,191 +80,9 @@ pub enum ValueReconstructResult {
     Missing,
 }
 
-#[derive(Debug)]
-pub struct LayerAccessStats(Mutex<LayerAccessStatsLocked>);
-
-/// This struct holds two instances of [`LayerAccessStatsInner`].
-/// Accesses are recorded to both instances.
-/// The `for_scraping_api`instance can be reset from the management API via [`LayerAccessStatsReset`].
-/// The `for_eviction_policy` is never reset.
-#[derive(Debug, Default, Clone)]
-struct LayerAccessStatsLocked {
-    for_scraping_api: LayerAccessStatsInner,
-    for_eviction_policy: LayerAccessStatsInner,
-}
-
-impl LayerAccessStatsLocked {
-    fn iter_mut(&mut self) -> impl Iterator<Item = &mut LayerAccessStatsInner> {
-        [&mut self.for_scraping_api, &mut self.for_eviction_policy].into_iter()
-    }
-}
-
-#[derive(Debug, Default, Clone)]
-struct LayerAccessStatsInner {
-    first_access: Option<LayerAccessStatFullDetails>,
-    count_by_access_kind: EnumMap<LayerAccessKind, u64>,
-    task_kind_flag: EnumSet<TaskKind>,
-    last_accesses: HistoryBufferWithDropCounter<LayerAccessStatFullDetails, 16>,
-    last_residence_changes: HistoryBufferWithDropCounter<LayerResidenceEvent, 16>,
-}
-
-#[derive(Debug, Clone, Copy)]
-pub(super) struct LayerAccessStatFullDetails {
-    pub(super) when: SystemTime,
-    pub(super) task_kind: TaskKind,
-    pub(super) access_kind: LayerAccessKind,
-}
-
-#[derive(Clone, Copy, strum_macros::EnumString)]
-pub enum LayerAccessStatsReset {
-    NoReset,
-    JustTaskKindFlags,
-    AllStats,
-}
-
-fn system_time_to_millis_since_epoch(ts: &SystemTime) -> u64 {
-    ts.duration_since(UNIX_EPOCH)
-        .expect("better to die in this unlikely case than report false stats")
-        .as_millis()
-        .try_into()
-        .expect("64 bits is enough for few more years")
-}
-
-impl LayerAccessStatFullDetails {
-    fn as_api_model(&self) -> pageserver_api::models::LayerAccessStatFullDetails {
-        let Self {
-            when,
-            task_kind,
-            access_kind,
-        } = self;
-        pageserver_api::models::LayerAccessStatFullDetails {
-            when_millis_since_epoch: system_time_to_millis_since_epoch(when),
-            task_kind: task_kind.into(), // into static str, powered by strum_macros
-            access_kind: *access_kind,
-        }
-    }
-}
-
-impl LayerAccessStats {
-    pub(crate) fn for_loading_layer(status: LayerResidenceStatus) -> Self {
-        let new = LayerAccessStats(Mutex::new(LayerAccessStatsLocked::default()));
-        new.record_residence_event(status, LayerResidenceEventReason::LayerLoad);
-        new
-    }
-
-    pub(crate) fn for_new_layer_file() -> Self {
-        let new = LayerAccessStats(Mutex::new(LayerAccessStatsLocked::default()));
-        new.record_residence_event(
-            LayerResidenceStatus::Resident,
-            LayerResidenceEventReason::LayerCreate,
-        );
-        new
-    }
-
-    /// Creates a clone of `self` and records `new_status` in the clone.
-    /// The `new_status` is not recorded in `self`
-    pub(crate) fn clone_for_residence_change(
-        &self,
-        new_status: LayerResidenceStatus,
-    ) -> LayerAccessStats {
-        let clone = {
-            let inner = self.0.lock().unwrap();
-            inner.clone()
-        };
-        let new = LayerAccessStats(Mutex::new(clone));
-        new.record_residence_event(new_status, LayerResidenceEventReason::ResidenceChange);
-        new
-    }
-
-    fn record_residence_event(
-        &self,
-        status: LayerResidenceStatus,
-        reason: LayerResidenceEventReason,
-    ) {
-        let mut locked = self.0.lock().unwrap();
-        locked.iter_mut().for_each(|inner| {
-            inner
-                .last_residence_changes
-                .write(LayerResidenceEvent::new(status, reason))
-        });
-    }
-
-    fn record_access(&self, access_kind: LayerAccessKind, task_kind: TaskKind) {
-        let this_access = LayerAccessStatFullDetails {
-            when: SystemTime::now(),
-            task_kind,
-            access_kind,
-        };
-
-        let mut locked = self.0.lock().unwrap();
-        locked.iter_mut().for_each(|inner| {
-            inner.first_access.get_or_insert(this_access);
-            inner.count_by_access_kind[access_kind] += 1;
-            inner.task_kind_flag |= task_kind;
-            inner.last_accesses.write(this_access);
-        })
-    }
-
-    fn as_api_model(
-        &self,
-        reset: LayerAccessStatsReset,
-    ) -> pageserver_api::models::LayerAccessStats {
-        let mut locked = self.0.lock().unwrap();
-        let inner = &mut locked.for_scraping_api;
-        let LayerAccessStatsInner {
-            first_access,
-            count_by_access_kind,
-            task_kind_flag,
-            last_accesses,
-            last_residence_changes,
-        } = inner;
-        let ret = pageserver_api::models::LayerAccessStats {
-            access_count_by_access_kind: count_by_access_kind
-                .iter()
-                .map(|(kind, count)| (kind, *count))
-                .collect(),
-            task_kind_access_flag: task_kind_flag
-                .iter()
-                .map(|task_kind| task_kind.into()) // into static str, powered by strum_macros
-                .collect(),
-            first: first_access.as_ref().map(|a| a.as_api_model()),
-            accesses_history: last_accesses.map(|m| m.as_api_model()),
-            residence_events_history: last_residence_changes.clone(),
-        };
-        match reset {
-            LayerAccessStatsReset::NoReset => (),
-            LayerAccessStatsReset::JustTaskKindFlags => {
-                inner.task_kind_flag.clear();
-            }
-            LayerAccessStatsReset::AllStats => {
-                *inner = LayerAccessStatsInner::default();
-            }
-        }
-        ret
-    }
-
-    pub(super) fn most_recent_access_or_residence_event(
-        &self,
-    ) -> Either<LayerAccessStatFullDetails, LayerResidenceEvent> {
-        let locked = self.0.lock().unwrap();
-        let inner = &locked.for_eviction_policy;
-        match inner.last_accesses.recent() {
-            Some(a) => Either::Left(*a),
-            None => match inner.last_residence_changes.recent() {
-                Some(e) => Either::Right(e.clone()),
-                None => unreachable!("constructors for LayerAccessStats ensure that there's always a residence change event"),
-            }
-        }
-    }
-}
-
 /// Supertrait of the [`Layer`] trait that captures the bare minimum interface
 /// required by [`LayerMap`].
-///
-/// All layers should implement a minimal `std::fmt::Debug` without tenant or
-/// timeline names, because those are known in the context of which the layers
-/// are used in (timeline).
-pub trait Layer: std::fmt::Debug + Send + Sync {
+pub trait Layer: Send + Sync {
     /// Range of keys that this layer covers
     fn get_key_range(&self) -> Range<Key>;
 
@@ -311,14 +117,13 @@ pub trait Layer: std::fmt::Debug + Send + Sync {
         key: Key,
         lsn_range: Range<Lsn>,
         reconstruct_data: &mut ValueReconstructState,
-        ctx: &RequestContext,
     ) -> Result<ValueReconstructResult>;
 
     /// A short ID string that uniquely identifies the given layer within a [`LayerMap`].
     fn short_id(&self) -> String;
 
     /// Dump summary of the contents of the layer to stdout
-    fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()>;
+    fn dump(&self, verbose: bool) -> Result<()>;
 }
 
 /// Returned by [`Layer::iter`]
@@ -339,7 +144,8 @@ pub type LayerKeyIter<'i> = Box<dyn Iterator<Item = (Key, Lsn, u64)> + 'i>;
 /// Furthermore, there are two kinds of on-disk layers: delta and image layers.
 /// A delta layer contains all modifications within a range of LSNs and keys.
 /// An image layer is a snapshot of all the data in a key-range, at a single
-/// LSN.
+/// LSN
+///
 pub trait PersistentLayer: Layer {
     fn get_tenant_id(&self) -> TenantId;
 
@@ -355,11 +161,11 @@ pub trait PersistentLayer: Layer {
     fn local_path(&self) -> Option<PathBuf>;
 
     /// Iterate through all keys and values stored in the layer
-    fn iter(&self, ctx: &RequestContext) -> Result<LayerIter<'_>>;
+    fn iter(&self) -> Result<LayerIter<'_>>;
 
     /// Iterate through all keys stored in the layer. Returns key, lsn and value size
     /// It is used only for compaction and so is currently implemented only for DeltaLayer
-    fn key_iter(&self, _ctx: &RequestContext) -> Result<LayerKeyIter<'_>> {
+    fn key_iter(&self) -> Result<LayerKeyIter<'_>> {
         panic!("Not implemented")
     }
 
@@ -379,10 +185,6 @@ pub trait PersistentLayer: Layer {
     /// Should not change over the lifetime of the layer object because
     /// current_physical_size is computed as the som of this value.
     fn file_size(&self) -> Option<u64>;
-
-    fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo;
-
-    fn access_stats(&self) -> &LayerAccessStats;
 }
 
 pub fn downcast_remote_layer(
@@ -395,11 +197,15 @@ pub fn downcast_remote_layer(
     }
 }
 
+impl std::fmt::Debug for dyn Layer {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Layer")
+            .field("short_id", &self.short_id())
+            .finish()
+    }
+}
+
 /// Holds metadata about a layer without any content. Used mostly for testing.
-///
-/// To use filenames as fixtures, parse them as [`LayerFileName`] then convert from that to a
-/// LayerDescriptor.
-#[derive(Clone, Debug)]
 pub struct LayerDescriptor {
     pub key: Range<Key>,
     pub lsn: Range<Lsn>,
@@ -425,7 +231,6 @@ impl Layer for LayerDescriptor {
         _key: Key,
         _lsn_range: Range<Lsn>,
         _reconstruct_data: &mut ValueReconstructState,
-        _ctx: &RequestContext,
     ) -> Result<ValueReconstructResult> {
         todo!("This method shouldn't be part of the Layer trait")
     }
@@ -434,65 +239,7 @@ impl Layer for LayerDescriptor {
         self.short_id.clone()
     }
 
-    fn dump(&self, _verbose: bool, _ctx: &RequestContext) -> Result<()> {
+    fn dump(&self, _verbose: bool) -> Result<()> {
         todo!()
     }
 }
-
-impl From<DeltaFileName> for LayerDescriptor {
-    fn from(value: DeltaFileName) -> Self {
-        let short_id = value.to_string();
-        LayerDescriptor {
-            key: value.key_range,
-            lsn: value.lsn_range,
-            is_incremental: true,
-            short_id,
-        }
-    }
-}
-
-impl From<ImageFileName> for LayerDescriptor {
-    fn from(value: ImageFileName) -> Self {
-        let short_id = value.to_string();
-        let lsn = value.lsn_as_range();
-        LayerDescriptor {
-            key: value.key_range,
-            lsn,
-            is_incremental: false,
-            short_id,
-        }
-    }
-}
-
-impl From<LayerFileName> for LayerDescriptor {
-    fn from(value: LayerFileName) -> Self {
-        match value {
-            LayerFileName::Delta(d) => Self::from(d),
-            LayerFileName::Image(i) => Self::from(i),
-        }
-    }
-}
-
-/// Helper enum to hold a PageServerConf, or a path
-///
-/// This is used by DeltaLayer and ImageLayer. Normally, this holds a reference to the
-/// global config, and paths to layer files are constructed using the tenant/timeline
-/// path from the config. But in the 'pageserver_binutils' binary, we need to construct a Layer
-/// struct for a file on disk, without having a page server running, so that we have no
-/// config. In that case, we use the Path variant to hold the full path to the file on
-/// disk.
-enum PathOrConf {
-    Path(PathBuf),
-    Conf(&'static PageServerConf),
-}
-
-/// Range wrapping newtype, which uses display to render Debug.
-///
-/// Useful with `Key`, which has too verbose `{:?}` for printing multiple layers.
-struct RangeDisplayDebug<'a, T: std::fmt::Display>(&'a Range<T>);
-
-impl<'a, T: std::fmt::Display> std::fmt::Debug for RangeDisplayDebug<'a, T> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}..{}", self.0.start, self.0.end)
-    }
-}

pageserver/src/tenant/storage_layer/delta_layer.rs

@@ -24,7 +24,6 @@
 //! "values" part.
 //!
 use crate::config::PageServerConf;
-use crate::context::RequestContext;
 use crate::page_cache::{PageReadGuard, PAGE_SZ};
 use crate::repository::{Key, Value, KEY_SIZE};
 use crate::tenant::blob_io::{BlobCursor, BlobWriter, WriteBlobWriter};
@@ -37,7 +36,6 @@ use crate::virtual_file::VirtualFile;
 use crate::{walrecord, TEMP_FILE_SUFFIX};
 use crate::{DELTA_FILE_MAGIC, STORAGE_FORMAT_VERSION};
 use anyhow::{bail, ensure, Context, Result};
-use pageserver_api::models::{HistoricLayerInfo, LayerAccessKind};
 use rand::{distributions::Alphanumeric, Rng};
 use serde::{Deserialize, Serialize};
 use std::fs::{self, File};
@@ -55,10 +53,7 @@ use utils::{
     lsn::Lsn,
 };
 
-use super::{
-    DeltaFileName, Layer, LayerAccessStats, LayerAccessStatsReset, LayerFileName, LayerIter,
-    LayerKeyIter, LayerResidenceStatus, PathOrConf,
-};
+use super::{DeltaFileName, Layer, LayerFileName, LayerIter, LayerKeyIter, PathOrConf};
 
 ///
 /// Header stored in the beginning of the file
@@ -67,7 +62,7 @@ use super::{
 /// the 'index' starts at the block indicated by 'index_start_blk'
 ///
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
-pub struct Summary {
+struct Summary {
     /// Magic value to identify this as a neon delta file. Always DELTA_FILE_MAGIC.
     magic: u16,
     format_version: u16,
@@ -78,9 +73,9 @@ pub struct Summary {
     lsn_range: Range<Lsn>,
 
     /// Block number where the 'index' part of the file begins.
-    pub index_start_blk: u32,
+    index_start_blk: u32,
     /// Block within the 'index', where the B-tree root page is stored
-    pub index_root_blk: u32,
+    index_root_blk: u32,
 }
 
 impl From<&DeltaLayer> for Summary {
@@ -130,7 +125,7 @@ impl BlobRef {
     }
 }
 
-pub const DELTA_KEY_SIZE: usize = KEY_SIZE + 8;
+const DELTA_KEY_SIZE: usize = KEY_SIZE + 8;
 struct DeltaKey([u8; DELTA_KEY_SIZE]);
 
 ///
@@ -170,13 +165,14 @@ impl DeltaKey {
     }
 }
 
-/// DeltaLayer is the in-memory data structure associated with an on-disk delta
-/// file.
 ///
-/// We keep a DeltaLayer in memory for each file, in the LayerMap. If a layer
-/// is in "loaded" state, we have a copy of the index in memory, in 'inner'.
-/// Otherwise the struct is just a placeholder for a file that exists on disk,
-/// and it needs to be loaded before using it in queries.
+/// DeltaLayer is the in-memory data structure associated with an
+/// on-disk delta file.  We keep a DeltaLayer in memory for each
+/// file, in the LayerMap. If a layer is in "loaded" state, we have a
+/// copy of the index in memory, in 'inner'. Otherwise the struct is
+/// just a placeholder for a file that exists on disk, and it needs to
+/// be loaded before using it in queries.
+///
 pub struct DeltaLayer {
     path_or_conf: PathOrConf,
 
@@ -187,24 +183,9 @@ pub struct DeltaLayer {
 
     pub file_size: u64,
 
-    access_stats: LayerAccessStats,
-
     inner: RwLock<DeltaLayerInner>,
 }
 
-impl std::fmt::Debug for DeltaLayer {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        use super::RangeDisplayDebug;
-
-        f.debug_struct("DeltaLayer")
-            .field("key_range", &RangeDisplayDebug(&self.key_range))
-            .field("lsn_range", &self.lsn_range)
-            .field("file_size", &self.file_size)
-            .field("inner", &self.inner)
-            .finish()
-    }
-}
-
 pub struct DeltaLayerInner {
     /// If false, the fields below have not been loaded into memory yet.
     loaded: bool,
@@ -217,16 +198,6 @@ pub struct DeltaLayerInner {
     file: Option<FileBlockReader<VirtualFile>>,
 }
 
-impl std::fmt::Debug for DeltaLayerInner {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("DeltaLayerInner")
-            .field("loaded", &self.loaded)
-            .field("index_start_blk", &self.index_start_blk)
-            .field("index_root_blk", &self.index_root_blk)
-            .finish()
-    }
-}
-
 impl Layer for DeltaLayer {
     fn get_key_range(&self) -> Range<Key> {
         self.key_range.clone()
@@ -243,7 +214,7 @@ impl Layer for DeltaLayer {
         self.filename().file_name()
     }
     /// debugging function to print out the contents of the layer
-    fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()> {
+    fn dump(&self, verbose: bool) -> Result<()> {
         println!(
             "----- delta layer for ten {} tli {} keys {}-{} lsn {}-{} ----",
             self.tenant_id,
@@ -258,7 +229,7 @@ impl Layer for DeltaLayer {
             return Ok(());
         }
 
-        let inner = self.load(LayerAccessKind::Dump, ctx)?;
+        let inner = self.load()?;
 
         println!(
             "index_start_blk: {}, root {}",
@@ -322,7 +293,6 @@ impl Layer for DeltaLayer {
         key: Key,
         lsn_range: Range<Lsn>,
         reconstruct_state: &mut ValueReconstructState,
-        ctx: &RequestContext,
     ) -> anyhow::Result<ValueReconstructResult> {
         ensure!(lsn_range.start >= self.lsn_range.start);
         let mut need_image = true;
@@ -331,7 +301,7 @@ impl Layer for DeltaLayer {
 
         {
             // Open the file and lock the metadata in memory
-            let inner = self.load(LayerAccessKind::GetValueReconstructData, ctx)?;
+            let inner = self.load()?;
 
             // Scan the page versions backwards, starting from `lsn`.
             let file = inner.file.as_ref().unwrap();
@@ -421,18 +391,16 @@ impl PersistentLayer for DeltaLayer {
         Some(self.path())
     }
 
-    fn iter(&self, ctx: &RequestContext) -> Result<LayerIter<'_>> {
-        let inner = self
-            .load(LayerAccessKind::KeyIter, ctx)
-            .context("load delta layer")?;
+    fn iter(&self) -> Result<LayerIter<'_>> {
+        let inner = self.load().context("load delta layer")?;
         Ok(match DeltaValueIter::new(inner) {
             Ok(iter) => Box::new(iter),
             Err(err) => Box::new(std::iter::once(Err(err))),
         })
     }
 
-    fn key_iter(&self, ctx: &RequestContext) -> Result<LayerKeyIter<'_>> {
-        let inner = self.load(LayerAccessKind::KeyIter, ctx)?;
+    fn key_iter(&self) -> Result<LayerKeyIter<'_>> {
+        let inner = self.load()?;
         Ok(Box::new(
             DeltaKeyIter::new(inner).context("Layer index is corrupted")?,
         ))
@@ -447,26 +415,6 @@ impl PersistentLayer for DeltaLayer {
     fn file_size(&self) -> Option<u64> {
         Some(self.file_size)
     }
-
-    fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo {
-        let layer_file_name = self.filename().file_name();
-        let lsn_range = self.get_lsn_range();
-
-        let access_stats = self.access_stats.as_api_model(reset);
-
-        HistoricLayerInfo::Delta {
-            layer_file_name,
-            layer_file_size: Some(self.file_size),
-            lsn_start: lsn_range.start,
-            lsn_end: lsn_range.end,
-            remote: false,
-            access_stats,
-        }
-    }
-
-    fn access_stats(&self) -> &LayerAccessStats {
-        &self.access_stats
-    }
 }
 
 impl DeltaLayer {
@@ -511,13 +459,7 @@ impl DeltaLayer {
     /// Open the underlying file and read the metadata into memory, if it's
     /// not loaded already.
     ///
-    fn load(
-        &self,
-        access_kind: LayerAccessKind,
-        ctx: &RequestContext,
-    ) -> Result<RwLockReadGuard<DeltaLayerInner>> {
-        self.access_stats
-            .record_access(access_kind, ctx.task_kind());
+    fn load(&self) -> Result<RwLockReadGuard<DeltaLayerInner>> {
         loop {
             // Quick exit if already loaded
             let inner = self.inner.read().unwrap();
@@ -598,7 +540,6 @@ impl DeltaLayer {
         tenant_id: TenantId,
         filename: &DeltaFileName,
         file_size: u64,
-        access_stats: LayerAccessStats,
     ) -> DeltaLayer {
         DeltaLayer {
             path_or_conf: PathOrConf::Conf(conf),
@@ -607,7 +548,6 @@ impl DeltaLayer {
             key_range: filename.key_range.clone(),
             lsn_range: filename.lsn_range.clone(),
             file_size,
-            access_stats,
             inner: RwLock::new(DeltaLayerInner {
                 loaded: false,
                 file: None,
@@ -637,7 +577,6 @@ impl DeltaLayer {
             key_range: summary.key_range,
             lsn_range: summary.lsn_range,
             file_size: metadata.len(),
-            access_stats: LayerAccessStats::for_loading_layer(LayerResidenceStatus::Resident),
             inner: RwLock::new(DeltaLayerInner {
                 loaded: false,
                 file: None,
@@ -808,7 +747,6 @@ impl DeltaLayerWriterInner {
             key_range: self.key_start..key_end,
             lsn_range: self.lsn_range.clone(),
             file_size: metadata.len(),
-            access_stats: LayerAccessStats::for_new_layer_file(),
             inner: RwLock::new(DeltaLayerInner {
                 loaded: false,
                 file: None,

pageserver/src/tenant/storage_layer/filename.rs

@@ -1,32 +1,23 @@
 //!
 //! Helper functions for dealing with filenames of the image and delta layer files.
 //!
+use crate::config::PageServerConf;
 use crate::repository::Key;
 use std::cmp::Ordering;
 use std::fmt;
 use std::ops::Range;
+use std::path::PathBuf;
 use std::str::FromStr;
 
 use utils::lsn::Lsn;
 
 // Note: Timeline::load_layer_map() relies on this sort order
-#[derive(PartialEq, Eq, Clone, Hash)]
+#[derive(Debug, PartialEq, Eq, Clone, Hash)]
 pub struct DeltaFileName {
     pub key_range: Range<Key>,
     pub lsn_range: Range<Lsn>,
 }
 
-impl std::fmt::Debug for DeltaFileName {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        use super::RangeDisplayDebug;
-
-        f.debug_struct("DeltaFileName")
-            .field("key_range", &RangeDisplayDebug(&self.key_range))
-            .field("lsn_range", &self.lsn_range)
-            .finish()
-    }
-}
-
 impl PartialOrd for DeltaFileName {
     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
         Some(self.cmp(other))
@@ -111,23 +102,12 @@ impl fmt::Display for DeltaFileName {
     }
 }
 
-#[derive(PartialEq, Eq, Clone, Hash)]
+#[derive(Debug, PartialEq, Eq, Clone, Hash)]
 pub struct ImageFileName {
     pub key_range: Range<Key>,
     pub lsn: Lsn,
 }
 
-impl std::fmt::Debug for ImageFileName {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        use super::RangeDisplayDebug;
-
-        f.debug_struct("ImageFileName")
-            .field("key_range", &RangeDisplayDebug(&self.key_range))
-            .field("lsn", &self.lsn)
-            .finish()
-    }
-}
-
 impl PartialOrd for ImageFileName {
     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
         Some(self.cmp(other))
@@ -150,13 +130,6 @@ impl Ord for ImageFileName {
     }
 }
 
-impl ImageFileName {
-    pub fn lsn_as_range(&self) -> Range<Lsn> {
-        // Saves from having to copypaste this all over
-        self.lsn..(self.lsn + 1)
-    }
-}
-
 ///
 /// Represents the filename of an ImageLayer
 ///
@@ -204,32 +177,49 @@ impl fmt::Display for ImageFileName {
 pub enum LayerFileName {
     Image(ImageFileName),
     Delta(DeltaFileName),
+    #[cfg(test)]
+    Test(String),
 }
 
 impl LayerFileName {
     pub fn file_name(&self) -> String {
         match self {
-            Self::Image(fname) => fname.to_string(),
-            Self::Delta(fname) => fname.to_string(),
+            LayerFileName::Image(fname) => format!("{fname}"),
+            LayerFileName::Delta(fname) => format!("{fname}"),
+            #[cfg(test)]
+            LayerFileName::Test(fname) => fname.to_string(),
         }
     }
+    #[cfg(test)]
+    pub(crate) fn new_test(name: &str) -> LayerFileName {
+        LayerFileName::Test(name.to_owned())
+    }
 }
 
 impl From<ImageFileName> for LayerFileName {
     fn from(fname: ImageFileName) -> Self {
-        Self::Image(fname)
+        LayerFileName::Image(fname)
     }
 }
 impl From<DeltaFileName> for LayerFileName {
     fn from(fname: DeltaFileName) -> Self {
-        Self::Delta(fname)
+        LayerFileName::Delta(fname)
     }
 }
 
+// include a `/` in the name as an additional layer of robustness
+// because `/` chars are not allowed in UNIX paths
+#[cfg(test)]
+const LAYER_FILE_NAME_TEST_PREFIX: &str = "LAYER_FILE_NAME::test/";
+
 impl FromStr for LayerFileName {
     type Err = String;
 
     fn from_str(value: &str) -> Result<Self, Self::Err> {
+        #[cfg(test)]
+        if let Some(value) = value.strip_prefix(LAYER_FILE_NAME_TEST_PREFIX) {
+            return Ok(LayerFileName::Test(value.to_owned()));
+        }
         let delta = DeltaFileName::parse_str(value);
         let image = ImageFileName::parse_str(value);
         let ok = match (delta, image) {
@@ -238,8 +228,8 @@ impl FromStr for LayerFileName {
                     "neither delta nor image layer file name: {value:?}"
                 ))
             }
-            (Some(delta), None) => Self::Delta(delta),
-            (None, Some(image)) => Self::Image(image),
+            (Some(delta), None) => LayerFileName::Delta(delta),
+            (None, Some(image)) => LayerFileName::Image(image),
             (Some(_), Some(_)) => unreachable!(),
         };
         Ok(ok)
@@ -252,8 +242,12 @@ impl serde::Serialize for LayerFileName {
         S: serde::Serializer,
     {
         match self {
-            Self::Image(fname) => serializer.serialize_str(&fname.to_string()),
-            Self::Delta(fname) => serializer.serialize_str(&fname.to_string()),
+            LayerFileName::Image(fname) => serializer.serialize_str(&format!("{}", fname)),
+            LayerFileName::Delta(fname) => serializer.serialize_str(&format!("{}", fname)),
+            #[cfg(test)]
+            LayerFileName::Test(t) => {
+                serializer.serialize_str(&format!("{LAYER_FILE_NAME_TEST_PREFIX}{t}"))
+            }
         }
     }
 }
@@ -276,3 +270,16 @@ impl<'de> serde::de::Visitor<'de> for LayerFileNameVisitor {
         v.parse().map_err(|e| E::custom(e))
     }
 }
+
+/// Helper enum to hold a PageServerConf, or a path
+///
+/// This is used by DeltaLayer and ImageLayer. Normally, this holds a reference to the
+/// global config, and paths to layer files are constructed using the tenant/timeline
+/// path from the config. But in the 'pageserver_binutils' binary, we need to construct a Layer
+/// struct for a file on disk, without having a page server running, so that we have no
+/// config. In that case, we use the Path variant to hold the full path to the file on
+/// disk.
+pub enum PathOrConf {
+    Path(PathBuf),
+    Conf(&'static PageServerConf),
+}

pageserver/src/tenant/storage_layer/image_layer.rs

@@ -20,21 +20,19 @@
 //! mapping from Key to an offset in the "values" part.  The
 //! actual page images are stored in the "values" part.
 use crate::config::PageServerConf;
-use crate::context::RequestContext;
 use crate::page_cache::PAGE_SZ;
 use crate::repository::{Key, KEY_SIZE};
 use crate::tenant::blob_io::{BlobCursor, BlobWriter, WriteBlobWriter};
 use crate::tenant::block_io::{BlockBuf, BlockReader, FileBlockReader};
 use crate::tenant::disk_btree::{DiskBtreeBuilder, DiskBtreeReader, VisitDirection};
 use crate::tenant::storage_layer::{
-    LayerAccessStats, PersistentLayer, ValueReconstructResult, ValueReconstructState,
+    PersistentLayer, ValueReconstructResult, ValueReconstructState,
 };
 use crate::virtual_file::VirtualFile;
 use crate::{IMAGE_FILE_MAGIC, STORAGE_FORMAT_VERSION, TEMP_FILE_SUFFIX};
 use anyhow::{bail, ensure, Context, Result};
 use bytes::Bytes;
 use hex;
-use pageserver_api::models::{HistoricLayerInfo, LayerAccessKind};
 use rand::{distributions::Alphanumeric, Rng};
 use serde::{Deserialize, Serialize};
 use std::fs::{self, File};
@@ -52,8 +50,8 @@ use utils::{
     lsn::Lsn,
 };
 
-use super::filename::{ImageFileName, LayerFileName};
-use super::{Layer, LayerAccessStatsReset, LayerIter, LayerResidenceStatus, PathOrConf};
+use super::filename::{ImageFileName, LayerFileName, PathOrConf};
+use super::{Layer, LayerIter};
 
 ///
 /// Header stored in the beginning of the file
@@ -95,13 +93,13 @@ impl From<&ImageLayer> for Summary {
     }
 }
 
-/// ImageLayer is the in-memory data structure associated with an on-disk image
-/// file.
 ///
-/// We keep an ImageLayer in memory for each file, in the LayerMap. If a layer
-/// is in "loaded" state, we have a copy of the index in memory, in 'inner'.
+/// ImageLayer is the in-memory data structure associated with an on-disk image
+/// file.  We keep an ImageLayer in memory for each file, in the LayerMap. If a
+/// layer is in "loaded" state, we have a copy of the index in memory, in 'inner'.
 /// Otherwise the struct is just a placeholder for a file that exists on disk,
 /// and it needs to be loaded before using it in queries.
+///
 pub struct ImageLayer {
     path_or_conf: PathOrConf,
     pub tenant_id: TenantId,
@@ -112,24 +110,9 @@ pub struct ImageLayer {
     // This entry contains an image of all pages as of this LSN
     pub lsn: Lsn,
 
-    access_stats: LayerAccessStats,
-
     inner: RwLock<ImageLayerInner>,
 }
 
-impl std::fmt::Debug for ImageLayer {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        use super::RangeDisplayDebug;
-
-        f.debug_struct("ImageLayer")
-            .field("key_range", &RangeDisplayDebug(&self.key_range))
-            .field("file_size", &self.file_size)
-            .field("lsn", &self.lsn)
-            .field("inner", &self.inner)
-            .finish()
-    }
-}
-
 pub struct ImageLayerInner {
     /// If false, the 'index' has not been loaded into memory yet.
     loaded: bool,
@@ -142,16 +125,6 @@ pub struct ImageLayerInner {
     file: Option<FileBlockReader<VirtualFile>>,
 }
 
-impl std::fmt::Debug for ImageLayerInner {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("ImageLayerInner")
-            .field("loaded", &self.loaded)
-            .field("index_start_blk", &self.index_start_blk)
-            .field("index_root_blk", &self.index_root_blk)
-            .finish()
-    }
-}
-
 impl Layer for ImageLayer {
     fn get_key_range(&self) -> Range<Key> {
         self.key_range.clone()
@@ -170,7 +143,7 @@ impl Layer for ImageLayer {
     }
 
     /// debugging function to print out the contents of the layer
-    fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()> {
+    fn dump(&self, verbose: bool) -> Result<()> {
         println!(
             "----- image layer for ten {} tli {} key {}-{} at {} ----",
             self.tenant_id, self.timeline_id, self.key_range.start, self.key_range.end, self.lsn
@@ -180,7 +153,7 @@ impl Layer for ImageLayer {
             return Ok(());
         }
 
-        let inner = self.load(LayerAccessKind::Dump, ctx)?;
+        let inner = self.load()?;
         let file = inner.file.as_ref().unwrap();
         let tree_reader =
             DiskBtreeReader::<_, KEY_SIZE>::new(inner.index_start_blk, inner.index_root_blk, file);
@@ -201,13 +174,12 @@ impl Layer for ImageLayer {
         key: Key,
         lsn_range: Range<Lsn>,
         reconstruct_state: &mut ValueReconstructState,
-        ctx: &RequestContext,
     ) -> anyhow::Result<ValueReconstructResult> {
         assert!(self.key_range.contains(&key));
         assert!(lsn_range.start >= self.lsn);
         assert!(lsn_range.end >= self.lsn);
 
-        let inner = self.load(LayerAccessKind::GetValueReconstructData, ctx)?;
+        let inner = self.load()?;
 
         let file = inner.file.as_ref().unwrap();
         let tree_reader = DiskBtreeReader::new(inner.index_start_blk, inner.index_root_blk, file);
@@ -248,7 +220,7 @@ impl PersistentLayer for ImageLayer {
     fn get_timeline_id(&self) -> TimelineId {
         self.timeline_id
     }
-    fn iter(&self, _ctx: &RequestContext) -> Result<LayerIter<'_>> {
+    fn iter(&self) -> Result<LayerIter<'_>> {
         unimplemented!();
     }
 
@@ -261,23 +233,6 @@ impl PersistentLayer for ImageLayer {
     fn file_size(&self) -> Option<u64> {
         Some(self.file_size)
     }
-
-    fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo {
-        let layer_file_name = self.filename().file_name();
-        let lsn_range = self.get_lsn_range();
-
-        HistoricLayerInfo::Image {
-            layer_file_name,
-            layer_file_size: Some(self.file_size),
-            lsn_start: lsn_range.start,
-            remote: false,
-            access_stats: self.access_stats.as_api_model(reset),
-        }
-    }
-
-    fn access_stats(&self) -> &LayerAccessStats {
-        &self.access_stats
-    }
 }
 
 impl ImageLayer {
@@ -315,13 +270,7 @@ impl ImageLayer {
     /// Open the underlying file and read the metadata into memory, if it's
     /// not loaded already.
     ///
-    fn load(
-        &self,
-        access_kind: LayerAccessKind,
-        ctx: &RequestContext,
-    ) -> Result<RwLockReadGuard<ImageLayerInner>> {
-        self.access_stats
-            .record_access(access_kind, ctx.task_kind());
+    fn load(&self) -> Result<RwLockReadGuard<ImageLayerInner>> {
         loop {
             // Quick exit if already loaded
             let inner = self.inner.read().unwrap();
@@ -401,7 +350,6 @@ impl ImageLayer {
         tenant_id: TenantId,
         filename: &ImageFileName,
         file_size: u64,
-        access_stats: LayerAccessStats,
     ) -> ImageLayer {
         ImageLayer {
             path_or_conf: PathOrConf::Conf(conf),
@@ -410,7 +358,6 @@ impl ImageLayer {
             key_range: filename.key_range.clone(),
             lsn: filename.lsn,
             file_size,
-            access_stats,
             inner: RwLock::new(ImageLayerInner {
                 loaded: false,
                 file: None,
@@ -438,7 +385,6 @@ impl ImageLayer {
             key_range: summary.key_range,
             lsn: summary.lsn,
             file_size: metadata.len(),
-            access_stats: LayerAccessStats::for_loading_layer(LayerResidenceStatus::Resident),
             inner: RwLock::new(ImageLayerInner {
                 file: None,
                 loaded: false,
@@ -598,7 +544,6 @@ impl ImageLayerWriterInner {
             key_range: self.key_range.clone(),
             lsn: self.lsn,
             file_size: metadata.len(),
-            access_stats: LayerAccessStats::for_new_layer_file(),
             inner: RwLock::new(ImageLayerInner {
                 loaded: false,
                 file: None,

pageserver/src/tenant/storage_layer/inmemory_layer.rs

@@ -5,7 +5,6 @@
 //! its position in the file, is kept in memory, though.
 //!
 use crate::config::PageServerConf;
-use crate::context::RequestContext;
 use crate::repository::{Key, Value};
 use crate::tenant::blob_io::{BlobCursor, BlobWriter};
 use crate::tenant::block_io::BlockReader;
@@ -13,7 +12,6 @@ use crate::tenant::ephemeral_file::EphemeralFile;
 use crate::tenant::storage_layer::{ValueReconstructResult, ValueReconstructState};
 use crate::walrecord;
 use anyhow::{ensure, Result};
-use pageserver_api::models::InMemoryLayerInfo;
 use std::cell::RefCell;
 use std::collections::HashMap;
 use tracing::*;
@@ -53,15 +51,6 @@ pub struct InMemoryLayer {
     inner: RwLock<InMemoryLayerInner>,
 }
 
-impl std::fmt::Debug for InMemoryLayer {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("InMemoryLayer")
-            .field("start_lsn", &self.start_lsn)
-            .field("inner", &self.inner)
-            .finish()
-    }
-}
-
 pub struct InMemoryLayerInner {
     /// Frozen layers have an exclusive end LSN.
     /// Writes are only allowed when this is None
@@ -80,14 +69,6 @@ pub struct InMemoryLayerInner {
     file: EphemeralFile,
 }
 
-impl std::fmt::Debug for InMemoryLayerInner {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("InMemoryLayerInner")
-            .field("end_lsn", &self.end_lsn)
-            .finish()
-    }
-}
-
 impl InMemoryLayerInner {
     fn assert_writeable(&self) {
         assert!(self.end_lsn.is_none());
@@ -98,16 +79,6 @@ impl InMemoryLayer {
     pub fn get_timeline_id(&self) -> TimelineId {
         self.timeline_id
     }
-
-    pub fn info(&self) -> InMemoryLayerInfo {
-        let lsn_start = self.start_lsn;
-        let lsn_end = self.inner.read().unwrap().end_lsn;
-
-        match lsn_end {
-            Some(lsn_end) => InMemoryLayerInfo::Frozen { lsn_start, lsn_end },
-            None => InMemoryLayerInfo::Open { lsn_start },
-        }
-    }
 }
 
 impl Layer for InMemoryLayer {
@@ -139,7 +110,7 @@ impl Layer for InMemoryLayer {
     }
 
     /// debugging function to print out the contents of the layer
-    fn dump(&self, verbose: bool, _ctx: &RequestContext) -> Result<()> {
+    fn dump(&self, verbose: bool) -> Result<()> {
         let inner = self.inner.read().unwrap();
 
         let end_str = inner
@@ -195,7 +166,6 @@ impl Layer for InMemoryLayer {
         key: Key,
         lsn_range: Range<Lsn>,
         reconstruct_state: &mut ValueReconstructState,
-        _ctx: &RequestContext,
     ) -> anyhow::Result<ValueReconstructResult> {
         ensure!(lsn_range.start >= self.start_lsn);
         let mut need_image = true;

pageserver/src/tenant/storage_layer/remote_layer.rs

@@ -2,12 +2,10 @@
 //! in remote storage.
 //!
 use crate::config::PageServerConf;
-use crate::context::RequestContext;
 use crate::repository::Key;
 use crate::tenant::remote_timeline_client::index::LayerFileMetadata;
 use crate::tenant::storage_layer::{Layer, ValueReconstructResult, ValueReconstructState};
 use anyhow::{bail, Result};
-use pageserver_api::models::HistoricLayerInfo;
 use std::ops::Range;
 use std::path::PathBuf;
 use std::sync::Arc;
@@ -19,19 +17,9 @@ use utils::{
 
 use super::filename::{DeltaFileName, ImageFileName, LayerFileName};
 use super::image_layer::ImageLayer;
-use super::{
-    DeltaLayer, LayerAccessStats, LayerAccessStatsReset, LayerIter, LayerKeyIter,
-    LayerResidenceStatus, PersistentLayer,
-};
+use super::{DeltaLayer, LayerIter, LayerKeyIter, PersistentLayer};
 
-/// RemoteLayer is a not yet downloaded [`ImageLayer`] or
-/// [`crate::storage_layer::DeltaLayer`].
-///
-/// RemoteLayer might be downloaded on-demand during operations which are
-/// allowed download remote layers and during which, it gets replaced with a
-/// concrete `DeltaLayer` or `ImageLayer`.
-///
-/// See: [`crate::context::RequestContext`] for authorization to download
+#[derive(Debug)]
 pub struct RemoteLayer {
     tenantid: TenantId,
     timelineid: TimelineId,
@@ -46,30 +34,7 @@ pub struct RemoteLayer {
 
     is_incremental: bool,
 
-    access_stats: LayerAccessStats,
-
     pub(crate) ongoing_download: Arc<tokio::sync::Semaphore>,
-
-    /// Has `LayerMap::replace` failed for this (true) or not (false).
-    ///
-    /// Used together with [`ongoing_download`] semaphore in `Timeline::download_remote_layer`.
-    /// The field is used to mark a RemoteLayer permanently (until restart or ignore+load)
-    /// unprocessable, because a LayerMap::replace failed.
-    ///
-    /// It is very unlikely to accumulate these in the Timeline's LayerMap, but having this avoids
-    /// a possible fast loop between `Timeline::get_reconstruct_data` and
-    /// `Timeline::download_remote_layer`, which also logs.
-    pub(crate) download_replacement_failure: std::sync::atomic::AtomicBool,
-}
-
-impl std::fmt::Debug for RemoteLayer {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("RemoteLayer")
-            .field("file_name", &self.file_name)
-            .field("layer_metadata", &self.layer_metadata)
-            .field("is_incremental", &self.is_incremental)
-            .finish()
-    }
 }
 
 impl Layer for RemoteLayer {
@@ -86,7 +51,6 @@ impl Layer for RemoteLayer {
         _key: Key,
         _lsn_range: Range<Lsn>,
         _reconstruct_state: &mut ValueReconstructState,
-        _ctx: &RequestContext,
     ) -> Result<ValueReconstructResult> {
         bail!(
             "layer {} needs to be downloaded",
@@ -99,7 +63,7 @@ impl Layer for RemoteLayer {
     }
 
     /// debugging function to print out the contents of the layer
-    fn dump(&self, _verbose: bool, _ctx: &RequestContext) -> Result<()> {
+    fn dump(&self, _verbose: bool) -> Result<()> {
         println!(
             "----- remote layer for ten {} tli {} keys {}-{} lsn {}-{} ----",
             self.tenantid,
@@ -147,11 +111,11 @@ impl PersistentLayer for RemoteLayer {
         None
     }
 
-    fn iter(&self, _ctx: &RequestContext) -> Result<LayerIter<'_>> {
+    fn iter(&self) -> Result<LayerIter<'_>> {
         bail!("cannot iterate a remote layer");
     }
 
-    fn key_iter(&self, _ctx: &RequestContext) -> Result<LayerKeyIter<'_>> {
+    fn key_iter(&self) -> Result<LayerKeyIter<'_>> {
         bail!("cannot iterate a remote layer");
     }
 
@@ -170,34 +134,6 @@ impl PersistentLayer for RemoteLayer {
     fn file_size(&self) -> Option<u64> {
         self.layer_metadata.file_size()
     }
-
-    fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo {
-        let layer_file_name = self.filename().file_name();
-        let lsn_range = self.get_lsn_range();
-
-        if self.is_delta {
-            HistoricLayerInfo::Delta {
-                layer_file_name,
-                layer_file_size: self.layer_metadata.file_size(),
-                lsn_start: lsn_range.start,
-                lsn_end: lsn_range.end,
-                remote: true,
-                access_stats: self.access_stats.as_api_model(reset),
-            }
-        } else {
-            HistoricLayerInfo::Image {
-                layer_file_name,
-                layer_file_size: self.layer_metadata.file_size(),
-                lsn_start: lsn_range.start,
-                remote: true,
-                access_stats: self.access_stats.as_api_model(reset),
-            }
-        }
-    }
-
-    fn access_stats(&self) -> &LayerAccessStats {
-        &self.access_stats
-    }
 }
 
 impl RemoteLayer {
@@ -206,20 +142,17 @@ impl RemoteLayer {
         timelineid: TimelineId,
         fname: &ImageFileName,
         layer_metadata: &LayerFileMetadata,
-        access_stats: LayerAccessStats,
     ) -> RemoteLayer {
         RemoteLayer {
             tenantid,
             timelineid,
             key_range: fname.key_range.clone(),
-            lsn_range: fname.lsn_as_range(),
+            lsn_range: fname.lsn..(fname.lsn + 1),
             is_delta: false,
             is_incremental: false,
             file_name: fname.to_owned().into(),
             layer_metadata: layer_metadata.clone(),
             ongoing_download: Arc::new(tokio::sync::Semaphore::new(1)),
-            download_replacement_failure: std::sync::atomic::AtomicBool::default(),
-            access_stats,
         }
     }
 
@@ -228,7 +161,6 @@ impl RemoteLayer {
         timelineid: TimelineId,
         fname: &DeltaFileName,
         layer_metadata: &LayerFileMetadata,
-        access_stats: LayerAccessStats,
     ) -> RemoteLayer {
         RemoteLayer {
             tenantid,
@@ -240,8 +172,6 @@ impl RemoteLayer {
             file_name: fname.to_owned().into(),
             layer_metadata: layer_metadata.clone(),
             ongoing_download: Arc::new(tokio::sync::Semaphore::new(1)),
-            download_replacement_failure: std::sync::atomic::AtomicBool::default(),
-            access_stats,
         }
     }
 
@@ -262,8 +192,6 @@ impl RemoteLayer {
                 self.tenantid,
                 &fname,
                 file_size,
-                self.access_stats
-                    .clone_for_residence_change(LayerResidenceStatus::Resident),
             ))
         } else {
             let fname = ImageFileName {
@@ -276,8 +204,6 @@ impl RemoteLayer {
                 self.tenantid,
                 &fname,
                 file_size,
-                self.access_stats
-                    .clone_for_residence_change(LayerResidenceStatus::Resident),
             ))
         }
     }

pageserver/src/tenant/tasks.rs

@@ -3,7 +3,7 @@
 
 use std::ops::ControlFlow;
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use std::time::Duration;
 
 use crate::context::{DownloadBehavior, RequestContext};
 use crate::metrics::TENANT_TASK_EVENTS;
@@ -11,7 +11,6 @@ use crate::task_mgr;
 use crate::task_mgr::{TaskKind, BACKGROUND_RUNTIME};
 use crate::tenant::mgr;
 use crate::tenant::{Tenant, TenantState};
-use tokio_util::sync::CancellationToken;
 use tracing::*;
 use utils::id::TenantId;
 
@@ -54,55 +53,37 @@ async fn compaction_loop(tenant_id: TenantId) {
     info!("starting");
     TENANT_TASK_EVENTS.with_label_values(&["start"]).inc();
     async {
-        let cancel = task_mgr::shutdown_token();
         let ctx = RequestContext::todo_child(TaskKind::Compaction, DownloadBehavior::Download);
-        let mut first = true;
         loop {
             trace!("waking up");
 
             let tenant = tokio::select! {
-                _ = cancel.cancelled() => {
+                _ = task_mgr::shutdown_watcher() => {
                     info!("received cancellation request");
-                    return;
+                return;
                 },
                 tenant_wait_result = wait_for_active_tenant(tenant_id, wait_duration) => match tenant_wait_result {
                     ControlFlow::Break(()) => return,
                     ControlFlow::Continue(tenant) => tenant,
                 },
-            };
+        };
 
-            let period = tenant.get_compaction_period();
-
-            // TODO: we shouldn't need to await to find tenant and this could be moved outside of
-            // loop, #3501. There are also additional "allowed_errors" in tests.
-            if first {
-                first = false;
-                if random_init_delay(period, &cancel).await.is_err() {
-                    break;
-                }
-            }
-
-            let started_at = Instant::now();
-
-            let sleep_duration = if period == Duration::ZERO {
+            let mut sleep_duration = tenant.get_compaction_period();
+            if sleep_duration == Duration::ZERO {
                 info!("automatic compaction is disabled");
                 // check again in 10 seconds, in case it's been enabled again.
-                Duration::from_secs(10)
+                sleep_duration = Duration::from_secs(10);
             } else {
                 // Run compaction
                 if let Err(e) = tenant.compaction_iteration(&ctx).await {
-                    error!("Compaction failed, retrying in {:?}: {e:?}", wait_duration);
-                    wait_duration
-                } else {
-                    period
+                    sleep_duration = wait_duration;
+                    error!("Compaction failed, retrying in {:?}: {e:?}", sleep_duration);
                 }
-            };
-
-            warn_when_period_overrun(started_at.elapsed(), period, "compaction");
+            }
 
             // Sleep
             tokio::select! {
-                _ = cancel.cancelled() => {
+                _ = task_mgr::shutdown_watcher() => {
                     info!("received cancellation request during idling");
                     break;
                 },
@@ -124,16 +105,14 @@ async fn gc_loop(tenant_id: TenantId) {
     info!("starting");
     TENANT_TASK_EVENTS.with_label_values(&["start"]).inc();
     async {
-        let cancel = task_mgr::shutdown_token();
         // GC might require downloading, to find the cutoff LSN that corresponds to the
         // cutoff specified as time.
         let ctx = RequestContext::todo_child(TaskKind::GarbageCollector, DownloadBehavior::Download);
-        let mut first = true;
         loop {
             trace!("waking up");
 
             let tenant = tokio::select! {
-                _ = cancel.cancelled() => {
+                _ = task_mgr::shutdown_watcher() => {
                     info!("received cancellation request");
                     return;
                 },
@@ -143,38 +122,27 @@ async fn gc_loop(tenant_id: TenantId) {
                 },
             };
 
-            let period = tenant.get_gc_period();
-
-            if first {
-                first = false;
-                if random_init_delay(period, &cancel).await.is_err() {
-                    break;
+            let gc_period = tenant.get_gc_period();
+            let gc_horizon = tenant.get_gc_horizon();
+            let mut sleep_duration = gc_period;
+            if sleep_duration == Duration::ZERO {
+                info!("automatic GC is disabled");
+                // check again in 10 seconds, in case it's been enabled again.
+                sleep_duration = Duration::from_secs(10);
+            } else {
+                // Run gc
+                if gc_horizon > 0 {
+                    if let Err(e) = tenant.gc_iteration(None, gc_horizon, tenant.get_pitr_interval(), &ctx).await
+                    {
+                        sleep_duration = wait_duration;
+                        error!("Gc failed, retrying in {:?}: {e:?}", sleep_duration);
+                    }
                 }
             }
 
-            let started_at = Instant::now();
-
-            let gc_horizon = tenant.get_gc_horizon();
-            let sleep_duration = if period == Duration::ZERO || gc_horizon == 0 {
-                info!("automatic GC is disabled");
-                // check again in 10 seconds, in case it's been enabled again.
-                Duration::from_secs(10)
-            } else {
-                // Run gc
-                let res = tenant.gc_iteration(None, gc_horizon, tenant.get_pitr_interval(), &ctx).await;
-                if let Err(e) = res {
-                    error!("Gc failed, retrying in {:?}: {e:?}", wait_duration);
-                    wait_duration
-                } else {
-                    period
-                }
-            };
-
-            warn_when_period_overrun(started_at.elapsed(), period, "gc");
-
             // Sleep
             tokio::select! {
-                _ = cancel.cancelled() => {
+                _ = task_mgr::shutdown_watcher() => {
                     info!("received cancellation request during idling");
                     break;
                 },
@@ -229,49 +197,3 @@ async fn wait_for_active_tenant(
         }
     }
 }
-
-#[derive(thiserror::Error, Debug)]
-#[error("cancelled")]
-pub(crate) struct Cancelled;
-
-/// Provide a random delay for background task initialization.
-///
-/// This delay prevents a thundering herd of background tasks and will likely keep them running on
-/// different periods for more stable load.
-pub(crate) async fn random_init_delay(
-    period: Duration,
-    cancel: &CancellationToken,
-) -> Result<(), Cancelled> {
-    use rand::Rng;
-
-    let d = {
-        let mut rng = rand::thread_rng();
-
-        // gen_range asserts that the range cannot be empty, which it could be because period can
-        // be set to zero to disable gc or compaction, so lets set it to be at least 10s.
-        let period = std::cmp::max(period, Duration::from_secs(10));
-
-        // semi-ok default as the source of jitter
-        rng.gen_range(Duration::ZERO..=period)
-    };
-
-    tokio::select! {
-        _ = cancel.cancelled() => Err(Cancelled),
-        _ = tokio::time::sleep(d) => Ok(()),
-    }
-}
-
-pub(crate) fn warn_when_period_overrun(elapsed: Duration, period: Duration, task: &str) {
-    // Duration::ZERO will happen because it's the "disable [bgtask]" value.
-    if elapsed >= period && period != Duration::ZERO {
-        // humantime does no significant digits clamping whereas Duration's debug is a bit more
-        // intelligent. however it makes sense to keep the "configuration format" for period, even
-        // though there's no way to output the actual config value.
-        warn!(
-            ?elapsed,
-            period = %humantime::format_duration(period),
-            task,
-            "task iteration took longer than the configured period"
-        );
-    }
-}

pageserver/src/tenant/timeline/eviction_task.rs

@@ -1,219 +0,0 @@
-//! The per-timeline layer eviction task.
-
-use std::{
-    ops::ControlFlow,
-    sync::Arc,
-    time::{Duration, SystemTime},
-};
-
-use either::Either;
-use tokio::time::Instant;
-use tokio_util::sync::CancellationToken;
-use tracing::{debug, error, info, instrument, warn};
-
-use crate::{
-    task_mgr::{self, TaskKind, BACKGROUND_RUNTIME},
-    tenant::{
-        config::{EvictionPolicy, EvictionPolicyLayerAccessThreshold},
-        storage_layer::PersistentLayer,
-    },
-};
-
-use super::Timeline;
-
-impl Timeline {
-    pub(super) fn launch_eviction_task(self: &Arc<Self>) {
-        let self_clone = Arc::clone(self);
-        task_mgr::spawn(
-            BACKGROUND_RUNTIME.handle(),
-            TaskKind::Eviction,
-            Some(self.tenant_id),
-            Some(self.timeline_id),
-            &format!("layer eviction for {}/{}", self.tenant_id, self.timeline_id),
-            false,
-            async move {
-                self_clone.eviction_task(task_mgr::shutdown_token()).await;
-                info!("eviction task finishing");
-                Ok(())
-            },
-        );
-    }
-
-    #[instrument(skip_all, fields(tenant_id = %self.tenant_id, timeline_id = %self.timeline_id))]
-    async fn eviction_task(self: Arc<Self>, cancel: CancellationToken) {
-        use crate::tenant::tasks::random_init_delay;
-        {
-            let policy = self.get_eviction_policy();
-            let period = match policy {
-                EvictionPolicy::LayerAccessThreshold(lat) => lat.period,
-                EvictionPolicy::NoEviction => Duration::from_secs(10),
-            };
-            if random_init_delay(period, &cancel).await.is_err() {
-                info!("shutting down");
-                return;
-            }
-        }
-
-        loop {
-            let policy = self.get_eviction_policy();
-            let cf = self.eviction_iteration(&policy, cancel.clone()).await;
-
-            match cf {
-                ControlFlow::Break(()) => break,
-                ControlFlow::Continue(sleep_until) => {
-                    tokio::select! {
-                        _ = cancel.cancelled() => {
-                            info!("shutting down");
-                            break;
-                        }
-                        _ = tokio::time::sleep_until(sleep_until) => { }
-                    }
-                }
-            }
-        }
-    }
-
-    #[instrument(skip_all, fields(policy_kind = policy.discriminant_str()))]
-    async fn eviction_iteration(
-        self: &Arc<Self>,
-        policy: &EvictionPolicy,
-        cancel: CancellationToken,
-    ) -> ControlFlow<(), Instant> {
-        debug!("eviction iteration: {policy:?}");
-        match policy {
-            EvictionPolicy::NoEviction => {
-                // check again in 10 seconds; XXX config watch mechanism
-                ControlFlow::Continue(Instant::now() + Duration::from_secs(10))
-            }
-            EvictionPolicy::LayerAccessThreshold(p) => {
-                let start = Instant::now();
-                match self.eviction_iteration_threshold(p, cancel).await {
-                    ControlFlow::Break(()) => return ControlFlow::Break(()),
-                    ControlFlow::Continue(()) => (),
-                }
-                let elapsed = start.elapsed();
-                crate::tenant::tasks::warn_when_period_overrun(elapsed, p.period, "eviction");
-                ControlFlow::Continue(start + p.period)
-            }
-        }
-    }
-
-    async fn eviction_iteration_threshold(
-        self: &Arc<Self>,
-        p: &EvictionPolicyLayerAccessThreshold,
-        cancel: CancellationToken,
-    ) -> ControlFlow<()> {
-        let now = SystemTime::now();
-
-        #[allow(dead_code)]
-        #[derive(Debug, Default)]
-        struct EvictionStats {
-            candidates: usize,
-            evicted: usize,
-            errors: usize,
-            not_evictable: usize,
-            skipped_for_shutdown: usize,
-        }
-        let mut stats = EvictionStats::default();
-        // Gather layers for eviction.
-        // NB: all the checks can be invalidated as soon as we release the layer map lock.
-        // We don't want to hold the layer map lock during eviction.
-        // So, we just need to deal with this.
-        let candidates: Vec<Arc<dyn PersistentLayer>> = {
-            let layers = self.layers.read().unwrap();
-            let mut candidates = Vec::new();
-            for hist_layer in layers.iter_historic_layers() {
-                if hist_layer.is_remote_layer() {
-                    continue;
-                }
-                let last_activity_ts = match hist_layer
-                    .access_stats()
-                    .most_recent_access_or_residence_event()
-                {
-                    Either::Left(mra) => mra.when,
-                    Either::Right(re) => re.timestamp,
-                };
-                let no_activity_for = match now.duration_since(last_activity_ts) {
-                    Ok(d) => d,
-                    Err(_e) => {
-                        // We reach here if `now` < `last_activity_ts`, which can legitimately
-                        // happen if there is an access between us getting `now`, and us getting
-                        // the access stats from the layer.
-                        //
-                        // The other reason why it can happen is system clock skew because
-                        // SystemTime::now() is not monotonic, so, even if there is no access
-                        // to the layer after we get `now` at the beginning of this function,
-                        // it could be that `now`  < `last_activity_ts`.
-                        //
-                        // To distinguish the cases, we would need to record `Instant`s in the
-                        // access stats (i.e., monotonic timestamps), but then, the timestamps
-                        // values in the access stats would need to be `Instant`'s, and hence
-                        // they would be meaningless outside of the pageserver process.
-                        // At the time of writing, the trade-off is that access stats are more
-                        // valuable than detecting clock skew.
-                        continue;
-                    }
-                };
-                if no_activity_for > p.threshold {
-                    candidates.push(hist_layer)
-                }
-            }
-            candidates
-        };
-        stats.candidates = candidates.len();
-
-        let remote_client = match self.remote_client.as_ref() {
-            None => {
-                error!(
-                    num_candidates = candidates.len(),
-                    "no remote storage configured, cannot evict layers"
-                );
-                return ControlFlow::Continue(());
-            }
-            Some(c) => c,
-        };
-
-        let results = match self
-            .evict_layer_batch(remote_client, &candidates[..], cancel)
-            .await
-        {
-            Err(pre_err) => {
-                stats.errors += candidates.len();
-                error!("could not do any evictions: {pre_err:#}");
-                return ControlFlow::Continue(());
-            }
-            Ok(results) => results,
-        };
-        assert_eq!(results.len(), candidates.len());
-        for (l, result) in candidates.iter().zip(results) {
-            match result {
-                None => {
-                    stats.skipped_for_shutdown += 1;
-                }
-                Some(Ok(true)) => {
-                    debug!("evicted layer {l:?}");
-                    stats.evicted += 1;
-                }
-                Some(Ok(false)) => {
-                    debug!("layer is not evictable: {l:?}");
-                    stats.not_evictable += 1;
-                }
-                Some(Err(e)) => {
-                    // This variant is the case where an unexpected error happened during eviction.
-                    // Expected errors that result in non-eviction are `Some(Ok(false))`.
-                    // So, dump Debug here to gather as much info as possible in this rare case.
-                    warn!("failed to evict layer {l:?}: {e:?}");
-                    stats.errors += 1;
-                }
-            }
-        }
-        if stats.candidates == stats.not_evictable {
-            debug!(stats=?stats, "eviction iteration complete");
-        } else if stats.errors > 0 || stats.not_evictable > 0 {
-            warn!(stats=?stats, "eviction iteration complete");
-        } else {
-            info!(stats=?stats, "eviction iteration complete");
-        }
-        ControlFlow::Continue(())
-    }
-}

pageserver/src/tenant/timeline/walreceiver/connection_manager.rs

@@ -13,7 +13,7 @@ use std::{collections::HashMap, num::NonZeroU64, ops::ControlFlow, sync::Arc, ti
 
 use super::TaskStateUpdate;
 use crate::broker_client::get_broker_client;
-use crate::context::{DownloadBehavior, RequestContext};
+use crate::context::RequestContext;
 use crate::task_mgr::WALRECEIVER_RUNTIME;
 use crate::task_mgr::{self, TaskKind};
 use crate::tenant::Timeline;
@@ -413,7 +413,7 @@ impl WalreceiverState {
         let timeline = Arc::clone(&self.timeline);
         let ctx = ctx.detached_child(
             TaskKind::WalReceiverConnectionHandler,
-            DownloadBehavior::Download,
+            ctx.download_behavior(),
         );
         let connection_handle = TaskHandle::spawn(move |events_sender, cancellation| {
             async move {

pageserver/src/walredo.rs

@@ -27,7 +27,8 @@ use std::fs::OpenOptions;
 use std::io::prelude::*;
 use std::io::{Error, ErrorKind};
 use std::ops::{Deref, DerefMut};
-use std::os::unix::io::{AsRawFd, RawFd};
+use std::os::fd::RawFd;
+use std::os::unix::io::AsRawFd;
 use std::os::unix::prelude::CommandExt;
 use std::path::PathBuf;
 use std::process::Stdio;

pgxn/neon/file_cache.c

@@ -132,7 +132,7 @@ lfc_shmem_request(void)
 	RequestNamedLWLockTranche("lfc_lock", 1);
 }
 
-static bool
+bool
 lfc_check_limit_hook(int *newval, void **extra, GucSource source)
 {
 	if (*newval > lfc_max_size)
@@ -143,7 +143,7 @@ lfc_check_limit_hook(int *newval, void **extra, GucSource source)
 	return true;
 }
 
-static void
+void
 lfc_change_limit_hook(int newval, void *extra)
 {
 	uint32 new_size = SIZE_MB_TO_CHUNKS(newval);
@@ -213,7 +213,7 @@ lfc_init(void)
 							INT_MAX,
 							PGC_SIGHUP,
 							GUC_UNIT_MB,
-							lfc_check_limit_hook,
+							NULL,
 							lfc_change_limit_hook,
 							NULL);
 
@@ -472,6 +472,7 @@ local_cache_pages(PG_FUNCTION_ARGS)
         HASH_SEQ_STATUS status;
 		FileCacheEntry* entry;
 		uint32 n_pages = 0;
+		uint32 i;
 
 		funcctx = SRF_FIRSTCALL_INIT();
 

proxy/Cargo.toml

@@ -6,65 +6,58 @@ license.workspace = true
 
 [dependencies]
 anyhow.workspace = true
-async-trait.workspace = true
 atty.workspace = true
 base64.workspace = true
 bstr.workspace = true
-bytes = { workspace = true, features = ["serde"] }
-chrono.workspace = true
+bytes = {workspace = true, features = ['serde'] }
 clap.workspace = true
+chrono.workspace = true
 consumption_metrics.workspace = true
 futures.workspace = true
 git-version.workspace = true
 hashbrown.workspace = true
-hashlink.workspace = true
 hex.workspace = true
 hmac.workspace = true
-hostname.workspace = true
-humantime.workspace = true
-hyper-tungstenite.workspace = true
 hyper.workspace = true
+hyper-tungstenite.workspace = true
 itertools.workspace = true
 md5.workspace = true
-metrics.workspace = true
 once_cell.workspace = true
-opentelemetry.workspace = true
 parking_lot.workspace = true
 pin-project-lite.workspace = true
-pq_proto.workspace = true
-prometheus.workspace = true
 rand.workspace = true
 regex.workspace = true
-reqwest = { workspace = true, features = ["json"] }
-reqwest-middleware.workspace = true
-reqwest-tracing.workspace = true
+reqwest = { workspace = true, features = [ "json" ] }
 routerify.workspace = true
-rustls-pemfile.workspace = true
 rustls.workspace = true
+rustls-pemfile.workspace = true
 scopeguard.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 sha2.workspace = true
 socket2.workspace = true
-sync_wrapper.workspace = true
 thiserror.workspace = true
-tls-listener.workspace = true
+tokio.workspace = true
 tokio-postgres.workspace = true
 tokio-rustls.workspace = true
-tokio = { workspace = true, features = ["signal"] }
-tracing-opentelemetry.workspace = true
-tracing-subscriber.workspace = true
-tracing-utils.workspace = true
+tls-listener.workspace = true
 tracing.workspace = true
+tracing-subscriber.workspace = true
 url.workspace = true
-utils.workspace = true
 uuid.workspace = true
 webpki-roots.workspace = true
 x509-parser.workspace = true
+metrics.workspace = true
+pq_proto.workspace = true
+utils.workspace = true
+prometheus.workspace = true
+humantime.workspace = true
+hostname.workspace = true
 
 workspace_hack.workspace = true
 
 [dev-dependencies]
+async-trait.workspace = true
 rcgen.workspace = true
 rstest.workspace = true
 tokio-postgres-rustls.workspace = true

proxy/src/auth.rs

@@ -1,7 +1,7 @@
 //! Client authentication mechanisms.
 
 pub mod backend;
-pub use backend::BackendType;
+pub use backend::{BackendType, ConsoleReqExtra};
 
 mod credentials;
 pub use credentials::ClientCredentials;
@@ -12,7 +12,7 @@ use password_hack::PasswordHackPayload;
 mod flow;
 pub use flow::*;
 
-use crate::{console, error::UserFacingError};
+use crate::error::UserFacingError;
 use std::io;
 use thiserror::Error;
 
@@ -26,10 +26,10 @@ pub enum AuthErrorImpl {
     Link(#[from] backend::LinkAuthError),
 
     #[error(transparent)]
-    GetAuthInfo(#[from] console::errors::GetAuthInfoError),
+    GetAuthInfo(#[from] backend::GetAuthInfoError),
 
     #[error(transparent)]
-    WakeCompute(#[from] console::errors::WakeComputeError),
+    WakeCompute(#[from] backend::WakeComputeError),
 
     /// SASL protocol errors (includes [SCRAM](crate::scram)).
     #[error(transparent)]

proxy/src/auth/backend.rs

@@ -1,41 +1,48 @@
-mod classic;
-mod hacks;
-mod link;
+mod postgres;
 
+mod link;
 pub use link::LinkAuthError;
 
+mod console;
+pub use console::{GetAuthInfoError, WakeComputeError};
+
 use crate::{
-    auth::{self, ClientCredentials},
-    console::{
-        self,
-        provider::{CachedNodeInfo, ConsoleReqExtra},
-        Api,
-    },
-    stream, url,
+    auth::{self, AuthFlow, ClientCredentials},
+    compute,
+    console::messages::MetricsAuxInfo,
+    http, mgmt, stream, url,
+    waiters::{self, Waiter, Waiters},
 };
-use futures::TryFutureExt;
+use once_cell::sync::Lazy;
 use std::borrow::Cow;
 use tokio::io::{AsyncRead, AsyncWrite};
-use tracing::info;
+use tracing::{info, warn};
 
-/// A product of successful authentication.
-pub struct AuthSuccess<T> {
-    /// Did we send [`pq_proto::BeMessage::AuthenticationOk`] to client?
-    pub reported_auth_ok: bool,
-    /// Something to be considered a positive result.
-    pub value: T,
+static CPLANE_WAITERS: Lazy<Waiters<mgmt::ComputeReady>> = Lazy::new(Default::default);
+
+/// Give caller an opportunity to wait for the cloud's reply.
+pub async fn with_waiter<R, T, E>(
+    psql_session_id: impl Into<String>,
+    action: impl FnOnce(Waiter<'static, mgmt::ComputeReady>) -> R,
+) -> Result<T, E>
+where
+    R: std::future::Future<Output = Result<T, E>>,
+    E: From<waiters::RegisterError>,
+{
+    let waiter = CPLANE_WAITERS.register(psql_session_id.into())?;
+    action(waiter).await
 }
 
-impl<T> AuthSuccess<T> {
-    /// Very similar to [`std::option::Option::map`].
-    /// Maps [`AuthSuccess<T>`] to [`AuthSuccess<R>`] by applying
-    /// a function to a contained value.
-    pub fn map<R>(self, f: impl FnOnce(T) -> R) -> AuthSuccess<R> {
-        AuthSuccess {
-            reported_auth_ok: self.reported_auth_ok,
-            value: f(self.value),
-        }
-    }
+pub fn notify(psql_session_id: &str, msg: mgmt::ComputeReady) -> Result<(), waiters::NotifyError> {
+    CPLANE_WAITERS.notify(psql_session_id, msg)
+}
+
+/// Extra query params we'd like to pass to the console.
+pub struct ConsoleReqExtra<'a> {
+    /// A unique identifier for a connection.
+    pub session_id: uuid::Uuid,
+    /// Name of client application, if set.
+    pub application_name: Option<&'a str>,
 }
 
 /// This type serves two purposes:
@@ -46,11 +53,12 @@ impl<T> AuthSuccess<T> {
 /// * However, when we substitute `T` with [`ClientCredentials`],
 ///   this helps us provide the credentials only to those auth
 ///   backends which require them for the authentication process.
+#[derive(Debug)]
 pub enum BackendType<'a, T> {
     /// Current Cloud API (V2).
-    Console(Cow<'a, console::provider::neon::Api>, T),
+    Console(Cow<'a, http::Endpoint>, T),
     /// Local mock of Cloud API (V2).
-    Postgres(Cow<'a, console::provider::mock::Api>, T),
+    Postgres(Cow<'a, url::ApiUrl>, T),
     /// Authentication via a web browser.
     Link(Cow<'a, url::ApiUrl>),
 }
@@ -59,8 +67,14 @@ impl std::fmt::Display for BackendType<'_, ()> {
     fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         use BackendType::*;
         match self {
-            Console(endpoint, _) => fmt.debug_tuple("Console").field(&endpoint.url()).finish(),
-            Postgres(endpoint, _) => fmt.debug_tuple("Postgres").field(&endpoint.url()).finish(),
+            Console(endpoint, _) => fmt
+                .debug_tuple("Console")
+                .field(&endpoint.url().as_str())
+                .finish(),
+            Postgres(endpoint, _) => fmt
+                .debug_tuple("Postgres")
+                .field(&endpoint.as_str())
+                .finish(),
             Link(url) => fmt.debug_tuple("Link").field(&url.as_str()).finish(),
         }
     }
@@ -106,96 +120,149 @@ impl<'a, T, E> BackendType<'a, Result<T, E>> {
     }
 }
 
-/// True to its name, this function encapsulates our current auth trade-offs.
-/// Here, we choose the appropriate auth flow based on circumstances.
-async fn auth_quirks(
-    api: &impl console::Api,
-    extra: &ConsoleReqExtra<'_>,
-    creds: &mut ClientCredentials<'_>,
-    client: &mut stream::PqStream<impl AsyncRead + AsyncWrite + Unpin>,
-    allow_cleartext: bool,
-) -> auth::Result<AuthSuccess<CachedNodeInfo>> {
-    // If there's no project so far, that entails that client doesn't
-    // support SNI or other means of passing the endpoint (project) name.
-    // We now expect to see a very specific payload in the place of password.
-    if creds.project.is_none() {
-        // Password will be checked by the compute node later.
-        return hacks::password_hack(api, extra, creds, client).await;
-    }
+/// A product of successful authentication.
+pub struct AuthSuccess<T> {
+    /// Did we send [`pq_proto::BeMessage::AuthenticationOk`] to client?
+    pub reported_auth_ok: bool,
+    /// Something to be considered a positive result.
+    pub value: T,
+}
 
-    // Password hack should set the project name.
-    // TODO: make `creds.project` more type-safe.
-    assert!(creds.project.is_some());
-
-    // Perform cleartext auth if we're allowed to do that.
-    // Currently, we use it for websocket connections (latency).
-    if allow_cleartext {
-        // Password will be checked by the compute node later.
-        return hacks::cleartext_hack(api, extra, creds, client).await;
-    }
-
-    // Finally, proceed with the main auth flow (SCRAM-based).
-    classic::authenticate(api, extra, creds, client).await
+/// Info for establishing a connection to a compute node.
+/// This is what we get after auth succeeded, but not before!
+pub struct NodeInfo {
+    /// Compute node connection params.
+    pub config: compute::ConnCfg,
+    /// Labels for proxy's metrics.
+    pub aux: MetricsAuxInfo,
 }
 
 impl BackendType<'_, ClientCredentials<'_>> {
-    /// Authenticate the client via the requested backend, possibly using credentials.
-    #[tracing::instrument(fields(allow_cleartext), skip_all)]
-    pub async fn authenticate(
+    /// Do something special if user didn't provide the `project` parameter.
+    async fn try_password_hack(
         &mut self,
         extra: &ConsoleReqExtra<'_>,
-        client: &mut stream::PqStream<impl AsyncRead + AsyncWrite + Unpin>,
-        allow_cleartext: bool,
-    ) -> auth::Result<AuthSuccess<CachedNodeInfo>> {
+        client: &mut stream::PqStream<impl AsyncRead + AsyncWrite + Unpin + Send>,
+    ) -> auth::Result<Option<AuthSuccess<NodeInfo>>> {
         use BackendType::*;
 
+        // If there's no project so far, that entails that client doesn't
+        // support SNI or other means of passing the project name.
+        // We now expect to see a very specific payload in the place of password.
+        let fetch_magic_payload = |client| async {
+            warn!("project name not specified, resorting to the password hack auth flow");
+            let payload = AuthFlow::new(client)
+                .begin(auth::PasswordHack)
+                .await?
+                .authenticate()
+                .await?;
+
+            info!(project = &payload.project, "received missing parameter");
+            auth::Result::Ok(payload)
+        };
+
+        // If we want to use cleartext password flow, we can read the password
+        // from the client and pretend that it's a magic payload (PasswordHack hack).
+        let fetch_plaintext_password = |client| async {
+            info!("using cleartext password flow");
+            let payload = AuthFlow::new(client)
+                .begin(auth::CleartextPassword)
+                .await?
+                .authenticate()
+                .await?;
+
+            auth::Result::Ok(auth::password_hack::PasswordHackPayload {
+                project: String::new(),
+                password: payload,
+            })
+        };
+
+        // TODO: find a proper way to merge those very similar blocks.
+        let (mut node, payload) = match self {
+            Console(endpoint, creds) if creds.project.is_none() => {
+                let payload = fetch_magic_payload(client).await?;
+
+                let mut creds = creds.as_ref();
+                creds.project = Some(payload.project.as_str().into());
+                let node = console::Api::new(endpoint, extra, &creds)
+                    .wake_compute()
+                    .await?;
+
+                (node, payload)
+            }
+            Console(endpoint, creds) if creds.use_cleartext_password_flow => {
+                // This is a hack to allow cleartext password in secure connections (wss).
+                let payload = fetch_plaintext_password(client).await?;
+                let creds = creds.as_ref();
+                let node = console::Api::new(endpoint, extra, &creds)
+                    .wake_compute()
+                    .await?;
+
+                (node, payload)
+            }
+            Postgres(endpoint, creds) if creds.project.is_none() => {
+                let payload = fetch_magic_payload(client).await?;
+
+                let mut creds = creds.as_ref();
+                creds.project = Some(payload.project.as_str().into());
+                let node = postgres::Api::new(endpoint, &creds).wake_compute().await?;
+
+                (node, payload)
+            }
+            _ => return Ok(None),
+        };
+
+        node.config.password(payload.password);
+        Ok(Some(AuthSuccess {
+            reported_auth_ok: false,
+            value: node,
+        }))
+    }
+
+    /// Authenticate the client via the requested backend, possibly using credentials.
+    pub async fn authenticate(
+        mut self,
+        extra: &ConsoleReqExtra<'_>,
+        client: &mut stream::PqStream<impl AsyncRead + AsyncWrite + Unpin + Send>,
+    ) -> auth::Result<AuthSuccess<NodeInfo>> {
+        use BackendType::*;
+
+        // Handle cases when `project` is missing in `creds`.
+        // TODO: type safety: return `creds` with irrefutable `project`.
+        if let Some(res) = self.try_password_hack(extra, client).await? {
+            info!("user successfully authenticated (using the password hack)");
+            return Ok(res);
+        }
+
         let res = match self {
-            Console(api, creds) => {
+            Console(endpoint, creds) => {
                 info!(
                     user = creds.user,
                     project = creds.project(),
                     "performing authentication using the console"
                 );
 
-                let api = api.as_ref();
-                auth_quirks(api, extra, creds, client, allow_cleartext).await?
+                assert!(creds.project.is_some());
+                console::Api::new(&endpoint, extra, &creds)
+                    .handle_user(client)
+                    .await?
             }
-            Postgres(api, creds) => {
-                info!(
-                    user = creds.user,
-                    project = creds.project(),
-                    "performing authentication using a local postgres instance"
-                );
+            Postgres(endpoint, creds) => {
+                info!("performing mock authentication using a local postgres instance");
 
-                let api = api.as_ref();
-                auth_quirks(api, extra, creds, client, allow_cleartext).await?
+                assert!(creds.project.is_some());
+                postgres::Api::new(&endpoint, &creds)
+                    .handle_user(client)
+                    .await?
             }
             // NOTE: this auth backend doesn't use client credentials.
             Link(url) => {
                 info!("performing link authentication");
-
-                link::authenticate(url, client)
-                    .await?
-                    .map(CachedNodeInfo::new_uncached)
+                link::handle_user(&url, client).await?
             }
         };
 
         info!("user successfully authenticated");
         Ok(res)
     }
-
-    /// When applicable, wake the compute node, gaining its connection info in the process.
-    /// The link auth flow doesn't support this, so we return [`None`] in that case.
-    pub async fn wake_compute(
-        &self,
-        extra: &ConsoleReqExtra<'_>,
-    ) -> Result<Option<CachedNodeInfo>, console::errors::WakeComputeError> {
-        use BackendType::*;
-
-        match self {
-            Console(api, creds) => api.wake_compute(extra, creds).map_ok(Some).await,
-            Postgres(api, creds) => api.wake_compute(extra, creds).map_ok(Some).await,
-            Link(_) => Ok(None),
-        }
-    }
 }

proxy/src/auth/backend/classic.rs

@@ -1,61 +0,0 @@
-use super::AuthSuccess;
-use crate::{
-    auth::{self, AuthFlow, ClientCredentials},
-    compute,
-    console::{self, AuthInfo, CachedNodeInfo, ConsoleReqExtra},
-    sasl, scram,
-    stream::PqStream,
-};
-use tokio::io::{AsyncRead, AsyncWrite};
-use tracing::info;
-
-pub(super) async fn authenticate(
-    api: &impl console::Api,
-    extra: &ConsoleReqExtra<'_>,
-    creds: &ClientCredentials<'_>,
-    client: &mut PqStream<impl AsyncRead + AsyncWrite + Unpin>,
-) -> auth::Result<AuthSuccess<CachedNodeInfo>> {
-    info!("fetching user's authentication info");
-    let info = api.get_auth_info(extra, creds).await?.unwrap_or_else(|| {
-        // If we don't have an authentication secret, we mock one to
-        // prevent malicious probing (possible due to missing protocol steps).
-        // This mocked secret will never lead to successful authentication.
-        info!("authentication info not found, mocking it");
-        AuthInfo::Scram(scram::ServerSecret::mock(creds.user, rand::random()))
-    });
-
-    let flow = AuthFlow::new(client);
-    let scram_keys = match info {
-        AuthInfo::Md5(_) => {
-            info!("auth endpoint chooses MD5");
-            return Err(auth::AuthError::bad_auth_method("MD5"));
-        }
-        AuthInfo::Scram(secret) => {
-            info!("auth endpoint chooses SCRAM");
-            let scram = auth::Scram(&secret);
-            let client_key = match flow.begin(scram).await?.authenticate().await? {
-                sasl::Outcome::Success(key) => key,
-                sasl::Outcome::Failure(reason) => {
-                    info!("auth backend failed with an error: {reason}");
-                    return Err(auth::AuthError::auth_failed(creds.user));
-                }
-            };
-
-            Some(compute::ScramKeys {
-                client_key: client_key.as_bytes(),
-                server_key: secret.server_key.as_bytes(),
-            })
-        }
-    };
-
-    let mut node = api.wake_compute(extra, creds).await?;
-    if let Some(keys) = scram_keys {
-        use tokio_postgres::config::AuthKeys;
-        node.config.auth_keys(AuthKeys::ScramSha256(keys));
-    }
-
-    Ok(AuthSuccess {
-        reported_auth_ok: false,
-        value: node,
-    })
-}

proxy/src/auth/backend/console.rs

@@ -0,0 +1,365 @@
+//! Cloud API V2.
+
+use super::{AuthSuccess, ConsoleReqExtra, NodeInfo};
+use crate::{
+    auth::{self, AuthFlow, ClientCredentials},
+    compute,
+    console::messages::{ConsoleError, GetRoleSecret, WakeCompute},
+    error::{io_error, UserFacingError},
+    http, sasl, scram,
+    stream::PqStream,
+};
+use futures::TryFutureExt;
+use reqwest::StatusCode as HttpStatusCode;
+use std::future::Future;
+use thiserror::Error;
+use tokio::io::{AsyncRead, AsyncWrite};
+use tracing::{error, info, info_span, warn, Instrument};
+
+/// A go-to error message which doesn't leak any detail.
+const REQUEST_FAILED: &str = "Console request failed";
+
+/// Common console API error.
+#[derive(Debug, Error)]
+pub enum ApiError {
+    /// Error returned by the console itself.
+    #[error("{REQUEST_FAILED} with {}: {}", .status, .text)]
+    Console {
+        status: HttpStatusCode,
+        text: Box<str>,
+    },
+
+    /// Various IO errors like broken pipe or malformed payload.
+    #[error("{REQUEST_FAILED}: {0}")]
+    Transport(#[from] std::io::Error),
+}
+
+impl ApiError {
+    /// Returns HTTP status code if it's the reason for failure.
+    fn http_status_code(&self) -> Option<HttpStatusCode> {
+        use ApiError::*;
+        match self {
+            Console { status, .. } => Some(*status),
+            _ => None,
+        }
+    }
+}
+
+impl UserFacingError for ApiError {
+    fn to_string_client(&self) -> String {
+        use ApiError::*;
+        match self {
+            // To minimize risks, only select errors are forwarded to users.
+            // Ask @neondatabase/control-plane for review before adding more.
+            Console { status, .. } => match *status {
+                HttpStatusCode::NOT_FOUND => {
+                    // Status 404: failed to get a project-related resource.
+                    format!("{REQUEST_FAILED}: endpoint cannot be found")
+                }
+                HttpStatusCode::NOT_ACCEPTABLE => {
+                    // Status 406: endpoint is disabled (we don't allow connections).
+                    format!("{REQUEST_FAILED}: endpoint is disabled")
+                }
+                HttpStatusCode::LOCKED => {
+                    // Status 423: project might be in maintenance mode (or bad state).
+                    format!("{REQUEST_FAILED}: endpoint is temporary unavailable")
+                }
+                _ => REQUEST_FAILED.to_owned(),
+            },
+            _ => REQUEST_FAILED.to_owned(),
+        }
+    }
+}
+
+// Helps eliminate graceless `.map_err` calls without introducing another ctor.
+impl From<reqwest::Error> for ApiError {
+    fn from(e: reqwest::Error) -> Self {
+        io_error(e).into()
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum GetAuthInfoError {
+    // We shouldn't include the actual secret here.
+    #[error("Console responded with a malformed auth secret")]
+    BadSecret,
+
+    #[error(transparent)]
+    ApiError(ApiError),
+}
+
+// This allows more useful interactions than `#[from]`.
+impl<E: Into<ApiError>> From<E> for GetAuthInfoError {
+    fn from(e: E) -> Self {
+        Self::ApiError(e.into())
+    }
+}
+
+impl UserFacingError for GetAuthInfoError {
+    fn to_string_client(&self) -> String {
+        use GetAuthInfoError::*;
+        match self {
+            // We absolutely should not leak any secrets!
+            BadSecret => REQUEST_FAILED.to_owned(),
+            // However, API might return a meaningful error.
+            ApiError(e) => e.to_string_client(),
+        }
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum WakeComputeError {
+    #[error("Console responded with a malformed compute address: {0}")]
+    BadComputeAddress(Box<str>),
+
+    #[error(transparent)]
+    ApiError(ApiError),
+}
+
+// This allows more useful interactions than `#[from]`.
+impl<E: Into<ApiError>> From<E> for WakeComputeError {
+    fn from(e: E) -> Self {
+        Self::ApiError(e.into())
+    }
+}
+
+impl UserFacingError for WakeComputeError {
+    fn to_string_client(&self) -> String {
+        use WakeComputeError::*;
+        match self {
+            // We shouldn't show user the address even if it's broken.
+            // Besides, user is unlikely to care about this detail.
+            BadComputeAddress(_) => REQUEST_FAILED.to_owned(),
+            // However, API might return a meaningful error.
+            ApiError(e) => e.to_string_client(),
+        }
+    }
+}
+
+/// Auth secret which is managed by the cloud.
+pub enum AuthInfo {
+    /// Md5 hash of user's password.
+    Md5([u8; 16]),
+
+    /// [SCRAM](crate::scram) authentication info.
+    Scram(scram::ServerSecret),
+}
+
+#[must_use]
+pub(super) struct Api<'a> {
+    endpoint: &'a http::Endpoint,
+    extra: &'a ConsoleReqExtra<'a>,
+    creds: &'a ClientCredentials<'a>,
+}
+
+impl<'a> AsRef<ClientCredentials<'a>> for Api<'a> {
+    fn as_ref(&self) -> &ClientCredentials<'a> {
+        self.creds
+    }
+}
+
+impl<'a> Api<'a> {
+    /// Construct an API object containing the auth parameters.
+    pub(super) fn new(
+        endpoint: &'a http::Endpoint,
+        extra: &'a ConsoleReqExtra<'a>,
+        creds: &'a ClientCredentials,
+    ) -> Self {
+        Self {
+            endpoint,
+            extra,
+            creds,
+        }
+    }
+
+    /// Authenticate the existing user or throw an error.
+    pub(super) async fn handle_user(
+        &'a self,
+        client: &mut PqStream<impl AsyncRead + AsyncWrite + Unpin + Send>,
+    ) -> auth::Result<AuthSuccess<NodeInfo>> {
+        handle_user(client, self, Self::get_auth_info, Self::wake_compute).await
+    }
+}
+
+impl Api<'_> {
+    async fn get_auth_info(&self) -> Result<Option<AuthInfo>, GetAuthInfoError> {
+        let request_id = uuid::Uuid::new_v4().to_string();
+        async {
+            let request = self
+                .endpoint
+                .get("proxy_get_role_secret")
+                .header("X-Request-ID", &request_id)
+                .query(&[("session_id", self.extra.session_id)])
+                .query(&[
+                    ("application_name", self.extra.application_name),
+                    ("project", Some(self.creds.project().expect("impossible"))),
+                    ("role", Some(self.creds.user)),
+                ])
+                .build()?;
+
+            info!(url = request.url().as_str(), "sending http request");
+            let response = self.endpoint.execute(request).await?;
+            let body = match parse_body::<GetRoleSecret>(response).await {
+                Ok(body) => body,
+                // Error 404 is special: it's ok not to have a secret.
+                Err(e) => match e.http_status_code() {
+                    Some(HttpStatusCode::NOT_FOUND) => return Ok(None),
+                    _otherwise => return Err(e.into()),
+                },
+            };
+
+            let secret = scram::ServerSecret::parse(&body.role_secret)
+                .map(AuthInfo::Scram)
+                .ok_or(GetAuthInfoError::BadSecret)?;
+
+            Ok(Some(secret))
+        }
+        .map_err(crate::error::log_error)
+        .instrument(info_span!("get_auth_info", id = request_id))
+        .await
+    }
+
+    /// Wake up the compute node and return the corresponding connection info.
+    pub async fn wake_compute(&self) -> Result<NodeInfo, WakeComputeError> {
+        let request_id = uuid::Uuid::new_v4().to_string();
+        async {
+            let request = self
+                .endpoint
+                .get("proxy_wake_compute")
+                .header("X-Request-ID", &request_id)
+                .query(&[("session_id", self.extra.session_id)])
+                .query(&[
+                    ("application_name", self.extra.application_name),
+                    ("project", Some(self.creds.project().expect("impossible"))),
+                ])
+                .build()?;
+
+            info!(url = request.url().as_str(), "sending http request");
+            let response = self.endpoint.execute(request).await?;
+            let body = parse_body::<WakeCompute>(response).await?;
+
+            // Unfortunately, ownership won't let us use `Option::ok_or` here.
+            let (host, port) = match parse_host_port(&body.address) {
+                None => return Err(WakeComputeError::BadComputeAddress(body.address)),
+                Some(x) => x,
+            };
+
+            let mut config = compute::ConnCfg::new();
+            config
+                .host(host)
+                .port(port)
+                .dbname(self.creds.dbname)
+                .user(self.creds.user);
+
+            Ok(NodeInfo {
+                config,
+                aux: body.aux,
+            })
+        }
+        .map_err(crate::error::log_error)
+        .instrument(info_span!("wake_compute", id = request_id))
+        .await
+    }
+}
+
+/// Common logic for user handling in API V2.
+/// We reuse this for a mock API implementation in [`super::postgres`].
+pub(super) async fn handle_user<'a, Endpoint, GetAuthInfo, WakeCompute>(
+    client: &mut PqStream<impl AsyncRead + AsyncWrite + Unpin>,
+    endpoint: &'a Endpoint,
+    get_auth_info: impl FnOnce(&'a Endpoint) -> GetAuthInfo,
+    wake_compute: impl FnOnce(&'a Endpoint) -> WakeCompute,
+) -> auth::Result<AuthSuccess<NodeInfo>>
+where
+    Endpoint: AsRef<ClientCredentials<'a>>,
+    GetAuthInfo: Future<Output = Result<Option<AuthInfo>, GetAuthInfoError>>,
+    WakeCompute: Future<Output = Result<NodeInfo, WakeComputeError>>,
+{
+    let creds = endpoint.as_ref();
+
+    info!("fetching user's authentication info");
+    let info = get_auth_info(endpoint).await?.unwrap_or_else(|| {
+        // If we don't have an authentication secret, we mock one to
+        // prevent malicious probing (possible due to missing protocol steps).
+        // This mocked secret will never lead to successful authentication.
+        info!("authentication info not found, mocking it");
+        AuthInfo::Scram(scram::ServerSecret::mock(creds.user, rand::random()))
+    });
+
+    let flow = AuthFlow::new(client);
+    let scram_keys = match info {
+        AuthInfo::Md5(_) => {
+            info!("auth endpoint chooses MD5");
+            return Err(auth::AuthError::bad_auth_method("MD5"));
+        }
+        AuthInfo::Scram(secret) => {
+            info!("auth endpoint chooses SCRAM");
+            let scram = auth::Scram(&secret);
+            let client_key = match flow.begin(scram).await?.authenticate().await? {
+                sasl::Outcome::Success(key) => key,
+                sasl::Outcome::Failure(reason) => {
+                    info!("auth backend failed with an error: {reason}");
+                    return Err(auth::AuthError::auth_failed(creds.user));
+                }
+            };
+
+            Some(compute::ScramKeys {
+                client_key: client_key.as_bytes(),
+                server_key: secret.server_key.as_bytes(),
+            })
+        }
+    };
+
+    let mut node = wake_compute(endpoint).await?;
+    if let Some(keys) = scram_keys {
+        use tokio_postgres::config::AuthKeys;
+        node.config.auth_keys(AuthKeys::ScramSha256(keys));
+    }
+
+    Ok(AuthSuccess {
+        reported_auth_ok: false,
+        value: node,
+    })
+}
+
+/// Parse http response body, taking status code into account.
+async fn parse_body<T: for<'a> serde::Deserialize<'a>>(
+    response: reqwest::Response,
+) -> Result<T, ApiError> {
+    let status = response.status();
+    if status.is_success() {
+        // We shouldn't log raw body because it may contain secrets.
+        info!("request succeeded, processing the body");
+        return Ok(response.json().await?);
+    }
+
+    // Don't throw an error here because it's not as important
+    // as the fact that the request itself has failed.
+    let body = response.json().await.unwrap_or_else(|e| {
+        warn!("failed to parse error body: {e}");
+        ConsoleError {
+            error: "reason unclear (malformed error message)".into(),
+        }
+    });
+
+    let text = body.error;
+    error!("console responded with an error ({status}): {text}");
+    Err(ApiError::Console { status, text })
+}
+
+fn parse_host_port(input: &str) -> Option<(&str, u16)> {
+    let (host, port) = input.split_once(':')?;
+    Some((host, port.parse().ok()?))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_host_port() {
+        let (host, port) = parse_host_port("127.0.0.1:5432").expect("failed to parse");
+        assert_eq!(host, "127.0.0.1");
+        assert_eq!(port, 5432);
+    }
+}

proxy/src/auth/backend/hacks.rs

@@ -1,66 +0,0 @@
-use super::AuthSuccess;
-use crate::{
-    auth::{self, AuthFlow, ClientCredentials},
-    console::{
-        self,
-        provider::{CachedNodeInfo, ConsoleReqExtra},
-    },
-    stream,
-};
-use tokio::io::{AsyncRead, AsyncWrite};
-use tracing::{info, warn};
-
-/// Compared to [SCRAM](crate::scram), cleartext password auth saves
-/// one round trip and *expensive* computations (>= 4096 HMAC iterations).
-/// These properties are benefical for serverless JS workers, so we
-/// use this mechanism for websocket connections.
-pub async fn cleartext_hack(
-    api: &impl console::Api,
-    extra: &ConsoleReqExtra<'_>,
-    creds: &mut ClientCredentials<'_>,
-    client: &mut stream::PqStream<impl AsyncRead + AsyncWrite + Unpin>,
-) -> auth::Result<AuthSuccess<CachedNodeInfo>> {
-    warn!("cleartext auth flow override is enabled, proceeding");
-    let password = AuthFlow::new(client)
-        .begin(auth::CleartextPassword)
-        .await?
-        .authenticate()
-        .await?;
-
-    let mut node = api.wake_compute(extra, creds).await?;
-    node.config.password(password);
-
-    // Report tentative success; compute node will check the password anyway.
-    Ok(AuthSuccess {
-        reported_auth_ok: false,
-        value: node,
-    })
-}
-
-/// Workaround for clients which don't provide an endpoint (project) name.
-/// Very similar to [`cleartext_hack`], but there's a specific password format.
-pub async fn password_hack(
-    api: &impl console::Api,
-    extra: &ConsoleReqExtra<'_>,
-    creds: &mut ClientCredentials<'_>,
-    client: &mut stream::PqStream<impl AsyncRead + AsyncWrite + Unpin>,
-) -> auth::Result<AuthSuccess<CachedNodeInfo>> {
-    warn!("project not specified, resorting to the password hack auth flow");
-    let payload = AuthFlow::new(client)
-        .begin(auth::PasswordHack)
-        .await?
-        .authenticate()
-        .await?;
-
-    info!(project = &payload.project, "received missing parameter");
-    creds.project = Some(payload.project.into());
-
-    let mut node = api.wake_compute(extra, creds).await?;
-    node.config.password(payload.password);
-
-    // Report tentative success; compute node will check the password anyway.
-    Ok(AuthSuccess {
-        reported_auth_ok: false,
-        value: node,
-    })
-}