debug-log the buggy key

pageserver/src/page_service.rs

@@ -736,7 +736,10 @@ impl PageServerHandler {
                         }
                         // the vectored get currently only supports a single LSN, so, bounce as soon
                         // as the effective request_lsn changes
-                        return *accum_lsn == this_lsn;
+                        if (*accum_lsn != this_lsn) {
+                            return false;
+                        }
+                        return true;
                     }
                     .await =>
                     {

pageserver/src/tenant/storage_layer/delta_layer.rs

@@ -842,6 +842,7 @@ impl DeltaLayerInner {
     // can be further optimised to visit the index only once.
     pub(super) async fn get_values_reconstruct_data(
         &self,
+        self_desc: PersistentLayerDesc,
         keyspace: KeySpace,
         lsn_range: Range<Lsn>,
         reconstruct_state: &mut ValuesReconstructState,
@@ -864,6 +865,7 @@ impl DeltaLayerInner {
         let data_end_offset = self.index_start_offset();
 
         let reads = Self::plan_reads(
+            self_desc,
             &keyspace,
             lsn_range.clone(),
             data_end_offset,
@@ -884,6 +886,7 @@ impl DeltaLayerInner {
     }
 
     async fn plan_reads<Reader>(
+        self_desc: PersistentLayerDesc,
         keyspace: &KeySpace,
         lsn_range: Range<Lsn>,
         data_end_offset: u64,
@@ -912,6 +915,10 @@ impl DeltaLayerInner {
                 let lsn = DeltaKey::extract_lsn_from_buf(&raw_key);
                 let blob_ref = BlobRef(value);
 
+                if key == Key::from_hex("000000067F000000050000400600000543D3").unwrap() {
+                    info!(file = %self_desc.layer_name(), %key, %lsn, will_init = blob_ref.will_init(), "delta layer found key")
+                }
+
                 // Lsns are not monotonically increasing across keys, so we don't assert on them.
                 assert!(key >= range.start);
 

pageserver/src/tenant/storage_layer/image_layer.rs

@@ -441,12 +441,13 @@ impl ImageLayerInner {
     // the reconstruct state with whatever is found.
     pub(super) async fn get_values_reconstruct_data(
         &self,
+        self_desc: PersistentLayerDesc,
         keyspace: KeySpace,
         reconstruct_state: &mut ValuesReconstructState,
         ctx: &RequestContext,
     ) -> Result<(), GetVectoredError> {
         let reads = self
-            .plan_reads(keyspace, None, ctx)
+            .plan_reads(self_desc, keyspace, None, ctx)
             .await
             .map_err(GetVectoredError::Other)?;
 
@@ -465,6 +466,7 @@ impl ImageLayerInner {
     /// this shard.
     async fn plan_reads(
         &self,
+        self_desc: PersistentLayerDesc,
         keyspace: KeySpace,
         shard_identity: Option<&ShardIdentity>,
         ctx: &RequestContext,
@@ -508,6 +510,10 @@ impl ImageLayerInner {
                     BlobFlag::None
                 };
 
+                if key == Key::from_hex("000000067F000000050000400600000543D3").unwrap() {
+                    info!(file = %self_desc.layer_name(), %key, %self.lsn, will_init=true, "image layer found key")
+                }
+
                 if key >= range.end {
                     planner.handle_range_end(offset);
                     range_end_handled = true;
@@ -537,6 +543,7 @@ impl ImageLayerInner {
         // Fragment the range into the regions owned by this ShardIdentity
         let plan = self
             .plan_reads(
+                todo!(),
                 KeySpace {
                     // If asked for the total key space, plan_reads will give us all the keys in the layer
                     ranges: vec![Key::MIN..Key::MAX],

pageserver/src/tenant/storage_layer/inmemory_layer.rs

@@ -469,6 +469,10 @@ impl InMemoryLayer {
                     senders.insert((key, *entry_lsn), tx);
                     reconstruct_state.update_key(&key, *entry_lsn, will_init, rx);
 
+                    if key == Key::from_hex("000000067F000000050000400600000543D3").unwrap() {
+                        info!(%key, %entry_lsn, will_init, "inmemory layer found key")
+                    }
+
                     if will_init {
                         break;
                     }

pageserver/src/tenant/storage_layer/layer.rs

@@ -1755,11 +1755,17 @@ impl DownloadedLayer {
             .map_err(GetVectoredError::Other)?
         {
             Delta(d) => {
-                d.get_values_reconstruct_data(keyspace, lsn_range, reconstruct_data, ctx)
-                    .await
+                d.get_values_reconstruct_data(
+                    owner.desc.clone(),
+                    keyspace,
+                    lsn_range,
+                    reconstruct_data,
+                    ctx,
+                )
+                .await
             }
             Image(i) => {
-                i.get_values_reconstruct_data(keyspace, reconstruct_data, ctx)
+                i.get_values_reconstruct_data(owner.desc.clone(), keyspace, reconstruct_data, ctx)
                     .await
             }
         }