fix test failure

Signed-off-by: Alex Chi Z <chi@neon.tech>

pageserver/src/tenant.rs

@@ -8225,6 +8225,13 @@ mod tests {
         assert_eq!(images.len(), 0); // the image layer should not contain tombstones, or it is not created
     }
 
+    fn key_max_minus_1() -> Key {
+        Key {
+            field6: u32::MAX - 1,
+            ..Key::MAX
+        }
+    }
+
     #[tokio::test]
     async fn test_simple_bottom_most_compaction_images() -> anyhow::Result<()> {
         let harness = TenantHarness::create("test_simple_bottom_most_compaction_images").await?;
@@ -8405,7 +8412,7 @@ mod tests {
             vec![
                 // Image layer at GC horizon
                 PersistentLayerKey {
-                    key_range: Key::MIN..Key::MAX,
+                    key_range: Key::MIN..key_max_minus_1(),
                     lsn_range: Lsn(0x30)..Lsn(0x31),
                     is_delta: false
                 },
@@ -10777,7 +10784,7 @@ mod tests {
             vec![
                 // The compacted image layer (full key range)
                 PersistentLayerKey {
-                    key_range: Key::MIN..Key::MAX,
+                    key_range: Key::MIN..key_max_minus_1(),
                     lsn_range: Lsn(0x10)..Lsn(0x11),
                     is_delta: false,
                 },
@@ -11124,7 +11131,7 @@ mod tests {
             vec![
                 // The compacted image layer (full key range)
                 PersistentLayerKey {
-                    key_range: Key::MIN..Key::MAX,
+                    key_range: Key::MIN..key_max_minus_1(),
                     lsn_range: Lsn(0x10)..Lsn(0x11),
                     is_delta: false,
                 },

pageserver/src/tenant/timeline/compaction.rs

@@ -436,6 +436,16 @@ impl KeyHistoryRetention {
         if dry_run {
             return true;
         }
+        if key.key_range.end == Key::MAX {
+            warn!(
+                key=%key,
+                "discard layer due to layer key range being Key::MAX, which should not happen",
+            );
+            if cfg!(feature = "testing") {
+                panic!("Key::MAX should not be part of the key space");
+            }
+            return true;
+        }
         let layer_generation;
         {
             let guard = tline.layers.read().await;
@@ -2760,7 +2770,15 @@ impl Timeline {
 
         let produced_image_layers = if let Some(writer) = image_layer_writer {
             if !dry_run {
-                let end_key = job_desc.compaction_key_range.end;
+                let mut end_key = job_desc.compaction_key_range.end;
+                if end_key == Key::MAX {
+                    // There's a potential bug to-be-resolved when end_key is Key::MAX, so we need to subtract 1 to workaround it.
+                    // Note that we never write Key::MAX into the image layer, as this is not part of the key space.
+                    end_key = Key {
+                        field6: u32::MAX - 1,
+                        ..Key::MAX
+                    };
+                }
                 writer
                     .finish_with_discard_fn(self, ctx, end_key, discard)
                     .await?