WIP: fast-import: change postgres database to neondb

.gitignore

@@ -25,5 +25,3 @@ compaction-suite-results.*
 
 # pgindent typedef lists
 *.list
-
-venv/

compute_tools/src/bin/fast_import.rs

@@ -189,12 +189,26 @@ pub(crate) async fn main() -> anyhow::Result<()> {
         format!("host=localhost port=5432 user={superuser} dbname=postgres");
     loop {
         let res = tokio_postgres::connect(&restore_pg_connstring, tokio_postgres::NoTls).await;
-        if res.is_ok() {
+        if let Ok((client, connection)) = res {
             info!("postgres is ready, could connect to it");
+
+            tokio::spawn(async move {
+                if let Err(e) = connection.await {
+                    eprintln!("connection error: {}", e);
+                }
+            });
+
+            client
+                .execute("CREATE DATABASE neondb", &[])
+                .await
+                .context("create database neondb")?;
+
             break;
         }
     }
 
+    let restore_pg_connstring = restore_pg_connstring.replace("dbname=postgres", "dbname=neondb");
+
     //
     // Decrypt connection string
     //

control_plane/src/local_env.rs

@@ -180,7 +180,7 @@ impl NeonStorageControllerConf {
     const DEFAULT_MAX_WARMING_UP_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
 
     // Very tight heartbeat interval to speed up tests
-    const DEFAULT_HEARTBEAT_INTERVAL: std::time::Duration = std::time::Duration::from_millis(1000);
+    const DEFAULT_HEARTBEAT_INTERVAL: std::time::Duration = std::time::Duration::from_millis(100);
 }
 
 impl Default for NeonStorageControllerConf {

docker-compose.yaml

@@ -1,96 +0,0 @@
-version: '3.8'
-
-x-build-args-bullseye: &build-args-bullseye
-  DEBIAN_VERSION: bullseye
-  GIT_VERSION: local # seems to be not used in compute node though
-  BUILD_TAG: ${BUILD_TAG:-local}
-
-x-build-args-bookworm: &build-args-bookworm
-  DEBIAN_VERSION: bookworm
-  GIT_VERSION: local # seems to be not used in compute node though
-  BUILD_TAG: ${BUILD_TAG:-local}
-
-services:
-  compute-node-v14: &compute-node-v14-base
-    image: neondatabase/compute-node-v14:${IMAGE_TAG:-local}
-    build:
-      context: .
-      dockerfile: compute/compute-node.Dockerfile
-      args:
-        <<: *build-args-bullseye
-        PG_VERSION: v14
-      cache_from:
-        - neondatabase/compute-node-v14:${CACHE_FROM_TAG:-latest}
-
-  compute-node-v14-amd64:
-    <<: *compute-node-v14-base
-    platform: linux/amd64
-    image: neondatabase/compute-node-v14:${IMAGE_TAG:-local}-amd64
-
-  compute-node-v14-arm64:
-    <<: *compute-node-v14-base
-    platform: linux/arm64
-    image: neondatabase/compute-node-v14:${IMAGE_TAG:-local}-arm64
-
-  compute-node-v15: &compute-node-v15-base
-    image: neondatabase/compute-node-v15:${IMAGE_TAG:-local}
-    build:
-      context: .
-      dockerfile: compute/compute-node.Dockerfile
-      args:
-        <<: *build-args-bullseye
-        PG_VERSION: v15
-      cache_from:
-        - neondatabase/compute-node-v15:${CACHE_FROM_TAG:-latest}
-
-  compute-node-v15-amd64:
-    <<: *compute-node-v15-base
-    platform: linux/amd64
-    image: neondatabase/compute-node-v15:${IMAGE_TAG:-local}-amd64
-
-  compute-node-v15-arm64:
-    <<: *compute-node-v15-base
-    platform: linux/arm64
-    image: neondatabase/compute-node-v15:${IMAGE_TAG:-local}-arm64
-
-  compute-node-v16: &compute-node-v16-base
-    image: neondatabase/compute-node-v16:${IMAGE_TAG:-local}
-    build:
-      context: .
-      dockerfile: compute/compute-node.Dockerfile
-      args:
-        <<: *build-args-bullseye
-        PG_VERSION: v16
-      cache_from:
-        - neondatabase/compute-node-v16:${CACHE_FROM_TAG:-latest}
-
-  compute-node-v16-amd64:
-    <<: *compute-node-v16-base
-    platform: linux/amd64
-    image: neondatabase/compute-node-v16:${IMAGE_TAG:-local}-amd64
-
-  compute-node-v16-arm64:
-    <<: *compute-node-v16-base
-    platform: linux/arm64
-    image: neondatabase/compute-node-v16:${IMAGE_TAG:-local}-arm64
-
-  compute-node-v17: &compute-node-v17-base
-    image: neondatabase/compute-node-v17:${IMAGE_TAG:-local}
-    build:
-      context: .
-      dockerfile: compute/compute-node.Dockerfile
-      args:
-        <<: *build-args-bookworm
-        PG_VERSION: v17
-      cache_from:
-        - neondatabase/compute-node-v17:${CACHE_FROM_TAG:-latest}
-
-  compute-node-v17-amd64:
-    <<: *compute-node-v17-base
-    platform: linux/amd64
-    image: neondatabase/compute-node-v17:${IMAGE_TAG:-local}-amd64
-
-  compute-node-v17-arm64:
-    <<: *compute-node-v17-base
-    platform: linux/arm64
-    image: neondatabase/compute-node-v17:${IMAGE_TAG:-local}-arm64

pageserver/src/tenant/checks.rs

@@ -1,15 +1,12 @@
 use std::collections::BTreeSet;
 
 use itertools::Itertools;
-use pageserver_compaction::helpers::overlaps_with;
 
 use super::storage_layer::LayerName;
 
 /// Checks whether a layer map is valid (i.e., is a valid result of the current compaction algorithm if nothing goes wrong).
 ///
-/// The function implements a fast path check and a slow path check.
-///
-/// The fast path checks if we can split the LSN range of a delta layer only at the LSNs of the delta layers. For example,
+/// The function checks if we can split the LSN range of a delta layer only at the LSNs of the delta layers. For example,
 ///
 /// ```plain
 /// |       |                 |       |
@@ -28,47 +25,31 @@ use super::storage_layer::LayerName;
 /// |       |    |   4   |    |       |
 ///
 /// If layer 2 and 4 contain the same single key, this is also a valid layer map.
-///
-/// However, if a partial compaction is still going on, it is possible that we get a layer map not satisfying the above condition.
-/// Therefore, we fallback to simply check if any of the two delta layers overlap. (See "A slow path...")
 pub fn check_valid_layermap(metadata: &[LayerName]) -> Option<String> {
     let mut lsn_split_point = BTreeSet::new(); // TODO: use a better data structure (range tree / range set?)
     let mut all_delta_layers = Vec::new();
     for name in metadata {
         if let LayerName::Delta(layer) = name {
-            all_delta_layers.push(layer.clone());
+            if layer.key_range.start.next() != layer.key_range.end {
+                all_delta_layers.push(layer.clone());
+            }
         }
     }
     for layer in &all_delta_layers {
-        if layer.key_range.start.next() != layer.key_range.end {
-            let lsn_range = &layer.lsn_range;
-            lsn_split_point.insert(lsn_range.start);
-            lsn_split_point.insert(lsn_range.end);
-        }
+        let lsn_range = &layer.lsn_range;
+        lsn_split_point.insert(lsn_range.start);
+        lsn_split_point.insert(lsn_range.end);
     }
-    for (idx, layer) in all_delta_layers.iter().enumerate() {
-        if layer.key_range.start.next() == layer.key_range.end {
-            continue;
-        }
+    for layer in &all_delta_layers {
         let lsn_range = layer.lsn_range.clone();
         let intersects = lsn_split_point.range(lsn_range).collect_vec();
         if intersects.len() > 1 {
-            // A slow path to check if the layer intersects with any other delta layer.
-            for (other_idx, other_layer) in all_delta_layers.iter().enumerate() {
-                if other_idx == idx {
-                    // do not check self intersects with self
-                    continue;
-                }
-                if overlaps_with(&layer.lsn_range, &other_layer.lsn_range)
-                    && overlaps_with(&layer.key_range, &other_layer.key_range)
-                {
-                    let err = format!(
-                            "layer violates the layer map LSN split assumption: layer {} intersects with layer {}",
-                            layer, other_layer
-                        );
-                    return Some(err);
-                }
-            }
+            let err = format!(
+                "layer violates the layer map LSN split assumption: layer {} intersects with LSN [{}]",
+                layer,
+                intersects.into_iter().map(|lsn| lsn.to_string()).join(", ")
+            );
+            return Some(err);
         }
     }
     None

pageserver/src/tenant/timeline/compaction.rs

@@ -29,7 +29,6 @@ use utils::id::TimelineId;
 use crate::context::{AccessStatsBehavior, RequestContext, RequestContextBuilder};
 use crate::page_cache;
 use crate::statvfs::Statvfs;
-use crate::tenant::checks::check_valid_layermap;
 use crate::tenant::remote_timeline_client::WaitCompletionError;
 use crate::tenant::storage_layer::batch_split_writer::{
     BatchWriterResult, SplitDeltaLayerWriter, SplitImageLayerWriter,
@@ -2157,14 +2156,15 @@ impl Timeline {
 
         // Step 1: construct a k-merge iterator over all layers.
         // Also, verify if the layer map can be split by drawing a horizontal line at every LSN start/end split point.
-        let layer_names = job_desc
-            .selected_layers
-            .iter()
-            .map(|layer| layer.layer_desc().layer_name())
-            .collect_vec();
-        if let Some(err) = check_valid_layermap(&layer_names) {
-            bail!("gc-compaction layer map check failed because {}, cannot proceed with compaction due to potential data loss", err);
-        }
+        // disable the check for now because we need to adjust the check for partial compactions, will enable later.
+        // let layer_names = job_desc
+        //     .selected_layers
+        //     .iter()
+        //     .map(|layer| layer.layer_desc().layer_name())
+        //     .collect_vec();
+        // if let Some(err) = check_valid_layermap(&layer_names) {
+        //     warn!("gc-compaction layer map check failed because {}, this is normal if partial compaction is not finished yet", err);
+        // }
         // The maximum LSN we are processing in this compaction loop
         let end_lsn = job_desc
             .selected_layers
@@ -2546,48 +2546,13 @@ impl Timeline {
         );
 
         // Step 3: Place back to the layer map.
-
-        // First, do a sanity check to ensure the newly-created layer map does not contain overlaps.
-        let all_layers = {
-            let guard = self.layers.read().await;
-            let layer_map = guard.layer_map()?;
-            layer_map.iter_historic_layers().collect_vec()
-        };
-
-        let mut final_layers = all_layers
-            .iter()
-            .map(|layer| layer.layer_name())
-            .collect::<HashSet<_>>();
-        for layer in &layer_selection {
-            final_layers.remove(&layer.layer_desc().layer_name());
-        }
-        for layer in &compact_to {
-            final_layers.insert(layer.layer_desc().layer_name());
-        }
-        let final_layers = final_layers.into_iter().collect_vec();
-
-        // TODO: move this check before we call `finish` on image layer writers. However, this will require us to get the layer name before we finish
-        // the writer, so potentially, we will need a function like `ImageLayerBatchWriter::get_all_pending_layer_keys` to get all the keys that are
-        // in the writer before finalizing the persistent layers. Now we would leave some dangling layers on the disk if the check fails.
-        if let Some(err) = check_valid_layermap(&final_layers) {
-            bail!("gc-compaction layer map check failed after compaction because {}, compaction result not applied to the layer map due to potential data loss", err);
-        }
-
-        // Between the sanity check and this compaction update, there could be new layers being flushed, but it should be fine because we only
-        // operate on L1 layers.
         {
+            // TODO: sanity check if the layer map is valid (i.e., should not have overlaps)
             let mut guard = self.layers.write().await;
             guard
                 .open_mut()?
                 .finish_gc_compaction(&layer_selection, &compact_to, &self.metrics)
         };
-
-        // Schedule an index-only upload to update the `latest_gc_cutoff` in the index_part.json.
-        // Otherwise, after restart, the index_part only contains the old `latest_gc_cutoff` and
-        // find_gc_cutoffs will try accessing things below the cutoff. TODO: ideally, this should
-        // be batched into `schedule_compaction_update`.
-        let disk_consistent_lsn = self.disk_consistent_lsn.load();
-        self.schedule_uploads(disk_consistent_lsn, None)?;
         self.remote_client
             .schedule_compaction_update(&layer_selection, &compact_to)?;