vadim change

Dockerfile.compute-node

@@ -776,6 +776,13 @@ ARG PG_VERSION
 ARG BUILD_TAG
 RUN apt update && apt install -y zstd
 
+# Define extension build numbers
+# NOTE: it is *not* necessary for the build_tag to be BUILD_TAG. In particular,
+# you should update the build_tag for each extension only when you build it
+RUN echo "kq_imcx 5670669815" >>  build_tags.txt && \ 
+    echo "anon 5670669815" >>  build_tags.txt && \
+    echo "postgis 5670669815" >>  build_tags.txt
+
 # copy the control files here
 COPY --from=kq-imcx-pg-build /extensions/ /extensions/
 COPY --from=pg-anon-pg-build /extensions/ /extensions/

control_plane/src/endpoint.rs

@@ -564,7 +564,9 @@ impl Endpoint {
                 }
                 Err(e) => {
                     if attempt == MAX_ATTEMPTS {
-                        return Err(e).context("timed out waiting to connect to compute_ctl HTTP");
+                        return Err(e).context(
+                            "timed out waiting to connect to compute_ctl HTTP; last error: {e}",
+                        );
                     }
                 }
             }

pageserver/src/tenant.rs

@@ -46,6 +46,7 @@ use std::sync::{Mutex, RwLock};
 use std::time::{Duration, Instant};
 
 use self::config::TenantConf;
+use self::delete::DeleteTimelineFlow;
 use self::metadata::LoadMetadataError;
 use self::metadata::TimelineMetadata;
 use self::remote_timeline_client::RemoteTimelineClient;
@@ -69,7 +70,6 @@ use crate::tenant::storage_layer::ImageLayer;
 use crate::tenant::storage_layer::Layer;
 use crate::InitializationOrder;
 
-use crate::tenant::timeline::delete::DeleteTimelineFlow;
 use crate::tenant::timeline::uninit::cleanup_timeline_directory;
 use crate::virtual_file::VirtualFile;
 use crate::walredo::PostgresRedoManager;
@@ -117,6 +117,7 @@ mod remote_timeline_client;
 pub mod storage_layer;
 
 pub mod config;
+pub mod delete;
 pub mod mgr;
 pub mod tasks;
 pub mod upload_queue;

pageserver/src/tenant/delete.rs

@@ -15,17 +15,15 @@ use utils::{
 use crate::{
     config::PageServerConf,
     task_mgr::{self, TaskKind},
-    tenant::{
-        metadata::TimelineMetadata,
-        remote_timeline_client::{
-            self, PersistIndexPartWithDeletedFlagError, RemoteTimelineClient,
-        },
-        CreateTimelineCause, DeleteTimelineError, Tenant,
-    },
+    tenant::{remote_timeline_client, DeleteTimelineError},
     InitializationOrder,
 };
 
-use super::Timeline;
+use super::{
+    metadata::TimelineMetadata,
+    remote_timeline_client::{PersistIndexPartWithDeletedFlagError, RemoteTimelineClient},
+    CreateTimelineCause, Tenant, Timeline,
+};
 
 /// Now that the Timeline is in Stopping state, request all the related tasks to shut down.
 async fn stop_tasks(timeline: &Timeline) -> Result<(), DeleteTimelineError> {

pageserver/src/tenant/mgr.rs

@@ -26,7 +26,7 @@ use crate::{InitializationOrder, IGNORED_TENANT_FILE_NAME};
 use utils::fs_ext::PathExt;
 use utils::id::{TenantId, TimelineId};
 
-use super::timeline::delete::DeleteTimelineFlow;
+use super::delete::DeleteTimelineFlow;
 
 /// The tenants known to the pageserver.
 /// The enum variants are used to distinguish the different states that the pageserver can be in.

pageserver/src/tenant/timeline.rs

@@ -1,4 +1,3 @@
-pub mod delete;
 mod eviction_task;
 pub mod layer_manager;
 mod logical_size;
@@ -80,7 +79,6 @@ use crate::METADATA_FILE_NAME;
 use crate::ZERO_PAGE;
 use crate::{is_temporary, task_mgr};
 
-use self::delete::DeleteTimelineFlow;
 pub(super) use self::eviction_task::EvictionTaskTenantState;
 use self::eviction_task::EvictionTaskTimelineState;
 use self::layer_manager::LayerManager;
@@ -88,6 +86,7 @@ use self::logical_size::LogicalSize;
 use self::walreceiver::{WalReceiver, WalReceiverConf};
 
 use super::config::TenantConf;
+use super::delete::DeleteTimelineFlow;
 use super::remote_timeline_client::index::IndexPart;
 use super::remote_timeline_client::RemoteTimelineClient;
 use super::storage_layer::{

scripts/combine_control_files.py

@@ -4,6 +4,7 @@
 # for all extensions in extensions subdir.
 import argparse
 import json
+import shutil
 import subprocess
 from pathlib import Path
 
@@ -45,11 +46,18 @@ if __name__ == "__main__":
     BUILD_TAG = args.BUILD_TAG
     public_ext_list = args.public_extensions.split(",")
 
+    build_tags = {}
+    with open("build_tags.txt", "r") as f:
+        for line in f:
+            ext, build_tag = line.strip().split(" ")
+            build_tags[ext] = build_tag
+
     ext_index = {}
     library_index = {}
     EXT_PATH = Path("extensions")
     for extension in EXT_PATH.iterdir():
         if extension.is_dir():
+            build_tag = build_tags[extension.name]
             control_data = {}
             for control_file in extension.glob("*.control"):
                 if control_file.suffix != ".control":
@@ -58,8 +66,13 @@ if __name__ == "__main__":
                     control_data[control_file.name] = f.read()
             ext_index[extension.name] = {
                 "control_data": control_data,
-                "archive_path": f"{BUILD_TAG}/{pg_version}/extensions/{extension.name}.tar.zst",
+                "archive_path": f"{build_tag}/{pg_version}/extensions/{extension.name}.tar.zst",
             }
+            # if we didn't build the extension for this build tag
+            # then we don't need to re-upload it. so we delete it
+            if build_tag != BUILD_TAG:
+                shutil.rmtree(extension)
+
         elif extension.suffix == ".zst":
             file_list = (
                 str(subprocess.check_output(["tar", "tf", str(extension)]), "utf-8")

test_runner/fixtures/neon_fixtures.py

@@ -1504,7 +1504,6 @@ class NeonCli(AbstractNeonCli):
         safekeepers: Optional[List[int]] = None,
         tenant_id: Optional[TenantId] = None,
         lsn: Optional[Lsn] = None,
-        branch_name: Optional[str] = None,
     ) -> "subprocess.CompletedProcess[str]":
         args = [
             "endpoint",
@@ -1518,11 +1517,8 @@ class NeonCli(AbstractNeonCli):
             args.append(f"--lsn={lsn}")
         args.extend(["--pg-port", str(pg_port)])
         args.extend(["--http-port", str(http_port)])
-
         if safekeepers is not None:
             args.extend(["--safekeepers", (",".join(map(str, safekeepers)))])
-        if branch_name is not None:
-            args.extend(["--branch-name", branch_name])
         if endpoint_id is not None:
             args.append(endpoint_id)
 

test_runner/regress/test_neon_local_cli.py

@@ -16,13 +16,11 @@ def test_neon_cli_basics(neon_env_builder: NeonEnvBuilder, port_distributor: Por
             endpoint_id="ep-basic-main", pg_port=pg_port, http_port=http_port
         )
 
-        branch_name = "migration-check"
-
-        env.neon_cli.create_branch(new_branch_name=branch_name)
+        env.neon_cli.create_branch(new_branch_name="migration_check")
         pg_port = port_distributor.get_port()
         http_port = port_distributor.get_port()
         env.neon_cli.endpoint_start(
-            f"ep-{branch_name}", pg_port, http_port, branch_name=branch_name
+            endpoint_id="ep-migration_check", pg_port=pg_port, http_port=http_port
         )
     finally:
         env.neon_cli.stop()

test_runner/regress/test_ondemand_download.py

@@ -712,10 +712,10 @@ def test_ondemand_download_failure_to_replace(
     env.pageserver.allowed_errors.append(actual_message)
 
     env.pageserver.allowed_errors.append(
-        ".* ERROR .*Error processing HTTP request: InternalServerError\\(get local timeline info.*"
+        ".* ERROR .*Error processing HTTP request: InternalServerError\\(get local timeline info"
     )
     # this might get to run and attempt on-demand, but not always
-    env.pageserver.allowed_errors.append(".* ERROR .*Task 'initial size calculation'.*")
+    env.pageserver.allowed_errors.append(".* ERROR .*Task 'initial size calculation'")
 
     # if the above returned, then we didn't have a livelock, and all is well
 

test_runner/regress/test_remote_storage.py

@@ -348,9 +348,6 @@ def test_remote_storage_upload_queue_retries(
     # XXX: should vary this test to selectively fail just layer uploads, index uploads, deletions
     #      but how do we validate the result after restore?
 
-    # these are always possible when we do an immediate stop. perhaps something with compacting has changed since.
-    env.pageserver.allowed_errors.append(r".*found future (delta|image) layer.*")
-
     env.pageserver.stop(immediate=True)
     env.endpoints.stop_all()