WIP mgiration

This reverts commit 9796aeaada.

pageserver/src/bin/pageserver.rs

@@ -280,6 +280,23 @@ fn start_pageserver(
     // We need to release the lock file only when the process exits.
     std::mem::forget(lock_file);
 
+    let skip_upgrade_tenant_conf: bool = {
+        match std::env::var("PAGESERVER_SKIP_UPGRADE_TENANT_CONF") {
+            Ok(val) => {
+                if val == "1" {
+                    info!("Skipping upgrade of tenant configs as per env var PAGESERVER_SKIP_UPGRADE_TENANT_CONF");
+                    true
+                } else {
+                    false
+                }
+            }
+            Err(VarError::NotPresent) => false,
+            Err(VarError::NotUnicode(_)) => {
+                panic!("env var PAGESERVER_SKIP_UPGRADE_TENANT_CONF is present but not unicode")
+            }
+        }
+    };
+
     // Bind the HTTP and libpq ports early, so that if they are in use by some other
     // process, we error out early.
     let http_addr = &conf.listen_http_addr;
@@ -404,6 +421,7 @@ fn start_pageserver(
         },
         order,
         shutdown_pageserver.clone(),
+        skip_upgrade_tenant_conf,
     ))?;
 
     BACKGROUND_RUNTIME.spawn({

pageserver/src/tenant.rs

@@ -2108,6 +2108,14 @@ where
     Ok(result)
 }
 
+#[derive(Debug, thiserror::Error)]
+pub(super) enum LoadTenantConfigError {
+    #[error("tenant config file does not exist")]
+    ConfigFileDoesNotExist,
+    #[error(transparent)]
+    Other(#[from] anyhow::Error),
+}
+
 impl Tenant {
     pub fn tenant_specific_overrides(&self) -> TenantConfOpt {
         *self.tenant_conf.read().unwrap()
@@ -2352,19 +2360,15 @@ impl Tenant {
     pub(super) fn load_tenant_config(
         conf: &'static PageServerConf,
         tenant_id: &TenantId,
-    ) -> anyhow::Result<TenantConfOpt> {
+    ) -> Result<TenantConfOpt, LoadTenantConfigError> {
         let target_config_path = conf.tenant_config_path(tenant_id);
         let target_config_display = target_config_path.display();
 
         info!("loading tenantconf from {target_config_display}");
 
-        // FIXME If the config file is not found, assume that we're attaching
-        // a detached tenant and config is passed via attach command.
-        // https://github.com/neondatabase/neon/issues/1555
-        // OR: we're loading after incomplete deletion that managed to remove config.
         if !target_config_path.exists() {
             info!("tenant config not found in {target_config_display}");
-            return Ok(TenantConfOpt::default());
+            return Err(LoadTenantConfigError::ConfigFileDoesNotExist);
         }
 
         // load and parse file
@@ -2384,8 +2388,7 @@ impl Tenant {
                         format!("Failed to parse config from file '{target_config_display}' as pageserver config")
                     })?;
                 }
-                _ => bail!("config file {target_config_display} has unrecognized pageserver option '{key}'"),
-
+                _ => return Err(LoadTenantConfigError::Other(anyhow::anyhow!("config file {target_config_display} has unrecognized pageserver option '{key}'"))),
             }
         }
 

pageserver/src/tenant/mgr.rs

@@ -3,6 +3,7 @@
 
 use rand::{distributions::Alphanumeric, Rng};
 use std::collections::{hash_map, HashMap};
+use std::env::VarError;
 use std::ffi::OsStr;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
@@ -27,7 +28,9 @@ use crate::deletion_queue::DeletionQueueClient;
 use crate::task_mgr::{self, TaskKind};
 use crate::tenant::config::TenantConfOpt;
 use crate::tenant::delete::DeleteTenantFlow;
-use crate::tenant::{create_tenant_files, CreateTenantFilesMode, Tenant, TenantState};
+use crate::tenant::{
+    create_tenant_files, CreateTenantFilesMode, LoadTenantConfigError, Tenant, TenantState,
+};
 use crate::{InitializationOrder, IGNORED_TENANT_FILE_NAME, TEMP_FILE_SUFFIX};
 
 use utils::crashsafe::path_with_suffix_extension;
@@ -111,6 +114,7 @@ pub async fn init_tenant_mgr(
     resources: TenantSharedResources,
     init_order: InitializationOrder,
     cancel: CancellationToken,
+    skip_upgrade_tenant_conf: bool,
 ) -> anyhow::Result<()> {
     // Scan local filesystem for attached tenants
     let tenants_dir = conf.tenants_path();
@@ -184,12 +188,6 @@ pub async fn init_tenant_mgr(
                         continue;
                     }
 
-                    let tenant_ignore_mark_file = tenant_dir_path.join(IGNORED_TENANT_FILE_NAME);
-                    if tenant_ignore_mark_file.exists() {
-                        info!("Found an ignore mark file {tenant_ignore_mark_file:?}, skipping the tenant");
-                        continue;
-                    }
-
                     let tenant_id = match tenant_dir_path
                         .file_name()
                         .and_then(OsStr::to_str)
@@ -206,6 +204,34 @@ pub async fn init_tenant_mgr(
                         }
                     };
 
+                    // Upgrade legacy on-disk state: before
+                    // - attach-time-tenant config (#4255) and
+                    // - tenant deletion markers (#4552),
+                    // it was possible that we could end up with a tenant directory but no tenant config file.
+                    // We handled that by loading the default tenant config, without persisting it.
+                    //
+                    // This step here moves these legacy tenants into the present, so we can make tenant config
+                    // file presence required in the future.
+                    if !skip_upgrade_tenant_conf {
+                                let target_config_path = conf.tenant_config_path(&tenant_id);
+                                if !target_config_path.try_exists().expect("filesystme error determining existence of tenant config file") {
+                                Tenant::persist_tenant_config(
+                                    &tenant_id,
+                                    &target_config_path,
+                                    TenantConfOpt::default(),
+                                )
+                                .await
+                                .context("upgrade tenant config on-disk state");
+                           }
+                       }
+                    }
+
+                    let tenant_ignore_mark_file = tenant_dir_path.join(IGNORED_TENANT_FILE_NAME);
+                    if tenant_ignore_mark_file.exists() {
+                        info!("Found an ignore mark file {tenant_ignore_mark_file:?}, skipping the tenant");
+                        continue;
+                    }
+
                     let generation = if let Some(generations) = &tenant_generations {
                         // We have a generation map: treat it as the authority for whether
                         // this tenant is really attached.