mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-10-26 21:56:17 +00:00
feat: support old-stage datanode config overlays (#8647)
* feat: support old-stage datanode config overlays Signed-off-by: discord9 <discord9@163.com> * fix: derive compat overlay policy from WAL config Signed-off-by: discord9 <discord9@163.com> --------- Signed-off-by: discord9 <discord9@163.com>
This commit is contained in:
discord9
+669
-118
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -22,7 +23,10 @@ use sqlness::interceptor::{InterceptorRef, Registry};
|
||||
|
||||
use crate::cmd::bare::ServerAddr;
|
||||
use crate::cmd::compat_case::{self, CompatCase, try_infer_version, version_matches_range};
|
||||
use crate::env::bare::{Env, StoreConfig, WalConfig};
|
||||
use crate::cmd::datanode_overlay::{
|
||||
DatanodeOverlay, DatanodeProtectionPolicy, PreparedDatanodeOverlay,
|
||||
};
|
||||
use crate::env::bare::{Env, GreptimeDB, StoreConfig, WalConfig};
|
||||
use crate::protocol_interceptor::{self, POSTGRES, PROTOCOL_KEY};
|
||||
use crate::util;
|
||||
|
||||
@@ -326,16 +330,12 @@ impl CompatCommand {
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(expected_cases) = self.expect_cases {
|
||||
assert_eq!(
|
||||
cases.len(),
|
||||
expected_cases,
|
||||
"Expected {expected_cases} compatibility cases after all filtering, found {}",
|
||||
cases.len()
|
||||
);
|
||||
}
|
||||
|
||||
if cases.is_empty() {
|
||||
if let Some(expected_cases) = self.expect_cases {
|
||||
panic!(
|
||||
"Expected {expected_cases} compatibility cases after all filtering, found 0"
|
||||
);
|
||||
}
|
||||
if dry_run {
|
||||
println!("DRY-RUN: no compat cases would run after version-range filtering");
|
||||
} else {
|
||||
@@ -344,8 +344,25 @@ impl CompatCommand {
|
||||
return;
|
||||
}
|
||||
|
||||
let wal_config = WalConfig::RaftEngine;
|
||||
let profiles =
|
||||
prepare_compat_profiles(cases, &wal_config).unwrap_or_else(|error| panic!("{error}"));
|
||||
emit_protected_path_warnings(&profiles);
|
||||
|
||||
if let Some(expected_cases) = self.expect_cases {
|
||||
assert_eq!(
|
||||
profiles.case_count(),
|
||||
expected_cases,
|
||||
"Expected {expected_cases} compatibility cases after all filtering, found {}",
|
||||
profiles.case_count()
|
||||
);
|
||||
}
|
||||
|
||||
if dry_run {
|
||||
println!("DRY-RUN: would run {} compat case(s)", cases.len());
|
||||
println!(
|
||||
"DRY-RUN: would run {} compat case(s)",
|
||||
profiles.case_count()
|
||||
);
|
||||
println!(" topology: {}", topology.as_str());
|
||||
println!(
|
||||
" from version: {}",
|
||||
@@ -359,22 +376,30 @@ impl CompatCommand {
|
||||
.as_deref()
|
||||
.unwrap_or("unknown (use --to-bins-dir or build debug binary)")
|
||||
);
|
||||
if pre_filter_count != cases.len() {
|
||||
if pre_filter_count != profiles.case_count() {
|
||||
println!();
|
||||
println!(
|
||||
"Version-range filtering reduced {} → {} cases (see 'Skipping case' messages above)",
|
||||
pre_filter_count,
|
||||
cases.len()
|
||||
profiles.case_count()
|
||||
);
|
||||
}
|
||||
println!();
|
||||
for c in &cases {
|
||||
for c in profiles.cases() {
|
||||
println!(" case: {}", c.metadata.name);
|
||||
println!(" namespace: {}", c.namespace);
|
||||
println!(" from_range: {:?}", c.metadata.from_range);
|
||||
println!(" to_range: {:?}", c.metadata.to_range);
|
||||
println!(" features: {:?}", c.metadata.features);
|
||||
}
|
||||
println!("DRY-RUN: compatibility profiles:");
|
||||
for profile in profiles.iter() {
|
||||
println!(" profile: {}", profile.profile_id());
|
||||
println!(" cases: {}", profile.case_names().join(", "));
|
||||
for source in profile.sources() {
|
||||
println!(" sidecar: {}", source.display());
|
||||
}
|
||||
}
|
||||
println!();
|
||||
println!("Dry run complete. Remove --dry-run to execute.");
|
||||
return;
|
||||
@@ -382,133 +407,499 @@ impl CompatCommand {
|
||||
|
||||
println!(
|
||||
"Running {} compat case(s) with topology {}:",
|
||||
cases.len(),
|
||||
profiles.case_count(),
|
||||
topology.as_str()
|
||||
);
|
||||
for c in &cases {
|
||||
for c in profiles.cases() {
|
||||
println!(
|
||||
" - {} (namespace: {}, topologies: {:?})",
|
||||
c.metadata.name, c.namespace, c.metadata.topologies
|
||||
);
|
||||
}
|
||||
|
||||
// ---- 6. Create temp directory (after filtering so early exits don't leave empty dirs) ----
|
||||
let temp_dir = tempfile::Builder::new()
|
||||
.prefix("sqlness-compat")
|
||||
.tempdir()
|
||||
.unwrap();
|
||||
let sqlness_home = temp_dir.keep();
|
||||
unsafe {
|
||||
std::env::set_var(
|
||||
"SQLNESS_HOME",
|
||||
sqlness_home.join("copy").display().to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
// ---- 7. Build interceptor registry ----
|
||||
// ---- 6. Build interceptor registry ----
|
||||
let interceptor_registry = create_interceptor_registry();
|
||||
|
||||
// ---- 7b. Create Env for the selected topology ----
|
||||
let setup_etcd = topology == CompatTopology::Distributed && self.setup_etcd;
|
||||
let store_config = StoreConfig {
|
||||
store_addrs: if setup_etcd {
|
||||
vec!["127.0.0.1:2379".to_string()]
|
||||
} else {
|
||||
vec![]
|
||||
},
|
||||
setup_etcd,
|
||||
setup_pg: None,
|
||||
setup_mysql: None,
|
||||
enable_flat_format: false,
|
||||
enable_gc: false,
|
||||
};
|
||||
|
||||
let env = Env::new(
|
||||
sqlness_home.clone(),
|
||||
ServerAddr::default(),
|
||||
WalConfig::RaftEngine,
|
||||
self.pull_version_on_need,
|
||||
let to_bins_dir = to_bins_dir.expect("to_bins_dir must be resolved in non-dry-run mode");
|
||||
let profile_config = ProfileRunConfig {
|
||||
interceptor_registry: &interceptor_registry,
|
||||
from_bins_dir,
|
||||
store_config,
|
||||
vec![],
|
||||
);
|
||||
|
||||
// ---- 7c. Etcd cleanup guard ----
|
||||
// Arm this only immediately before starting the cluster. Earlier validation
|
||||
// failures should not stop an unrelated local container named `etcd`.
|
||||
let mut etcd_guard = if setup_etcd {
|
||||
Some(EtcdGuard::new())
|
||||
} else {
|
||||
None
|
||||
to_bins_dir,
|
||||
pull_version_on_need: self.pull_version_on_need,
|
||||
setup_etcd: self.setup_etcd,
|
||||
fail_fast: self.fail_fast,
|
||||
topology,
|
||||
preserve_state: self.preserve_state,
|
||||
wal_config,
|
||||
};
|
||||
|
||||
// ---- 8. Run setup phase on the from-version cluster ----
|
||||
println!("Starting from-version {} cluster...", topology.as_str());
|
||||
let mut db = match topology {
|
||||
CompatTopology::Distributed => env.compat_start_distributed(0).await,
|
||||
CompatTopology::Standalone => env.compat_start_standalone(0).await,
|
||||
};
|
||||
|
||||
println!("Running setup phase...");
|
||||
for case in &cases {
|
||||
run_compat_phase(&db, case, &interceptor_registry, CompatPhase::Setup)
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("Setup failed for case '{}': {e}", case.metadata.name));
|
||||
println!(" Setup: {} - OK", case.metadata.name);
|
||||
let mut failures = Vec::new();
|
||||
let mut preserved_states = Vec::new();
|
||||
for profile in profiles.iter() {
|
||||
let outcome = run_profile(profile, &profile_config).await;
|
||||
failures.extend(outcome.failures);
|
||||
if let Some(state) = outcome.preserved_state {
|
||||
preserved_states.push(state);
|
||||
}
|
||||
if outcome.stop_remaining_profiles {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 9. Switch to the to-version binary and restart cluster ----
|
||||
// to_bins_dir was already resolved during version-range filtering
|
||||
println!("Restarting cluster with to-version binary on preserved state...");
|
||||
let to_bins_dir = to_bins_dir.expect("to_bins_dir must be resolved in non-dry-run mode");
|
||||
env.compat_restart(&db, to_bins_dir).await;
|
||||
for state in &preserved_states {
|
||||
println!("Preserved compat profile state: {}", state.display());
|
||||
}
|
||||
|
||||
if failures.is_empty() {
|
||||
println!("\n\x1b[32mAll compat tests passed!\x1b[0m");
|
||||
} else {
|
||||
panic!("\n\x1b[31mFailed cases: {}\x1b[0m", failures.join(", "));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cases sharing one old-stage datanode configuration lifecycle.
|
||||
#[derive(Debug)]
|
||||
enum CompatProfile {
|
||||
Baseline {
|
||||
cases: Vec<CompatCase>,
|
||||
},
|
||||
Overlay {
|
||||
overlay: Arc<PreparedDatanodeOverlay>,
|
||||
cases: Vec<CompatCase>,
|
||||
sources: Vec<PathBuf>,
|
||||
},
|
||||
}
|
||||
|
||||
impl CompatProfile {
|
||||
fn profile_id(&self) -> &str {
|
||||
match self {
|
||||
Self::Baseline { .. } => "baseline",
|
||||
Self::Overlay { overlay, .. } => overlay.profile_id(),
|
||||
}
|
||||
}
|
||||
|
||||
fn cases(&self) -> &[CompatCase] {
|
||||
match self {
|
||||
Self::Baseline { cases } | Self::Overlay { cases, .. } => cases,
|
||||
}
|
||||
}
|
||||
|
||||
fn case_names(&self) -> Vec<&str> {
|
||||
self.cases()
|
||||
.iter()
|
||||
.map(|case| case.metadata.name.as_str())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn sources(&self) -> &[PathBuf] {
|
||||
match self {
|
||||
Self::Baseline { .. } => &[],
|
||||
Self::Overlay { sources, .. } => sources,
|
||||
}
|
||||
}
|
||||
|
||||
fn overlay(&self) -> Option<Arc<PreparedDatanodeOverlay>> {
|
||||
match self {
|
||||
Self::Baseline { .. } => None,
|
||||
Self::Overlay { overlay, .. } => Some(Arc::clone(overlay)),
|
||||
}
|
||||
}
|
||||
|
||||
fn touched_protected_paths(&self) -> &[crate::cmd::datanode_overlay::DottedPath] {
|
||||
match self {
|
||||
Self::Baseline { .. } => &[],
|
||||
Self::Overlay { overlay, .. } => overlay.touched_protected_paths(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Deterministically ordered compatibility profiles.
|
||||
#[derive(Debug)]
|
||||
struct CompatProfiles(Vec<CompatProfile>);
|
||||
|
||||
impl CompatProfiles {
|
||||
fn iter(&self) -> impl Iterator<Item = &CompatProfile> {
|
||||
self.0.iter()
|
||||
}
|
||||
|
||||
fn cases(&self) -> impl Iterator<Item = &CompatCase> {
|
||||
self.0.iter().flat_map(CompatProfile::cases)
|
||||
}
|
||||
|
||||
fn case_count(&self) -> usize {
|
||||
self.0.iter().map(|profile| profile.cases().len()).sum()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct OverlayProfileGroup {
|
||||
overlay: Arc<PreparedDatanodeOverlay>,
|
||||
cases: Vec<CompatCase>,
|
||||
sources: Vec<PathBuf>,
|
||||
}
|
||||
|
||||
fn prepare_compat_profiles(
|
||||
cases: Vec<CompatCase>,
|
||||
wal_config: &WalConfig,
|
||||
) -> Result<CompatProfiles, String> {
|
||||
let protection = DatanodeProtectionPolicy::for_wal(wal_config);
|
||||
let mut baseline_cases = Vec::new();
|
||||
let mut overlays: BTreeMap<[u8; 32], OverlayProfileGroup> = BTreeMap::new();
|
||||
|
||||
for case in cases {
|
||||
let Some(reference) = case.metadata.old_datanode_overlay() else {
|
||||
baseline_cases.push(case);
|
||||
continue;
|
||||
};
|
||||
let overlay = DatanodeOverlay::load(&case.dir, reference).map_err(|error| {
|
||||
format!(
|
||||
"Failed to load old datanode overlay for compatibility case '{}': {error}",
|
||||
case.metadata.name
|
||||
)
|
||||
})?;
|
||||
let prepared = Arc::new(overlay.prepare(&protection).map_err(|error| {
|
||||
format!(
|
||||
"Failed to prepare old datanode overlay for compatibility case '{}': {error}",
|
||||
case.metadata.name
|
||||
)
|
||||
})?);
|
||||
let key = *prepared.profile_key();
|
||||
let source = prepared.source().to_path_buf();
|
||||
let entry = overlays.entry(key).or_insert_with(|| OverlayProfileGroup {
|
||||
overlay: Arc::clone(&prepared),
|
||||
cases: Vec::new(),
|
||||
sources: Vec::new(),
|
||||
});
|
||||
entry.cases.push(case);
|
||||
entry.sources.push(source);
|
||||
}
|
||||
|
||||
baseline_cases.sort_by(|left, right| left.metadata.name.cmp(&right.metadata.name));
|
||||
let mut profiles = Vec::new();
|
||||
if !baseline_cases.is_empty() {
|
||||
profiles.push(CompatProfile::Baseline {
|
||||
cases: baseline_cases,
|
||||
});
|
||||
}
|
||||
for (
|
||||
_,
|
||||
OverlayProfileGroup {
|
||||
overlay,
|
||||
mut cases,
|
||||
mut sources,
|
||||
},
|
||||
) in overlays
|
||||
{
|
||||
cases.sort_by(|left, right| left.metadata.name.cmp(&right.metadata.name));
|
||||
sources.sort();
|
||||
sources.dedup();
|
||||
profiles.push(CompatProfile::Overlay {
|
||||
overlay,
|
||||
cases,
|
||||
sources,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(CompatProfiles(profiles))
|
||||
}
|
||||
|
||||
fn emit_protected_path_warnings(profiles: &CompatProfiles) {
|
||||
for profile in profiles.iter() {
|
||||
let paths: Vec<_> = profile
|
||||
.touched_protected_paths()
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect();
|
||||
if paths.is_empty() {
|
||||
continue;
|
||||
}
|
||||
println!(
|
||||
"{}",
|
||||
format_protected_path_warning(profile.profile_id(), profile.case_names(), paths)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn format_protected_path_warning(
|
||||
profile_id: &str,
|
||||
mut case_names: Vec<&str>,
|
||||
mut paths: Vec<String>,
|
||||
) -> String {
|
||||
case_names.sort_unstable();
|
||||
paths.sort_unstable();
|
||||
format!(
|
||||
"Warning: datanode overlay profile {profile_id} touches runner-owned paths [{}] for cases [{}]",
|
||||
paths.join(", "),
|
||||
case_names.join(", ")
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ProfileOutcome {
|
||||
failures: Vec<String>,
|
||||
stop_remaining_profiles: bool,
|
||||
preserved_state: Option<PathBuf>,
|
||||
}
|
||||
|
||||
/// Pure compatibility profile progress policy, independent of process ownership.
|
||||
#[derive(Default)]
|
||||
struct ProfileProgress {
|
||||
successful_setup_indexes: Vec<usize>,
|
||||
failures: Vec<String>,
|
||||
fail_fast_setup_failure: bool,
|
||||
fail_fast_verify_failure: bool,
|
||||
cleanup_failed: bool,
|
||||
}
|
||||
|
||||
impl ProfileProgress {
|
||||
fn record_setup_success(&mut self, case_index: usize) {
|
||||
self.successful_setup_indexes.push(case_index);
|
||||
}
|
||||
|
||||
fn record_setup_failure(&mut self, failure: String, fail_fast: bool) -> bool {
|
||||
self.failures.push(failure);
|
||||
self.fail_fast_setup_failure = fail_fast;
|
||||
fail_fast
|
||||
}
|
||||
|
||||
fn record_verify_failure(&mut self, failure: String, fail_fast: bool) -> bool {
|
||||
self.failures.push(failure);
|
||||
self.fail_fast_verify_failure = fail_fast;
|
||||
fail_fast
|
||||
}
|
||||
|
||||
fn record_cleanup_failure(&mut self, failure: String) {
|
||||
self.failures.push(failure);
|
||||
self.cleanup_failed = true;
|
||||
}
|
||||
|
||||
fn should_transition_to_current(&self) -> bool {
|
||||
!self.successful_setup_indexes.is_empty() && !self.fail_fast_setup_failure
|
||||
}
|
||||
|
||||
fn should_stop_remaining_profiles(&self) -> bool {
|
||||
self.fail_fast_setup_failure || self.fail_fast_verify_failure || self.cleanup_failed
|
||||
}
|
||||
}
|
||||
|
||||
struct ProfileRunConfig<'a> {
|
||||
interceptor_registry: &'a Registry,
|
||||
from_bins_dir: Option<PathBuf>,
|
||||
to_bins_dir: PathBuf,
|
||||
pull_version_on_need: bool,
|
||||
setup_etcd: bool,
|
||||
fail_fast: bool,
|
||||
topology: CompatTopology,
|
||||
preserve_state: bool,
|
||||
wal_config: WalConfig,
|
||||
}
|
||||
|
||||
async fn run_profile(profile: &CompatProfile, config: &ProfileRunConfig<'_>) -> ProfileOutcome {
|
||||
println!("Running compatibility profile {}", profile.profile_id());
|
||||
let temp_dir = tempfile::Builder::new()
|
||||
.prefix(&format!("sqlness-compat-{}-", profile.profile_id()))
|
||||
.tempdir()
|
||||
.unwrap();
|
||||
let profile_state = ProfileStateGuard::new(temp_dir, config.preserve_state);
|
||||
let sqlness_home = profile_state.path().to_path_buf();
|
||||
// Standalone runs need no etcd.
|
||||
let setup_etcd = config.topology == CompatTopology::Distributed && config.setup_etcd;
|
||||
unsafe {
|
||||
std::env::set_var(
|
||||
"SQLNESS_HOME",
|
||||
sqlness_home.join("copy").display().to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
let store_config = StoreConfig {
|
||||
store_addrs: setup_etcd
|
||||
.then(|| "127.0.0.1:2379".to_string())
|
||||
.into_iter()
|
||||
.collect(),
|
||||
setup_etcd,
|
||||
setup_pg: None,
|
||||
setup_mysql: None,
|
||||
enable_flat_format: false,
|
||||
enable_gc: false,
|
||||
};
|
||||
let env = Env::new(
|
||||
sqlness_home.clone(),
|
||||
ServerAddr::default(),
|
||||
config.wal_config.clone(),
|
||||
config.pull_version_on_need,
|
||||
config.from_bins_dir.clone(),
|
||||
store_config,
|
||||
vec![],
|
||||
);
|
||||
if let Some(overlay) = profile.overlay() {
|
||||
env.activate_compat_old(overlay);
|
||||
}
|
||||
|
||||
// Arm immediately before starting the profile so earlier preflight failures
|
||||
// cannot remove a developer-owned etcd container.
|
||||
let etcd_guard = setup_etcd.then(EtcdGuard::new);
|
||||
println!(
|
||||
"Starting old-version {} cluster...",
|
||||
config.topology.as_str()
|
||||
);
|
||||
let db = match config.topology {
|
||||
CompatTopology::Distributed => env.compat_start_distributed(0).await,
|
||||
CompatTopology::Standalone => env.compat_start_standalone(0).await,
|
||||
};
|
||||
|
||||
let mut progress = ProfileProgress::default();
|
||||
for (case_index, case) in profile.cases().iter().enumerate() {
|
||||
match run_compat_phase(&db, case, config.interceptor_registry, CompatPhase::Setup).await {
|
||||
Ok(()) => {
|
||||
println!(" Setup: {} - OK", case.metadata.name);
|
||||
progress.record_setup_success(case_index);
|
||||
}
|
||||
Err(error) => {
|
||||
println!(" Setup: {} - FAILED: {error}", case.metadata.name);
|
||||
if progress.record_setup_failure(
|
||||
format!("{} (setup): {error}", case.metadata.name),
|
||||
config.fail_fast,
|
||||
) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if progress.should_transition_to_current() {
|
||||
println!("Restarting cluster with new-version binary on preserved state...");
|
||||
env.activate_compat_current();
|
||||
env.compat_restart(&db, config.to_bins_dir.clone()).await;
|
||||
|
||||
// ---- 10. Run verify phase on the to-version cluster ----
|
||||
println!("Running verify phase...");
|
||||
let mut failed = Vec::new();
|
||||
for case in &cases {
|
||||
match run_compat_phase(&db, case, &interceptor_registry, CompatPhase::Verify).await {
|
||||
for case_index in progress.successful_setup_indexes.clone() {
|
||||
let case = &profile.cases()[case_index];
|
||||
match run_compat_phase(&db, case, config.interceptor_registry, CompatPhase::Verify)
|
||||
.await
|
||||
{
|
||||
Ok(()) => println!(" Verify: {} - PASSED", case.metadata.name),
|
||||
Err(e) => {
|
||||
println!(" Verify: {} - FAILED: {e}", case.metadata.name);
|
||||
failed.push(case.metadata.name.clone());
|
||||
if self.fail_fast {
|
||||
Err(error) => {
|
||||
println!(" Verify: {} - FAILED: {error}", case.metadata.name);
|
||||
if progress.record_verify_failure(
|
||||
format!("{} (verify): {error}", case.metadata.name),
|
||||
config.fail_fast,
|
||||
) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if progress.successful_setup_indexes.is_empty() {
|
||||
println!("Skipping current-version restart: no setup cases succeeded");
|
||||
}
|
||||
|
||||
// ---- 11. Stop cluster ----
|
||||
db.compat_stop();
|
||||
let cleanup = cleanup_profile(db, etcd_guard, setup_etcd, profile_state).await;
|
||||
for failure in cleanup.failures {
|
||||
progress.record_cleanup_failure(failure);
|
||||
}
|
||||
let stop_remaining_profiles = progress.should_stop_remaining_profiles();
|
||||
ProfileOutcome {
|
||||
failures: progress.failures,
|
||||
stop_remaining_profiles,
|
||||
preserved_state: cleanup.preserved_state,
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 12. Cleanup ----
|
||||
// Etcd is always cleaned up; --preserve-state only preserves sqlness_home.
|
||||
if setup_etcd {
|
||||
println!("Stopping etcd");
|
||||
util::stop_rm_etcd();
|
||||
struct CleanupOutcome {
|
||||
failures: Vec<String>,
|
||||
preserved_state: Option<PathBuf>,
|
||||
}
|
||||
|
||||
/// Owns one profile state directory until normal cleanup or unwind finalization.
|
||||
struct ProfileStateGuard {
|
||||
temp_dir: Option<tempfile::TempDir>,
|
||||
preserve_state: bool,
|
||||
}
|
||||
|
||||
impl ProfileStateGuard {
|
||||
fn new(temp_dir: tempfile::TempDir, preserve_state: bool) -> Self {
|
||||
Self {
|
||||
temp_dir: Some(temp_dir),
|
||||
preserve_state,
|
||||
}
|
||||
}
|
||||
|
||||
fn path(&self) -> &std::path::Path {
|
||||
self.temp_dir.as_ref().unwrap().path()
|
||||
}
|
||||
|
||||
fn finalize(mut self) -> CleanupOutcome {
|
||||
let temp_dir = self.temp_dir.take().unwrap();
|
||||
let mut failures = Vec::new();
|
||||
if self.preserve_state {
|
||||
let path = temp_dir.keep();
|
||||
if let Err(error) = std::fs::metadata(&path) {
|
||||
failures.push(format!(
|
||||
"failed to confirm preserved profile state {}: {error}",
|
||||
path.display()
|
||||
));
|
||||
}
|
||||
return CleanupOutcome {
|
||||
failures,
|
||||
preserved_state: Some(path),
|
||||
};
|
||||
}
|
||||
|
||||
if !self.preserve_state {
|
||||
println!("Removing state in {:?}", sqlness_home);
|
||||
tokio::fs::remove_dir_all(sqlness_home)
|
||||
.await
|
||||
.unwrap_or_else(|e| println!("Warning: failed to clean up temp dir: {e}"));
|
||||
println!("Removing state in {}", temp_dir.path().display());
|
||||
if let Err(error) = temp_dir.close() {
|
||||
failures.push(format!("failed to remove profile state: {error}"));
|
||||
}
|
||||
CleanupOutcome {
|
||||
failures,
|
||||
preserved_state: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Disarm the etcd guard now that we've done normal cleanup.
|
||||
if let Some(mut guard) = etcd_guard.take() {
|
||||
guard.disarm();
|
||||
impl Drop for ProfileStateGuard {
|
||||
fn drop(&mut self) {
|
||||
let Some(temp_dir) = self.temp_dir.take() else {
|
||||
return;
|
||||
};
|
||||
if self.preserve_state {
|
||||
let path = temp_dir.keep();
|
||||
println!(
|
||||
"Preserved compat profile state after abnormal exit: {}",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if failed.is_empty() {
|
||||
println!("\n\x1b[32mAll compat tests passed!\x1b[0m");
|
||||
} else {
|
||||
println!("\n\x1b[31mFailed cases: {}\x1b[0m", failed.join(", "));
|
||||
// Explicitly drop the guard before exit so it doesn't double-cleanup.
|
||||
std::process::exit(1);
|
||||
async fn cleanup_profile(
|
||||
mut db: GreptimeDB,
|
||||
mut etcd_guard: Option<EtcdGuard>,
|
||||
setup_etcd: bool,
|
||||
profile_state: ProfileStateGuard,
|
||||
) -> CleanupOutcome {
|
||||
db.compat_stop();
|
||||
drop(db);
|
||||
let mut failures = Vec::new();
|
||||
if setup_etcd {
|
||||
println!("Stopping etcd");
|
||||
match util::stop_rm_etcd_checked() {
|
||||
Ok(()) => {
|
||||
if let Some(guard) = etcd_guard.as_mut() {
|
||||
guard.disarm();
|
||||
}
|
||||
}
|
||||
Err(error) => failures.push(format!("profile etcd cleanup: {error}")),
|
||||
}
|
||||
}
|
||||
// On failed checked cleanup this Drop performs one best-effort retry before
|
||||
// profile state is finalized.
|
||||
drop(etcd_guard);
|
||||
|
||||
let mut state_outcome = profile_state.finalize();
|
||||
failures.append(&mut state_outcome.failures);
|
||||
CleanupOutcome {
|
||||
failures,
|
||||
preserved_state: state_outcome.preserved_state,
|
||||
}
|
||||
}
|
||||
|
||||
/// Guard that stops/removes Docker etcd on drop (panic or early exit).
|
||||
@@ -546,13 +937,9 @@ impl Drop for EtcdGuard {
|
||||
fn drop(&mut self) {
|
||||
if self.active {
|
||||
println!("EtcdGuard: emergency etcd cleanup (panic or early exit)");
|
||||
// Best-effort: don't panic in Drop
|
||||
let _ = std::process::Command::new("docker")
|
||||
.args(["container", "stop", "etcd"])
|
||||
.status();
|
||||
let _ = std::process::Command::new("docker")
|
||||
.args(["container", "rm", "etcd"])
|
||||
.status();
|
||||
if let Err(error) = util::stop_rm_etcd_checked() {
|
||||
println!("EtcdGuard: emergency etcd cleanup failed: {error}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -942,7 +1329,36 @@ fn simple_diff(expected: &str, actual: &str) -> String {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::trim_trailing_blank_lines;
|
||||
use std::path::Path;
|
||||
|
||||
use super::*;
|
||||
use crate::cmd::compat_case::{CaseMetadata, OldConfigMetadata};
|
||||
|
||||
fn test_case(temp_dir: &Path, name: &str, overlay: Option<&str>) -> CompatCase {
|
||||
let case_dir = temp_dir.join(name);
|
||||
std::fs::create_dir_all(&case_dir).unwrap();
|
||||
if let Some(overlay) = overlay {
|
||||
std::fs::write(case_dir.join("overlay.toml"), overlay).unwrap();
|
||||
}
|
||||
CompatCase {
|
||||
metadata: CaseMetadata {
|
||||
name: name.to_string(),
|
||||
reason: "test".to_string(),
|
||||
introduced_by: "test".to_string(),
|
||||
topologies: vec![CompatTopology::Distributed.as_str().to_string()],
|
||||
from_range: vec!["*".to_string()],
|
||||
to_range: vec!["*".to_string()],
|
||||
features: vec!["table".to_string()],
|
||||
owner: "test".to_string(),
|
||||
namespace: None,
|
||||
old_config: overlay.map(|_| OldConfigMetadata {
|
||||
datanode: PathBuf::from("overlay.toml"),
|
||||
}),
|
||||
},
|
||||
dir: case_dir,
|
||||
namespace: name.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_trim_trailing_blank_lines_preserves_single_final_newline() {
|
||||
@@ -951,4 +1367,139 @@ mod tests {
|
||||
|
||||
assert_eq!(output, "SELECT 1;\n\n+---+\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profiles_are_baseline_first_and_group_by_full_semantic_digest() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let profiles = prepare_compat_profiles(
|
||||
vec![
|
||||
test_case(temp_dir.path(), "overlay_b", Some("[x]\nb = 2\na = 1\n")),
|
||||
test_case(temp_dir.path(), "baseline", None),
|
||||
test_case(temp_dir.path(), "overlay_a", Some("[x]\na = 1\nb = 2\n")),
|
||||
test_case(temp_dir.path(), "overlay_c", Some("value = 3\n")),
|
||||
],
|
||||
&WalConfig::RaftEngine,
|
||||
)
|
||||
.unwrap();
|
||||
let profiles: Vec<_> = profiles.iter().collect();
|
||||
|
||||
assert!(matches!(profiles[0], CompatProfile::Baseline { .. }));
|
||||
assert_eq!(profiles[0].case_names(), ["baseline"]);
|
||||
assert_eq!(profiles[1].case_names(), ["overlay_a", "overlay_b"]);
|
||||
assert_eq!(profiles[1].sources().len(), 2);
|
||||
let overlay_keys: Vec<_> = profiles[1..]
|
||||
.iter()
|
||||
.map(|profile| match profile {
|
||||
CompatProfile::Overlay { overlay, .. } => *overlay.profile_key(),
|
||||
CompatProfile::Baseline { .. } => unreachable!(),
|
||||
})
|
||||
.collect();
|
||||
assert!(overlay_keys.windows(2).all(|keys| keys[0] < keys[1]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_fail_fast_setup_verifies_only_successful_cases() {
|
||||
let mut progress = ProfileProgress::default();
|
||||
progress.record_setup_success(0);
|
||||
assert!(!progress.record_setup_failure("case_b (setup): failed".to_string(), false));
|
||||
progress.record_setup_success(2);
|
||||
|
||||
assert!(progress.should_transition_to_current());
|
||||
assert_eq!(progress.successful_setup_indexes, [0, 2]);
|
||||
assert!(!progress.should_stop_remaining_profiles());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fail_fast_setup_blocks_current_and_later_profiles() {
|
||||
let mut progress = ProfileProgress::default();
|
||||
progress.record_setup_success(0);
|
||||
|
||||
assert!(progress.record_setup_failure("case_b (setup): failed".to_string(), true));
|
||||
assert!(!progress.should_transition_to_current());
|
||||
assert!(progress.should_stop_remaining_profiles());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_successful_setups_skips_current() {
|
||||
let mut progress = ProfileProgress::default();
|
||||
progress.record_setup_failure("case_a (setup): failed".to_string(), false);
|
||||
|
||||
assert!(!progress.should_transition_to_current());
|
||||
assert!(!progress.should_stop_remaining_profiles());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_fail_fast_stops_later_profiles_after_cleanup() {
|
||||
let mut progress = ProfileProgress::default();
|
||||
progress.record_setup_success(0);
|
||||
|
||||
assert!(progress.record_verify_failure("case_a (verify): failed".to_string(), true));
|
||||
assert!(progress.should_stop_remaining_profiles());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_fail_fast_aggregates_failures_but_cleanup_failure_stops_profiles() {
|
||||
let mut progress = ProfileProgress::default();
|
||||
progress.record_setup_failure("case_a (setup): failed".to_string(), false);
|
||||
progress.record_setup_success(1);
|
||||
progress.record_verify_failure("case_b (verify): failed".to_string(), false);
|
||||
|
||||
assert!(!progress.should_stop_remaining_profiles());
|
||||
progress.record_cleanup_failure("failed to remove profile state".to_string());
|
||||
assert_eq!(progress.failures.len(), 3);
|
||||
assert!(progress.should_stop_remaining_profiles());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn protected_path_warning_is_sorted_and_value_free() {
|
||||
let warning = format_protected_path_warning(
|
||||
"abcdef123456",
|
||||
vec!["case_z", "case_a"],
|
||||
vec!["wal.provider".to_string(), "mode".to_string()],
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
warning,
|
||||
"Warning: datanode overlay profile abcdef123456 touches runner-owned paths [mode, wal.provider] for cases [case_a, case_z]"
|
||||
);
|
||||
assert!(!warning.contains("secret-overlay-value"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_state_guard_preserves_state_on_forced_unwind() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let path = temp_dir.path().to_path_buf();
|
||||
|
||||
let unwind = std::panic::catch_unwind(|| {
|
||||
let _state = ProfileStateGuard::new(temp_dir, true);
|
||||
panic!("forced profile unwind");
|
||||
});
|
||||
|
||||
assert!(unwind.is_err());
|
||||
assert!(path.is_dir());
|
||||
std::fs::remove_dir_all(path).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_state_guard_removes_state_on_drop_without_preservation() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let path = temp_dir.path().to_path_buf();
|
||||
|
||||
drop(ProfileStateGuard::new(temp_dir, false));
|
||||
|
||||
assert!(!path.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_state_guard_normal_finalization_transfers_ownership_once() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let path = temp_dir.path().to_path_buf();
|
||||
|
||||
let cleanup = ProfileStateGuard::new(temp_dir, true).finalize();
|
||||
|
||||
assert!(cleanup.failures.is_empty());
|
||||
assert_eq!(cleanup.preserved_state.as_deref(), Some(path.as_path()));
|
||||
assert!(path.is_dir());
|
||||
std::fs::remove_dir_all(path).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,6 +43,17 @@ pub struct CaseMetadata {
|
||||
/// Must match `[a-z0-9_]+`.
|
||||
#[serde(default)]
|
||||
pub namespace: Option<String>,
|
||||
/// Optional old-stage server configuration sidecars.
|
||||
#[serde(default)]
|
||||
pub old_config: Option<OldConfigMetadata>,
|
||||
}
|
||||
|
||||
/// Optional configuration sidecars for the old compatibility stage.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct OldConfigMetadata {
|
||||
/// Datanode TOML sidecar, resolved relative to the compatibility case directory.
|
||||
pub datanode: PathBuf,
|
||||
}
|
||||
|
||||
impl CaseMetadata {
|
||||
@@ -53,6 +64,13 @@ impl CaseMetadata {
|
||||
.clone()
|
||||
.unwrap_or_else(|| sanitize_namespace(case_dir_name))
|
||||
}
|
||||
|
||||
/// Returns the old-stage datanode sidecar reference, if configured.
|
||||
pub fn old_datanode_overlay(&self) -> Option<&Path> {
|
||||
self.old_config
|
||||
.as_ref()
|
||||
.map(|config| config.datanode.as_path())
|
||||
}
|
||||
}
|
||||
|
||||
/// A loaded compatibility case (metadata + file paths).
|
||||
@@ -469,6 +487,74 @@ mod tests {
|
||||
assert_eq!(sanitize_namespace("a.b-c"), "a_b_c");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_case_metadata_parses_old_datanode_overlay() {
|
||||
let metadata: CaseMetadata = toml::from_str(
|
||||
r#"
|
||||
name = "case"
|
||||
reason = "reason"
|
||||
introduced_by = "test"
|
||||
topologies = ["distributed"]
|
||||
from_range = ["*"]
|
||||
to_range = ["*"]
|
||||
features = ["table"]
|
||||
owner = "team"
|
||||
|
||||
[old_config]
|
||||
datanode = "old-datanode.overlay.toml"
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
metadata.old_datanode_overlay(),
|
||||
Some(Path::new("old-datanode.overlay.toml"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_case_metadata_rejects_empty_old_config() {
|
||||
let error = toml::from_str::<CaseMetadata>(
|
||||
r#"
|
||||
name = "case"
|
||||
reason = "reason"
|
||||
introduced_by = "test"
|
||||
topologies = ["distributed"]
|
||||
from_range = ["*"]
|
||||
to_range = ["*"]
|
||||
features = ["table"]
|
||||
owner = "team"
|
||||
|
||||
[old_config]
|
||||
"#,
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(error.to_string().contains("missing field `datanode`"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_case_metadata_rejects_unknown_old_config_fields() {
|
||||
let error = toml::from_str::<CaseMetadata>(
|
||||
r#"
|
||||
name = "case"
|
||||
reason = "reason"
|
||||
introduced_by = "test"
|
||||
topologies = ["distributed"]
|
||||
from_range = ["*"]
|
||||
to_range = ["*"]
|
||||
features = ["table"]
|
||||
owner = "team"
|
||||
|
||||
[old_config]
|
||||
unsupported = "value"
|
||||
"#,
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(error.to_string().contains("unknown field `unsupported`"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_cases_metadata_rejects_empty_required_vectors() {
|
||||
let case = CompatCase {
|
||||
@@ -482,6 +568,7 @@ mod tests {
|
||||
features: vec!["table".to_string()],
|
||||
owner: "team".to_string(),
|
||||
namespace: None,
|
||||
old_config: None,
|
||||
},
|
||||
dir: PathBuf::from("case"),
|
||||
namespace: "case".to_string(),
|
||||
@@ -503,6 +590,7 @@ mod tests {
|
||||
features: vec!["table".to_string()],
|
||||
owner: "test".to_string(),
|
||||
namespace: None,
|
||||
old_config: None,
|
||||
},
|
||||
dir: PathBuf::from("bad_constraint"),
|
||||
namespace: "bad_constraint".to_string(),
|
||||
@@ -524,6 +612,7 @@ mod tests {
|
||||
features: vec!["table".to_string()],
|
||||
owner: "test".to_string(),
|
||||
namespace: None,
|
||||
old_config: None,
|
||||
},
|
||||
dir: PathBuf::from("case_a"),
|
||||
namespace: "shared_name".to_string(),
|
||||
@@ -539,6 +628,7 @@ mod tests {
|
||||
features: vec!["table".to_string()],
|
||||
owner: "test".to_string(),
|
||||
namespace: None,
|
||||
old_config: None,
|
||||
},
|
||||
dir: PathBuf::from("case_b"),
|
||||
namespace: "shared_name".to_string(),
|
||||
@@ -563,6 +653,7 @@ mod tests {
|
||||
features: vec!["table".to_string()],
|
||||
owner: "test".to_string(),
|
||||
namespace: Some("shared_name".to_string()),
|
||||
old_config: None,
|
||||
},
|
||||
dir: PathBuf::from("case_a"),
|
||||
namespace: "shared_name".to_string(),
|
||||
@@ -578,6 +669,7 @@ mod tests {
|
||||
features: vec!["table".to_string()],
|
||||
owner: "test".to_string(),
|
||||
namespace: Some("shared_name".to_string()),
|
||||
old_config: None,
|
||||
},
|
||||
dir: PathBuf::from("case_b"),
|
||||
namespace: "shared_name".to_string(),
|
||||
@@ -599,6 +691,7 @@ mod tests {
|
||||
features: vec!["table".to_string()],
|
||||
owner: "team".to_string(),
|
||||
namespace: None,
|
||||
old_config: None,
|
||||
},
|
||||
dir: PathBuf::from("case"),
|
||||
namespace: "case".to_string(),
|
||||
|
||||
@@ -0,0 +1,637 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io::Read;
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
use toml::value::Table;
|
||||
|
||||
use crate::env::bare::WalConfig;
|
||||
/// A dotted TOML path owned by the compatibility runner.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub(crate) struct DottedPath(Vec<String>);
|
||||
|
||||
impl DottedPath {
|
||||
fn new(parts: &[&str]) -> Self {
|
||||
Self(parts.iter().map(|part| (*part).to_string()).collect())
|
||||
}
|
||||
|
||||
fn parts(&self) -> impl Iterator<Item = &str> {
|
||||
self.0.iter().map(String::as_str)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for DottedPath {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.0.join("."))
|
||||
}
|
||||
}
|
||||
|
||||
/// Runner-owned datanode configuration paths that an overlay cannot override.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct DatanodeProtectionPolicy {
|
||||
protected_paths: Vec<DottedPath>,
|
||||
}
|
||||
|
||||
impl DatanodeProtectionPolicy {
|
||||
/// Builds the policy for the runner-selected WAL configuration.
|
||||
pub(crate) fn for_wal(wal: &WalConfig) -> Self {
|
||||
let mut protected_paths = vec![
|
||||
DottedPath::new(&["mode"]),
|
||||
DottedPath::new(&["node_id"]),
|
||||
DottedPath::new(&["storage", "data_home"]),
|
||||
DottedPath::new(&["meta_client_options", "metasrv_addrs"]),
|
||||
DottedPath::new(&["wal", "provider"]),
|
||||
];
|
||||
protected_paths.push(match wal {
|
||||
WalConfig::RaftEngine => DottedPath::new(&["wal", "dir"]),
|
||||
WalConfig::Kafka { .. } => DottedPath::new(&["wal", "broker_endpoints"]),
|
||||
});
|
||||
protected_paths.sort();
|
||||
|
||||
Self { protected_paths }
|
||||
}
|
||||
}
|
||||
|
||||
/// A parsed old-stage datanode sidecar, loaded from a confined case directory.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct DatanodeOverlay {
|
||||
source: PathBuf,
|
||||
value: toml::Value,
|
||||
profile_key: [u8; 32],
|
||||
profile_id: String,
|
||||
}
|
||||
|
||||
impl DatanodeOverlay {
|
||||
/// Loads and parses a datanode sidecar referenced relative to `case_dir`.
|
||||
///
|
||||
/// This is intended to prevent accidental case-directory escapes; it is not
|
||||
/// an adversarial concurrent-filesystem security boundary.
|
||||
pub(crate) fn load(case_dir: &Path, relative_ref: &Path) -> Result<Self, String> {
|
||||
validate_relative_reference(relative_ref)?;
|
||||
|
||||
let canonical_case_dir = case_dir.canonicalize().map_err(|error| {
|
||||
format!(
|
||||
"Failed to canonicalize compatibility case directory {}: {error}",
|
||||
case_dir.display()
|
||||
)
|
||||
})?;
|
||||
let requested_path = case_dir.join(relative_ref);
|
||||
let canonical_target = requested_path.canonicalize().map_err(|error| {
|
||||
format!(
|
||||
"Failed to resolve datanode overlay {} relative to case directory {}: {error}",
|
||||
relative_ref.display(),
|
||||
case_dir.display()
|
||||
)
|
||||
})?;
|
||||
|
||||
if !canonical_target.starts_with(&canonical_case_dir) {
|
||||
return Err(format!(
|
||||
"Datanode overlay {} resolves outside compatibility case directory {}",
|
||||
relative_ref.display(),
|
||||
canonical_case_dir.display()
|
||||
));
|
||||
}
|
||||
|
||||
let mut file = File::open(&canonical_target).map_err(|error| {
|
||||
format!(
|
||||
"Failed to open datanode overlay {}: {error}",
|
||||
canonical_target.display()
|
||||
)
|
||||
})?;
|
||||
let metadata = file.metadata().map_err(|error| {
|
||||
format!(
|
||||
"Failed to inspect datanode overlay {}: {error}",
|
||||
canonical_target.display()
|
||||
)
|
||||
})?;
|
||||
if !metadata.is_file() {
|
||||
return Err(format!(
|
||||
"Datanode overlay {} must be a regular file",
|
||||
canonical_target.display()
|
||||
));
|
||||
}
|
||||
|
||||
let mut content = String::new();
|
||||
file.read_to_string(&mut content).map_err(|error| {
|
||||
format!(
|
||||
"Failed to read datanode overlay {}: {error}",
|
||||
canonical_target.display()
|
||||
)
|
||||
})?;
|
||||
let value: toml::Value = toml::from_str(&content).map_err(|error| {
|
||||
format!(
|
||||
"Failed to parse datanode overlay {}: {error}",
|
||||
canonical_target.display()
|
||||
)
|
||||
})?;
|
||||
if !value.is_table() {
|
||||
return Err(format!(
|
||||
"Datanode overlay {} must have a TOML table at its root",
|
||||
canonical_target.display()
|
||||
));
|
||||
}
|
||||
|
||||
let profile_key = semantic_profile_key(&value);
|
||||
let profile_id = hex::encode(profile_key)[..12].to_string();
|
||||
|
||||
Ok(Self {
|
||||
source: canonical_target,
|
||||
value,
|
||||
profile_key,
|
||||
profile_id,
|
||||
})
|
||||
}
|
||||
|
||||
/// Validates runner-owned path conflicts and records overridden protected paths.
|
||||
pub(crate) fn prepare(
|
||||
self,
|
||||
protection: &DatanodeProtectionPolicy,
|
||||
) -> Result<PreparedDatanodeOverlay, String> {
|
||||
let mut touched_protected_paths = Vec::new();
|
||||
for path in &protection.protected_paths {
|
||||
validate_protected_ancestors(&self.value, path).map_err(|error| {
|
||||
format!(
|
||||
"Datanode overlay {} (profile {}): {error}",
|
||||
self.source.display(),
|
||||
self.profile_id
|
||||
)
|
||||
})?;
|
||||
if value_at_path(&self.value, path).is_some() {
|
||||
touched_protected_paths.push(path.clone());
|
||||
}
|
||||
}
|
||||
touched_protected_paths.sort();
|
||||
|
||||
Ok(PreparedDatanodeOverlay {
|
||||
source: self.source,
|
||||
value: self.value,
|
||||
profile_key: self.profile_key,
|
||||
profile_id: self.profile_id,
|
||||
protected_paths: protection.protected_paths.clone(),
|
||||
touched_protected_paths,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// A validated datanode overlay ready for application to a rendered baseline.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct PreparedDatanodeOverlay {
|
||||
source: PathBuf,
|
||||
value: toml::Value,
|
||||
profile_key: [u8; 32],
|
||||
profile_id: String,
|
||||
protected_paths: Vec<DottedPath>,
|
||||
touched_protected_paths: Vec<DottedPath>,
|
||||
}
|
||||
|
||||
impl PreparedDatanodeOverlay {
|
||||
/// Returns the canonical full SHA-256 profile key used for grouping.
|
||||
pub(crate) fn profile_key(&self) -> &[u8; 32] {
|
||||
&self.profile_key
|
||||
}
|
||||
|
||||
/// Returns a truncated profile ID suitable only for diagnostics.
|
||||
pub(crate) fn profile_id(&self) -> &str {
|
||||
&self.profile_id
|
||||
}
|
||||
|
||||
/// Returns the canonical sidecar path used for diagnostics.
|
||||
pub(crate) fn source(&self) -> &Path {
|
||||
&self.source
|
||||
}
|
||||
|
||||
/// Returns protected paths declared by the sidecar, in dotted-path order.
|
||||
pub(crate) fn touched_protected_paths(&self) -> &[DottedPath] {
|
||||
&self.touched_protected_paths
|
||||
}
|
||||
|
||||
/// Merges the sidecar into a rendered baseline and restores runner-owned fields.
|
||||
pub(crate) fn apply_to_rendered_baseline(&self, baseline: &str) -> Result<String, String> {
|
||||
let baseline_value: toml::Value = toml::from_str(baseline)
|
||||
.map_err(|error| format!("Failed to parse rendered datanode baseline: {error}"))?;
|
||||
if !baseline_value.is_table() {
|
||||
return Err(
|
||||
"Rendered datanode baseline must have a TOML table at its root".to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
let mut merged = baseline_value.clone();
|
||||
merge_toml_values(&mut merged, &self.value);
|
||||
|
||||
for path in &self.protected_paths {
|
||||
restore_protected_path(&mut merged, &baseline_value, path)?;
|
||||
}
|
||||
|
||||
toml::to_string(&merged)
|
||||
.map_err(|error| format!("Failed to serialize merged datanode configuration: {error}"))
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_relative_reference(relative_ref: &Path) -> Result<(), String> {
|
||||
if relative_ref.as_os_str().is_empty()
|
||||
|| relative_ref
|
||||
.components()
|
||||
.any(|component| !matches!(component, Component::Normal(_)))
|
||||
{
|
||||
return Err(format!(
|
||||
"Datanode overlay reference {} must be a non-empty relative path with normal components only",
|
||||
relative_ref.display()
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_protected_ancestors(value: &toml::Value, path: &DottedPath) -> Result<(), String> {
|
||||
let mut current = value;
|
||||
let parts: Vec<_> = path.parts().collect();
|
||||
for (index, part) in parts.iter().enumerate().take(parts.len().saturating_sub(1)) {
|
||||
let Some(table) = current.as_table() else {
|
||||
return Err(format!(
|
||||
"Datanode overlay makes ancestor {} of protected path {} non-table",
|
||||
parts[..index].join("."),
|
||||
path
|
||||
));
|
||||
};
|
||||
let Some(next) = table.get(*part) else {
|
||||
return Ok(());
|
||||
};
|
||||
if !next.is_table() {
|
||||
return Err(format!(
|
||||
"Datanode overlay makes ancestor {} of protected path {} non-table",
|
||||
parts[..=index].join("."),
|
||||
path
|
||||
));
|
||||
}
|
||||
current = next;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn value_at_path<'a>(value: &'a toml::Value, path: &DottedPath) -> Option<&'a toml::Value> {
|
||||
let mut current = value;
|
||||
for part in path.parts() {
|
||||
current = current.as_table()?.get(part)?;
|
||||
}
|
||||
Some(current)
|
||||
}
|
||||
|
||||
fn merge_toml_values(baseline: &mut toml::Value, overlay: &toml::Value) {
|
||||
match (baseline, overlay) {
|
||||
(toml::Value::Table(baseline), toml::Value::Table(overlay)) => {
|
||||
for (key, overlay_value) in overlay {
|
||||
match baseline.get_mut(key) {
|
||||
Some(baseline_value) => merge_toml_values(baseline_value, overlay_value),
|
||||
None => {
|
||||
baseline.insert(key.clone(), overlay_value.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
(baseline, overlay) => *baseline = overlay.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn restore_protected_path(
|
||||
merged: &mut toml::Value,
|
||||
baseline: &toml::Value,
|
||||
path: &DottedPath,
|
||||
) -> Result<(), String> {
|
||||
let Some(merged_table) = merged.as_table_mut() else {
|
||||
return Err("Merged datanode configuration must have a TOML table at its root".to_string());
|
||||
};
|
||||
let baseline_value = value_at_path(baseline, path).cloned();
|
||||
restore_path_in_table(merged_table, &path.0, baseline_value);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn restore_path_in_table(table: &mut Table, parts: &[String], baseline_value: Option<toml::Value>) {
|
||||
let Some((part, remaining)) = parts.split_first() else {
|
||||
return;
|
||||
};
|
||||
if remaining.is_empty() {
|
||||
match baseline_value {
|
||||
Some(value) => {
|
||||
table.insert(part.clone(), value);
|
||||
}
|
||||
None => {
|
||||
table.remove(part);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(value) = table.get_mut(part) else {
|
||||
return;
|
||||
};
|
||||
if let Some(table) = value.as_table_mut() {
|
||||
restore_path_in_table(table, remaining, baseline_value);
|
||||
}
|
||||
}
|
||||
|
||||
fn semantic_profile_key(value: &toml::Value) -> [u8; 32] {
|
||||
let mut encoded = Vec::new();
|
||||
encode_semantic_value(&mut encoded, value);
|
||||
Sha256::digest(encoded).into()
|
||||
}
|
||||
|
||||
fn encode_semantic_value(output: &mut Vec<u8>, value: &toml::Value) {
|
||||
match value {
|
||||
toml::Value::String(value) => encode_bytes(output, b"string", value.as_bytes()),
|
||||
toml::Value::Integer(value) => {
|
||||
encode_bytes(output, b"integer", &value.to_be_bytes());
|
||||
}
|
||||
toml::Value::Float(value) => {
|
||||
encode_bytes(output, b"float", &value.to_bits().to_be_bytes());
|
||||
}
|
||||
toml::Value::Boolean(value) => {
|
||||
encode_bytes(output, b"boolean", &[u8::from(*value)]);
|
||||
}
|
||||
toml::Value::Datetime(value) => {
|
||||
encode_bytes(output, b"datetime", value.to_string().as_bytes())
|
||||
}
|
||||
toml::Value::Array(values) => {
|
||||
let mut payload = Vec::new();
|
||||
encode_length(&mut payload, values.len());
|
||||
for value in values {
|
||||
encode_semantic_value(&mut payload, value);
|
||||
}
|
||||
encode_bytes(output, b"array", &payload);
|
||||
}
|
||||
toml::Value::Table(table) => {
|
||||
let mut payload = Vec::new();
|
||||
encode_length(&mut payload, table.len());
|
||||
let mut entries: Vec<_> = table.iter().collect();
|
||||
entries.sort_unstable_by(|(left, _), (right, _)| left.as_bytes().cmp(right.as_bytes()));
|
||||
for (key, value) in entries {
|
||||
encode_bytes(&mut payload, b"key", key.as_bytes());
|
||||
encode_semantic_value(&mut payload, value);
|
||||
}
|
||||
encode_bytes(output, b"table", &payload);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_bytes(output: &mut Vec<u8>, tag: &[u8], value: &[u8]) {
|
||||
encode_length(output, tag.len());
|
||||
output.extend_from_slice(tag);
|
||||
encode_length(output, value.len());
|
||||
output.extend_from_slice(value);
|
||||
}
|
||||
|
||||
fn encode_length(output: &mut Vec<u8>, length: usize) {
|
||||
output.extend_from_slice(&(length as u64).to_be_bytes());
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn load_overlay(content: &str) -> DatanodeOverlay {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let path = temp_dir.path().join("overlay.toml");
|
||||
std::fs::write(&path, content).unwrap();
|
||||
let overlay = DatanodeOverlay::load(temp_dir.path(), Path::new("overlay.toml")).unwrap();
|
||||
// `load` owns the parsed TOML, so the temporary source need not outlive it.
|
||||
overlay
|
||||
}
|
||||
|
||||
fn prepared_overlay(content: &str, wal: &WalConfig) -> PreparedDatanodeOverlay {
|
||||
load_overlay(content)
|
||||
.prepare(&DatanodeProtectionPolicy::for_wal(wal))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merges_recursive_tables_and_replaces_non_tables_atomically() {
|
||||
let prepared = prepared_overlay(
|
||||
r#"
|
||||
scalar = "replacement"
|
||||
primitive_array = [3, 2, 1]
|
||||
array_of_tables = [{ name = "new" }]
|
||||
[nested]
|
||||
new_key = true
|
||||
old_key = "replacement"
|
||||
"#,
|
||||
&WalConfig::RaftEngine,
|
||||
);
|
||||
let merged = prepared
|
||||
.apply_to_rendered_baseline(
|
||||
r#"
|
||||
scalar = 1
|
||||
primitive_array = [1, 2]
|
||||
array_of_tables = [{ name = "old" }, { name = "older" }]
|
||||
untouched = "kept"
|
||||
[nested]
|
||||
old_key = 1
|
||||
retained = "kept"
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let value: toml::Value = toml::from_str(&merged).unwrap();
|
||||
|
||||
assert_eq!(value["scalar"].as_str(), Some("replacement"));
|
||||
assert_eq!(value["primitive_array"].as_array().unwrap().len(), 3);
|
||||
assert_eq!(value["array_of_tables"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(value["nested"]["old_key"].as_str(), Some("replacement"));
|
||||
assert_eq!(value["nested"]["new_key"].as_bool(), Some(true));
|
||||
assert_eq!(value["nested"]["retained"].as_str(), Some("kept"));
|
||||
assert_eq!(value["untouched"].as_str(), Some("kept"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restores_protected_paths_and_deletes_absent_raft_wal_dir() {
|
||||
let prepared = prepared_overlay(
|
||||
r#"
|
||||
mode = "standalone"
|
||||
node_id = 99
|
||||
[storage]
|
||||
data_home = "/overlay/data"
|
||||
[meta_client_options]
|
||||
metasrv_addrs = ["overlay:3002"]
|
||||
[wal]
|
||||
provider = "kafka"
|
||||
dir = "/overlay/wal"
|
||||
tuning = 42
|
||||
"#,
|
||||
&WalConfig::RaftEngine,
|
||||
);
|
||||
let merged = prepared
|
||||
.apply_to_rendered_baseline(
|
||||
r#"
|
||||
mode = "distributed"
|
||||
node_id = 1
|
||||
[storage]
|
||||
data_home = "/runner/data"
|
||||
[meta_client_options]
|
||||
metasrv_addrs = ["runner:3002"]
|
||||
[wal]
|
||||
provider = "raft_engine"
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let value: toml::Value = toml::from_str(&merged).unwrap();
|
||||
|
||||
assert_eq!(value["mode"].as_str(), Some("distributed"));
|
||||
assert_eq!(value["node_id"].as_integer(), Some(1));
|
||||
assert_eq!(value["storage"]["data_home"].as_str(), Some("/runner/data"));
|
||||
assert_eq!(
|
||||
value["meta_client_options"]["metasrv_addrs"][0].as_str(),
|
||||
Some("runner:3002")
|
||||
);
|
||||
assert_eq!(value["wal"]["provider"].as_str(), Some("raft_engine"));
|
||||
assert!(value["wal"].get("dir").is_none());
|
||||
assert_eq!(value["wal"]["tuning"].as_integer(), Some(42));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kafka_policy_restores_broker_endpoints() {
|
||||
let prepared = prepared_overlay(
|
||||
r#"
|
||||
[wal]
|
||||
broker_endpoints = ["overlay:9092"]
|
||||
provider = "raft_engine"
|
||||
"#,
|
||||
&WalConfig::Kafka {
|
||||
needs_kafka_cluster: false,
|
||||
broker_endpoints: vec![],
|
||||
},
|
||||
);
|
||||
let merged = prepared
|
||||
.apply_to_rendered_baseline(
|
||||
r#"
|
||||
[wal]
|
||||
provider = "kafka"
|
||||
broker_endpoints = ["runner:9092"]
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let value: toml::Value = toml::from_str(&merged).unwrap();
|
||||
|
||||
assert_eq!(value["wal"]["provider"].as_str(), Some("kafka"));
|
||||
assert_eq!(
|
||||
value["wal"]["broker_endpoints"][0].as_str(),
|
||||
Some("runner:9092")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_non_table_protected_ancestor() {
|
||||
let overlay = load_overlay("wal = \"not a table\"");
|
||||
let error = overlay
|
||||
.prepare(&DatanodeProtectionPolicy::for_wal(&WalConfig::RaftEngine))
|
||||
.unwrap_err();
|
||||
|
||||
assert!(error.contains("ancestor wal"));
|
||||
assert!(error.contains("profile"));
|
||||
assert!(error.contains("wal.dir") || error.contains("wal.provider"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collects_sorted_touched_protected_paths() {
|
||||
let prepared = prepared_overlay(
|
||||
r#"
|
||||
node_id = 4
|
||||
[wal]
|
||||
dir = "/overlay/wal"
|
||||
provider = "kafka"
|
||||
"#,
|
||||
&WalConfig::RaftEngine,
|
||||
);
|
||||
|
||||
let paths: Vec<_> = prepared
|
||||
.touched_protected_paths()
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect();
|
||||
assert_eq!(paths, ["node_id", "wal.dir", "wal.provider"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn semantic_identity_ignores_formatting_and_table_order() {
|
||||
let first = load_overlay("[settings]\nb = 2\na = \"value\"\n");
|
||||
let second = load_overlay("# comment\n[settings]\na = \"value\"\nb = 2\n");
|
||||
|
||||
assert_eq!(first.profile_key, second.profile_key);
|
||||
assert_eq!(first.profile_id, second.profile_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn semantic_identity_preserves_scalar_types_and_array_order() {
|
||||
let integer = load_overlay("value = 1");
|
||||
let float = load_overlay("value = 1.0");
|
||||
let first_order = load_overlay("values = [1, 2]");
|
||||
let second_order = load_overlay("values = [2, 1]");
|
||||
|
||||
assert_ne!(integer.profile_key, float.profile_key);
|
||||
assert_ne!(first_order.profile_key, second_order.profile_key);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepared_overlay_exposes_profile_and_source_for_runner_grouping() {
|
||||
let overlay = load_overlay("value = 1");
|
||||
let expected_key = overlay.profile_key;
|
||||
let expected_id = overlay.profile_id.clone();
|
||||
let prepared = overlay
|
||||
.prepare(&DatanodeProtectionPolicy::for_wal(&WalConfig::RaftEngine))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(prepared.profile_key(), &expected_key);
|
||||
assert_eq!(prepared.profile_id(), expected_id);
|
||||
assert!(prepared.source().ends_with("overlay.toml"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_absolute_and_traversal_references() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let absolute = temp_dir.path().join("overlay.toml");
|
||||
std::fs::write(&absolute, "value = 1").unwrap();
|
||||
|
||||
assert!(DatanodeOverlay::load(temp_dir.path(), &absolute).is_err());
|
||||
assert!(DatanodeOverlay::load(temp_dir.path(), Path::new("../overlay.toml")).is_err());
|
||||
assert!(DatanodeOverlay::load(temp_dir.path(), Path::new("./overlay.toml")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_directories_and_parse_errors() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir(temp_dir.path().join("directory.toml")).unwrap();
|
||||
std::fs::write(temp_dir.path().join("broken.toml"), "[broken").unwrap();
|
||||
|
||||
let directory_error =
|
||||
DatanodeOverlay::load(temp_dir.path(), Path::new("directory.toml")).unwrap_err();
|
||||
assert!(directory_error.contains("regular file"));
|
||||
let parse_error =
|
||||
DatanodeOverlay::load(temp_dir.path(), Path::new("broken.toml")).unwrap_err();
|
||||
assert!(parse_error.contains("Failed to parse datanode overlay"));
|
||||
assert!(parse_error.contains("broken.toml"));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn rejects_symlink_escape() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let case_dir = tempfile::tempdir().unwrap();
|
||||
let outside_dir = tempfile::tempdir().unwrap();
|
||||
let outside_file = outside_dir.path().join("outside.toml");
|
||||
std::fs::write(&outside_file, "value = 1").unwrap();
|
||||
symlink(&outside_file, case_dir.path().join("escape.toml")).unwrap();
|
||||
|
||||
let error = DatanodeOverlay::load(case_dir.path(), Path::new("escape.toml")).unwrap_err();
|
||||
assert!(error.contains("outside compatibility case directory"));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user