diff --git a/src/common/datasource/src/compression.rs b/src/common/datasource/src/compression.rs index b7eb2d8b644..2e72decdd26 100644 --- a/src/common/datasource/src/compression.rs +++ b/src/common/datasource/src/compression.rs @@ -16,6 +16,7 @@ use std::fmt::Display; use std::io; use std::str::FromStr; +use async_compression::Level; use async_compression::tokio::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZstdDecoder}; use async_compression::tokio::write; use bytes::Bytes; @@ -88,6 +89,15 @@ impl CompressionType { Self::Uncompressed => "", } } + + /// Encodes `content` as a gzip stream at level 0, which stores it without compression. + pub async fn encode_gzip_stored(content: &[u8]) -> io::Result> { + let mut buffer = Vec::with_capacity(content.len()); + let mut encoder = write::GzipEncoder::with_quality(&mut buffer, Level::Precise(0)); + encoder.write_all(content).await?; + encoder.shutdown().await?; + Ok(buffer) + } } macro_rules! impl_compression_type { diff --git a/src/mito2/AGENTS.md b/src/mito2/AGENTS.md index d39cb56f617..ddb7c7c69b8 100644 --- a/src/mito2/AGENTS.md +++ b/src/mito2/AGENTS.md @@ -103,6 +103,9 @@ Tests live next to the code as `*_test.rs` (e.g. `src/mito2/src/engine/flush_tes - Sequence numbers are strictly increasing per region; dedup and snapshot reads depend on this. Do not change assignment lightly. - Manifest version is monotonic — never reset or skip it. +- In regions whose writes require a lease and whose WAL is the object store WAL, + each manifest version has one fixed object name and only one writer can create + it (`manifest/storage.rs`). - Lock ordering: take the manifest lock before updating `version_control`; the reverse deadlocks against concurrent flush/compaction. - All region I/O runs on tokio workers; never `block_on` inside a worker. diff --git a/src/mito2/src/compaction/compactor.rs b/src/mito2/src/compaction/compactor.rs index 0e425906c14..dadcce8a85c 100644 --- a/src/mito2/src/compaction/compactor.rs +++ b/src/mito2/src/compaction/compactor.rs @@ -292,6 +292,7 @@ impl CompactionRegionOpener { manifest_manager, RegionRoleState::Leader(RegionLeaderState::Writable), hook, + Default::default(), )); let file_purger = { diff --git a/src/mito2/src/compaction/scheduler_test.rs b/src/mito2/src/compaction/scheduler_test.rs index 08626f60f7c..d7f7c7d8def 100644 --- a/src/mito2/src/compaction/scheduler_test.rs +++ b/src/mito2/src/compaction/scheduler_test.rs @@ -2334,6 +2334,7 @@ async fn test_compaction_bypass_in_staging_mode() { checkpoint_distance: 10, remove_file_options: Default::default(), manifest_cache: None, + conditional_publication: false, }, FormatType::PrimaryKey, false, @@ -2345,6 +2346,7 @@ async fn test_compaction_bypass_in_staging_mode() { manager, RegionRoleState::Leader(RegionLeaderState::Staging), None, + Default::default(), )) }; diff --git a/src/mito2/src/engine/object_store_wal_test.rs b/src/mito2/src/engine/object_store_wal_test.rs index 5a61d8b6733..6a600958bf9 100644 --- a/src/mito2/src/engine/object_store_wal_test.rs +++ b/src/mito2/src/engine/object_store_wal_test.rs @@ -17,8 +17,11 @@ use std::collections::HashMap; use std::sync::Arc; use std::sync::atomic::Ordering; +use std::time::{Duration, Instant}; -use api::v1::Rows; +use api::v1::region::{StrictWindow, compact_request}; +use api::v1::{ColumnSchema, Rows}; +use common_datasource::compression::CompressionType; use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; use common_recordbatch::RecordBatches; @@ -26,19 +29,24 @@ use common_wal::config::object_store::{ObjectStoreWalConfig, STANDALONE_GENERATI use common_wal::options::{ObjectStoreWalOptions, WAL_OPTIONS_KEY, WalOptions}; use futures::TryStreamExt; use log_store::ObjectStoreLogStore; +use object_store::ObjectStore; +use rstest::rstest; use store_api::logstore::LogStore; use store_api::logstore::provider::Provider; -use store_api::region_engine::RegionEngine; +use store_api::region_engine::{RegionEngine, RegionRole}; use store_api::region_request::{ - PathType, RegionCleanUpRequest, RegionCloseRequest, RegionDropRequest, RegionOpenRequest, - RegionRequest, + PathType, RegionCleanUpRequest, RegionCloseRequest, RegionCompactRequest, RegionDropRequest, + RegionFlushRequest, RegionOpenRequest, RegionRequest, RegionTruncateRequest, }; use store_api::storage::{RegionId, ScanRequest}; use crate::config::MitoConfig; use crate::engine::MitoEngine; +use crate::manifest::storage::{delta_file, gen_path, manifest_dir}; +use crate::sst::location::region_dir_from_table_dir; use crate::test_util::{ - CreateRequestBuilder, TestEnv, build_rows, flush_region, put_rows, reopen_region, rows_schema, + CheckpointTaskBlocker, CreateRequestBuilder, TestEnv, build_rows, flush_region, nop_action, + put_rows, reopen_region, rows_schema, }; use crate::wal::EntryId; @@ -377,3 +385,187 @@ async fn test_object_store_wal_options_reject_raft_engine_log_store() { .unwrap_err(); assert_eq!(StatusCode::InvalidArguments, err.status_code()); } + +/// Returns a config that starts a checkpoint on every commit. +fn checkpoint_every_commit_config() -> MitoConfig { + MitoConfig { + manifest_checkpoint_distance: 1, + ..Default::default() + } +} + +/// Creates a region whose writes require a lease. Returns the lease deadline, which the frozen +/// clock of the lease checks has not reached. +async fn create_leased_region( + env: &mut TestEnv, + region_id: RegionId, +) -> ( + MitoEngine, + Arc, + String, + Vec, + Instant, +) { + let (engine, log_store) = env + .create_engine_with_object_store_wal(checkpoint_every_commit_config(), &wal_config()) + .await; + engine.require_write_lease(); + engine.freeze_lease_clock(Instant::now()); + let (table_dir, schema) = create_region(&engine, region_id).await; + let deadline = Instant::now() + Duration::from_secs(10); + engine + .set_region_role_with_lease(region_id, RegionRole::Leader, Instant::now(), deadline) + .unwrap(); + (engine, log_store, table_dir, schema, deadline) +} + +async fn count_ssts(store: &ObjectStore, region_dir: &str) -> usize { + let entries = store.list(region_dir).await.unwrap(); + entries + .iter() + .filter(|e| e.name().ends_with(".parquet")) + .count() +} + +#[rstest] +#[case::flush(RegionRequest::Flush(RegionFlushRequest::default()), 1)] +#[case::compaction( + RegionRequest::Compact(RegionCompactRequest { + options: compact_request::Options::StrictWindow(StrictWindow { window_seconds: 3600 }), + ..Default::default() + }), + 1 +)] +#[case::truncate(RegionRequest::Truncate(RegionTruncateRequest::All), 0)] +#[tokio::test] +async fn test_stale_commit_on_deleted_delta_name_conflicts( + #[case] commit: RegionRequest, + #[case] new_ssts: usize, +) { + common_telemetry::init_default_ut_logging(); + // The creation is version 0 and the flushes below are versions 1 and 2. + const VERSION: u64 = 3; + let (blocker, layer) = CheckpointTaskBlocker::block_delta_write(VERSION); + let mut env = TestEnv::with_prefix("stale-manifest-commit") + .await + .with_mock_layer(layer); + let region_id = RegionId::new(1, 1); + let (old_leader, log_store, table_dir, schema, deadline) = + create_leased_region(&mut env, region_id).await; + let rows = |start| Rows { + schema: schema.clone(), + rows: build_rows(start, start + 3), + }; + for start in [0, 3] { + put_rows(&old_leader, region_id, rows(start)).await; + flush_region(&old_leader, region_id, None).await; + } + put_rows(&old_leader, region_id, rows(6)).await; + + let region = old_leader.get_region(region_id).unwrap(); + let (manifest, checkpoint) = { + let mut manager = region.manifest_ctx.manifest_manager.write().await; + manager.wait_for_pending_checkpoint().await; + ( + manager.manifest(), + manager.checkpointer().last_checkpoint_version(), + ) + }; + let store = env.get_object_store().unwrap(); + let region_dir = region_dir_from_table_dir(&table_dir, region_id, PathType::Bare); + let ssts = count_ssts(&store, ®ion_dir).await; + + let engine = old_leader.clone(); + let commit = tokio::spawn(async move { engine.handle_request(region_id, commit).await }); + blocker.wait_until_blocked().await; + let version = region.version_control.current().version; + let entry_ids = stored_entry_ids(&log_store, region_id).await; + + let new_holder = env + .create_engine_with_log_store(checkpoint_every_commit_config(), log_store.clone()) + .await; + new_holder.require_write_lease(); + let open = RegionRequest::Open(open_request(table_dir.clone())); + new_holder.handle_request(region_id, open).await.unwrap(); + let delta = gen_path( + &manifest_dir(®ion_dir), + &delta_file(VERSION), + CompressionType::Gzip, + ); + { + let held = new_holder.get_region(region_id).unwrap(); + let mut manager = held.manifest_ctx.manifest_manager.write().await; + assert_eq!(VERSION - 1, manager.last_version()); + while manager.last_version() < VERSION || store.exists(&delta).await.unwrap() { + manager + .update_with_verification(nop_action(), false, true, || false) + .await + .unwrap(); + manager.wait_for_pending_checkpoint().await; + } + } + + old_leader.freeze_lease_clock(deadline); + blocker.release(); + let error = commit.await.unwrap().unwrap_err(); + assert!(error.retry_hint().is_retryable(), "{error:?}"); + assert!(format!("{error:?}").contains("conflicts"), "{error:?}"); + + let manager = region.manifest_ctx.manifest_manager.read().await; + assert_eq!(VERSION - 1, manager.last_version()); + assert!(Arc::ptr_eq(&manifest, &manager.manifest())); + assert_eq!(checkpoint, manager.checkpointer().last_checkpoint_version()); + assert!(!manager.checkpointer().is_doing_checkpoint()); + drop(manager); + assert!(Arc::ptr_eq( + &version, + ®ion.version_control.current().version + )); + assert_eq!(ssts + new_ssts, count_ssts(&store, ®ion_dir).await); + assert!(store.exists(&delta).await.unwrap()); + assert_eq!(entry_ids, stored_entry_ids(&log_store, region_id).await); + + reopen_region(&new_holder, region_id, table_dir, false, wal_options()).await; + assert_eq!(9, count_rows(&new_holder, region_id).await); +} + +#[tokio::test] +async fn test_commit_crossing_deadline_applies_without_checkpoint() { + common_telemetry::init_default_ut_logging(); + let (blocker, layer) = CheckpointTaskBlocker::block_delta_write(1); + let mut env = TestEnv::with_prefix("commit-crossing-deadline") + .await + .with_mock_layer(layer); + let region_id = RegionId::new(1, 1); + let (engine, log_store, table_dir, schema, deadline) = + create_leased_region(&mut env, region_id).await; + let rows = Rows { + schema, + rows: build_rows(0, 3), + }; + put_rows(&engine, region_id, rows).await; + + let flush = { + let engine = engine.clone(); + tokio::spawn(async move { + let request = RegionRequest::Flush(RegionFlushRequest::default()); + engine.handle_request(region_id, request).await + }) + }; + blocker.wait_until_blocked().await; + engine.freeze_lease_clock(deadline); + blocker.release(); + flush.await.unwrap().unwrap(); + + let region = engine.get_region(region_id).unwrap(); + let manager = region.manifest_ctx.manifest_manager.read().await; + assert_eq!(1, manager.last_version()); + assert_eq!(1, region.version().ssts.levels()[0].files.len()); + assert_eq!(0, manager.checkpointer().last_checkpoint_version()); + assert!(!manager.checkpointer().is_doing_checkpoint()); + assert!(stored_entry_ids(&log_store, region_id).await.is_empty()); + drop(manager); + + reopen_region(&engine, region_id, table_dir, false, wal_options()).await; + assert_eq!(3, count_rows(&engine, region_id).await); +} diff --git a/src/mito2/src/engine/region_hook.rs b/src/mito2/src/engine/region_hook.rs index c8b736d2894..51028b855bc 100644 --- a/src/mito2/src/engine/region_hook.rs +++ b/src/mito2/src/engine/region_hook.rs @@ -120,8 +120,8 @@ //! immediately after SST files are written but **before** the manifest is committed. //! //! `on_manifest_updated` is funneled through [`ManifestContext::update_locked`], -//! the sole caller of the low-level [`RegionManifestManager::update`], which -//! packages each successful write into a [`PendingManifestHook`]. The caller +//! which calls the low-level [`RegionManifestManager::update_with_verification`] +//! and packages each successful write into a [`PendingManifestHook`]. The caller //! owns the write lock, drops it, and *then* fires the receipt — the hook must //! never run under the lock. [`ManifestContext::update_manifest`] is the common //! case: it acquires the lock, delegates to `update_locked`, and fires the @@ -148,7 +148,7 @@ //! [`on_region_dropped`]: RegionHook::on_region_dropped //! [`on_region_files_removed`]: RegionHook::on_region_files_removed //! [`on_region_gc`]: RegionHook::on_region_gc -//! [`RegionManifestManager::update`]: crate::manifest::manager::RegionManifestManager::update +//! [`RegionManifestManager::update_with_verification`]: crate::manifest::manager::RegionManifestManager::update_with_verification //! [`ManifestContext::update_locked`]: crate::region::ManifestContext::update_locked //! [`ManifestContext::update_manifest`]: crate::region::ManifestContext::update_manifest diff --git a/src/mito2/src/error.rs b/src/mito2/src/error.rs index 74282dd4356..9a6181e7dc0 100644 --- a/src/mito2/src/error.rs +++ b/src/mito2/src/error.rs @@ -147,6 +147,25 @@ pub enum Error { location: Location, }, + #[snafu(display("Manifest version {version} conflicts in {manifest_dir}: {reason}"))] + ManifestVersionConflict { + version: ManifestVersion, + manifest_dir: String, + reason: String, + #[snafu(implicit)] + location: Location, + }, + + #[snafu(display( + "Manifest checkpoint {version} in {manifest_dir} was removed after it was listed" + ))] + ManifestCheckpointVanished { + version: ManifestVersion, + manifest_dir: String, + #[snafu(implicit)] + location: Location, + }, + #[snafu(display("Failed to serialize manifest, region_id: {}", region_id))] SerializeManifest { region_id: RegionId, @@ -1561,9 +1580,11 @@ impl ErrorExt for Error { MultiRegionCompactionResource { .. } => StatusCode::RuntimeResourcesExhausted, SstWriteMemoryExceeded { .. } => StatusCode::RuntimeResourcesExhausted, DataTypeMismatch { source, .. } => source.status_code(), - OpenDal { .. } | ManifestDeltaNotFound { .. } | ReadParquet { .. } => { - StatusCode::StorageUnavailable - } + OpenDal { .. } + | ManifestDeltaNotFound { .. } + | ManifestVersionConflict { .. } + | ManifestCheckpointVanished { .. } + | ReadParquet { .. } => StatusCode::StorageUnavailable, WriteWal { source, .. } | ReadWal { source, .. } | DeleteWal { source, .. } @@ -1784,7 +1805,9 @@ impl ErrorExt for Error { | RegionBusy { .. } | ManualCompactionAlreadyRunning { .. } | FlushableRegionState { .. } - | ManifestDeltaNotFound { .. } => RetryHint::Retryable, + | ManifestDeltaNotFound { .. } + | ManifestVersionConflict { .. } + | ManifestCheckpointVanished { .. } => RetryHint::Retryable, OpenDal { error, .. } | DeleteSsts { error, .. } @@ -2002,5 +2025,14 @@ mod tests { // This can be raised while applying an edit after its manifest file was saved. let error = RegionMetadataNotFoundSnafu {}.build(); assert!(error.may_have_persisted_manifest_update()); + + let error = ManifestVersionConflictSnafu { + version: 1u64, + manifest_dir: "manifest/", + reason: "test", + } + .build(); + assert!(error.may_have_persisted_manifest_update()); + assert!(error.retry_hint().is_retryable()); } } diff --git a/src/mito2/src/manifest/manager.rs b/src/mito2/src/manifest/manager.rs index 972fa310e93..aabfa18ff68 100644 --- a/src/mito2/src/manifest/manager.rs +++ b/src/mito2/src/manifest/manager.rs @@ -55,6 +55,9 @@ pub struct RegionManifestOptions { pub remove_file_options: RemoveFileOptions, /// Optional cache for manifest files. pub manifest_cache: Option, + /// Whether two writers can race for a version, so that deltas are published by conditional + /// create under a fixed name and recovery starts from the highest listed checkpoint. + pub(crate) conditional_publication: bool, } impl RegionManifestOptions { @@ -71,8 +74,16 @@ impl RegionManifestOptions { enable_gc: config.gc.enable, }, manifest_cache: None, + conditional_publication: false, } } + + /// Sets whether deltas are published by conditional create and recovery starts from the + /// highest listed checkpoint. + pub(crate) fn with_conditional_publication(mut self, enabled: bool) -> Self { + self.conditional_publication = enabled; + self + } } /// Options for updating `removed_files` field in [RegionManifest]. @@ -181,7 +192,8 @@ impl RegionManifestManager { stats.total_manifest_size.clone(), stats.metric().cloned(), options.manifest_cache.clone(), - ); + ) + .with_conditional_publication(options.conditional_publication)?; let manifest_version = stats.manifest_version.clone(); info!( @@ -269,7 +281,8 @@ impl RegionManifestManager { stats.total_manifest_size.clone(), stats.metric().cloned(), options.manifest_cache.clone(), - ); + ) + .with_conditional_publication(options.conditional_publication)?; let manifest_version = stats.manifest_version.clone(); // recover from storage @@ -556,32 +569,30 @@ impl RegionManifestManager { } /// Updates the manifest. Returns the current manifest version number. + /// + /// The caller has no Leader deadline to show that its commit returned in time, so a normal + /// commit under conditional publication is always verified and starts no checkpoint. pub async fn update( &mut self, action_list: RegionMetaActionList, is_staging: bool, ) -> Result { - self.update_inner(action_list, is_staging, !is_staging) + self.update_with_verification(action_list, is_staging, !is_staging, || true) .await } - /// Updates the normal manifest without starting a checkpoint. + /// Updates the manifest. Returns the current manifest version number. /// - /// This is only used while a leader is downgrading. The final flush still - /// publishes its manifest edit, but must not start cleanup after the - /// downgrade checkpoint barrier. - pub(crate) async fn update_normal_without_checkpoint( - &mut self, - action_list: RegionMetaActionList, - ) -> Result { - self.update_inner(action_list, false, false).await - } - - async fn update_inner( + /// `allow_checkpoint` tells whether a normal commit may start a checkpoint. `must_verify` is + /// evaluated after the save returns, for a normal commit under conditional publication. If it + /// returns true, the commit may have landed on a name that checkpoint cleanup freed, so it + /// only applies when LIST finds no checkpoint that covers it, and it starts no checkpoint. + pub(crate) async fn update_with_verification( &mut self, action_list: RegionMetaActionList, is_staging: bool, - allow_checkpoint: bool, + mut allow_checkpoint: bool, + must_verify: impl FnOnce() -> bool, ) -> Result { let _t = MANIFEST_OP_ELAPSED .with_label_values(&["update"]) @@ -594,10 +605,22 @@ impl RegionManifestManager { } ); - let version = self.increase_version(); + let conditional = self.store.is_conditional_publication(); + let version = if conditional { + self.last_version() + 1 + } else { + self.increase_version() + }; self.store .save(version, &action_list.encode()?, is_staging) .await?; + if conditional { + if !is_staging && must_verify() { + self.store.ensure_version_not_checkpointed(version).await?; + allow_checkpoint = false; + } + self.set_version(version); + } // For a staging region, the manifest is initially inherited from the previous manifest(i.e., `self.manifest`). // When the staging manifest becomes available, it will be used to construct the new manifest. diff --git a/src/mito2/src/manifest/storage.rs b/src/mito2/src/manifest/storage.rs index 9b534c8f21b..7de29a319df 100644 --- a/src/mito2/src/manifest/storage.rs +++ b/src/mito2/src/manifest/storage.rs @@ -24,22 +24,23 @@ use std::sync::Arc; use std::sync::atomic::AtomicU64; use common_datasource::compression::CompressionType; +use common_error::ext::ErrorExt; use common_telemetry::{debug, warn}; use crc32fast::Hasher; +use futures::TryStreamExt; use lazy_static::lazy_static; use object_store::util::join_dir; use object_store::{Lister, ObjectStore, util}; use regex::Regex; -#[cfg(test)] -use snafu::ResultExt; -use snafu::ensure; -use store_api::ManifestVersion; +use snafu::{ResultExt, ensure}; use store_api::storage::RegionId; +use store_api::{MIN_VERSION, ManifestVersion}; use crate::cache::manifest_cache::ManifestCache; -#[cfg(test)] -use crate::error::OpenDalSnafu; -use crate::error::{ChecksumMismatchSnafu, Result}; +use crate::error::{ + ChecksumMismatchSnafu, ManifestVersionConflictSnafu, OpenDalSnafu, Result, + UnsupportedOperationSnafu, +}; use crate::manifest::storage::checkpoint::CheckpointStorage; use crate::manifest::storage::delta::DeltaStorage; use crate::manifest::storage::size_tracker::{CheckpointTracker, DeltaTracker, SizeTracker}; @@ -167,6 +168,8 @@ pub struct ManifestObjectStore { delta_storage: DeltaStorage, /// The staging file storage. staging_storage: StagingStorage, + /// Whether deltas are created conditionally and the last checkpoint is found by LIST. + conditional_publication: bool, } impl ManifestObjectStore { @@ -227,9 +230,34 @@ impl ManifestObjectStore { checkpoint_storage, delta_storage, staging_storage, + conditional_publication: false, } } + /// Sets whether this store publishes deltas by conditional create and recovers from the + /// highest checkpoint found by LIST. + /// + /// Returns an error if the mode is enabled on a backend without conditional create. + pub(crate) fn with_conditional_publication(mut self, enabled: bool) -> Result { + let capability = self.object_store.info().capability(); + ensure!( + !enabled || capability.write_with_if_not_exists, + UnsupportedOperationSnafu { + err_msg: format!( + "conditional create is required for manifest {} of a region whose writes need a lease", + self.path + ), + } + ); + self.conditional_publication = enabled; + Ok(self) + } + + /// Returns whether this store publishes deltas by conditional create. + pub(crate) fn is_conditional_publication(&self) -> bool { + self.conditional_publication + } + /// Returns the manifest dir pub(crate) fn manifest_dir(&self) -> &str { &self.path @@ -391,11 +419,35 @@ impl ManifestObjectStore { ) -> Result<()> { if is_staging { self.staging_storage.save(version, bytes).await + } else if self.conditional_publication { + self.delta_storage.save_conditional(version, bytes).await } else { self.delta_storage.save(version, bytes).await } } + /// Fails with a version conflict unless LIST proves that no checkpoint at or above `version` + /// exists. + /// + /// Cleanup deletes a delta only after a checkpoint that covers it is saved, and never deletes + /// the highest checkpoint. So a late create that landed on a deleted name always finds one. + pub(crate) async fn ensure_version_not_checkpointed( + &self, + version: ManifestVersion, + ) -> Result<()> { + let reason = match self.list_highest_checkpoint(version).await { + Ok(None) => return Ok(()), + Ok(Some((checkpoint, _))) => format!("covered by checkpoint {checkpoint}"), + Err(e) => format!("cannot list the checkpoints: {}", e.output_msg()), + }; + ManifestVersionConflictSnafu { + version, + manifest_dir: &self.path, + reason, + } + .fail() + } + /// Save the checkpoint manifest file. pub(crate) async fn save_checkpoint( &self, @@ -411,7 +463,21 @@ impl ManifestObjectStore { /// Return manifest version and the raw [RegionCheckpoint](crate::manifest::action::RegionCheckpoint) content if any /// Also applies the checkpoint retention rule to the tracked size. pub async fn load_last_checkpoint(&mut self) -> Result)>> { - let checkpoint = self.checkpoint_storage.load_last_checkpoint().await?; + let checkpoint = if self.conditional_publication { + match self.list_highest_checkpoint(MIN_VERSION).await { + Ok(Some((version, compress_type))) => Some( + self.checkpoint_storage + .load_listed_checkpoint(version, compress_type) + .await?, + ), + Ok(None) => None, + // A region that does not exist yet can have no manifest directory. + Err(e) if e.is_object_not_found() => None, + Err(e) => return Err(e), + } + } else { + self.checkpoint_storage.load_last_checkpoint().await? + }; if let Some((version, _)) = &checkpoint { self.retain_manifest_size_after(*version); } @@ -469,6 +535,31 @@ impl ManifestObjectStore { pub(crate) fn staging_storage(&self) -> &StagingStorage { &self.staging_storage } + + /// Returns the version and compression of the highest checkpoint at or above `min_version` + /// found by LIST. + async fn list_highest_checkpoint( + &self, + min_version: ManifestVersion, + ) -> Result> { + let mut lister = self + .object_store + .lister_with(&self.path) + .start_after(&list_start_after(&self.path, min_version)) + .await + .context(OpenDalSnafu)?; + let mut highest = None; + while let Some(entry) = lister.try_next().await.context(OpenDalSnafu)? { + let file_name = entry.name(); + if is_checkpoint_file(file_name) { + let version = file_version(file_name); + if version >= min_version && highest.is_none_or(|(v, _)| v < version) { + highest = Some((version, file_compress_type(file_name))); + } + } + } + Ok(highest) + } } #[cfg(test)] diff --git a/src/mito2/src/manifest/storage/checkpoint.rs b/src/mito2/src/manifest/storage/checkpoint.rs index 769d8fc897b..9016195ce11 100644 --- a/src/mito2/src/manifest/storage/checkpoint.rs +++ b/src/mito2/src/manifest/storage/checkpoint.rs @@ -19,12 +19,13 @@ use common_datasource::compression::CompressionType; use common_telemetry::{debug, warn}; use object_store::{ErrorKind, ObjectStore}; use serde::{Deserialize, Serialize}; -use snafu::ResultExt; +use snafu::{OptionExt, ResultExt}; use store_api::ManifestVersion; use crate::cache::manifest_cache::ManifestCache; use crate::error::{ - CompressObjectSnafu, DecompressObjectSnafu, OpenDalSnafu, Result, SerdeJsonSnafu, Utf8Snafu, + CompressObjectSnafu, DecompressObjectSnafu, ManifestCheckpointVanishedSnafu, OpenDalSnafu, + Result, SerdeJsonSnafu, Utf8Snafu, }; use crate::manifest::storage::size_tracker::Tracker; use crate::manifest::storage::utils::{get_from_cache_decoded, write_and_put_cache}; @@ -200,6 +201,44 @@ impl CheckpointStorage { self.load_checkpoint(metadata).await } + /// Loads the checkpoint that LIST found at `version` with `compress_type`. + /// + /// The checkpoint pointer only provides the checksum, when it points at the same version: the + /// checkpoint task of a stale writer can leave it at an older one. + /// + /// Fails if cleanup removed the checkpoint after the LIST, because the deltas it covered can + /// be gone too. + pub(crate) async fn load_listed_checkpoint( + &self, + version: ManifestVersion, + compress_type: CompressionType, + ) -> Result<(ManifestVersion, Vec)> { + let checksum = self + .read_last_checkpoint_metadata() + .await? + .filter(|metadata| metadata.version == version) + .and_then(|metadata| metadata.checksum); + let mut storage = Self::new( + self.path.clone(), + self.object_store.clone(), + compress_type, + self.manifest_cache.clone(), + self.size_tracker.clone(), + ); + storage + .load_checkpoint(CheckpointMetadata { + size: 0, + version, + checksum, + extend_metadata: HashMap::new(), + }) + .await? + .context(ManifestCheckpointVanishedSnafu { + version, + manifest_dir: &self.path, + }) + } + /// Reads the small checkpoint pointer without loading its payload. pub(crate) async fn read_last_checkpoint_metadata(&self) -> Result> { let last_checkpoint_path = self.last_checkpoint_path(); diff --git a/src/mito2/src/manifest/storage/delta.rs b/src/mito2/src/manifest/storage/delta.rs index 79c6da14e9e..8b3a0a1d34a 100644 --- a/src/mito2/src/manifest/storage/delta.rs +++ b/src/mito2/src/manifest/storage/delta.rs @@ -18,7 +18,7 @@ use common_datasource::compression::CompressionType; use common_telemetry::{debug, warn}; use futures::TryStreamExt; use futures::future::try_join_all; -use object_store::{Entry, ErrorKind, Lister, ObjectStore}; +use object_store::{Buffer, Entry, ErrorKind, Lister, ObjectStore}; use snafu::{ResultExt, ensure}; use store_api::ManifestVersion; use store_api::storage::RegionId; @@ -27,7 +27,7 @@ use tokio::sync::Semaphore; use crate::cache::manifest_cache::ManifestCache; use crate::error::{ CompressObjectSnafu, DecompressObjectSnafu, InvalidScanIndexSnafu, ManifestDeltaNotFoundSnafu, - OpenDalSnafu, Result, + ManifestVersionConflictSnafu, OpenDalSnafu, Result, }; use crate::manifest::storage::size_tracker::Tracker; use crate::manifest::storage::utils::{ @@ -290,6 +290,90 @@ impl DeltaStorage { Ok(()) } + + /// Creates the delta of `version` under its fixed `.json.gz` name, so that only one writer can + /// commit the version. + /// + /// An existing object with the same decoded content is a retry of an earlier write that + /// landed. Any other existing object fails with a version conflict. + pub(crate) async fn save_conditional( + &mut self, + version: ManifestVersion, + bytes: &[u8], + ) -> Result<()> { + let path = gen_path(&self.path, &delta_file(version), CompressionType::Gzip); + // Every writer has to use one object name, so `compress_manifest = false` is honoured + // with gzip level 0, which stores the bytes without compressing them. + let data = if self.compress_type.is_compressed() { + CompressionType::Gzip.encode(bytes).await + } else { + CompressionType::encode_gzip_stored(bytes).await + } + .context(CompressObjectSnafu { + compress_type: CompressionType::Gzip, + path: &path, + })?; + let data = Buffer::from(data); + + let stored_size = match self + .object_store + .write_with(&path, data.clone()) + .if_not_exists(true) + .await + { + Ok(_) => data.len(), + Err(error) + if matches!( + error.kind(), + ErrorKind::AlreadyExists | ErrorKind::ConditionNotMatch + ) => + { + self.ensure_same_delta(version, &path, bytes).await? + } + Err(error) => return Err(error).context(OpenDalSnafu), + }; + + self.delta_tracker.record(version, stored_size as u64); + if let Some(cache) = &self.manifest_cache { + cache.put_file(path, data.to_vec()).await; + } + + Ok(()) + } + + /// Checks that the object at `path` decodes to `bytes`, reading it directly from storage. + /// Returns the size of the stored object. + async fn ensure_same_delta( + &self, + version: ManifestVersion, + path: &str, + bytes: &[u8], + ) -> Result { + let conflict = |reason: String| { + ManifestVersionConflictSnafu { + version, + manifest_dir: &self.path, + reason, + } + .build() + }; + let existing = self + .object_store + .read(path) + .await + .map_err(|e| conflict(format!("cannot read the existing delta: {e}")))?; + let stored_size = existing.len(); + let existing = CompressionType::Gzip + .decode(existing) + .await + .map_err(|e| conflict(format!("cannot decode the existing delta: {e}")))?; + if existing != bytes { + return Err(conflict( + "the existing delta has different content".to_string(), + )); + } + Ok(stored_size) + } } #[cfg(test)] diff --git a/src/mito2/src/manifest/tests.rs b/src/mito2/src/manifest/tests.rs index 01ab6e16a0c..20b7945ef84 100644 --- a/src/mito2/src/manifest/tests.rs +++ b/src/mito2/src/manifest/tests.rs @@ -13,4 +13,5 @@ // limitations under the License. mod checkpoint; +mod conditional; pub mod utils; diff --git a/src/mito2/src/manifest/tests/checkpoint.rs b/src/mito2/src/manifest/tests/checkpoint.rs index 20a0b07708c..72f4377221f 100644 --- a/src/mito2/src/manifest/tests/checkpoint.rs +++ b/src/mito2/src/manifest/tests/checkpoint.rs @@ -21,8 +21,7 @@ use common_datasource::compression::CompressionType; use common_error::ext::{ErrorExt, RetryHint}; use common_error::status_code::StatusCode; use object_store::layers::mock::{ - BytesRange, Error as MockError, ErrorKind, MockLayer, MockLayerBuilder, OpDelete, - Result as MockResult, RpRead, oio, + Error as MockError, ErrorKind, MockLayerBuilder, OpDelete, Result as MockResult, oio, }; use store_api::storage::{FileId, RegionId}; use strum::IntoEnumIterator; @@ -37,10 +36,10 @@ use crate::manifest::storage::{ checkpoint_file, delta_file, file_version, gen_path, is_delta_file, }; use crate::manifest::tests::utils::{ - basic_region_metadata, manifest_dir_usage, manifest_recovery_usage, + basic_region_metadata, fail_manifest_reads_layer, manifest_dir_usage, manifest_recovery_usage, }; use crate::sst::file::FileMeta; -use crate::test_util::{CheckpointTaskBlocker, TestEnv}; +use crate::test_util::{CheckpointTaskBlocker, TestEnv, nop_action}; async fn build_manager( checkpoint_distance: u64, @@ -80,43 +79,6 @@ async fn reopen_manager( .unwrap() } -fn nop_action() -> RegionMetaActionList { - RegionMetaActionList::new(vec![RegionMetaAction::Edit(RegionEdit { - files_to_add: vec![], - files_to_remove: vec![], - timestamp_ms: None, - compaction_time_window: None, - flushed_entry_id: None, - flushed_sequence: None, - committed_sequence: None, - })]) -} - -struct NotFoundReader; - -impl oio::StreamRead for NotFoundReader { - async fn open(&self, _: BytesRange) -> MockResult<(RpRead, Box)> { - Err(MockError::new( - ErrorKind::NotFound, - "mock listed manifest delta not found", - )) - } -} - -fn fail_manifest_delta_reads_layer() -> MockLayer { - MockLayerBuilder::default() - .reader_factory(Arc::new(|path, _args, inner| { - let file_name = path.rsplit('/').next().unwrap_or(path); - if is_delta_file(file_name) { - Box::new(oio::StreamReader::new(NotFoundReader)) - } else { - inner - } - })) - .build() - .unwrap() -} - #[tokio::test] async fn manager_without_checkpoint() { let (_env, mut manager) = build_manager(0, CompressionType::Uncompressed).await; @@ -651,7 +613,7 @@ async fn manifest_size_ignores_leftover_files_after_failed_delete(#[case] fail_d async fn open_preserves_listed_delta_not_found_retry_hint() { let env = TestEnv::new() .await - .with_mock_layer(fail_manifest_delta_reads_layer()); + .with_mock_layer(fail_manifest_reads_layer(is_delta_file)); let metadata = Arc::new(basic_region_metadata()); let mut manager = env .create_manifest_manager(CompressionType::Uncompressed, 0, Some(metadata.clone())) @@ -682,7 +644,7 @@ async fn open_preserves_listed_delta_not_found_retry_hint() { async fn install_preserves_listed_delta_not_found_retry_hint() { let env = TestEnv::new() .await - .with_mock_layer(fail_manifest_delta_reads_layer()); + .with_mock_layer(fail_manifest_reads_layer(is_delta_file)); let metadata = Arc::new(basic_region_metadata()); let mut manager = env .create_manifest_manager(CompressionType::Uncompressed, 0, Some(metadata)) @@ -1000,6 +962,7 @@ async fn manifest_size_warm_cache_matches_recovery_bytes( checkpoint_distance: 3, remove_file_options: Default::default(), manifest_cache: Some(cache.clone()), + conditional_publication: false, }; let expected = manifest_recovery_usage(env.data_home().join("manifest")).await; // First open fills the cache; the second reuses the cached checkpoint and deltas. @@ -1101,6 +1064,7 @@ async fn manifest_cache_write_through_survives_reopen(#[case] compression: Compr checkpoint_distance: 3, remove_file_options: Default::default(), manifest_cache: Some(cache.clone()), + conditional_publication: false, }; let mut writer = RegionManifestManager::new( Arc::new(basic_region_metadata()), diff --git a/src/mito2/src/manifest/tests/conditional.rs b/src/mito2/src/manifest/tests/conditional.rs new file mode 100644 index 00000000000..e270d205475 --- /dev/null +++ b/src/mito2/src/manifest/tests/conditional.rs @@ -0,0 +1,372 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Tests for manifests published by conditional create. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use common_datasource::compression::CompressionType; +use common_error::ext::ErrorExt; +use common_error::status_code::StatusCode; +use common_test_util::temp_dir::{TempDir, create_temp_dir}; +use object_store::ObjectStore; +use object_store::layers::CapabilityOverrideLayer; +use object_store::layers::mock::{ + Error as MockError, ErrorKind, MockLayer, MockLayerBuilder, Result as MockResult, oio, +}; +use object_store::services::Fs; +use rstest::rstest; +use store_api::ManifestVersion; + +use crate::error::{Error, Result}; +use crate::manifest::manager::{RegionManifestManager, RegionManifestOptions}; +use crate::manifest::storage::{ + LAST_CHECKPOINT_FILE, ManifestObjectStore, delta_file, gen_path, is_checkpoint_file, + is_delta_file, +}; +use crate::manifest::tests::utils::{basic_region_metadata, fail_manifest_reads_layer}; +use crate::sst::FormatType; +use crate::test_util::nop_action; + +fn temp_store() -> (TempDir, ObjectStore) { + let dir = create_temp_dir(""); + let store = ObjectStore::new(Fs::default().root(dir.path().to_str().unwrap())).unwrap(); + (dir, store) +} + +fn options( + store: &ObjectStore, + compress_type: CompressionType, + checkpoint_distance: u64, + conditional_publication: bool, +) -> RegionManifestOptions { + RegionManifestOptions { + manifest_dir: "/".to_string(), + object_store: store.clone(), + compress_type, + checkpoint_distance, + remove_file_options: Default::default(), + manifest_cache: None, + conditional_publication, + } +} + +async fn create_manager(options: RegionManifestOptions) -> RegionManifestManager { + RegionManifestManager::new( + Arc::new(basic_region_metadata()), + 0, + options, + FormatType::PrimaryKey, + false, + &Default::default(), + ) + .await + .unwrap() +} + +async fn open_manager(options: RegionManifestOptions) -> RegionManifestManager { + RegionManifestManager::open(options, &Default::default()) + .await + .unwrap() + .unwrap() +} + +async fn commit(manager: &mut RegionManifestManager) -> Result { + manager + .update_with_verification(nop_action(), false, true, || false) + .await +} + +fn delta_key(version: ManifestVersion) -> String { + gen_path("/", &delta_file(version), CompressionType::Gzip) +} + +fn conditional_store(store: &ObjectStore, compress_type: CompressionType) -> ManifestObjectStore { + ManifestObjectStore::new("/", store.clone(), compress_type, Default::default(), None) + .with_conditional_publication(true) + .unwrap() +} + +/// Returns a store that counts object writes and LIST requests. +fn counting_store() -> (TempDir, ObjectStore, Arc, Arc) { + let (dir, store) = temp_store(); + let writes = Arc::new(AtomicUsize::new(0)); + let lists = Arc::new(AtomicUsize::new(0)); + let (observed_writes, observed_lists) = (writes.clone(), lists.clone()); + let layer = MockLayerBuilder::default() + .writer_factory(Arc::new(move |_, _, inner| { + observed_writes.fetch_add(1, Ordering::Relaxed); + inner + })) + .lister_factory(Arc::new(move |_, _, inner| { + observed_lists.fetch_add(1, Ordering::Relaxed); + inner + })) + .build() + .unwrap(); + (dir, store.layer(layer), writes, lists) +} + +struct FailingLister(ErrorKind); + +impl oio::List for FailingLister { + async fn next(&mut self) -> MockResult> { + Err(MockError::new(self.0, "mock list failure")) + } +} + +fn failing_list_layer(kind: ErrorKind) -> MockLayer { + MockLayerBuilder::default() + .lister_factory(Arc::new(move |_, _, _| Box::new(FailingLister(kind)))) + .build() + .unwrap() +} + +fn assert_conflict(error: &Error) { + assert!( + matches!(error, Error::ManifestVersionConflict { .. }), + "{error:?}" + ); +} + +#[rstest] +#[case::conditional(true)] +#[case::legacy(false)] +#[tokio::test] +async fn test_ordinary_commit_writes_once_without_listing(#[case] conditional: bool) { + let (_dir, store, writes, lists) = counting_store(); + let mut manager = create_manager(options(&store, CompressionType::Gzip, 0, conditional)).await; + writes.store(0, Ordering::Relaxed); + + manager + .update_with_verification(nop_action(), false, true, || !conditional) + .await + .unwrap(); + + assert_eq!(1, writes.load(Ordering::Relaxed)); + assert_eq!(0, lists.load(Ordering::Relaxed)); +} + +#[tokio::test] +async fn test_commit_without_deadline_is_verified() { + let (_dir, store, _writes, lists) = counting_store(); + let mut manager = create_manager(options(&store, CompressionType::Gzip, 1, true)).await; + + manager.update(nop_action(), false).await.unwrap(); + manager.wait_for_pending_checkpoint().await; + + assert_eq!(1, lists.load(Ordering::Relaxed)); + assert_eq!(1, manager.last_version()); + assert_eq!(0, manager.checkpointer().last_checkpoint_version()); +} + +#[rstest] +#[tokio::test] +async fn test_list_error_during_verification_conflicts( + #[values(ErrorKind::Unexpected, ErrorKind::NotFound)] kind: ErrorKind, +) { + let (_dir, store) = temp_store(); + let store = store.layer(failing_list_layer(kind)); + let mut manager = create_manager(options(&store, CompressionType::Gzip, 1, true)).await; + + let error = manager + .update_with_verification(nop_action(), false, true, || true) + .await + .unwrap_err(); + + assert_conflict(&error); + assert!(error.to_string().contains("mock list failure"), "{error}"); + assert_eq!(0, manager.last_version()); + assert_eq!(0, manager.manifest().manifest_version); + assert!(store.exists(&delta_key(1)).await.unwrap()); +} + +#[tokio::test] +async fn test_recovery_treats_missing_directory_as_no_checkpoint() { + let (_dir, store) = temp_store(); + let store = store.layer(failing_list_layer(ErrorKind::NotFound)); + + let checkpoint = conditional_store(&store, CompressionType::Gzip) + .load_last_checkpoint() + .await; + + assert!(checkpoint.unwrap().is_none()); +} + +#[tokio::test] +async fn test_compression_settings_share_one_name() { + let (_dir, store) = temp_store(); + let mut stored = conditional_store(&store, CompressionType::Uncompressed); + let mut compressed = conditional_store(&store, CompressionType::Gzip); + let data = vec![b'a'; 1024]; + + stored.save(1, &data, false).await.unwrap(); + let stored_object = store.read(&delta_key(1)).await.unwrap().to_vec(); + assert!(stored_object.len() > data.len()); + + // A retry by a writer with the other setting succeeds and leaves the object alone. + compressed.save(1, &data, false).await.unwrap(); + let retried_object = store.read(&delta_key(1)).await.unwrap().to_vec(); + assert_eq!(stored_object, retried_object); + assert_eq!(stored_object.len() as u64, compressed.total_manifest_size()); + + compressed.save(2, &data, false).await.unwrap(); + assert!(store.read(&delta_key(2)).await.unwrap().len() < data.len()); + assert_eq!( + vec![(1, data.clone()), (2, data)], + stored.fetch_manifests(1, 3).await.unwrap() + ); +} + +#[rstest] +#[case::different_content(false, false)] +#[case::undecodable(true, false)] +#[case::unreadable(false, true)] +#[tokio::test] +async fn test_existing_delta_conflicts(#[case] undecodable: bool, #[case] unreadable: bool) { + let (_dir, store) = temp_store(); + let existing = if undecodable { + b"not gzip".to_vec() + } else { + let other = b"other".as_slice(); + CompressionType::Gzip.encode(other).await.unwrap() + }; + store.write(&delta_key(1), existing.clone()).await.unwrap(); + let writer_store = if unreadable { + store + .clone() + .layer(fail_manifest_reads_layer(is_delta_file)) + } else { + store.clone() + }; + + let error = conditional_store(&writer_store, CompressionType::Gzip) + .save(1, b"mine", false) + .await + .unwrap_err(); + + assert_conflict(&error); + assert!(error.may_have_persisted_manifest_update()); + assert_eq!(existing, store.read(&delta_key(1)).await.unwrap().to_vec()); +} + +#[tokio::test] +async fn test_failed_write_does_not_skip_version() { + let (_dir, store) = temp_store(); + let mut manager = create_manager(options(&store, CompressionType::Gzip, 0, true)).await; + let key = delta_key(1); + store.write(&key, b"not gzip".to_vec()).await.unwrap(); + + assert_conflict(&commit(&mut manager).await.unwrap_err()); + assert_eq!(0, manager.last_version()); + + store.delete(&key).await.unwrap(); + assert_eq!(1, commit(&mut manager).await.unwrap()); + assert_eq!(1, manager.manifest().manifest_version); +} + +#[tokio::test] +async fn test_stale_checkpoint_pointer_does_not_hide_newer_edits() { + let (_dir, store) = temp_store(); + let pointer = format!("/{LAST_CHECKPOINT_FILE}"); + let region_options = options(&store, CompressionType::Uncompressed, 1, true); + let mut manager = create_manager(region_options.clone()).await; + commit(&mut manager).await.unwrap(); + manager.wait_for_pending_checkpoint().await; + let stale_pointer = store.read(&pointer).await.unwrap().to_vec(); + for _ in 0..2 { + commit(&mut manager).await.unwrap(); + manager.wait_for_pending_checkpoint().await; + } + manager + .update_with_verification(nop_action(), false, false, || false) + .await + .unwrap(); + store.write(&pointer, stale_pointer).await.unwrap(); + + let reopened = open_manager(region_options).await; + + assert_eq!(4, reopened.last_version()); + assert_eq!(3, reopened.checkpointer().last_checkpoint_version()); +} + +#[tokio::test] +async fn test_recovery_fails_when_listed_checkpoint_is_gone() { + let (_dir, store) = temp_store(); + let compress_type = CompressionType::Gzip; + let mut manager = create_manager(options(&store, compress_type, 1, true)).await; + commit(&mut manager).await.unwrap(); + manager.wait_for_pending_checkpoint().await; + let unreadable = store.layer(fail_manifest_reads_layer(is_checkpoint_file)); + + let conditional = options(&unreadable, compress_type, 1, true); + let error = RegionManifestManager::open(conditional, &Default::default()) + .await + .unwrap_err(); + + assert!( + matches!(error, Error::ManifestCheckpointVanished { .. }), + "{error:?}" + ); + assert_eq!(StatusCode::StorageUnavailable, error.status_code()); + assert!(error.retry_hint().is_retryable()); +} + +#[tokio::test] +async fn test_backend_without_conditional_create_is_refused_at_open() { + let (_dir, store) = temp_store(); + let compress_type = CompressionType::Gzip; + create_manager(options(&store, compress_type, 0, false)).await; + let unsupported = store.layer(CapabilityOverrideLayer::new(|mut capability| { + capability.write_with_if_not_exists = false; + capability + })); + + let conditional = options(&unsupported, compress_type, 0, true); + let error = RegionManifestManager::open(conditional, &Default::default()) + .await + .unwrap_err(); + + assert_eq!(StatusCode::Unsupported, error.status_code()); + open_manager(options(&unsupported, compress_type, 0, false)).await; +} + +#[tokio::test] +async fn test_open_writes_nothing() { + let (_dir, store, writes, _lists) = counting_store(); + let region_options = options(&store, CompressionType::Gzip, 0, true); + create_manager(region_options.clone()).await; + writes.store(0, Ordering::Relaxed); + + open_manager(region_options).await; + + assert_eq!(0, writes.load(Ordering::Relaxed)); +} + +#[rstest] +#[tokio::test] +async fn test_deltas_are_readable_across_modes( + #[values(false, true)] written_conditionally: bool, + #[values(CompressionType::Uncompressed, CompressionType::Gzip)] compress_type: CompressionType, +) { + let (_dir, store) = temp_store(); + let mut writer = create_manager(options(&store, compress_type, 0, written_conditionally)).await; + commit(&mut writer).await.unwrap(); + + let mut reader = open_manager(options(&store, compress_type, 0, !written_conditionally)).await; + + assert_eq!(1, reader.last_version()); + assert_eq!(2, commit(&mut reader).await.unwrap()); +} diff --git a/src/mito2/src/manifest/tests/utils.rs b/src/mito2/src/manifest/tests/utils.rs index 743c088e0d4..8629d6b413b 100644 --- a/src/mito2/src/manifest/tests/utils.rs +++ b/src/mito2/src/manifest/tests/utils.rs @@ -13,10 +13,15 @@ // limitations under the License. use std::path::Path; +use std::sync::Arc; use api::v1::SemanticType; use datatypes::prelude::ConcreteDataType; use datatypes::schema::ColumnSchema; +use object_store::layers::mock::{ + BytesRange, Error as MockError, ErrorKind, MockLayer, MockLayerBuilder, Result as MockResult, + RpRead, oio, +}; use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataBuilder}; use store_api::storage::RegionId; @@ -51,6 +56,22 @@ pub fn basic_region_metadata() -> RegionMetadata { builder.build().unwrap() } +/// Returns a layer that fails reads of the files `is_target` selects with `NotFound` and leaves +/// other reads untouched. +pub(crate) fn fail_manifest_reads_layer(is_target: fn(&str) -> bool) -> MockLayer { + MockLayerBuilder::default() + .reader_factory(Arc::new(move |path, _args, inner| { + let file_name = path.rsplit('/').next().unwrap_or(path); + if is_target(file_name) { + Box::new(oio::StreamReader::new(NotFoundReader)) + } else { + inner + } + })) + .build() + .unwrap() +} + /// Measures stored delta and checkpoint bytes independently of the size tracker. pub async fn manifest_dir_usage(path: impl AsRef) -> u64 { let mut entries = tokio::fs::read_dir(path).await.unwrap(); @@ -103,3 +124,14 @@ pub async fn manifest_recovery_usage(path: impl AsRef) -> u64 { } total } + +struct NotFoundReader; + +impl oio::StreamRead for NotFoundReader { + async fn open(&self, _: BytesRange) -> MockResult<(RpRead, Box)> { + Err(MockError::new( + ErrorKind::NotFound, + "mock listed manifest file not found", + )) + } +} diff --git a/src/mito2/src/region.rs b/src/mito2/src/region.rs index ed24421d023..003a2240108 100644 --- a/src/mito2/src/region.rs +++ b/src/mito2/src/region.rs @@ -60,7 +60,7 @@ use crate::error::{ FlushableRegionStateSnafu, InvalidPartitionExprSnafu, RegionNotFoundSnafu, RegionStateSnafu, RegionTruncatedSnafu, Result, UnexpectedSnafu, UpdateManifestSnafu, }; -use crate::lease::Deadline; +use crate::lease::{Deadline, NodeLease}; use crate::manifest::action::{ RegionChange, RegionManifest, RegionMetaAction, RegionMetaActionList, }; @@ -1264,6 +1264,8 @@ pub(crate) struct ManifestContext { /// long the Leader role stays valid. The engine extends it from those leases, and writes check /// it. pub(crate) leader_deadline: Deadline, + /// The clock of [`Self::leader_deadline`]. + node_lease: Arc, /// Partition info of the region in staging mode. /// /// During the staging mode, the region metadata in [`VersionControlRef`] is not updated, @@ -1278,12 +1280,14 @@ impl ManifestContext { manager: RegionManifestManager, state: RegionRoleState, hook: Option, + node_lease: Arc, ) -> Self { ManifestContext { manifest_manager: tokio::sync::RwLock::new(manager), state: AtomicCell::new(state), downgrade_started_at: Mutex::new(Instant::now()), leader_deadline: Deadline::default(), + node_lease, staging_partition_info: Mutex::new(None), hook, } @@ -1507,9 +1511,9 @@ impl ManifestContext { /// Performs a manifest write under a caller-held write lock and returns a /// [`PendingManifestHook`] to [`fire`](PendingManifestHook::fire) after - /// dropping the lock. This is the sole caller of - /// [`RegionManifestManager::update`], so it is the funnel through which all - /// logical manifest writes notify the hook. + /// dropping the lock. It calls + /// [`RegionManifestManager::update_with_verification`] and is the funnel + /// through which all logical manifest writes notify the hook. /// /// Does not validate state or edit applicability — the caller must do so /// while still holding the lock (see `update_manifest_with_state_check` @@ -1521,17 +1525,19 @@ impl ManifestContext { is_staging: bool, ) -> Result { let region_id = manager.manifest().metadata.region_id; - // Clone before `action_list` is moved into `update` so the hook still - // sees what was written. + // Clone before `action_list` is moved into `update_with_verification` + // so the hook still sees what was written. let action_list_for_hook = self.hook.as_ref().map(|_| action_list.clone()); - let version = if !is_staging - && self.state.load() == RegionRoleState::Leader(RegionLeaderState::Downgrading) - { - manager.update_normal_without_checkpoint(action_list).await - } else { - manager.update(action_list, is_staging).await - } - .inspect_err(|e| error!(e; "Failed to update manifest, region_id: {}", region_id))?; + // The final flush of a downgrading leader must not start cleanup after the downgrade + // checkpoint barrier. + let allow_checkpoint = !is_staging + && self.state.load() != RegionRoleState::Leader(RegionLeaderState::Downgrading); + let version = manager + .update_with_verification(action_list, is_staging, allow_checkpoint, || { + self.is_leader_lease_expired() + }) + .await + .inspect_err(|e| error!(e; "Failed to update manifest, region_id: {}", region_id))?; Ok(PendingManifestHook::new( region_id, @@ -1542,6 +1548,11 @@ impl ManifestContext { )) } + /// Returns whether the region's Leader deadline has passed. No deadline counts as passed. + fn is_leader_lease_expired(&self) -> bool { + self.node_lease.now() >= self.leader_deadline.get() + } + /// Updates the manifest using a caller-held write lock, releases the lock, /// and then fires the manifest hook. /// @@ -2342,6 +2353,7 @@ mod tests { checkpoint_distance: 10, remove_file_options: Default::default(), manifest_cache: None, + conditional_publication: false, }, FormatType::PrimaryKey, false, @@ -2354,6 +2366,7 @@ mod tests { manager, RegionRoleState::Leader(RegionLeaderState::Writable), None, + Default::default(), )); MitoRegion { @@ -2897,6 +2910,7 @@ mod tests { checkpoint_distance: 10, remove_file_options: Default::default(), manifest_cache: None, + conditional_publication: false, }, FormatType::PrimaryKey, false, @@ -2908,6 +2922,7 @@ mod tests { manager, RegionRoleState::Leader(RegionLeaderState::Staging), None, + Default::default(), )) }; @@ -2966,6 +2981,7 @@ mod tests { checkpoint_distance: 10, remove_file_options: Default::default(), manifest_cache: None, + conditional_publication: false, }, FormatType::PrimaryKey, false, @@ -2978,6 +2994,7 @@ mod tests { manager, RegionRoleState::Leader(RegionLeaderState::Writable), None, + Default::default(), )); let region = MitoRegion { diff --git a/src/mito2/src/region/opener.rs b/src/mito2/src/region/opener.rs index 17a32f1a43a..1bcf61b2ce5 100644 --- a/src/mito2/src/region/opener.rs +++ b/src/mito2/src/region/opener.rs @@ -55,6 +55,7 @@ use crate::error::{ DataTypeMismatchSnafu, EmptyRegionDirSnafu, InvalidMetadataSnafu, InvalidRegionOptionsSnafu, ObjectStoreNotFoundSnafu, RegionCorruptedSnafu, Result, StaleLogEntrySnafu, }; +use crate::lease::NodeLease; use crate::manifest::action::RegionManifest; use crate::manifest::manager::{RegionManifestManager, RegionManifestOptions}; use crate::memtable::bulk::part::BulkPart; @@ -157,6 +158,7 @@ pub(crate) struct RegionOpener { options: Option, cache_manager: Option, skip_wal_replay: bool, + node_lease: Arc, puffin_manager_factory: PuffinManagerFactory, intermediate_manager: IntermediateManager, time_provider: TimeProviderRef, @@ -178,6 +180,7 @@ impl RegionOpener { region_id: RegionId, table_dir: &str, path_type: PathType, + node_lease: Arc, memtable_builder_provider: MemtableBuilderProvider, object_store_manager: ObjectStoreManagerRef, purge_scheduler: SchedulerRef, @@ -198,6 +201,7 @@ impl RegionOpener { options: None, cache_manager: None, skip_wal_replay: false, + node_lease, puffin_manager_factory, intermediate_manager, time_provider, @@ -386,7 +390,10 @@ impl RegionOpener { wal.start_region_history(region_id, &provider)?; // Create a manifest manager for this region and writes regions to the manifest file. let mut region_manifest_options = - RegionManifestOptions::new(config, ®ion_dir, &object_store); + RegionManifestOptions::new(config, ®ion_dir, &object_store) + .with_conditional_publication( + options.requires_conditional_manifest_publication(self.node_lease.is_enabled()), + ); // Set manifest cache if available region_manifest_options.manifest_cache = self .cache_manager @@ -445,6 +452,7 @@ impl RegionOpener { manifest_manager, RegionRoleState::Leader(RegionLeaderState::Writable), self.hook.clone(), + self.node_lease.clone(), )), file_purger: create_file_purger( config.gc.enable, @@ -514,7 +522,11 @@ impl RegionOpener { let mut region_options = self.options.as_ref().unwrap().clone(); let object_storage = get_object_store(®ion_options.storage, &self.object_store_manager)?; let mut region_manifest_options = - RegionManifestOptions::new(config, &self.region_dir(), &object_storage); + RegionManifestOptions::new(config, &self.region_dir(), &object_storage) + .with_conditional_publication( + region_options + .requires_conditional_manifest_publication(self.node_lease.is_enabled()), + ); // Set manifest cache if available region_manifest_options.manifest_cache = self .cache_manager @@ -693,6 +705,7 @@ impl RegionOpener { manifest_manager, RegionRoleState::Follower, self.hook.clone(), + self.node_lease.clone(), )), file_purger, provider: provider.clone(), diff --git a/src/mito2/src/region/options.rs b/src/mito2/src/region/options.rs index 85cb79928cd..ca696137f40 100644 --- a/src/mito2/src/region/options.rs +++ b/src/mito2/src/region/options.rs @@ -265,6 +265,12 @@ impl RegionOptions { pub fn primary_key_encoding(&self) -> PrimaryKeyEncoding { self.primary_key_encoding.unwrap_or_default() } + + /// Returns whether the manifest of this region is published conditionally, which it is when + /// the region's writes require a lease and its WAL is the object store WAL. + pub(crate) fn requires_conditional_manifest_publication(&self, lease_enabled: bool) -> bool { + lease_enabled && matches!(self.wal_options, WalOptions::ObjectStore(_)) + } } impl RegionOptions { @@ -696,7 +702,7 @@ fn validate_enum_options( mod tests { use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; - use common_wal::options::KafkaWalOptions; + use common_wal::options::{KafkaWalOptions, ObjectStoreWalOptions}; use store_api::mito_engine_options::{ EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, SKIP_WAL_KEY, WRITE_BUFFER_SIZE_KEY, }; @@ -717,6 +723,21 @@ mod tests { assert_eq!(RegionOptions::default(), options); } + #[test] + fn test_conditional_manifest_requires_lease_and_object_store_wal() { + let options = |wal_options| RegionOptions { + wal_options, + ..Default::default() + }; + let object_store_wal = options(WalOptions::ObjectStore(ObjectStoreWalOptions::new( + "wal".to_string(), + ))); + + assert!(object_store_wal.requires_conditional_manifest_publication(true)); + assert!(!object_store_wal.requires_conditional_manifest_publication(false)); + assert!(!options(WalOptions::RaftEngine).requires_conditional_manifest_publication(true)); + } + #[test] fn test_float_field_encoding_defaults_and_parses() { let options = RegionOptions::try_from_options(RegionId::new(0, 0), &make_map(&[])).unwrap(); diff --git a/src/mito2/src/region/utils.rs b/src/mito2/src/region/utils.rs index 7dfaad72aa7..496668cfce0 100644 --- a/src/mito2/src/region/utils.rs +++ b/src/mito2/src/region/utils.rs @@ -39,14 +39,20 @@ use crate::sst::location; pub struct RegionMetadataLoader { config: Arc, object_store_manager: ObjectStoreManagerRef, + lease_enabled: bool, } impl RegionMetadataLoader { /// Creates a new `RegionMetadataLoader`. - pub fn new(config: Arc, object_store_manager: ObjectStoreManagerRef) -> Self { + pub fn new( + config: Arc, + object_store_manager: ObjectStoreManagerRef, + lease_enabled: bool, + ) -> Self { Self { config, object_store_manager, + lease_enabled, } } @@ -56,9 +62,7 @@ impl RegionMetadataLoader { region_dir: &str, region_options: &RegionOptions, ) -> Result> { - let manifest = self - .load_manifest(region_dir, ®ion_options.storage) - .await?; + let manifest = self.load_manifest(region_dir, region_options).await?; Ok(manifest.map(|m| m.metadata.clone())) } @@ -66,11 +70,14 @@ impl RegionMetadataLoader { pub async fn load_manifest( &self, region_dir: &str, - storage: &Option, + region_options: &RegionOptions, ) -> Result>> { - let object_store = get_object_store(storage, &self.object_store_manager)?; + let object_store = get_object_store(®ion_options.storage, &self.object_store_manager)?; let region_manifest_options = - RegionManifestOptions::new(&self.config, region_dir, &object_store); + RegionManifestOptions::new(&self.config, region_dir, &object_store) + .with_conditional_publication( + region_options.requires_conditional_manifest_publication(self.lease_enabled), + ); let Some(manifest_manager) = RegionManifestManager::open(region_manifest_options, &Default::default()).await? else { diff --git a/src/mito2/src/test_util.rs b/src/mito2/src/test_util.rs index f85274b6bfc..29718163528 100644 --- a/src/mito2/src/test_util.rs +++ b/src/mito2/src/test_util.rs @@ -63,6 +63,7 @@ use rskafka::client::partition::{Compression, UnknownTopicHandling}; use rskafka::client::{Client, ClientBuilder}; use rskafka::record::Record; use rstest_reuse::template; +use store_api::ManifestVersion; use store_api::logstore::LogStore; use store_api::metadata::{ColumnMetadata, RegionMetadataRef}; use store_api::region_engine::{RegionEngine, RegionRole}; @@ -79,8 +80,11 @@ use crate::engine::listener::EventListenerRef; use crate::engine::{MITO_ENGINE_NAME, MitoEngine}; use crate::error::Result; use crate::flush::{WriteBufferManager, WriteBufferManagerRef}; +use crate::manifest::action::{RegionEdit, RegionMetaAction, RegionMetaActionList}; use crate::manifest::manager::{RegionManifestManager, RegionManifestOptions}; -use crate::manifest::storage::{is_checkpoint_file, is_delta_file}; +use crate::manifest::storage::{ + LAST_CHECKPOINT_FILE, delta_file, is_checkpoint_file, is_delta_file, +}; use crate::read::{Batch, BatchBuilder}; use crate::region::opener::{PartitionExprFetcher, PartitionExprFetcherRef}; use crate::sst::FormatType; @@ -91,7 +95,20 @@ use crate::sst::index::puffin_manager::PuffinManagerFactory; use crate::time_provider::{StdTimeProvider, TimeProviderRef}; use crate::worker::WorkerGroup; -/// Controls a mock object-store layer that blocks a checkpoint task once. +/// Returns a manifest action list with one empty edit. +pub(crate) fn nop_action() -> RegionMetaActionList { + RegionMetaActionList::new(vec![RegionMetaAction::Edit(RegionEdit { + files_to_add: vec![], + files_to_remove: vec![], + timestamp_ms: None, + compaction_time_window: None, + flushed_entry_id: None, + flushed_sequence: None, + committed_sequence: None, + })]) +} + +/// Controls a mock object-store layer that blocks one manifest write or checkpoint cleanup. #[derive(Clone)] pub(crate) struct CheckpointTaskBlocker { entered: Arc, @@ -123,23 +140,12 @@ impl CheckpointTaskBlocker { /// Blocks publication of `_last_checkpoint` at writer close. pub(crate) fn block_last_checkpoint_write() -> (Self, MockLayer) { - let blocker = Self { - entered: Arc::new(Notify::new()), - release: Arc::new(Notify::new()), - armed: Arc::new(AtomicBool::new(true)), - }; - let factory_blocker = blocker.clone(); - let layer = MockLayerBuilder::default() - .writer_factory(Arc::new(move |path, _args, inner| { - Box::new(BlockingCheckpointWriter { - path: path.to_string(), - inner, - blocker: factory_blocker.clone(), - }) - })) - .build() - .unwrap(); - (blocker, layer) + Self::block_write(LAST_CHECKPOINT_FILE.to_string(), true) + } + + /// Blocks the first write of the delta of `version`, before any of its bytes reach storage. + pub(crate) fn block_delta_write(version: ManifestVersion) -> (Self, MockLayer) { + Self::block_write(delta_file(version), false) } pub(crate) async fn wait_until_blocked(&self) { @@ -154,6 +160,30 @@ impl CheckpointTaskBlocker { self.armed.store(true, Ordering::Release); } + fn block_write(file_name: String, at_close: bool) -> (Self, MockLayer) { + let blocker = Self { + entered: Arc::new(Notify::new()), + release: Arc::new(Notify::new()), + armed: Arc::new(AtomicBool::new(true)), + }; + let factory_blocker = blocker.clone(); + let layer = MockLayerBuilder::default() + .writer_factory(Arc::new(move |path, _args, inner| { + if path.contains(&file_name) { + Box::new(BlockingManifestWriter { + inner, + blocker: factory_blocker.clone(), + at_close, + }) + } else { + inner + } + })) + .build() + .unwrap(); + (blocker, layer) + } + async fn block_once(&self) { if self.armed.swap(false, Ordering::AcqRel) { self.entered.notify_one(); @@ -195,19 +225,22 @@ fn is_manifest_checkpoint_file(path: &str) -> bool { is_delta_file(file_name) || is_checkpoint_file(file_name) } -struct BlockingCheckpointWriter { - path: String, +struct BlockingManifestWriter { inner: Writer, blocker: CheckpointTaskBlocker, + at_close: bool, } -impl object_store::layers::mock::Write for BlockingCheckpointWriter { +impl object_store::layers::mock::Write for BlockingManifestWriter { async fn write(&mut self, bs: Buffer) -> MockResult<()> { + if !self.at_close { + self.blocker.block_once().await; + } self.inner.write(bs).await } async fn close(&mut self) -> MockResult { - if self.path.ends_with("_last_checkpoint") { + if self.at_close { self.blocker.block_once().await; } self.inner.close().await @@ -839,6 +872,7 @@ impl TestEnv { checkpoint_distance, remove_file_options: Default::default(), manifest_cache: None, + conditional_publication: false, }; if let Some(metadata) = initial_metadata { diff --git a/src/mito2/src/test_util/scheduler_util.rs b/src/mito2/src/test_util/scheduler_util.rs index 8a109d0d3e2..3249104978a 100644 --- a/src/mito2/src/test_util/scheduler_util.rs +++ b/src/mito2/src/test_util/scheduler_util.rs @@ -155,6 +155,7 @@ impl SchedulerEnv { checkpoint_distance: 10, remove_file_options: Default::default(), manifest_cache: None, + conditional_publication: false, }, FormatType::PrimaryKey, false, @@ -164,6 +165,7 @@ impl SchedulerEnv { .unwrap(), RegionRoleState::Leader(RegionLeaderState::Writable), None, + Default::default(), )) } diff --git a/src/mito2/src/worker/handle_catchup.rs b/src/mito2/src/worker/handle_catchup.rs index 0ca51b98a11..26229b66e93 100644 --- a/src/mito2/src/worker/handle_catchup.rs +++ b/src/mito2/src/worker/handle_catchup.rs @@ -135,6 +135,7 @@ impl RegionWorkerLoop { region_id, region.table_dir(), region.access_layer.path_type(), + self.node_lease.clone(), self.memtable_builder_provider.clone(), self.object_store_manager.clone(), self.purge_scheduler.clone(), diff --git a/src/mito2/src/worker/handle_copy_region.rs b/src/mito2/src/worker/handle_copy_region.rs index 710bf694f13..34ef8404ca6 100644 --- a/src/mito2/src/worker/handle_copy_region.rs +++ b/src/mito2/src/worker/handle_copy_region.rs @@ -64,8 +64,11 @@ impl RegionWorkerLoop { return; } - let region_metadata_loader = - RegionMetadataLoader::new(self.config.clone(), self.object_store_manager.clone()); + let region_metadata_loader = RegionMetadataLoader::new( + self.config.clone(), + self.object_store_manager.clone(), + self.node_lease.is_enabled(), + ); let worker_sender = self.sender.clone(); common_runtime::spawn_global(async move { @@ -154,7 +157,7 @@ impl RegionWorkerLoop { "Loading source region manifest from region dir: {region_dir}, target region: {target_region_id}" ); let source_region_manifest = region_metadata_loader - .load_manifest(®ion_dir, ®ion.version().options.storage) + .load_manifest(®ion_dir, ®ion.version().options) .await? .context(MissingManifestSnafu { region_id: source_region_id, diff --git a/src/mito2/src/worker/handle_create.rs b/src/mito2/src/worker/handle_create.rs index d2eab4a3f59..5c7da5779f3 100644 --- a/src/mito2/src/worker/handle_create.rs +++ b/src/mito2/src/worker/handle_create.rs @@ -61,6 +61,7 @@ impl RegionWorkerLoop { region_id, &request.table_dir, request.path_type, + self.node_lease.clone(), self.memtable_builder_provider.clone(), self.object_store_manager.clone(), self.purge_scheduler.clone(), diff --git a/src/mito2/src/worker/handle_open.rs b/src/mito2/src/worker/handle_open.rs index b734d7e9b86..28a612fad8d 100644 --- a/src/mito2/src/worker/handle_open.rs +++ b/src/mito2/src/worker/handle_open.rs @@ -185,6 +185,7 @@ impl RegionWorkerLoop { region_id, &request.table_dir, request.path_type, + self.node_lease.clone(), self.memtable_builder_provider.clone(), self.object_store_manager.clone(), self.purge_scheduler.clone(), diff --git a/src/mito2/src/worker/handle_remap.rs b/src/mito2/src/worker/handle_remap.rs index 639b9eef3e9..24480f0f490 100644 --- a/src/mito2/src/worker/handle_remap.rs +++ b/src/mito2/src/worker/handle_remap.rs @@ -58,8 +58,11 @@ impl RegionWorkerLoop { return; } - let region_metadata_loader = - RegionMetadataLoader::new(self.config.clone(), self.object_store_manager.clone()); + let region_metadata_loader = RegionMetadataLoader::new( + self.config.clone(), + self.object_store_manager.clone(), + self.node_lease.is_enabled(), + ); common_runtime::spawn_global(async move { let result = Self::fetch_and_remap_manifests( region, @@ -92,11 +95,11 @@ impl RegionWorkerLoop { let now = Instant::now(); for input_region in &input_regions { let region_dir = region_dir_from_table_dir(table_dir, *input_region, path_type); - let storage = region_options.storage.clone(); + let region_options = region_options.clone(); let moved_region_metadata_loader = region_metadata_loader.clone(); tasks.push(async move { moved_region_metadata_loader - .load_manifest(®ion_dir, &storage) + .load_manifest(®ion_dir, ®ion_options) .await }); }