fix(mito2): cancel compaction merges at batch boundaries (#10411)

* fix(mito2): cancel compaction merges at batch boundaries

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix(mito2): preserve upload errors and expose merge cancellation

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix(mito2): normalize encoder cancellation at SST writer boundary

Signed-off-by: WenyXu <wenymedia@gmail.com>

* style(mito2): use imports for SST write types

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix(mito2): share cooperative upload cancellation with tokens

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix(mito2): classify nested upload cancellation errors

Signed-off-by: WenyXu <wenymedia@gmail.com>

---------

Signed-off-by: WenyXu <wenymedia@gmail.com>
This commit is contained in:
Weny Xu
2026-11-16 08:32:11 +00:00
committed by GitHub
parent 21ada9de09
commit 89e14ab936
20 changed files with 1676 additions and 265 deletions
+2
View File
@@ -104,6 +104,8 @@ where
#[derive(Copy, Clone, Debug)]
pub struct Cancelled;
impl std::error::Error for Cancelled {}
impl Display for Cancelled {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "Future has been cancelled")
+114 -16
View File
@@ -51,7 +51,7 @@ use crate::sst::index::registry::IndexRegistry;
use crate::sst::index::{IndexerBuilderImpl, build_index_from_parquet_bytes};
use crate::sst::location::{self, region_dir_from_table_dir};
use crate::sst::parquet::reader::ParquetReaderBuilder;
use crate::sst::parquet::writer::ParquetWriter;
use crate::sst::parquet::writer::{ParquetWriter, SstWriteContext};
use crate::sst::parquet::{SstInfo, WriteOptions};
use crate::sst::{DEFAULT_WRITE_CONCURRENCY, FormatType};
@@ -376,18 +376,13 @@ impl AccessLayer {
.await
}
/// Writes an SST whose output is registered with `context.guard`.
///
/// The write cache is bypassed, upload concurrency is fixed to one, and
/// the guard runs before either the SST or its index becomes visible.
/// `context.limits` bounds the encoder and footer workspace.
#[cfg(feature = "enterprise")]
pub(crate) async fn write_sst_with_guard(
/// Writes SSTs with explicit ownership and cooperative cancellation.
pub(crate) async fn write_sst_with_context(
&self,
request: SstWriteRequest,
write_opts: &WriteOptions,
metrics: &mut Metrics,
context: crate::sst::parquet::writer::SstWriteContext,
context: SstWriteContext,
) -> Result<SstInfoArray> {
self.write_sst_inner(request, write_opts, metrics, Some(context))
.await
@@ -398,10 +393,12 @@ impl AccessLayer {
request: SstWriteRequest,
write_opts: &WriteOptions,
metrics: &mut Metrics,
context: Option<crate::sst::parquet::writer::SstWriteContext>,
context: Option<SstWriteContext>,
) -> Result<SstInfoArray> {
let limits = context.as_ref().map(|context| context.limits);
let guard = context.map(|context| context.guard);
let context = context.unwrap_or_default();
let limits = context.limits;
let guard = context.guard.clone();
let cancellation = context.cancellation.clone();
let op_type = request.op_type;
let region_id = request.metadata.region_id;
let region_metadata = request.metadata.clone();
@@ -413,11 +410,11 @@ impl AccessLayer {
};
let sst_info = if let Some(write_cache) = cache_manager.write_cache()
&& guard.is_none()
&& limits.is_none()
{
// Write to the write cache.
write_cache
.write_and_upload_sst(
.write_and_upload_sst_with_context(
request,
SstUploadRequest {
dest_path_provider: RegionFilePathFactory::new(
@@ -428,10 +425,12 @@ impl AccessLayer {
},
write_opts,
metrics,
context,
)
.await?
} else {
// Write cache is disabled, or bypassed by an SST write guard.
// The write cache does not enforce SstWriteLimits, so bounded writes
// bypass it.
let store = self.object_store.clone();
let path_provider = RegionFilePathFactory::new(self.table_dir.clone(), self.path_type);
let puffin_factory = if limits.is_some() {
@@ -469,12 +468,13 @@ impl AccessLayer {
)
.await
.with_file_cleaner(cleaner)
.with_write_concurrency(if guard.is_some() {
.with_write_concurrency(if limits.is_some() {
1
} else {
crate::sst::DEFAULT_WRITE_CONCURRENCY
})
.with_write_guard(guard)
.with_cancellation(cancellation)
.with_memory_limits(limits);
match request.sst_write_format {
FormatType::PrimaryKey => {
@@ -1104,6 +1104,104 @@ mod tests {
assert_eq!(1, warm.mem_cache_hit);
}
#[rstest::rstest]
#[tokio::test]
async fn test_cancelled_write_cleans_rolled_ssts(
#[values(false, true)] enable_write_cache: bool,
#[values(FormatType::Flat, FormatType::PrimaryKey)] format: FormatType,
) {
use crate::read::FlatSource;
use crate::sst::file::UncommittedSsts;
use crate::test_util::sst_util::{new_record_batch_by_range, sst_region_metadata};
use tokio_util::sync::CancellationToken;
let mut env = TestEnv::new().await;
let store = env.init_object_store_manager();
let local_dir = create_temp_dir("cancelled-sst-cache");
let local_store = new_fs_store(local_dir.path().to_str().unwrap());
let write_cache = if enable_write_cache {
Some(
env.create_write_cache(local_store.clone(), ReadableSize::mb(10))
.await,
)
} else {
None
};
let cache = Arc::new(CacheManager::builder().write_cache(write_cache).build());
let layer = Arc::new(AccessLayer::new(
"cancelled",
PathType::Bare,
store.clone(),
env.get_puffin_manager(),
env.get_intermediate_manager(),
));
let metadata = Arc::new(sst_region_metadata());
let uncommitted =
UncommittedSsts::new(metadata.region_id, layer.clone(), Some(cache.clone()));
let cancellation = CancellationToken::new();
let signal = cancellation.clone();
let tracked = uncommitted.clone();
let schema = new_record_batch_by_range(&["a", "b"], 0, 8).schema();
let source = FlatSource::new_stream(
schema,
Box::pin(async_stream::try_stream! {
for i in 0..64 {
if tracked.num_tracked_files() >= 3 {
signal.cancel();
}
let tag = format!("{i:04}");
yield new_record_batch_by_range(&[&tag, "b"], 0, 4096);
}
}),
);
let result = layer
.write_sst_with_context(
SstWriteRequest {
index_build_inputs: None,
op_type: OperationType::Compact,
metadata,
source,
cache_manager: cache,
max_sequence: None,
sst_write_format: format,
preserve_row_sequence: false,
index_options: Default::default(),
index_config: Default::default(),
inverted_index_config: Default::default(),
fulltext_index_config: Default::default(),
bloom_filter_index_config: Default::default(),
},
&WriteOptions {
max_file_size: Some(1),
row_group_size: 8,
..Default::default()
},
&mut Metrics::new(WriteType::Compaction),
SstWriteContext {
guard: Some(Arc::new(uncommitted.clone())),
cancellation: Some(cancellation),
..Default::default()
},
)
.await;
assert!(matches!(
result,
Err(crate::error::Error::SstWriteCancelled { .. })
));
assert!(
uncommitted.num_tracked_files() >= 2,
"must cover files finalized before cancellation"
);
uncommitted.cleanup_for_test().await.unwrap();
for object_store in [&store, &local_store] {
let entries = object_store.list_with("/").recursive(true).await.unwrap();
assert!(
entries.iter().all(|entry| entry.metadata().is_dir()),
"uncommitted files remain: {entries:?}"
);
}
}
#[tokio::test]
async fn test_new_fs_cache_store() {
let root = common_test_util::temp_dir::create_temp_dir("fs-cache-store");
+344 -27
View File
@@ -19,11 +19,11 @@ use std::time::{Duration, Instant};
use common_base::readable_size::ReadableSize;
use common_telemetry::{debug, info};
use futures::AsyncWriteExt;
use object_store::ObjectStore;
use snafu::ResultExt;
use snafu::{IntoError, ResultExt};
use store_api::storage::RegionId;
use tokio::sync::mpsc::{UnboundedSender, unbounded_channel};
use tokio_util::sync::CancellationToken;
use crate::access_layer::{
EncodedSstIndexBuildParams, FilePathProvider, Metrics, OperationType, RegionFilePathFactory,
@@ -40,8 +40,9 @@ use crate::sst::index::intermediate::IntermediateManager;
use crate::sst::index::puffin_manager::{PuffinManagerFactory, SstPuffinManager};
use crate::sst::index::registry::IndexRegistry;
use crate::sst::index::{IndexerBuilderImpl, build_index_from_parquet_bytes};
use crate::sst::parquet::writer::ParquetWriter;
use crate::sst::parquet::writer::{ParquetWriter, SstWriteContext};
use crate::sst::parquet::{SstInfo, WriteOptions};
use crate::sst::upload::{self, UploadError};
use crate::sst::{DEFAULT_WRITE_BUFFER_SIZE, DEFAULT_WRITE_CONCURRENCY};
/// Wraps the remote object store used by write cache uploads.
@@ -283,13 +284,14 @@ impl WriteCache {
&self.intermediate_manager
}
/// Writes SST to the cache and then uploads it to the remote object store.
pub(crate) async fn write_and_upload_sst(
/// Preserves write ownership and cancellation through local encoding and upload.
pub(crate) async fn write_and_upload_sst_with_context(
&self,
write_request: SstWriteRequest,
upload_request: SstUploadRequest,
write_opts: &WriteOptions,
metrics: &mut Metrics,
context: SstWriteContext,
) -> Result<SstInfoArray> {
let region_id = write_request.metadata.region_id;
let override_sequence = if write_request.preserve_row_sequence {
@@ -328,7 +330,9 @@ impl WriteCache {
metrics,
)
.await
.with_file_cleaner(cleaner);
.with_file_cleaner(cleaner)
.with_write_guard(context.guard)
.with_cancellation(context.cancellation.clone());
let sst_info = match write_request.sst_write_format {
crate::sst::FormatType::PrimaryKey => {
@@ -357,13 +361,21 @@ impl WriteCache {
let op_type = write_request.op_type;
let remote_store = &upload_request.remote_store;
for sst in &sst_info {
if context
.cancellation
.as_ref()
.is_some_and(|handle| handle.is_cancelled())
{
err = Some(error::SstWriteCancelledSnafu.build());
break;
}
let parquet_key = IndexKey::new(region_id, sst.file_id, FileType::Parquet);
let parquet_path = upload_request
.dest_path_provider
.build_sst_file_path(RegionFileId::new(region_id, sst.file_id));
let start = Instant::now();
if let Err(e) = self
.upload(
.upload_with_cancellation(
parquet_key,
&parquet_path,
remote_store,
@@ -371,6 +383,7 @@ impl WriteCache {
op_type,
write_buffer_size: write_opts.write_buffer_size,
},
context.cancellation.as_ref(),
)
.await
{
@@ -387,7 +400,7 @@ impl WriteCache {
.build_index_file_path(RegionFileId::new(region_id, sst.file_id));
let start = Instant::now();
if let Err(e) = self
.upload(
.upload_with_cancellation(
puffin_key,
&puffin_path,
remote_store,
@@ -395,6 +408,7 @@ impl WriteCache {
op_type,
write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
},
context.cancellation.as_ref(),
)
.await
{
@@ -467,10 +481,21 @@ impl WriteCache {
upload_path: &str,
remote_store: &ObjectStore,
options: UploadOptions,
) -> Result<()> {
self.upload_with_cancellation(index_key, upload_path, remote_store, options, None)
.await
}
async fn upload_with_cancellation(
&self,
index_key: IndexKey,
upload_path: &str,
remote_store: &ObjectStore,
options: UploadOptions,
cancellation: Option<&CancellationToken>,
) -> Result<()> {
let region_id = index_key.region_id;
let file_id = index_key.file_id;
let file_type = index_key.file_type;
let cache_path = self.file_cache.cache_file_path(index_key);
let start = Instant::now();
@@ -480,13 +505,14 @@ impl WriteCache {
.stat(&cache_path)
.await
.context(error::OpenDalSnafu)?;
let reader = self
let mut reader = self
.file_cache
.local_store()
.reader(&cache_path)
.reader_with(&cache_path)
.chunk(options.write_buffer_size.as_bytes().max(1) as usize)
.await
.context(error::OpenDalSnafu)?
.into_futures_async_read(0..cached_value.content_length())
.into_stream(0..cached_value.content_length())
.await
.context(error::OpenDalSnafu)?;
@@ -499,24 +525,50 @@ impl WriteCache {
.chunk(options.write_buffer_size.as_bytes() as usize)
.concurrent(DEFAULT_WRITE_CONCURRENCY)
.await
.context(error::OpenDalSnafu)?
.into_futures_async_write();
.context(error::OpenDalSnafu)?;
let bytes_written =
futures::io::copy(reader, &mut writer)
let result = async {
let bytes_written = upload::copy(
&mut reader,
&mut writer,
options.write_buffer_size.as_bytes() as usize,
cancellation,
)
.await
.map_err(|error| match error {
UploadError::Cancelled => error::SstWriteCancelledSnafu.build(),
UploadError::Read { source } => error::OpenDalSnafu.into_error(source),
UploadError::Write { source } => error::UploadSnafu {
region_id,
file_id,
file_type: index_key.file_type,
}
.into_error(std::io::Error::from(source)),
})?;
if cancellation.is_some_and(CancellationToken::is_cancelled) {
return error::SstWriteCancelledSnafu.fail();
}
writer
.close()
.await
.map_err(std::io::Error::from)
.context(error::UploadSnafu {
region_id,
file_id,
file_type,
file_type: index_key.file_type,
})?;
// Must close to upload all data.
writer.close().await.context(error::UploadSnafu {
region_id,
file_id,
file_type,
})?;
Ok(bytes_written)
}
.await;
let bytes_written = match result {
Ok(bytes_written) => bytes_written,
Err(error) => {
if let Err(abort_error) = writer.abort().await {
common_telemetry::warn!(abort_error; "Failed to abort SST upload, region: {}, file: {}", region_id, file_id);
}
return Err(error);
}
};
UPLOAD_BYTES_TOTAL.inc_by(bytes_written);
@@ -647,6 +699,253 @@ mod tests {
sst_file_handle_with_file_id, sst_region_metadata,
};
struct CancellingUploadWriter {
inner: object_store::layers::mock::oio::Writer,
cancellation: CancellationToken,
bytes_written: Arc<std::sync::atomic::AtomicUsize>,
aborted: Arc<std::sync::atomic::AtomicBool>,
}
impl object_store::layers::mock::oio::Write for CancellingUploadWriter {
async fn write(&mut self, buffer: object_store::Buffer) -> object_store::Result<()> {
let len = buffer.len();
self.inner.write(buffer).await?;
self.bytes_written
.fetch_add(len, std::sync::atomic::Ordering::Relaxed);
self.cancellation.cancel();
Ok(())
}
async fn close(
&mut self,
) -> object_store::layers::mock::Result<object_store::layers::mock::Metadata> {
self.inner.close().await
}
async fn abort(&mut self) -> object_store::Result<()> {
self.aborted
.store(true, std::sync::atomic::Ordering::Relaxed);
self.inner.abort().await
}
}
#[tokio::test]
async fn test_cancel_upload_aborts_incomplete_object() {
use object_store::layers::mock::MockLayerBuilder;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
let env = TestEnv::new().await;
let local_store = ObjectStore::new(Memory::default()).unwrap();
let cancellation = CancellationToken::new();
let bytes_written = Arc::new(AtomicUsize::new(0));
let aborted = Arc::new(AtomicBool::new(false));
let signal = cancellation.clone();
let bytes = bytes_written.clone();
let abort = aborted.clone();
let layer = MockLayerBuilder::default()
.writer_factory(Arc::new(move |_, _, inner| {
Box::new(CancellingUploadWriter {
inner,
cancellation: signal.clone(),
bytes_written: bytes.clone(),
aborted: abort.clone(),
})
}))
.build()
.unwrap();
let remote_store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
let cache = env
.create_write_cache(local_store.clone(), ReadableSize::mb(10))
.await;
let key = IndexKey::new(RegionId::new(1024, 1), FileId::random(), FileType::Parquet);
let data = vec![42; 1024 * 1024];
local_store
.write(&cache.file_cache.cache_file_path(key), data.clone())
.await
.unwrap();
let result = cache
.upload_with_cancellation(
key,
"cancelled.parquet",
&remote_store,
UploadOptions {
op_type: OperationType::Compact,
write_buffer_size: ReadableSize::kb(1),
},
Some(&cancellation),
)
.await;
assert!(matches!(
result,
Err(error::Error::SstWriteCancelled { .. })
));
assert!(aborted.load(Ordering::Relaxed));
let bytes = bytes_written.load(Ordering::Relaxed);
assert!(
bytes > 0 && bytes < data.len(),
"wrote {bytes} bytes before cancelling"
);
assert!(!remote_store.exists("cancelled.parquet").await.unwrap());
}
struct FailingUploadWriter {
inner: object_store::layers::mock::oio::Writer,
fail_close: bool,
fail_abort: bool,
successful_writes: usize,
}
impl object_store::layers::mock::oio::Write for FailingUploadWriter {
async fn write(&mut self, bytes: object_store::Buffer) -> object_store::Result<()> {
if !self.fail_close && self.successful_writes == 1 {
return Err(object_store::Error::new(
object_store::ErrorKind::NotFound,
"injected write failure",
));
}
self.inner.write(bytes).await?;
self.successful_writes += 1;
Ok(())
}
async fn close(&mut self) -> object_store::Result<object_store::layers::mock::Metadata> {
Err(object_store::Error::new(
object_store::ErrorKind::NotFound,
"injected close failure",
))
}
async fn abort(&mut self) -> object_store::Result<()> {
if self.fail_abort {
return Err(object_store::Error::new(
object_store::ErrorKind::Unexpected,
"injected abort failure",
));
}
self.inner.abort().await
}
}
#[rstest::rstest]
#[case::write(false, false)]
#[case::close(true, false)]
#[case::write_and_abort(false, true)]
#[tokio::test]
async fn test_upload_error_preserves_file_context(
#[case] fail_close: bool,
#[case] fail_abort: bool,
) {
use object_store::layers::mock::{MockLayerBuilder, ObservedWriter, WriterObservations};
let env = TestEnv::new().await;
let local_store = ObjectStore::new(Memory::default()).unwrap();
let observations = Arc::new(std::sync::Mutex::new(WriterObservations::default()));
let observed = observations.clone();
let layer = MockLayerBuilder::default()
.writer_factory(Arc::new(move |_, _, inner| {
Box::new(ObservedWriter::new(
Box::new(FailingUploadWriter {
inner,
fail_close,
fail_abort,
successful_writes: 0,
}),
observed.clone(),
))
}))
.build()
.unwrap();
let remote_store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
let cache = env
.create_write_cache(local_store.clone(), ReadableSize::mb(10))
.await;
let key = IndexKey::new(RegionId::new(1024, 1), FileId::random(), FileType::Parquet);
local_store
.write(&cache.file_cache.cache_file_path(key), vec![42; 4096])
.await
.unwrap();
assert!(!cache.file_cache.contains_key(&key));
let err = cache
.upload(
key,
"failed.parquet",
&remote_store,
UploadOptions {
op_type: OperationType::Compact,
write_buffer_size: ReadableSize::kb(1),
},
)
.await
.unwrap_err();
assert!(!err.is_object_not_found());
assert!(err.may_have_persisted_manifest_update());
match err {
error::Error::Upload {
region_id,
file_id,
file_type,
error,
..
} => {
assert_eq!(key.region_id, region_id);
assert_eq!(key.file_id, file_id);
assert_eq!(key.file_type, file_type);
assert_eq!(std::io::ErrorKind::NotFound, error.kind());
let message = if fail_close {
"injected close failure"
} else {
"injected write failure"
};
assert!(error.to_string().contains(message));
}
err => panic!("expected Upload error, got {err:?}"),
}
assert!(!cache.file_cache.contains_key(&key));
assert!(!remote_store.exists("failed.parquet").await.unwrap());
let observations = observations.lock().unwrap();
// The write failure follows one completed chunk, so partial progress is exercised.
assert_eq!(
vec![1024; if fail_close { 4 } else { 2 }],
observations.write_sizes
);
assert_eq!(1, observations.abort_calls);
assert_eq!(usize::from(fail_close), observations.close_calls);
}
#[tokio::test]
async fn test_upload_local_read_error() {
use object_store::layers::mock::MockLayerBuilder;
let env = TestEnv::new().await;
// The unit reader rejects reads after stat and reader creation succeed.
let layer = MockLayerBuilder::default()
.reader_factory(Arc::new(|_, _, _| Box::new(())))
.build()
.unwrap();
let local_store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
let remote_store = ObjectStore::new(Memory::default()).unwrap();
let cache = env
.create_write_cache(local_store.clone(), ReadableSize::mb(10))
.await;
let key = IndexKey::new(RegionId::new(1024, 1), FileId::random(), FileType::Parquet);
local_store
.write(&cache.file_cache.cache_file_path(key), vec![42; 4096])
.await
.unwrap();
let err = cache
.upload(
key,
"failed.parquet",
&remote_store,
UploadOptions {
op_type: OperationType::Compact,
write_buffer_size: ReadableSize::kb(1),
},
)
.await
.unwrap_err();
assert!(matches!(err, error::Error::OpenDal { .. }));
assert!(!remote_store.exists("failed.parquet").await.unwrap());
}
struct RedirectUploadStoreWrapper {
target_store: ObjectStore,
last_op_type: Mutex<Option<OperationType>>,
@@ -776,7 +1075,13 @@ mod tests {
// Write to cache and upload sst to mock remote store
let mut metrics = Metrics::new(WriteType::Flush);
let mut sst_infos = write_cache
.write_and_upload_sst(write_request, upload_request, &write_opts, &mut metrics)
.write_and_upload_sst_with_context(
write_request,
upload_request,
&write_opts,
&mut metrics,
SstWriteContext::default(),
)
.await
.unwrap();
let sst_info = sst_infos.remove(0);
@@ -890,7 +1195,13 @@ mod tests {
let mut metrics = Metrics::new(WriteType::Flush);
let mut sst_infos = write_cache
.write_and_upload_sst(write_request, upload_request, &write_opts, &mut metrics)
.write_and_upload_sst_with_context(
write_request,
upload_request,
&write_opts,
&mut metrics,
SstWriteContext::default(),
)
.await
.unwrap();
let sst_info = sst_infos.remove(0);
@@ -983,7 +1294,13 @@ mod tests {
let mut metrics = Metrics::new(WriteType::Flush);
write_cache
.write_and_upload_sst(write_request, upload_request, &write_opts, &mut metrics)
.write_and_upload_sst_with_context(
write_request,
upload_request,
&write_opts,
&mut metrics,
SstWriteContext::default(),
)
.await
.unwrap_err();
let atomic_write_dir = write_cache_dir.path().join(ATOMIC_WRITE_DIR);
+186 -30
View File
@@ -17,7 +17,6 @@ use std::sync::Arc;
use std::time::Duration;
use common_base::Plugins;
use common_base::cancellation::CancellationHandle;
use common_meta::key::SchemaMetadataManagerRef;
use common_telemetry::{info, warn};
use common_time::TimeToLive;
@@ -31,6 +30,7 @@ use store_api::ManifestVersion;
use store_api::metadata::RegionMetadataRef;
use store_api::region_request::PathType;
use store_api::storage::{RegionId, SequenceNumber};
use tokio_util::sync::CancellationToken;
use crate::access_layer::{
AccessLayer, AccessLayerRef, Metrics, OperationType, SstWriteRequest, WriteType,
@@ -61,6 +61,7 @@ use crate::sst::index::puffin_manager::PuffinManagerFactory;
use crate::sst::location::region_dir_from_table_dir;
use crate::sst::parquet::metadata::extract_primary_key_range;
use crate::sst::parquet::parallel_writer::EncodeRuntime;
use crate::sst::parquet::writer::SstWriteContext;
use crate::sst::parquet::{SstInfo, WriteOptions};
use crate::sst::version::{SstVersion, SstVersionRef};
@@ -555,11 +556,14 @@ pub trait Compactor: Send + Sync + 'static {
/// implementations in tests.
#[async_trait::async_trait]
pub trait SstMerger: Send + Sync + 'static {
/// Merges an output, observing cancellation at safe boundaries and draining
/// in-flight writers before returning.
async fn merge_single_output(
&self,
context: CompactionContext,
output: CompactionOutput,
write_opts: WriteOptions,
cancellation: CancellationToken,
) -> Result<(Vec<FileMeta>, Vec<SstInfo>)>;
}
@@ -618,6 +622,37 @@ impl SstMerger for DefaultSstMerger {
context: CompactionContext,
output: CompactionOutput,
write_opts: WriteOptions,
cancellation: CancellationToken,
) -> Result<(Vec<FileMeta>, Vec<SstInfo>)> {
let uncommitted = UncommittedSsts::new(
context.region_id,
context.access_layer.clone(),
Some(context.cache_manager.clone()),
);
let result = self
.merge_output(context, output, write_opts, &uncommitted, cancellation)
.await;
if result.is_err() {
uncommitted.cleanup().await;
} else {
// Successful outputs are transferred to the caller for publication.
uncommitted.disarm_cleanup();
}
match result {
Err(error::Error::SstWriteCancelled { .. }) => error::CompactionCancelledSnafu.fail(),
result => result,
}
}
}
impl DefaultSstMerger {
async fn merge_output(
&self,
context: CompactionContext,
output: CompactionOutput,
write_opts: WriteOptions,
uncommitted: &UncommittedSsts,
cancellation: CancellationToken,
) -> Result<(Vec<FileMeta>, Vec<SstInfo>)> {
let region_id = context.region_id;
validate_region_compaction_output(region_id, &output)?;
@@ -652,13 +687,18 @@ impl SstMerger for DefaultSstMerger {
merge_mode,
read_parallelism: write_opts.encode_parallelism,
};
let source = spawn_merge(builder.build_flat_sst_reader().await?);
let source = tokio::select! {
biased;
_ = cancellation.cancelled() => return error::CompactionCancelledSnafu.fail(),
source = builder.build_flat_sst_reader() => source?,
};
let source = spawn_merge(source);
let mut metrics = Metrics::new(WriteType::Compaction);
let region_metadata = context.region_metadata.clone();
let sst_infos = context
.access_layer
.write_sst(
.write_sst_with_context(
SstWriteRequest {
index_build_inputs: None,
op_type: OperationType::Compact,
@@ -683,6 +723,11 @@ impl SstMerger for DefaultSstMerger {
},
&write_opts,
&mut metrics,
SstWriteContext {
guard: Some(Arc::new(uncommitted.clone())),
cancellation: Some(cancellation),
..Default::default()
},
)
.await?;
// Convert partition expression once outside the map
@@ -746,7 +791,7 @@ impl SstMerger for DefaultSstMerger {
/// implementations in tests.
pub struct DefaultCompactor<M = DefaultSstMerger> {
merger: M,
cancel_handle: Arc<CancellationHandle>,
cancel_token: CancellationToken,
uncommitted: Option<UncommittedSsts>,
}
@@ -755,37 +800,37 @@ impl<M: SstMerger> DefaultCompactor<M> {
pub fn with_merger(merger: M) -> Self {
Self {
merger,
cancel_handle: Arc::new(CancellationHandle::default()),
cancel_token: CancellationToken::new(),
uncommitted: None,
}
}
}
impl DefaultCompactor {
/// Creates a new `DefaultCompactor` with the given cancel handle and no
/// Creates a new `DefaultCompactor` with the given cancellation token and no
/// uncommitted SSTs tracking.
///
/// This is the public entry point for external crates that want a
/// cancellable compactor without opting into uncommitted-SST tracking.
///
/// Cancellation stops new output batches. Already spawned outputs finish,
/// and their successful SSTs remain in the returned [`MergeOutput`]. This
/// can delay cancellation until the current batch completes.
pub fn new_with_cancel_handle(cancel_handle: Arc<CancellationHandle>) -> Self {
/// Cancellation stops new output batches and interrupts active outputs at
/// batch and upload-chunk boundaries. Successful complete outputs remain in
/// the returned [`MergeOutput`]; incomplete outputs are cleaned up.
pub fn new_with_cancel_token(cancel_token: CancellationToken) -> Self {
Self {
merger: DefaultSstMerger,
cancel_handle,
cancel_token,
uncommitted: None,
}
}
pub(crate) fn with_cancel_handle(
cancel_handle: Arc<CancellationHandle>,
pub(crate) fn with_cancel_token(
cancel_token: CancellationToken,
uncommitted: UncommittedSsts,
) -> Self {
Self {
merger: DefaultSstMerger,
cancel_handle,
cancel_token,
uncommitted: Some(uncommitted),
}
}
@@ -804,11 +849,11 @@ impl<M: SstMerger> DefaultCompactor<M> {
..Default::default()
};
for output in &unit.outputs {
if self.cancel_handle.is_cancelled() {
if self.cancel_token.is_cancelled() {
return error::CompactionCancelledSnafu.fail();
}
// Let the writer finish and register its files before observing cancellation.
// Dropping it mid-write can lose ownership of already finalized SSTs.
// The merger observes cancellation and drains its writers before returning.
// Keep awaiting it so cleanup cannot race with a late finalized SST.
let (files, infos) = self
.merger
.merge_single_output(
@@ -828,6 +873,7 @@ impl<M: SstMerger> DefaultCompactor<M> {
),
encode_runtime: EncodeRuntime::Compact,
},
self.cancel_token.clone(),
)
.await?;
if let Some(uncommitted) = &self.uncommitted {
@@ -836,7 +882,7 @@ impl<M: SstMerger> DefaultCompactor<M> {
result.files_to_add.extend(files);
result.sst_infos.extend(infos);
}
if self.cancel_handle.is_cancelled() {
if self.cancel_token.is_cancelled() {
return error::CompactionCancelledSnafu.fail();
}
Ok(result)
@@ -895,9 +941,10 @@ where
let merger = self.merger.clone();
let context = context.clone();
let uncommitted = self.uncommitted.clone();
let cancellation = self.cancel_token.clone();
let fut = async move {
let result = merger
.merge_single_output(context, output, write_opts)
.merge_single_output(context, output, write_opts, cancellation)
.await;
if let (Some(uncommitted), Ok((_, infos))) = (&uncommitted, &result) {
uncommitted.track(infos);
@@ -915,7 +962,7 @@ where
+ picker_output.expired_ssts.len(),
);
while !tasks.is_empty() && !self.cancel_handle.is_cancelled() {
while !tasks.is_empty() && !self.cancel_token.is_cancelled() {
let mut chunk: Vec<(Vec<FileMeta>, _)> = Vec::with_capacity(internal_parallelism);
for _ in 0..internal_parallelism {
if let Some(task) = tasks.pop() {
@@ -930,9 +977,8 @@ where
})
.collect();
// Like merge_unit, cancellation must not drop a writer mid-output:
// its blocking encoders would detach and finalized SSTs could lose
// their owner. Join the whole batch and retain every successful output.
// Each merger cancels cooperatively and owns its incomplete outputs.
// Join every sibling before returning so writers and cleanup cannot race.
let mut join_error = None;
while let Some((inputs, handle)) = spawned.pop() {
match handle.await {
@@ -943,6 +989,7 @@ where
}
compacted_inputs.extend(inputs);
}
Ok(Err(error::Error::CompactionCancelled { .. })) => {}
Ok(Err(e)) => {
warn!(
e; "Failed to merge compaction output for region: {}, inputs: [{}]",
@@ -966,7 +1013,7 @@ where
return Err(e).context(error::JoinSnafu);
}
if self.cancel_handle.is_cancelled() {
if self.cancel_token.is_cancelled() {
info!("Compaction merge cancelled for region: {}", region_id);
break;
}
@@ -1006,9 +1053,12 @@ mod tests {
use std::time::Duration;
use datatypes::arrow::record_batch::RecordBatch;
use object_store::ObjectStore;
use object_store::layers::mock::{self, MockLayerBuilder};
use object_store::services::Memory;
use parquet::arrow::ArrowWriter;
use store_api::storage::{FileId, RegionId};
use tokio::sync::{mpsc, oneshot};
use tokio::sync::{Notify, mpsc, oneshot};
use tokio::time::timeout;
use super::{DefaultCompactor, *};
@@ -1032,6 +1082,105 @@ mod tests {
FileHandle::new(meta, Arc::new(NoopFilePurger))
}
struct PendingCompactionReader {
started: Arc<Notify>,
}
impl mock::oio::Read for PendingCompactionReader {
async fn open(
&self,
_: mock::BytesRange,
) -> object_store::Result<(mock::RpRead, Box<dyn mock::oio::ReadStreamDyn>)> {
self.started.notify_one();
std::future::pending().await
}
async fn read(
&self,
_: mock::BytesRange,
) -> object_store::Result<(mock::RpRead, object_store::Buffer)> {
self.started.notify_one();
std::future::pending().await
}
}
#[rstest::rstest]
#[case::unit(true)]
#[case::outputs(false)]
#[tokio::test]
async fn test_default_merger_observes_compactor_cancellation(#[case] merge_unit: bool) {
let mut region = new_test_compaction_region().await;
let started = Arc::new(Notify::new());
let signal = started.clone();
let layer = MockLayerBuilder::default()
.reader_factory(Arc::new(move |_, _, _| {
Box::new(PendingCompactionReader {
started: signal.clone(),
})
}))
.build()
.unwrap();
let store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
let access = &region.access_layer;
region.access_layer = Arc::new(AccessLayer::new(
access.table_dir(),
access.path_type(),
store.clone(),
access.puffin_manager_factory().clone(),
access.intermediate_manager().clone(),
));
let input = new_file_handle(dummy_file_meta());
store
.write(
&input.file_path(
region.access_layer.table_dir(),
region.access_layer.path_type(),
),
vec![0; input.meta_ref().file_size as usize],
)
.await
.unwrap();
let picker_output = PickerOutput {
outputs: vec![CompactionOutput {
output_level: 1,
inputs: vec![input],
filter_deleted: false,
output_time_range: None,
}],
..Default::default()
};
let compactor = DefaultCompactor::with_merger(DefaultSstMerger);
let context = CompactionContext::from(&region);
let mut merge = Box::pin(async {
if merge_unit {
let unit = CompactionUnit::from_picker(picker_output)
.unwrap()
.pop()
.unwrap();
compactor.merge_unit(&context, &unit).await
} else {
compactor.merge_ssts(&region, picker_output).await
}
});
tokio::select! {
_ = started.notified() => {},
result = &mut merge => panic!("merge returned before cancellation: {result:?}"),
_ = tokio::time::sleep(Duration::from_secs(10)) => panic!("merge did not reach the reader"),
}
compactor.cancel_token.cancel();
let result = timeout(Duration::from_secs(10), merge).await.unwrap();
if merge_unit {
assert!(matches!(
result,
Err(error::Error::CompactionCancelled { .. })
));
} else {
let result = result.unwrap();
assert!(result.files_to_add.is_empty());
assert!(result.files_to_remove.is_empty());
}
}
#[test]
fn test_effective_target_trust_and_max_sequence() {
let target = RegionId::new(1, 1);
@@ -1420,7 +1569,12 @@ mod tests {
.await
.unwrap();
let (files, _) = DefaultSstMerger
.merge_single_output(context, output, WriteOptions::default())
.merge_single_output(
context,
output,
WriteOptions::default(),
CancellationToken::new(),
)
.await
.unwrap();
assert_eq!(1, files.len());
@@ -1481,6 +1635,7 @@ mod tests {
_context: CompactionContext,
_output: CompactionOutput,
_write_opts: WriteOptions,
_cancellation: CancellationToken,
) -> Result<(Vec<FileMeta>, Vec<SstInfo>)> {
let idx = self.call_idx.fetch_add(1, Ordering::SeqCst);
match self.results.lock().unwrap().get(idx) {
@@ -1713,6 +1868,7 @@ mod tests {
_context: CompactionContext,
output: CompactionOutput,
_write_opts: WriteOptions,
_cancellation: CancellationToken,
) -> Result<(Vec<FileMeta>, Vec<SstInfo>)> {
let input_id = output.inputs[0].meta_ref().file_id;
let mut file = dummy_file_meta();
@@ -1765,7 +1921,7 @@ mod tests {
.collect(),
..Default::default()
};
let cancel_handle = Arc::new(CancellationHandle::default());
let cancel_token = CancellationToken::new();
let (started_tx, mut started_rx) = mpsc::unbounded_channel();
let compactor = DefaultCompactor {
merger: GatedEncodingMerger {
@@ -1773,7 +1929,7 @@ mod tests {
// Outputs are popped from the back, then joined in reverse order.
panic_input: panic_merge.then_some(inputs[1].file_id),
},
cancel_handle: cancel_handle.clone(),
cancel_token: cancel_token.clone(),
uncommitted: None,
};
let memory = new_compaction_memory_manager(1024 * 1024);
@@ -1792,7 +1948,7 @@ mod tests {
.unwrap(),
);
}
cancel_handle.cancel();
cancel_token.cancel();
assert!(futures::poll!(merge.as_mut()).is_pending());
assert!(memory.try_acquire(memory.limit_bytes()).is_none());
assert!(
@@ -1855,7 +2011,7 @@ mod tests {
let merger = MockMerger::new(vec![Ok(vec![dummy_file_meta()])]);
let calls = merger.call_idx.clone();
let compactor = DefaultCompactor::with_merger(merger);
compactor.cancel_handle.cancel();
compactor.cancel_token.cancel();
let picker_output = PickerOutput {
outputs: vec![CompactionOutput {
output_level: 1,
+7 -5
View File
@@ -28,12 +28,12 @@ use std::collections::{BTreeMap, HashMap, HashSet};
use std::sync::Arc;
use std::sync::atomic::AtomicU64;
use common_base::cancellation::{CancellableFuture, CancellationHandle};
use common_time::Timestamp;
use partition::PartitionRuleRef;
use snafu::{OptionExt, ensure};
use store_api::metadata::RegionMetadataRef;
use store_api::storage::RegionId;
use tokio_util::sync::CancellationToken;
use crate::access_layer::AccessLayerRef;
use crate::cache::CacheManagerRef;
@@ -82,7 +82,7 @@ pub struct MultiRegionCompactionContext {
pub cache: CacheManagerRef,
pub config: Arc<MitoConfig>,
pub protection: Arc<dyn SstWriteGuard>,
pub cancellation: Arc<CancellationHandle>,
pub cancellation: CancellationToken,
}
/// Source I/O and lossless output accounting, excluding footer and query reads.
@@ -137,9 +137,11 @@ pub async fn compact_regions(
source::SharedSource::build(&plan, &context, metadata, counters.clone()).await?;
worker::execute(plan, context, source, counters).await
};
CancellableFuture::new(execution, cancellation)
.await
.map_err(|_| crate::error::CompactionCancelledSnafu.build())?
tokio::select! {
biased;
_ = cancellation.cancelled() => crate::error::CompactionCancelledSnafu.fail(),
result = execution => result,
}
}
fn validate_plan(plan: &MultiRegionCompactionPlan) -> Result<RegionMetadataRef> {
@@ -109,7 +109,7 @@ pub(crate) struct SharedSource {
window: (Timestamp, Timestamp),
pub(crate) limits: MemoryLimits,
peak_group_bytes: AtomicU64,
cancellation: Arc<common_base::cancellation::CancellationHandle>,
cancellation: CancellationToken,
stop: CancellationToken,
counters: Arc<SourceCounters>,
}
@@ -277,7 +277,7 @@ impl Fixture {
cache: Arc::new(CacheManager::default()),
config: self.config.clone(),
protection: protection.clone(),
cancellation: Arc::new(CancellationHandle::default()),
cancellation: CancellationToken::new(),
},
protection,
)
@@ -2002,7 +2002,7 @@ async fn test_interrupted_indexed_worker_cleans_tantivy_files(#[case] fail_sibli
}
}
struct CancelBeforeCreate(Arc<CancellationHandle>);
struct CancelBeforeCreate(CancellationToken);
#[async_trait::async_trait]
impl SstWriteGuard for CancelBeforeCreate {
@@ -16,6 +16,7 @@ use common_base::readable_size::ReadableSize;
use partition::expr::PartitionExpr;
use snafu::{ResultExt, ensure};
use store_api::codec::PrimaryKeyEncoding;
use tokio_util::sync::CancellationToken;
use crate::access_layer::{Metrics, OperationType, SstWriteRequest, WriteType};
use crate::compaction::multi_region::memory::{self, MemoryLimits};
@@ -27,6 +28,7 @@ use crate::sst::FormatType;
use crate::sst::file::FileMeta;
use crate::sst::parquet::metadata::extract_primary_key_range;
use crate::sst::parquet::parallel_writer::EncodeRuntime;
use crate::sst::parquet::writer::SstWriteContext;
use crate::sst::parquet::{SstInfo, WriteOptions};
pub(crate) struct TargetWriter<'a> {
@@ -71,13 +73,14 @@ impl<'a> TargetWriter<'a> {
let infos = self
.context
.access_layer
.write_sst_with_guard(
.write_sst_with_context(
request,
&self.options(),
&mut metrics,
crate::sst::parquet::writer::SstWriteContext {
guard: self.context.protection.clone(),
limits: self.limits.writer,
SstWriteContext {
guard: Some(self.context.protection.clone()),
limits: Some(self.limits.writer),
..Default::default()
},
)
.await?;
@@ -207,7 +210,7 @@ impl<'a> TargetWriter<'a> {
fn output_projection(
target: &MultiRegionTarget,
mut source: FlatSource,
cancellation: Arc<common_base::cancellation::CancellationHandle>,
cancellation: CancellationToken,
) -> Result<FlatSource> {
let projection: Vec<_> = source
.schema()
+1 -1
View File
@@ -2519,7 +2519,7 @@ async fn test_request_cancel_state_transitions() {
let state = status.start_local_task();
assert_eq!(status.request_cancel(), RequestCancelResult::CancelIssued);
assert!(state.cancel_handle().is_cancelled());
assert!(state.cancel_token().is_cancelled());
assert_eq!(
status.request_cancel(),
RequestCancelResult::AlreadyCancelling
+12 -14
View File
@@ -15,7 +15,6 @@
use std::fmt::{Debug, Formatter};
use std::sync::Arc;
use common_base::cancellation::CancellableFuture;
use common_memory_manager::OnExhaustedPolicy;
use common_telemetry::error;
use futures::FutureExt;
@@ -91,25 +90,24 @@ impl CompactionTask {
None
} else {
let _wait_timer = crate::metrics::COMPACTION_MEMORY_WAIT.start_timer();
let cancellation = self.state.cancel_token();
Some(
CancellableFuture::new(
self.memory_manager
.acquire_with_policy(bytes, self.memory_policy),
self.state.cancel_handle(),
)
.await
.map_err(|_| CompactionCancelledSnafu.build())?
.context(CompactionMemoryExhaustedSnafu {
region_id,
policy: format!("{:?}", self.memory_policy),
})?,
tokio::select! {
biased;
_ = cancellation.cancelled() => return CompactionCancelledSnafu.fail(),
memory = self.memory_manager.acquire_with_policy(bytes, self.memory_policy) => memory,
}
.context(CompactionMemoryExhaustedSnafu {
region_id,
policy: format!("{:?}", self.memory_policy),
})?,
)
};
let _timer = COMPACTION_STAGE_ELAPSED
.with_label_values(&["merge"])
.start_timer();
let compactor = DefaultCompactor::with_cancel_handle(
self.state.cancel_handle(),
let compactor = DefaultCompactor::with_cancel_token(
self.state.cancel_token(),
self.uncommitted.clone(),
);
let context = self.compaction_context.clone();
+7 -3
View File
@@ -1398,6 +1398,9 @@ pub enum Error {
#[snafu(display("Compaction is cancelled."))]
CompactionCancelled {},
#[snafu(display("SST write is cancelled."))]
SstWriteCancelled {},
#[snafu(display("Flush is cancelled."))]
FlushCancelled {},
@@ -1818,9 +1821,10 @@ impl ErrorExt for Error {
source.status_code()
}
ManualCompactionOverride {} | CompactionCancelled {} | FlushCancelled {} => {
StatusCode::Cancelled
}
ManualCompactionOverride {}
| CompactionCancelled {}
| FlushCancelled {}
| SstWriteCancelled {} => StatusCode::Cancelled,
// A concurrent manual compaction fails fast instead of being queued;
// the conflict is reported to the caller, which decides whether to
+13 -9
View File
@@ -21,7 +21,6 @@ use std::sync::{Arc, Mutex};
use std::time::Instant;
use bytes::Bytes;
use common_base::cancellation::CancellableFuture;
use common_telemetry::{debug, error, info};
use datatypes::arrow::datatypes::SchemaRef;
use partition::expr::PartitionExpr;
@@ -537,11 +536,11 @@ impl RegionFlushTask {
// that acknowledges appends before their entries are durable can have
// handed out `last_entry_id` for an entry that is still in its backlog.
// A DDL that cancels the flush must not wait behind that upload.
let durable = self.durability_barrier.wait(version_data.last_entry_id);
tokio::pin!(durable);
match CancellableFuture::new(durable.as_mut(), state.cancel_handle()).await {
Ok(result) => result?,
Err(_) => return FlushCancelledSnafu.fail(),
let cancellation = state.cancel_token();
tokio::select! {
biased;
_ = cancellation.cancelled() => return FlushCancelledSnafu.fail(),
result = self.durability_barrier.wait(version_data.last_entry_id) => result?,
}
let edit = RegionEdit {
@@ -803,12 +802,17 @@ impl RegionFlushTask {
.collect::<Vec<_>>();
let join_all = futures::future::join_all(tasks);
tokio::pin!(join_all);
let results = match CancellableFuture::new(join_all.as_mut(), state.cancel_handle()).await {
Ok(results) => results
let cancellation = state.cancel_token();
let results = match tokio::select! {
biased;
_ = cancellation.cancelled() => None,
results = join_all.as_mut() => Some(results),
} {
Some(results) => results
.into_iter()
.map(|result| result.context(JoinSnafu))
.collect::<Result<Vec<_>>>()?,
Err(_) => {
None => {
for handle in abort_handles {
handle.abort();
}
+9 -9
View File
@@ -15,7 +15,7 @@
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use common_base::cancellation::CancellationHandle;
use tokio_util::sync::CancellationToken;
pub mod remote_job_scheduler;
pub mod scheduler;
@@ -23,7 +23,7 @@ pub mod scheduler;
/// Shared state for cooperatively cancelling a task until it starts committing.
#[derive(Debug, Clone)]
pub(crate) struct CancellableTaskState {
cancel_handle: Arc<CancellationHandle>,
cancel_token: CancellationToken,
commit_started: Arc<Mutex<bool>>,
started: Arc<AtomicBool>,
}
@@ -31,7 +31,7 @@ pub(crate) struct CancellableTaskState {
impl CancellableTaskState {
pub(crate) fn new() -> Self {
Self {
cancel_handle: Arc::new(CancellationHandle::default()),
cancel_token: CancellationToken::new(),
commit_started: Arc::new(Mutex::new(false)),
started: Arc::new(AtomicBool::new(false)),
}
@@ -47,12 +47,12 @@ impl CancellableTaskState {
self.started.load(Ordering::Relaxed)
}
pub(crate) fn cancel_handle(&self) -> Arc<CancellationHandle> {
self.cancel_handle.clone()
pub(crate) fn cancel_token(&self) -> CancellationToken {
self.cancel_token.clone()
}
pub(crate) fn is_cancelled(&self) -> bool {
self.cancel_handle.is_cancelled()
self.cancel_token.is_cancelled()
}
/// Returns whether the task has crossed the non-cancellable publication boundary.
@@ -65,7 +65,7 @@ impl CancellableTaskState {
/// Returns false if cancellation was requested first.
pub(crate) fn mark_commit_started(&self) -> bool {
let mut commit_started = self.commit_started.lock().unwrap();
if self.cancel_handle.is_cancelled() {
if self.cancel_token.is_cancelled() {
return false;
}
*commit_started = true;
@@ -78,11 +78,11 @@ impl CancellableTaskState {
if *commit_started {
return RequestCancelResult::TooLateToCancel;
}
if self.cancel_handle.is_cancelled() {
if self.cancel_token.is_cancelled() {
return RequestCancelResult::AlreadyCancelling;
}
self.cancel_handle.cancel();
self.cancel_token.cancel();
RequestCancelResult::CancelIssued
}
}
+1
View File
@@ -50,6 +50,7 @@ pub mod location;
pub mod parquet;
pub(crate) mod primary_key;
pub mod range_index;
pub(crate) mod upload;
pub(crate) mod version;
/// Default write buffer size, it should be greater than the default minimum upload part of S3 (5mb).
+11 -1
View File
@@ -45,6 +45,7 @@ use crate::sst::file_ref::{FileReferenceManager, FileReferenceManagerRef};
use crate::sst::index::fulltext_index::FULLTEXT_BLOOM_VERSION;
use crate::sst::location;
use crate::sst::parquet::SstInfo;
use crate::sst::parquet::writer::SstWriteGuard;
use crate::sst::primary_key::PrimaryKeyRangeMapper;
/// Custom serde functions for Bytes fields serialized as base64 strings.
@@ -1042,7 +1043,7 @@ pub async fn delete_files(
Ok(())
}
/// Tracks finalized SSTs until their manifest edit is committed.
/// Tracks SSTs from creation until their manifest edit is committed.
///
/// Flush and local compaction jobs use this to remove files that were written successfully but
/// abandoned by cancellation or a later failure.
@@ -1054,6 +1055,15 @@ pub(crate) struct UncommittedSsts {
cache_manager: Option<CacheManagerRef>,
}
#[async_trait::async_trait]
impl SstWriteGuard for UncommittedSsts {
async fn before_create(&self, file_id: RegionFileId) -> crate::error::Result<()> {
// An index can be created before its final metadata is available.
self.track_before_upload(file_id.file_id(), true);
Ok(())
}
}
impl UncommittedSsts {
pub(crate) fn new(
region_id: RegionId,
+9 -56
View File
@@ -12,15 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use bytes::Bytes;
use datatypes::arrow::datatypes::SchemaRef;
use datatypes::arrow::record_batch::RecordBatch;
use futures::future::BoxFuture;
use object_store::{ObjectStore, Writer};
use object_store::ObjectStore;
use parquet::arrow::AsyncArrowWriter;
use parquet::arrow::async_writer::AsyncFileWriter;
use parquet::basic::{Compression, Encoding, ZstdLevel};
use parquet::errors::ParquetError;
use parquet::file::metadata::KeyValue;
use parquet::file::properties::WriterProperties;
use snafu::{OptionExt, ResultExt};
@@ -29,55 +25,9 @@ use crate::access_layer::TempFileCleaner;
use crate::error::{OpenDalSnafu, Result, UnexpectedSnafu, WriteParquetSnafu};
use crate::sst::{DEFAULT_WRITE_BUFFER_SIZE, DEFAULT_WRITE_CONCURRENCY};
type ArrowWriter = AsyncArrowWriter<AsyncWriter>;
use crate::sst::parquet::writer::ObjectStoreWriter;
/// Bridges an OpenDAL [`Writer`] with Parquet's [`AsyncFileWriter`] and tracks
/// the number of bytes successfully submitted to the object store.
struct AsyncWriter {
inner: Writer,
output_bytes: u64,
}
impl AsyncWriter {
fn new(inner: Writer) -> Self {
Self {
inner,
output_bytes: 0,
}
}
fn output_bytes(&self) -> u64 {
self.output_bytes
}
fn into_inner(self) -> Writer {
self.inner
}
}
impl AsyncFileWriter for AsyncWriter {
fn write(&mut self, bytes: Bytes) -> BoxFuture<'_, parquet::errors::Result<()>> {
Box::pin(async move {
let len = bytes.len() as u64;
self.inner
.write(bytes)
.await
.map_err(|error| ParquetError::External(Box::new(error)))?;
self.output_bytes += len;
Ok(())
})
}
fn complete(&mut self) -> BoxFuture<'_, parquet::errors::Result<()>> {
Box::pin(async move {
self.inner
.close()
.await
.map(|_| ())
.map_err(|error| ParquetError::External(Box::new(error)))
})
}
}
type ArrowWriter = AsyncArrowWriter<ObjectStoreWriter>;
/// Shared Parquet output and cleanup lifecycle for index writers.
pub(crate) struct ParquetIndexWriter {
@@ -112,9 +62,12 @@ impl ParquetIndexWriter {
.set_statistics_truncate_length(None)
.set_key_value_metadata(key_value_metadata)
.build();
let writer =
AsyncArrowWriter::try_new(AsyncWriter::new(output), schema.clone(), Some(properties))
.context(WriteParquetSnafu)?;
let writer = AsyncArrowWriter::try_new(
ObjectStoreWriter::new(output, DEFAULT_WRITE_BUFFER_SIZE.as_bytes() as usize, None),
schema.clone(),
Some(properties),
)
.context(WriteParquetSnafu)?;
Ok(Self {
name,
+37 -4
View File
@@ -19,6 +19,7 @@ use std::sync::Arc;
use std::time::{Duration, Instant};
use bytes::Bytes;
use common_base::cancellation::Cancelled;
use common_runtime::JoinHandle;
use datatypes::arrow::datatypes::SchemaRef;
use datatypes::arrow::record_batch::RecordBatch;
@@ -31,6 +32,7 @@ use parquet::errors::{ParquetError, Result};
use parquet::file::metadata::ParquetMetaData;
use parquet::file::properties::WriterProperties;
use parquet::file::writer::SerializedFileWriter;
use tokio_util::sync::CancellationToken;
/// Runtime whose blocking pool encodes the columns of a writer.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
@@ -90,6 +92,7 @@ pub struct ParallelArrowWriter<W> {
/// Tasks encoding the previous row group.
encoding: Vec<JoinHandle<Result<EncodedFields>>>,
num_row_groups: usize,
cancellation: Option<CancellationToken>,
}
impl<W: AsyncFileWriter> ParallelArrowWriter<W> {
@@ -137,13 +140,20 @@ impl<W: AsyncFileWriter> ParallelArrowWriter<W> {
buffered_rows: 0,
encoding: Vec::new(),
num_row_groups: 0,
cancellation: None,
})
}
pub(crate) fn with_cancellation(mut self, cancellation: Option<CancellationToken>) -> Self {
self.cancellation = cancellation;
self
}
/// Buffers `batch`, starting to encode each row group once it is full.
pub async fn write(&mut self, batch: &RecordBatch) -> Result<()> {
let mut offset = 0;
while offset < batch.num_rows() {
check_cancelled(self.cancellation.as_ref())?;
let len = (self.max_row_group_rows - self.buffered_rows).min(batch.num_rows() - offset);
self.buffered.push(batch.slice(offset, len));
self.buffered_rows += len;
@@ -165,6 +175,11 @@ impl<W: AsyncFileWriter> ParallelArrowWriter<W> {
/// Writes the remaining rows and the footer, then completes the async writer.
pub async fn close(mut self) -> Result<ParquetMetaData> {
self.finish().await
}
/// Finalizes the file while retaining the upload for abort if it fails.
pub(crate) async fn finish(&mut self) -> Result<ParquetMetaData> {
self.flush().await?;
let metadata = self.file_writer.finish()?;
self.write_buffered_bytes().await?;
@@ -173,9 +188,10 @@ impl<W: AsyncFileWriter> ParallelArrowWriter<W> {
}
/// Discards buffered input and waits for encoders without writing a failed SST.
pub(crate) async fn abort(mut self) {
pub(crate) async fn abort(mut self) -> W {
self.buffered.clear();
abort_encoding_tasks(&mut self.encoding).await;
self.async_writer
}
/// Installs controlled tasks to test SST write-error cleanup independently of codec timing.
@@ -191,6 +207,7 @@ impl<W: AsyncFileWriter> ParallelArrowWriter<W> {
/// Starts encoding the buffered rows once the previous row group is written.
async fn encode_buffered(&mut self) -> Result<()> {
self.finish_encoding().await?;
check_cancelled(self.cancellation.as_ref())?;
let batches: Arc<[RecordBatch]> = mem::take(&mut self.buffered).into();
self.buffered_rows = 0;
@@ -209,8 +226,9 @@ impl<W: AsyncFileWriter> ParallelArrowWriter<W> {
.map(|fields| {
let batches = batches.clone();
let schema = self.arrow_schema.clone();
let cancellation = self.cancellation.clone();
self.runtime
.spawn(move || encode_fields(&schema, &batches, fields))
.spawn(move || encode_fields(&schema, &batches, fields, cancellation.as_ref()))
})
.collect();
Ok(())
@@ -301,17 +319,26 @@ fn assign_fields(
.collect()
}
fn check_cancelled(cancellation: Option<&CancellationToken>) -> Result<()> {
if cancellation.is_some_and(CancellationToken::is_cancelled) {
return Err(ParquetError::External(Box::new(Cancelled)));
}
Ok(())
}
/// Encodes `fields` of all `batches` and closes their column writers.
fn encode_fields(
schema: &SchemaRef,
batches: &[RecordBatch],
fields: Vec<FieldWriters>,
cancellation: Option<&CancellationToken>,
) -> Result<EncodedFields> {
let mut chunks = Vec::new();
let mut costs = Vec::with_capacity(fields.len());
for (field, mut writers) in fields {
let start = Instant::now();
for batch in batches {
check_cancelled(cancellation)?;
let leaves = compute_leaves(schema.field(field), batch.column(field))?;
if leaves.len() != writers.len() {
return Err(ParquetError::General(format!(
@@ -322,10 +349,12 @@ fn encode_fields(
)));
}
for (leaf, (_, writer)) in leaves.iter().zip(writers.iter_mut()) {
check_cancelled(cancellation)?;
writer.write(leaf)?;
}
}
for (leaf, writer) in writers {
check_cancelled(cancellation)?;
chunks.push((leaf, writer.close()?));
}
costs.push((field, start.elapsed()));
@@ -534,8 +563,12 @@ mod tests {
let running = runtime.spawn_blocking(move || {
started_tx.send(()).unwrap();
release_rx.blocking_recv().unwrap();
let _encoded =
encode_fields(&schema(), &running_batches, vec![(0, vec![(0, column)])])?;
let _encoded = encode_fields(
&schema(),
&running_batches,
vec![(0, vec![(0, column)])],
None,
)?;
Err(ParquetError::General("later encoding failure".to_string()))
});
// With one blocking thread, this also proves the first task has ended.
+657 -75
View File
@@ -15,15 +15,15 @@
//! Parquet writer.
use std::collections::HashMap;
use std::error::Error as StdError;
use std::future::Future;
use std::mem;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::task::{Context, Poll};
use std::time::Instant;
use bytes::Bytes;
use common_base::cancellation::Cancelled;
use common_telemetry::debug;
use common_time::Timestamp;
use datatypes::arrow::array::{
@@ -34,23 +34,24 @@ use datatypes::arrow::compute::{max, min};
use datatypes::arrow::datatypes::{DataType, SchemaRef, TimeUnit};
use datatypes::arrow::record_batch::RecordBatch;
use datatypes::extension::json::is_json2_extension_type;
use object_store::{FuturesAsyncWriter, ObjectStore};
use futures::future::BoxFuture;
use object_store::{ObjectStore, Writer};
use parquet::arrow::AsyncArrowWriter;
use parquet::arrow::async_writer::AsyncFileWriter;
use parquet::basic::{Compression, Encoding, ZstdLevel};
use parquet::errors::ParquetError;
use parquet::file::metadata::{FileMetaData, KeyValue, ParquetMetaData};
use parquet::file::properties::WriterProperties;
use smallvec::smallvec;
use snafu::{OptionExt, ResultExt, ensure};
use snafu::{IntoError, OptionExt, ResultExt, ensure};
use store_api::metadata::RegionMetadataRef;
use store_api::storage::{FileId, SequenceNumber};
use tokio::io::AsyncWrite;
use tokio_util::compat::{Compat, FuturesAsyncWriteCompatExt};
use tokio_util::sync::CancellationToken;
use crate::access_layer::{FilePathProvider, Metrics, SstInfoArray, TempFileCleaner};
use crate::config::{IndexBuildMode, IndexConfig};
use crate::error::{
InvalidMetadataSnafu, InvalidRecordBatchSnafu, OpenDalSnafu, Result, UnexpectedSnafu,
Error, InvalidMetadataSnafu, InvalidRecordBatchSnafu, OpenDalSnafu, Result, UnexpectedSnafu,
WriteParquetSnafu,
};
use crate::read::FlatSource;
@@ -65,6 +66,7 @@ use crate::sst::parquet::{
COLUMN_INDEX_TRUNCATE_LENGTH, PARQUET_METADATA_KEY, SstInfo, WriteOptions,
apply_column_compression, apply_float_field_encoding, apply_internal_column_encoding,
};
use crate::sst::upload::{self, UploadError};
use crate::sst::{
DEFAULT_WRITE_CONCURRENCY, FlatSchemaOptions, SeriesEstimator, maybe_wrap_schema,
};
@@ -179,16 +181,17 @@ impl<W: AsyncFileWriter> SstArrowWriter<W> {
}
}
async fn close(self) -> parquet::errors::Result<ParquetMetaData> {
async fn finish(&mut self) -> parquet::errors::Result<ParquetMetaData> {
match self {
SstArrowWriter::Serial(writer) => writer.close().await,
SstArrowWriter::Parallel(writer) => writer.close().await,
SstArrowWriter::Serial(writer) => writer.finish().await,
SstArrowWriter::Parallel(writer) => writer.finish().await,
}
}
async fn abort(self) {
if let SstArrowWriter::Parallel(writer) = self {
writer.abort().await;
async fn abort(self) -> W {
match self {
SstArrowWriter::Serial(writer) => writer.into_inner(),
SstArrowWriter::Parallel(writer) => writer.abort().await,
}
}
}
@@ -248,28 +251,105 @@ pub struct ParquetWriter<'a, F: WriterFactory, I: IndexerBuilder, P: FilePathPro
/// Write metrics
metrics: &'a mut Metrics,
write_guard: Option<Arc<dyn SstWriteGuard>>,
cancellation: Option<CancellationToken>,
memory: WriteMemory,
}
/// Registers durable ownership before either an SST or its index is published to storage.
/// Registers output ownership before either an SST or its index is created in storage.
#[async_trait::async_trait]
pub trait SstWriteGuard: Send + Sync {
async fn before_create(&self, file_id: RegionFileId) -> Result<()>;
}
/// Protection and resource limits travel together for table-compaction output.
/// Optional ownership, cancellation, and resource limits for an SST write.
#[derive(Default)]
pub(crate) struct SstWriteContext {
pub(crate) guard: Arc<dyn SstWriteGuard>,
pub(crate) limits: SstWriteLimits,
/// Registers output ownership before creating files, including cached files.
pub(crate) guard: Option<Arc<dyn SstWriteGuard>>,
/// Resource bounds enforced by the direct writer, with one upload at a time.
/// The write cache does not enforce these limits, so bounded writes bypass it.
pub(crate) limits: Option<SstWriteLimits>,
/// Signals cancellation; the writer still drains encoders and aborts unfinished output.
pub(crate) cancellation: Option<CancellationToken>,
}
pub trait WriterFactory {
type Writer: AsyncWrite + Send + Unpin;
type Writer: AsyncFileWriter;
/// Opens output with the same cancellation signal as its encoders.
fn create(
&mut self,
file_path: &str,
write_buffer_size: usize,
cancellation: Option<&CancellationToken>,
) -> impl Future<Output = Result<Self::Writer>>;
/// Aborts unfinished output after all encoders have stopped using it.
fn abort(writer: Self::Writer) -> impl Future<Output = Result<()>>;
}
/// Bridges an OpenDAL [`Writer`] with Parquet's [`AsyncFileWriter`] and tracks
/// the number of bytes successfully submitted to the object store.
pub struct ObjectStoreWriter {
inner: Writer,
output_bytes: u64,
write_buffer_size: usize,
cancellation: Option<CancellationToken>,
}
impl ObjectStoreWriter {
pub(crate) fn new(
inner: Writer,
write_buffer_size: usize,
cancellation: Option<CancellationToken>,
) -> Self {
Self {
inner,
output_bytes: 0,
write_buffer_size,
cancellation,
}
}
pub(crate) fn output_bytes(&self) -> u64 {
self.output_bytes
}
pub(crate) fn into_inner(self) -> Writer {
self.inner
}
}
impl AsyncFileWriter for ObjectStoreWriter {
fn write(&mut self, bytes: Bytes) -> BoxFuture<'_, parquet::errors::Result<()>> {
Box::pin(async move {
let written = upload::write(
&mut self.inner,
bytes.into(),
self.write_buffer_size,
self.cancellation.as_ref(),
)
.await
.map_err(|error| match error {
UploadError::Cancelled => ParquetError::External(Box::new(Cancelled)),
UploadError::Read { source } | UploadError::Write { source } => {
ParquetError::External(Box::new(source))
}
})?;
self.output_bytes += written;
Ok(())
})
}
fn complete(&mut self) -> BoxFuture<'_, parquet::errors::Result<()>> {
Box::pin(async move {
self.inner
.close()
.await
.map(|_| ())
.map_err(|error| ParquetError::External(Box::new(error)))
})
}
}
pub struct ObjectStoreWriterFactory {
@@ -278,17 +358,26 @@ pub struct ObjectStoreWriterFactory {
}
impl WriterFactory for ObjectStoreWriterFactory {
type Writer = Compat<FuturesAsyncWriter>;
type Writer = ObjectStoreWriter;
async fn create(&mut self, file_path: &str, write_buffer_size: usize) -> Result<Self::Writer> {
async fn create(
&mut self,
file_path: &str,
write_buffer_size: usize,
cancellation: Option<&CancellationToken>,
) -> Result<Self::Writer> {
self.object_store
.writer_with(file_path)
.chunk(write_buffer_size)
.concurrent(self.concurrency)
.await
.map(|v| v.into_futures_async_write().compat_write())
.map(|writer| ObjectStoreWriter::new(writer, write_buffer_size, cancellation.cloned()))
.context(OpenDalSnafu)
}
async fn abort(writer: Self::Writer) -> Result<()> {
writer.into_inner().abort().await.context(OpenDalSnafu)
}
}
impl<'a, I, P> ParquetWriter<'a, ObjectStoreWriterFactory, I, P>
@@ -361,9 +450,26 @@ where
file_cleaner: None,
metrics,
write_guard: None,
cancellation: None,
}
}
pub(crate) fn with_cancellation(mut self, cancellation: Option<CancellationToken>) -> Self {
self.cancellation = cancellation;
self
}
fn check_cancelled(&self) -> Result<()> {
ensure!(
!self
.cancellation
.as_ref()
.is_some_and(|handle| handle.is_cancelled()),
crate::error::SstWriteCancelledSnafu
);
Ok(())
}
pub(crate) fn with_write_guard(mut self, guard: Option<Arc<dyn SstWriteGuard>>) -> Self {
self.write_guard = guard;
self
@@ -409,8 +515,9 @@ where
ssts: &mut SstInfoArray,
stats: &mut SourceStats,
) -> Result<()> {
self.check_cancelled()?;
// maybe_init_writer will re-create a new file.
if let Some(mut current_writer) = mem::take(&mut self.writer) {
if self.writer.is_some() {
let mut stats = mem::take(stats);
// At least one row has been written.
assert!(stats.num_rows > 0);
@@ -436,9 +543,28 @@ where
);
}
}
current_writer.flush().await.context(WriteParquetSnafu)?;
let parquet_metadata = current_writer.close().await.context(WriteParquetSnafu)?;
self.check_cancelled()?;
self.writer
.as_mut()
.context(UnexpectedSnafu {
reason: "Missing SST writer during finalization",
})?
.flush()
.await
.map_err(map_write_error)?;
self.check_cancelled()?;
// Keep the upload owned until finalization succeeds, so an error
// still reaches the common abort path.
let parquet_metadata = self
.writer
.as_mut()
.context(UnexpectedSnafu {
reason: "Missing SST writer during finalization",
})?
.finish()
.await
.map_err(map_write_error)?;
self.writer.take();
let file_size = self.bytes_written.load(Ordering::Relaxed) as u64;
if self.memory.limits.is_some() {
self.memory.completed = self.memory.completed.saturating_add(
@@ -517,6 +643,7 @@ where
);
let res = self.write_all_flat_inner(source, &converter, opts).await;
if res.is_err() {
self.abort_current_indexer().await;
self.abort_current_file().await;
}
res
@@ -542,6 +669,7 @@ where
};
let res = self.write_all_flat_inner(source, &converter, opts).await;
if res.is_err() {
self.abort_current_indexer().await;
self.abort_current_file().await;
}
res
@@ -558,7 +686,17 @@ where
loop {
let start = Instant::now();
let batch = match source.next_batch().await {
self.check_cancelled()?;
let next = if let Some(cancellation) = &self.cancellation {
tokio::select! {
biased;
_ = cancellation.cancelled() => return crate::error::SstWriteCancelledSnafu.fail(),
next = source.next_batch() => next,
}
} else {
source.next_batch().await
};
let batch = match next {
Ok(Some(batch)) => batch,
Ok(None) => break,
Err(e) => {
@@ -622,15 +760,18 @@ where
stats: &mut SourceStats,
) -> Result<()> {
let result = async {
self.check_cancelled()?;
let arrow_batch = converter.convert_batch(batch)?;
self.check_cancelled()?;
let start = Instant::now();
self.maybe_init_writer(arrow_batch.schema_ref(), opts)
.await?
.write(&arrow_batch)
.await
.context(WriteParquetSnafu)?;
.map_err(map_write_error)?;
self.metrics.write_batch += start.elapsed();
self.check_cancelled()?;
stats.update_flat(batch)?;
if matches!(self.index_config.build_mode, IndexBuildMode::Sync) {
let start = Instant::now();
@@ -664,7 +805,10 @@ where
/// Joins encoders before dropping the writer and deleting its failed output.
async fn abort_current_file(&mut self) {
if let Some(writer) = self.writer.take() {
writer.abort().await;
let output = writer.abort().await;
if let Err(error) = F::abort(output.inner).await {
common_telemetry::warn!(error; "Failed to abort SST writer, file: {}", self.current_file);
}
}
if let Some(cleaner) = &self.file_cleaner {
cleaner.clean_by_file_id(self.current_file).await;
@@ -672,7 +816,7 @@ where
}
async fn abort_current_indexer(&mut self) {
if let Some(indexer) = &mut self.current_indexer {
if let Some(mut indexer) = self.current_indexer.take() {
indexer.abort().await;
}
}
@@ -682,6 +826,7 @@ where
schema: &SchemaRef,
opts: &WriteOptions,
) -> Result<&mut SstArrowWriter<SizeAwareWriter<F::Writer>>> {
self.check_cancelled()?;
if let Some(ref mut w) = self.writer {
Ok(w)
} else {
@@ -719,7 +864,11 @@ where
}
let writer = SizeAwareWriter::new(
self.writer_factory
.create(&sst_file_path, opts.write_buffer_size.as_bytes() as usize)
.create(
&sst_file_path,
opts.write_buffer_size.as_bytes() as usize,
self.cancellation.as_ref(),
)
.await?,
self.bytes_written.clone(),
);
@@ -736,12 +885,13 @@ where
opts.encode_parallelism,
opts.encode_runtime,
)
.context(WriteParquetSnafu)?,
.map_err(map_write_error)?
.with_cancellation(self.cancellation.clone()),
)
} else {
SstArrowWriter::Serial(
AsyncArrowWriter::try_new(writer, schema, Some(writer_props))
.context(WriteParquetSnafu)?,
.map_err(map_write_error)?,
)
};
self.writer = Some(arrow_writer);
@@ -762,6 +912,19 @@ where
}
}
/// Keeps cancellation inside the SST write error contract, including nested encoder errors.
fn map_write_error(error: ParquetError) -> Error {
let cancelled = std::iter::successors(Some(&error as &(dyn StdError + 'static)), |source| {
(*source).source()
})
.any(|source| source.is::<Cancelled>());
if cancelled {
crate::error::SstWriteCancelledSnafu.build()
} else {
WriteParquetSnafu.into_error(error)
}
}
#[derive(Default)]
struct SourceStats {
/// Number of rows fetched.
@@ -875,47 +1038,34 @@ impl<W> SizeAwareWriter<W> {
}
}
impl<W> AsyncWrite for SizeAwareWriter<W>
where
W: AsyncWrite + Unpin,
{
fn poll_write(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<std::result::Result<usize, std::io::Error>> {
let this = self.as_mut().get_mut();
match Pin::new(&mut this.inner).poll_write(cx, buf) {
Poll::Ready(Ok(bytes_written)) => {
this.size.fetch_add(bytes_written, Ordering::Relaxed);
Poll::Ready(Ok(bytes_written))
}
other => other,
}
impl<W: AsyncFileWriter> AsyncFileWriter for SizeAwareWriter<W> {
fn write(&mut self, bytes: Bytes) -> BoxFuture<'_, parquet::errors::Result<()>> {
Box::pin(async move {
let len = bytes.len();
self.inner.write(bytes).await?;
self.size.fetch_add(len, Ordering::Relaxed);
Ok(())
})
}
fn poll_flush(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<std::result::Result<(), std::io::Error>> {
Pin::new(&mut self.inner).poll_flush(cx)
}
fn poll_shutdown(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<std::result::Result<(), std::io::Error>> {
Pin::new(&mut self.inner).poll_shutdown(cx)
fn complete(&mut self) -> BoxFuture<'_, parquet::errors::Result<()>> {
self.inner.complete()
}
}
#[cfg(test)]
mod tests {
use std::sync::Mutex;
use std::sync::atomic::AtomicBool;
use std::time::Duration;
use common_base::readable_size::ReadableSize;
use common_runtime::runtime::{BuilderBuild, RuntimeTrait};
use object_store::layers::mock::{
Metadata, MockLayerBuilder, ObservedWriter, WriterObservations, oio,
};
use object_store::services::Memory;
use object_store::{Buffer, Error as ObjectStoreError, ErrorKind, Result as ObjectStoreResult};
use parquet::arrow::ArrowWriter;
use parquet::errors::ParquetError;
use store_api::region_request::PathType;
@@ -925,20 +1075,445 @@ mod tests {
use super::*;
use crate::access_layer::{RegionFilePathFactory, WriteType};
use crate::compaction::memory_manager::new_compaction_memory_manager;
use crate::error::Error;
use crate::sst::FormatType;
use crate::sst::parquet::tests::NoopIndexBuilder;
use crate::test_util::TestEnv;
use crate::test_util::sst_util::{new_record_batch_by_range, sst_region_metadata};
struct GatedObjectStoreWriter {
inner: oio::Writer,
started: Option<oneshot::Sender<()>>,
release: Option<oneshot::Receiver<()>>,
completed: Arc<AtomicUsize>,
write_error: Option<ObjectStoreError>,
}
impl oio::Write for GatedObjectStoreWriter {
async fn write(&mut self, bytes: Buffer) -> ObjectStoreResult<()> {
if let Some(started) = self.started.take() {
started.send(()).unwrap();
self.release.take().unwrap().await.unwrap();
}
if let Some(error) = self.write_error.take() {
return Err(error);
}
self.inner.write(bytes).await?;
self.completed.fetch_add(1, Ordering::Relaxed);
Ok(())
}
async fn close(&mut self) -> ObjectStoreResult<Metadata> {
self.inner.close().await
}
async fn abort(&mut self) -> ObjectStoreResult<()> {
self.inner.abort().await
}
}
#[rstest::rstest]
#[tokio::test]
async fn test_object_store_writer_cancellation_between_chunks(
#[values(false, true)] observe_cancellation: bool,
) {
let cancellation = CancellationToken::new();
let observations = Arc::new(Mutex::new(WriterObservations::default()));
let completed = Arc::new(AtomicUsize::new(0));
let (started_tx, started_rx) = oneshot::channel();
let (release_tx, release_rx) = oneshot::channel();
let gate = Mutex::new(Some((started_tx, release_rx)));
let observed = observations.clone();
let completed_writes = completed.clone();
let layer = MockLayerBuilder::default()
.writer_factory(Arc::new(move |_, _, inner| {
let (started, release) = gate.lock().unwrap().take().unwrap();
Box::new(ObservedWriter::new(
Box::new(GatedObjectStoreWriter {
inner,
started: Some(started),
release: Some(release),
completed: completed_writes.clone(),
write_error: None,
}),
observed.clone(),
))
}))
.build()
.unwrap();
let store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
let inner = store
.writer_with("output.parquet")
.chunk(4)
.concurrent(1)
.await
.unwrap();
let mut writer =
ObjectStoreWriter::new(inner, 4, observe_cancellation.then(|| cancellation.clone()));
let data = Bytes::from(vec![42; 32]);
let mut write = Box::pin(writer.write(data.clone()));
assert!(futures::poll!(write.as_mut()).is_pending());
timeout(Duration::from_secs(5), started_rx)
.await
.unwrap()
.unwrap();
cancellation.cancel();
// A started storage write retains its owner until it completes.
assert!(futures::poll!(write.as_mut()).is_pending());
assert_eq!(0, completed.load(Ordering::Relaxed));
release_tx.send(()).unwrap();
let result = timeout(Duration::from_secs(5), write).await.unwrap();
{
let observed = observations.lock().unwrap();
assert_eq!(0, observed.abort_calls);
assert_eq!(0, observed.close_calls);
}
if observe_cancellation {
assert!(
matches!(result, Err(ParquetError::External(ref source)) if source.is::<Cancelled>())
);
assert_eq!(1, completed.load(Ordering::Relaxed));
assert_eq!(vec![4], observations.lock().unwrap().write_sizes);
// The caller owns abort; the adapter only reports cancellation.
writer.into_inner().abort().await.unwrap();
assert_eq!(1, observations.lock().unwrap().abort_calls);
assert!(!store.exists("output.parquet").await.unwrap());
} else {
result.unwrap();
writer.complete().await.unwrap();
assert_eq!(data, store.read("output.parquet").await.unwrap().to_bytes());
let observed = observations.lock().unwrap();
assert_eq!(1, observed.close_calls);
assert_eq!(0, observed.abort_calls);
assert_eq!(vec![4; 8], observed.write_sizes);
}
}
#[rstest::rstest]
#[tokio::test]
async fn test_parquet_write_error_aborts_upload(
#[values(1, 2)] parallelism: usize,
#[values(false, true)] cancel_during_write: bool,
) {
let cancellation = CancellationToken::new();
let observations = Arc::new(Mutex::new(WriterObservations::default()));
let completed = Arc::new(AtomicUsize::new(0));
let (started_tx, started_rx) = oneshot::channel();
let (release_tx, release_rx) = oneshot::channel();
let gate = Mutex::new(Some((started_tx, release_rx)));
let observed = observations.clone();
let completed_writes = completed.clone();
let layer = MockLayerBuilder::default()
.writer_factory(Arc::new(move |_, _, inner| {
let (started, release) = gate.lock().unwrap().take().unwrap();
Box::new(ObservedWriter::new(
Box::new(GatedObjectStoreWriter {
inner,
started: Some(started),
release: Some(release),
completed: completed_writes.clone(),
write_error: (!cancel_during_write).then(|| {
ObjectStoreError::new(
ErrorKind::PermissionDenied,
"injected upload failure",
)
}),
}),
observed.clone(),
))
}))
.build()
.unwrap();
let store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
let metadata = Arc::new(sst_region_metadata());
let mut metrics = Metrics::new(WriteType::Compaction);
let mut writer = ParquetWriter::new_with_object_store(
store,
metadata,
IndexConfig::default(),
NoopIndexBuilder,
RegionFilePathFactory::new(String::new(), PathType::Bare),
&mut metrics,
)
.await
.with_write_concurrency(1)
.with_cancellation(Some(cancellation.clone()));
let batch = new_record_batch_by_range(&["a", "b"], 0, 8);
let opts = WriteOptions {
// A partial group reaches the upload through finish_current_file.
row_group_size: batch.num_rows() + 1,
write_buffer_size: ReadableSize(1),
encode_parallelism: parallelism,
..Default::default()
};
let source_finished = Arc::new(AtomicBool::new(false));
let finished = source_finished.clone();
let source = FlatSource::new_stream(
batch.schema(),
Box::pin(async_stream::try_stream! {
yield batch;
finished.store(true, Ordering::Relaxed);
}),
);
let mut write = Box::pin(writer.write_all_flat(source, None, &opts));
tokio::select! {
started = started_rx => started.unwrap(),
result = &mut write => panic!("writer returned before the upload gate: {result:?}"),
_ = tokio::time::sleep(Duration::from_secs(5)) => panic!("writer did not reach the upload gate"),
}
assert!(source_finished.load(Ordering::Relaxed));
if cancel_during_write {
cancellation.cancel();
}
assert!(futures::poll!(write.as_mut()).is_pending());
assert_eq!(0, observations.lock().unwrap().abort_calls);
release_tx.send(()).unwrap();
let error = timeout(Duration::from_secs(5), write)
.await
.unwrap()
.unwrap_err();
if cancel_during_write {
assert!(
matches!(error, Error::SstWriteCancelled { .. }),
"{error:?}"
);
} else {
assert!(matches!(error, Error::WriteParquet { .. }), "{error:?}");
let source =
std::iter::successors(Some(&error as &(dyn StdError + 'static)), |error| {
(*error).source()
})
.find_map(|error| error.downcast_ref::<ObjectStoreError>())
.unwrap();
assert_eq!(ErrorKind::PermissionDenied, source.kind());
assert!(source.to_string().contains("injected upload failure"));
}
assert_eq!(
usize::from(cancel_during_write),
completed.load(Ordering::Relaxed)
);
let observed = observations.lock().unwrap();
assert_eq!(vec![1], observed.write_sizes);
assert_eq!(1, observed.abort_calls);
assert_eq!(0, observed.close_calls);
assert!(writer.writer.is_none());
assert!(writer.current_indexer.is_none());
}
#[rstest::rstest]
#[case::flat(FormatType::Flat)]
#[case::primary_key(FormatType::PrimaryKey)]
#[tokio::test]
async fn test_source_error_waits_for_encoding_cleanup(#[case] format: FormatType) {
async fn test_encoding_error_classification(
#[case] format: FormatType,
#[values(false, true)] finish_file: bool,
#[values(false, true)] cancelled: bool,
) {
let mut env = TestEnv::new().await;
let metadata = Arc::new(sst_region_metadata());
let mut metrics = Metrics::new(WriteType::Compaction);
let mut writer = ParquetWriter::new_with_object_store(
env.init_object_store_manager(),
metadata.clone(),
IndexConfig::default(),
NoopIndexBuilder,
RegionFilePathFactory::new(String::new(), PathType::Bare),
&mut metrics,
)
.await;
let batch = new_record_batch_by_range(&["a", "b"], 0, 4);
let converter = match format {
FormatType::Flat => FlatBatchConverter::Flat(FlatWriteFormat::new(
metadata.clone(),
&FlatSchemaOptions::default(),
)),
FormatType::PrimaryKey => FlatBatchConverter::PrimaryKey {
format: PrimaryKeyWriteFormat::new(metadata.clone()),
num_fields: metadata.field_columns().count(),
},
};
let schema = converter.convert_batch(&batch).unwrap().schema();
let opts = WriteOptions {
// A full row group joins the previous task in append_flat_batch;
// a partial group joins it in finish_current_file instead.
row_group_size: batch.num_rows() + usize::from(finish_file),
encode_parallelism: 2,
..Default::default()
};
let SstArrowWriter::Parallel(parallel) =
writer.maybe_init_writer(&schema, &opts).await.unwrap()
else {
panic!("expected parallel encoding");
};
let task = common_runtime::spawn_blocking_compact(move || {
let source: Box<dyn std::error::Error + Send + Sync> = if cancelled {
Box::new(Cancelled)
} else {
Box::new(std::io::Error::other("encoding failed"))
};
Err(ParquetError::External(source))
});
// No SST-level cancellation handle: only the encoder's error can report cancellation.
parallel.set_encoding_tasks_for_test(vec![task]);
let source =
FlatSource::new_stream(batch.schema(), Box::pin(futures::stream::iter([Ok(batch)])));
let error = match format {
FormatType::Flat => writer.write_all_flat(source, None, &opts).await,
FormatType::PrimaryKey => {
writer
.write_all_flat_as_primary_key(source, None, &opts)
.await
}
}
.unwrap_err();
if cancelled {
assert!(matches!(error, Error::SstWriteCancelled { .. }));
} else {
assert!(
matches!(error, Error::WriteParquet { error: ParquetError::External(ref source), .. }
if source.downcast_ref::<std::io::Error>().is_some_and(|error| error.to_string() == "encoding failed"))
);
}
assert!(writer.writer.is_none());
assert!(writer.current_indexer.is_none());
assert_eq!(0, writer.bytes_written.load(Ordering::Relaxed));
}
#[rstest::rstest]
#[case::flat(FormatType::Flat)]
#[case::primary_key(FormatType::PrimaryKey)]
#[tokio::test]
async fn test_cancel_pending_source(
#[case] format: FormatType,
#[values(1, 2)] parallelism: usize,
) {
let mut env = TestEnv::new().await;
let metadata = Arc::new(sst_region_metadata());
let cancellation = CancellationToken::new();
let mut metrics = Metrics::new(WriteType::Compaction);
let mut writer = ParquetWriter::new_with_object_store(
env.init_object_store_manager(),
metadata,
IndexConfig::default(),
NoopIndexBuilder,
RegionFilePathFactory::new(String::new(), PathType::Bare),
&mut metrics,
)
.await
.with_cancellation(Some(cancellation.clone()));
let batch = new_record_batch_by_range(&["a", "b"], 0, 4);
let schema = batch.schema();
let source = FlatSource::new_stream(
schema,
Box::pin(async_stream::try_stream! {
yield batch;
futures::future::pending::<()>().await;
}),
);
let opts = WriteOptions {
row_group_size: 8,
encode_parallelism: parallelism,
..Default::default()
};
let mut write = Box::pin(async {
match format {
FormatType::Flat => writer.write_all_flat(source, None, &opts).await,
FormatType::PrimaryKey => {
writer
.write_all_flat_as_primary_key(source, None, &opts)
.await
}
}
});
assert!(futures::poll!(write.as_mut()).is_pending());
cancellation.cancel();
let error = timeout(Duration::from_secs(5), write)
.await
.unwrap()
.unwrap_err();
assert!(matches!(error, Error::SstWriteCancelled { .. }));
assert!(writer.writer.is_none());
}
/// Controlled comparison: identical batches and cancellation point, changing
/// only whether the writer observes cancellation.
#[rstest::rstest]
#[case::flat(FormatType::Flat)]
#[case::primary_key(FormatType::PrimaryKey)]
#[tokio::test]
async fn test_batch_cancellation_ablation(
#[case] format: FormatType,
#[values(1, 2)] parallelism: usize,
) {
for enabled in [false, true] {
let mut env = TestEnv::new().await;
let cancellation = CancellationToken::new();
let mut metrics = Metrics::new(WriteType::Compaction);
let mut writer = ParquetWriter::new_with_object_store(
env.init_object_store_manager(),
Arc::new(sst_region_metadata()),
IndexConfig::default(),
NoopIndexBuilder,
RegionFilePathFactory::new(String::new(), PathType::Bare),
&mut metrics,
)
.await
.with_cancellation(enabled.then(|| cancellation.clone()));
let batch = new_record_batch_by_range(&["a", "b"], 0, 4);
let consumed = Arc::new(AtomicUsize::new(0));
let observed = consumed.clone();
let source = FlatSource::new_stream(
batch.schema(),
Box::pin(async_stream::try_stream! {
for i in 0..32 {
if i == 1 { cancellation.cancel(); }
observed.fetch_add(1, Ordering::Relaxed);
yield new_record_batch_by_range(&["a", "b"], i * 8, i * 8 + 8);
}
}),
);
let opts = WriteOptions {
row_group_size: 8,
encode_parallelism: parallelism,
..Default::default()
};
let result = match format {
FormatType::Flat => writer.write_all_flat(source, None, &opts).await,
FormatType::PrimaryKey => {
writer
.write_all_flat_as_primary_key(source, None, &opts)
.await
}
};
if enabled {
assert!(result.is_err());
assert_eq!(2, consumed.load(Ordering::Relaxed));
assert!(writer.writer.is_none());
} else {
assert_eq!(
256,
result
.unwrap()
.iter()
.map(|info| info.num_rows)
.sum::<usize>()
);
assert_eq!(32, consumed.load(Ordering::Relaxed));
}
}
}
#[rstest::rstest]
#[case::flat(FormatType::Flat)]
#[case::primary_key(FormatType::PrimaryKey)]
#[tokio::test]
async fn test_source_error_waits_for_encoding_cleanup(
#[case] format: FormatType,
#[values(false, true)] cancel_source: bool,
) {
let mut env = TestEnv::new().await;
let metadata = Arc::new(sst_region_metadata());
let cancellation = CancellationToken::new();
let mut metrics = Metrics::new(WriteType::Compaction);
let mut writer = ParquetWriter::new_with_object_store(
env.init_object_store_manager(),
metadata.clone(),
@@ -947,7 +1522,8 @@ mod tests {
RegionFilePathFactory::new("".to_string(), PathType::Bare),
&mut metrics,
)
.await;
.await
.with_cancellation(Some(cancellation.clone()));
let opts = WriteOptions {
row_group_size: 8,
encode_parallelism: 2,
@@ -1010,13 +1586,11 @@ mod tests {
parallel.set_encoding_tasks_for_test(vec![running, queued]);
let source = FlatSource::new_stream(
batch.schema(),
Box::pin(futures::stream::iter(vec![
Ok(batch),
InvalidRecordBatchSnafu {
reason: "source read failed",
}
.fail(),
])),
Box::pin(async_stream::try_stream! {
yield batch;
if cancel_source { futures::future::pending::<()>().await; }
Err(InvalidRecordBatchSnafu { reason: "source read failed" }.build())?;
}),
);
let memory = new_compaction_memory_manager(1024 * 1024);
let budget = memory.try_acquire(memory.limit_bytes()).unwrap();
@@ -1034,6 +1608,10 @@ mod tests {
// The next row group is buffered, then the source fails. No codec timing
// or I/O wait can make the old indexer-only cleanup pass this assertion.
assert!(futures::poll!(write.as_mut()).is_pending());
if cancel_source {
cancellation.cancel();
assert!(futures::poll!(write.as_mut()).is_pending());
}
assert!(memory.try_acquire(memory.limit_bytes()).is_none());
assert!(retained_field.upgrade().is_some());
release_tx.send(()).unwrap();
@@ -1041,9 +1619,13 @@ mod tests {
.await
.unwrap()
.unwrap_err();
assert!(
matches!(error, Error::InvalidRecordBatch { reason, .. } if reason == "source read failed")
);
if cancel_source {
assert!(matches!(error, Error::SstWriteCancelled { .. }));
} else {
assert!(
matches!(error, Error::InvalidRecordBatch { reason, .. } if reason == "source read failed")
);
}
assert!(!queued_started.load(Ordering::SeqCst));
assert!(retained_field.upgrade().is_none());
assert!(memory.try_acquire(memory.limit_bytes()).is_some());
+243
View File
@@ -0,0 +1,243 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Cooperative byte transfer for SST uploads. Callers own close, abort, and file cleanup.
use futures::{Stream, StreamExt};
use object_store::{Buffer, Error as ObjectStoreError, Result as ObjectStoreResult, Writer};
use snafu::Snafu;
use tokio_util::sync::CancellationToken;
/// Preserves read/write attribution until the caller adds its file context.
#[derive(Debug, Snafu)]
pub(crate) enum UploadError {
#[snafu(display("Upload cancelled"))]
Cancelled,
#[snafu(display("Failed to read upload source: {source}"))]
Read { source: ObjectStoreError },
#[snafu(display("Failed to write upload: {source}"))]
Write { source: ObjectStoreError },
}
/// Copies chunks without dropping an in-flight write when cancellation arrives.
pub(crate) async fn copy(
reader: &mut (impl Stream<Item = ObjectStoreResult<Buffer>> + Unpin),
writer: &mut Writer,
write_buffer_size: usize,
cancellation: Option<&CancellationToken>,
) -> Result<u64, UploadError> {
let mut written = 0;
loop {
let next = if let Some(cancellation) = cancellation {
tokio::select! {
biased;
_ = cancellation.cancelled() => return Err(UploadError::Cancelled),
next = reader.next() => next,
}
} else {
reader.next().await
};
let Some(bytes) = next else {
return Ok(written);
};
let bytes = bytes.map_err(|source| UploadError::Read { source })?;
written += write(writer, bytes, write_buffer_size, cancellation).await?;
}
}
/// Writes bounded, zero-copy slices, awaiting each started write to completion.
pub(crate) async fn write(
writer: &mut Writer,
mut bytes: Buffer,
write_buffer_size: usize,
cancellation: Option<&CancellationToken>,
) -> Result<u64, UploadError> {
let len = bytes.len() as u64;
// Keep each slice nonempty even when the configured size is zero.
let write_buffer_size = write_buffer_size.max(1);
while !bytes.is_empty() {
if cancellation.is_some_and(CancellationToken::is_cancelled) {
return Err(UploadError::Cancelled);
}
let chunk = bytes.split_to(bytes.len().min(write_buffer_size));
writer
.write(chunk)
.await
.map_err(|source| UploadError::Write { source })?;
}
if cancellation.is_some_and(CancellationToken::is_cancelled) {
return Err(UploadError::Cancelled);
}
Ok(len)
}
#[cfg(test)]
mod tests {
use std::sync::{Arc, Mutex};
use std::time::Duration;
use bytes::Bytes;
use futures::stream;
use object_store::ObjectStore;
use object_store::layers::mock::{
Metadata, MockLayerBuilder, ObservedWriter, WriterObservations, oio,
};
use object_store::services::Memory;
use tokio::sync::Notify;
use tokio::time::timeout;
use super::*;
#[tokio::test]
async fn test_copy_cancels_pending_read() {
let store = ObjectStore::new(Memory::default()).unwrap();
let mut writer = store.writer("pending").await.unwrap();
let mut reader = stream::pending();
let cancellation = CancellationToken::new();
let mut copy = Box::pin(super::copy(
&mut reader,
&mut writer,
4,
Some(&cancellation),
));
assert!(futures::poll!(copy.as_mut()).is_pending());
cancellation.cancel();
let error = timeout(Duration::from_secs(5), copy)
.await
.unwrap()
.unwrap_err();
assert!(matches!(error, UploadError::Cancelled));
writer.abort().await.unwrap();
assert!(!store.exists("pending").await.unwrap());
}
struct GatedWriter {
inner: oio::Writer,
started: Arc<Notify>,
release: Arc<Notify>,
}
impl oio::Write for GatedWriter {
async fn write(&mut self, bytes: Buffer) -> ObjectStoreResult<()> {
self.started.notify_one();
self.release.notified().await;
self.inner.write(bytes).await
}
async fn close(&mut self) -> ObjectStoreResult<Metadata> {
self.inner.close().await
}
async fn abort(&mut self) -> ObjectStoreResult<()> {
self.inner.abort().await
}
}
#[tokio::test]
async fn test_copy_cancellation_waits_for_started_write() {
let started = Arc::new(Notify::new());
let release = Arc::new(Notify::new());
let observations = Arc::new(Mutex::new(WriterObservations::default()));
let observer = observations.clone();
let started_write = started.clone();
let release_write = release.clone();
let layer = MockLayerBuilder::default()
.writer_factory(Arc::new(move |_, _, inner| {
Box::new(ObservedWriter::new(
Box::new(GatedWriter {
inner,
started: started_write.clone(),
release: release_write.clone(),
}),
observer.clone(),
))
}))
.build()
.unwrap();
let store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
// Leave the OpenDAL writer unbuffered so the observer sees helper boundaries.
let mut writer = store.writer("gated").await.unwrap();
let mut reader = stream::iter([Ok(Buffer::from(Bytes::from_static(b"abcdefghijkl")))]);
let cancellation = CancellationToken::new();
let mut copy = Box::pin(super::copy(
&mut reader,
&mut writer,
4,
Some(&cancellation),
));
assert!(futures::poll!(copy.as_mut()).is_pending());
timeout(Duration::from_secs(5), started.notified())
.await
.unwrap();
cancellation.cancel();
assert!(futures::poll!(copy.as_mut()).is_pending());
release.notify_one();
let error = timeout(Duration::from_secs(5), copy)
.await
.unwrap()
.unwrap_err();
assert!(matches!(error, UploadError::Cancelled));
{
let observations = observations.lock().unwrap();
assert_eq!(vec![4], observations.write_sizes);
assert_eq!(0, observations.close_calls);
assert_eq!(0, observations.abort_calls);
}
writer.abort().await.unwrap();
assert_eq!(1, observations.lock().unwrap().abort_calls);
assert!(!store.exists("gated").await.unwrap());
}
#[rstest::rstest]
#[tokio::test]
async fn test_copy_preserves_contents(#[values(false, true)] cancellable: bool) {
let observations = Arc::new(Mutex::new(WriterObservations::default()));
let observer = observations.clone();
let layer = MockLayerBuilder::default()
.writer_factory(Arc::new(move |_, _, inner| {
Box::new(ObservedWriter::new(inner, observer.clone()))
}))
.build()
.unwrap();
let store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
let mut writer = store.writer("complete").await.unwrap();
let mut reader = stream::iter([
Ok(Buffer::from(Bytes::from_static(b"abcde"))),
Ok(Buffer::from(Bytes::from_static(b"fghijkl"))),
]);
let cancellation = CancellationToken::new();
let copied = super::copy(
&mut reader,
&mut writer,
4,
cancellable.then_some(&cancellation),
)
.await
.unwrap();
assert_eq!(12, copied);
{
let observations = observations.lock().unwrap();
assert_eq!(vec![4, 1, 4, 3], observations.write_sizes);
assert_eq!(0, observations.close_calls);
assert_eq!(0, observations.abort_calls);
}
writer.close().await.unwrap();
assert_eq!(1, observations.lock().unwrap().close_calls);
assert_eq!(
Bytes::from_static(b"abcdefghijkl"),
store.read("complete").await.unwrap().to_bytes()
);
}
}
+12 -7
View File
@@ -18,7 +18,6 @@ use std::num::NonZeroU64;
use std::sync::Arc;
use std::time::Duration;
use common_base::cancellation::CancellableFuture;
use common_telemetry::{error, info, warn};
use futures::TryStreamExt;
use snafu::ensure;
@@ -416,16 +415,22 @@ impl<S: LogStore> TailReplayTask<S> {
/// Awaits `future` unless the task is cancelled first.
async fn unless_cancelled<T>(&self, future: impl Future<Output = Result<T>>) -> Result<T> {
CancellableFuture::new(future, self.state.cancel_handle())
.await
.unwrap_or_else(|_| FlushCancelledSnafu.fail())
let cancellation = self.state.cancel_token();
tokio::select! {
biased;
_ = cancellation.cancelled() => FlushCancelledSnafu.fail(),
result = future => result,
}
}
/// Sleeps for `delay`. Returns false if the task is cancelled first.
async fn sleep(&self, delay: Duration) -> bool {
CancellableFuture::new(tokio::time::sleep(delay), self.state.cancel_handle())
.await
.is_ok()
self.unless_cancelled(async {
tokio::time::sleep(delay).await;
Ok(())
})
.await
.is_ok()
}
}