mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-21 18:24:07 +00:00
fix(mito2): cancel compaction merges at batch boundaries (#10411)
* fix(mito2): cancel compaction merges at batch boundaries Signed-off-by: WenyXu <wenymedia@gmail.com> * fix(mito2): preserve upload errors and expose merge cancellation Signed-off-by: WenyXu <wenymedia@gmail.com> * fix(mito2): normalize encoder cancellation at SST writer boundary Signed-off-by: WenyXu <wenymedia@gmail.com> * style(mito2): use imports for SST write types Signed-off-by: WenyXu <wenymedia@gmail.com> * fix(mito2): share cooperative upload cancellation with tokens Signed-off-by: WenyXu <wenymedia@gmail.com> * fix(mito2): classify nested upload cancellation errors Signed-off-by: WenyXu <wenymedia@gmail.com> --------- Signed-off-by: WenyXu <wenymedia@gmail.com>
This commit is contained in:
@@ -104,6 +104,8 @@ where
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
pub struct Cancelled;
|
||||
|
||||
impl std::error::Error for Cancelled {}
|
||||
|
||||
impl Display for Cancelled {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "Future has been cancelled")
|
||||
|
||||
+114
-16
@@ -51,7 +51,7 @@ use crate::sst::index::registry::IndexRegistry;
|
||||
use crate::sst::index::{IndexerBuilderImpl, build_index_from_parquet_bytes};
|
||||
use crate::sst::location::{self, region_dir_from_table_dir};
|
||||
use crate::sst::parquet::reader::ParquetReaderBuilder;
|
||||
use crate::sst::parquet::writer::ParquetWriter;
|
||||
use crate::sst::parquet::writer::{ParquetWriter, SstWriteContext};
|
||||
use crate::sst::parquet::{SstInfo, WriteOptions};
|
||||
use crate::sst::{DEFAULT_WRITE_CONCURRENCY, FormatType};
|
||||
|
||||
@@ -376,18 +376,13 @@ impl AccessLayer {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Writes an SST whose output is registered with `context.guard`.
|
||||
///
|
||||
/// The write cache is bypassed, upload concurrency is fixed to one, and
|
||||
/// the guard runs before either the SST or its index becomes visible.
|
||||
/// `context.limits` bounds the encoder and footer workspace.
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub(crate) async fn write_sst_with_guard(
|
||||
/// Writes SSTs with explicit ownership and cooperative cancellation.
|
||||
pub(crate) async fn write_sst_with_context(
|
||||
&self,
|
||||
request: SstWriteRequest,
|
||||
write_opts: &WriteOptions,
|
||||
metrics: &mut Metrics,
|
||||
context: crate::sst::parquet::writer::SstWriteContext,
|
||||
context: SstWriteContext,
|
||||
) -> Result<SstInfoArray> {
|
||||
self.write_sst_inner(request, write_opts, metrics, Some(context))
|
||||
.await
|
||||
@@ -398,10 +393,12 @@ impl AccessLayer {
|
||||
request: SstWriteRequest,
|
||||
write_opts: &WriteOptions,
|
||||
metrics: &mut Metrics,
|
||||
context: Option<crate::sst::parquet::writer::SstWriteContext>,
|
||||
context: Option<SstWriteContext>,
|
||||
) -> Result<SstInfoArray> {
|
||||
let limits = context.as_ref().map(|context| context.limits);
|
||||
let guard = context.map(|context| context.guard);
|
||||
let context = context.unwrap_or_default();
|
||||
let limits = context.limits;
|
||||
let guard = context.guard.clone();
|
||||
let cancellation = context.cancellation.clone();
|
||||
let op_type = request.op_type;
|
||||
let region_id = request.metadata.region_id;
|
||||
let region_metadata = request.metadata.clone();
|
||||
@@ -413,11 +410,11 @@ impl AccessLayer {
|
||||
};
|
||||
|
||||
let sst_info = if let Some(write_cache) = cache_manager.write_cache()
|
||||
&& guard.is_none()
|
||||
&& limits.is_none()
|
||||
{
|
||||
// Write to the write cache.
|
||||
write_cache
|
||||
.write_and_upload_sst(
|
||||
.write_and_upload_sst_with_context(
|
||||
request,
|
||||
SstUploadRequest {
|
||||
dest_path_provider: RegionFilePathFactory::new(
|
||||
@@ -428,10 +425,12 @@ impl AccessLayer {
|
||||
},
|
||||
write_opts,
|
||||
metrics,
|
||||
context,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
// Write cache is disabled, or bypassed by an SST write guard.
|
||||
// The write cache does not enforce SstWriteLimits, so bounded writes
|
||||
// bypass it.
|
||||
let store = self.object_store.clone();
|
||||
let path_provider = RegionFilePathFactory::new(self.table_dir.clone(), self.path_type);
|
||||
let puffin_factory = if limits.is_some() {
|
||||
@@ -469,12 +468,13 @@ impl AccessLayer {
|
||||
)
|
||||
.await
|
||||
.with_file_cleaner(cleaner)
|
||||
.with_write_concurrency(if guard.is_some() {
|
||||
.with_write_concurrency(if limits.is_some() {
|
||||
1
|
||||
} else {
|
||||
crate::sst::DEFAULT_WRITE_CONCURRENCY
|
||||
})
|
||||
.with_write_guard(guard)
|
||||
.with_cancellation(cancellation)
|
||||
.with_memory_limits(limits);
|
||||
match request.sst_write_format {
|
||||
FormatType::PrimaryKey => {
|
||||
@@ -1104,6 +1104,104 @@ mod tests {
|
||||
assert_eq!(1, warm.mem_cache_hit);
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[tokio::test]
|
||||
async fn test_cancelled_write_cleans_rolled_ssts(
|
||||
#[values(false, true)] enable_write_cache: bool,
|
||||
#[values(FormatType::Flat, FormatType::PrimaryKey)] format: FormatType,
|
||||
) {
|
||||
use crate::read::FlatSource;
|
||||
use crate::sst::file::UncommittedSsts;
|
||||
use crate::test_util::sst_util::{new_record_batch_by_range, sst_region_metadata};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
let mut env = TestEnv::new().await;
|
||||
let store = env.init_object_store_manager();
|
||||
let local_dir = create_temp_dir("cancelled-sst-cache");
|
||||
let local_store = new_fs_store(local_dir.path().to_str().unwrap());
|
||||
let write_cache = if enable_write_cache {
|
||||
Some(
|
||||
env.create_write_cache(local_store.clone(), ReadableSize::mb(10))
|
||||
.await,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let cache = Arc::new(CacheManager::builder().write_cache(write_cache).build());
|
||||
let layer = Arc::new(AccessLayer::new(
|
||||
"cancelled",
|
||||
PathType::Bare,
|
||||
store.clone(),
|
||||
env.get_puffin_manager(),
|
||||
env.get_intermediate_manager(),
|
||||
));
|
||||
let metadata = Arc::new(sst_region_metadata());
|
||||
let uncommitted =
|
||||
UncommittedSsts::new(metadata.region_id, layer.clone(), Some(cache.clone()));
|
||||
let cancellation = CancellationToken::new();
|
||||
let signal = cancellation.clone();
|
||||
let tracked = uncommitted.clone();
|
||||
let schema = new_record_batch_by_range(&["a", "b"], 0, 8).schema();
|
||||
let source = FlatSource::new_stream(
|
||||
schema,
|
||||
Box::pin(async_stream::try_stream! {
|
||||
for i in 0..64 {
|
||||
if tracked.num_tracked_files() >= 3 {
|
||||
signal.cancel();
|
||||
}
|
||||
let tag = format!("{i:04}");
|
||||
yield new_record_batch_by_range(&[&tag, "b"], 0, 4096);
|
||||
}
|
||||
}),
|
||||
);
|
||||
let result = layer
|
||||
.write_sst_with_context(
|
||||
SstWriteRequest {
|
||||
index_build_inputs: None,
|
||||
op_type: OperationType::Compact,
|
||||
metadata,
|
||||
source,
|
||||
cache_manager: cache,
|
||||
max_sequence: None,
|
||||
sst_write_format: format,
|
||||
preserve_row_sequence: false,
|
||||
index_options: Default::default(),
|
||||
index_config: Default::default(),
|
||||
inverted_index_config: Default::default(),
|
||||
fulltext_index_config: Default::default(),
|
||||
bloom_filter_index_config: Default::default(),
|
||||
},
|
||||
&WriteOptions {
|
||||
max_file_size: Some(1),
|
||||
row_group_size: 8,
|
||||
..Default::default()
|
||||
},
|
||||
&mut Metrics::new(WriteType::Compaction),
|
||||
SstWriteContext {
|
||||
guard: Some(Arc::new(uncommitted.clone())),
|
||||
cancellation: Some(cancellation),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(crate::error::Error::SstWriteCancelled { .. })
|
||||
));
|
||||
assert!(
|
||||
uncommitted.num_tracked_files() >= 2,
|
||||
"must cover files finalized before cancellation"
|
||||
);
|
||||
uncommitted.cleanup_for_test().await.unwrap();
|
||||
for object_store in [&store, &local_store] {
|
||||
let entries = object_store.list_with("/").recursive(true).await.unwrap();
|
||||
assert!(
|
||||
entries.iter().all(|entry| entry.metadata().is_dir()),
|
||||
"uncommitted files remain: {entries:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_new_fs_cache_store() {
|
||||
let root = common_test_util::temp_dir::create_temp_dir("fs-cache-store");
|
||||
|
||||
Vendored
+344
-27
@@ -19,11 +19,11 @@ use std::time::{Duration, Instant};
|
||||
|
||||
use common_base::readable_size::ReadableSize;
|
||||
use common_telemetry::{debug, info};
|
||||
use futures::AsyncWriteExt;
|
||||
use object_store::ObjectStore;
|
||||
use snafu::ResultExt;
|
||||
use snafu::{IntoError, ResultExt};
|
||||
use store_api::storage::RegionId;
|
||||
use tokio::sync::mpsc::{UnboundedSender, unbounded_channel};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::access_layer::{
|
||||
EncodedSstIndexBuildParams, FilePathProvider, Metrics, OperationType, RegionFilePathFactory,
|
||||
@@ -40,8 +40,9 @@ use crate::sst::index::intermediate::IntermediateManager;
|
||||
use crate::sst::index::puffin_manager::{PuffinManagerFactory, SstPuffinManager};
|
||||
use crate::sst::index::registry::IndexRegistry;
|
||||
use crate::sst::index::{IndexerBuilderImpl, build_index_from_parquet_bytes};
|
||||
use crate::sst::parquet::writer::ParquetWriter;
|
||||
use crate::sst::parquet::writer::{ParquetWriter, SstWriteContext};
|
||||
use crate::sst::parquet::{SstInfo, WriteOptions};
|
||||
use crate::sst::upload::{self, UploadError};
|
||||
use crate::sst::{DEFAULT_WRITE_BUFFER_SIZE, DEFAULT_WRITE_CONCURRENCY};
|
||||
|
||||
/// Wraps the remote object store used by write cache uploads.
|
||||
@@ -283,13 +284,14 @@ impl WriteCache {
|
||||
&self.intermediate_manager
|
||||
}
|
||||
|
||||
/// Writes SST to the cache and then uploads it to the remote object store.
|
||||
pub(crate) async fn write_and_upload_sst(
|
||||
/// Preserves write ownership and cancellation through local encoding and upload.
|
||||
pub(crate) async fn write_and_upload_sst_with_context(
|
||||
&self,
|
||||
write_request: SstWriteRequest,
|
||||
upload_request: SstUploadRequest,
|
||||
write_opts: &WriteOptions,
|
||||
metrics: &mut Metrics,
|
||||
context: SstWriteContext,
|
||||
) -> Result<SstInfoArray> {
|
||||
let region_id = write_request.metadata.region_id;
|
||||
let override_sequence = if write_request.preserve_row_sequence {
|
||||
@@ -328,7 +330,9 @@ impl WriteCache {
|
||||
metrics,
|
||||
)
|
||||
.await
|
||||
.with_file_cleaner(cleaner);
|
||||
.with_file_cleaner(cleaner)
|
||||
.with_write_guard(context.guard)
|
||||
.with_cancellation(context.cancellation.clone());
|
||||
|
||||
let sst_info = match write_request.sst_write_format {
|
||||
crate::sst::FormatType::PrimaryKey => {
|
||||
@@ -357,13 +361,21 @@ impl WriteCache {
|
||||
let op_type = write_request.op_type;
|
||||
let remote_store = &upload_request.remote_store;
|
||||
for sst in &sst_info {
|
||||
if context
|
||||
.cancellation
|
||||
.as_ref()
|
||||
.is_some_and(|handle| handle.is_cancelled())
|
||||
{
|
||||
err = Some(error::SstWriteCancelledSnafu.build());
|
||||
break;
|
||||
}
|
||||
let parquet_key = IndexKey::new(region_id, sst.file_id, FileType::Parquet);
|
||||
let parquet_path = upload_request
|
||||
.dest_path_provider
|
||||
.build_sst_file_path(RegionFileId::new(region_id, sst.file_id));
|
||||
let start = Instant::now();
|
||||
if let Err(e) = self
|
||||
.upload(
|
||||
.upload_with_cancellation(
|
||||
parquet_key,
|
||||
&parquet_path,
|
||||
remote_store,
|
||||
@@ -371,6 +383,7 @@ impl WriteCache {
|
||||
op_type,
|
||||
write_buffer_size: write_opts.write_buffer_size,
|
||||
},
|
||||
context.cancellation.as_ref(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -387,7 +400,7 @@ impl WriteCache {
|
||||
.build_index_file_path(RegionFileId::new(region_id, sst.file_id));
|
||||
let start = Instant::now();
|
||||
if let Err(e) = self
|
||||
.upload(
|
||||
.upload_with_cancellation(
|
||||
puffin_key,
|
||||
&puffin_path,
|
||||
remote_store,
|
||||
@@ -395,6 +408,7 @@ impl WriteCache {
|
||||
op_type,
|
||||
write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
|
||||
},
|
||||
context.cancellation.as_ref(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -467,10 +481,21 @@ impl WriteCache {
|
||||
upload_path: &str,
|
||||
remote_store: &ObjectStore,
|
||||
options: UploadOptions,
|
||||
) -> Result<()> {
|
||||
self.upload_with_cancellation(index_key, upload_path, remote_store, options, None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn upload_with_cancellation(
|
||||
&self,
|
||||
index_key: IndexKey,
|
||||
upload_path: &str,
|
||||
remote_store: &ObjectStore,
|
||||
options: UploadOptions,
|
||||
cancellation: Option<&CancellationToken>,
|
||||
) -> Result<()> {
|
||||
let region_id = index_key.region_id;
|
||||
let file_id = index_key.file_id;
|
||||
let file_type = index_key.file_type;
|
||||
let cache_path = self.file_cache.cache_file_path(index_key);
|
||||
|
||||
let start = Instant::now();
|
||||
@@ -480,13 +505,14 @@ impl WriteCache {
|
||||
.stat(&cache_path)
|
||||
.await
|
||||
.context(error::OpenDalSnafu)?;
|
||||
let reader = self
|
||||
let mut reader = self
|
||||
.file_cache
|
||||
.local_store()
|
||||
.reader(&cache_path)
|
||||
.reader_with(&cache_path)
|
||||
.chunk(options.write_buffer_size.as_bytes().max(1) as usize)
|
||||
.await
|
||||
.context(error::OpenDalSnafu)?
|
||||
.into_futures_async_read(0..cached_value.content_length())
|
||||
.into_stream(0..cached_value.content_length())
|
||||
.await
|
||||
.context(error::OpenDalSnafu)?;
|
||||
|
||||
@@ -499,24 +525,50 @@ impl WriteCache {
|
||||
.chunk(options.write_buffer_size.as_bytes() as usize)
|
||||
.concurrent(DEFAULT_WRITE_CONCURRENCY)
|
||||
.await
|
||||
.context(error::OpenDalSnafu)?
|
||||
.into_futures_async_write();
|
||||
.context(error::OpenDalSnafu)?;
|
||||
|
||||
let bytes_written =
|
||||
futures::io::copy(reader, &mut writer)
|
||||
let result = async {
|
||||
let bytes_written = upload::copy(
|
||||
&mut reader,
|
||||
&mut writer,
|
||||
options.write_buffer_size.as_bytes() as usize,
|
||||
cancellation,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
UploadError::Cancelled => error::SstWriteCancelledSnafu.build(),
|
||||
UploadError::Read { source } => error::OpenDalSnafu.into_error(source),
|
||||
UploadError::Write { source } => error::UploadSnafu {
|
||||
region_id,
|
||||
file_id,
|
||||
file_type: index_key.file_type,
|
||||
}
|
||||
.into_error(std::io::Error::from(source)),
|
||||
})?;
|
||||
if cancellation.is_some_and(CancellationToken::is_cancelled) {
|
||||
return error::SstWriteCancelledSnafu.fail();
|
||||
}
|
||||
writer
|
||||
.close()
|
||||
.await
|
||||
.map_err(std::io::Error::from)
|
||||
.context(error::UploadSnafu {
|
||||
region_id,
|
||||
file_id,
|
||||
file_type,
|
||||
file_type: index_key.file_type,
|
||||
})?;
|
||||
|
||||
// Must close to upload all data.
|
||||
writer.close().await.context(error::UploadSnafu {
|
||||
region_id,
|
||||
file_id,
|
||||
file_type,
|
||||
})?;
|
||||
Ok(bytes_written)
|
||||
}
|
||||
.await;
|
||||
let bytes_written = match result {
|
||||
Ok(bytes_written) => bytes_written,
|
||||
Err(error) => {
|
||||
if let Err(abort_error) = writer.abort().await {
|
||||
common_telemetry::warn!(abort_error; "Failed to abort SST upload, region: {}, file: {}", region_id, file_id);
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
|
||||
UPLOAD_BYTES_TOTAL.inc_by(bytes_written);
|
||||
|
||||
@@ -647,6 +699,253 @@ mod tests {
|
||||
sst_file_handle_with_file_id, sst_region_metadata,
|
||||
};
|
||||
|
||||
struct CancellingUploadWriter {
|
||||
inner: object_store::layers::mock::oio::Writer,
|
||||
cancellation: CancellationToken,
|
||||
bytes_written: Arc<std::sync::atomic::AtomicUsize>,
|
||||
aborted: Arc<std::sync::atomic::AtomicBool>,
|
||||
}
|
||||
|
||||
impl object_store::layers::mock::oio::Write for CancellingUploadWriter {
|
||||
async fn write(&mut self, buffer: object_store::Buffer) -> object_store::Result<()> {
|
||||
let len = buffer.len();
|
||||
self.inner.write(buffer).await?;
|
||||
self.bytes_written
|
||||
.fetch_add(len, std::sync::atomic::Ordering::Relaxed);
|
||||
self.cancellation.cancel();
|
||||
Ok(())
|
||||
}
|
||||
async fn close(
|
||||
&mut self,
|
||||
) -> object_store::layers::mock::Result<object_store::layers::mock::Metadata> {
|
||||
self.inner.close().await
|
||||
}
|
||||
async fn abort(&mut self) -> object_store::Result<()> {
|
||||
self.aborted
|
||||
.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
self.inner.abort().await
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cancel_upload_aborts_incomplete_object() {
|
||||
use object_store::layers::mock::MockLayerBuilder;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
|
||||
let env = TestEnv::new().await;
|
||||
let local_store = ObjectStore::new(Memory::default()).unwrap();
|
||||
let cancellation = CancellationToken::new();
|
||||
let bytes_written = Arc::new(AtomicUsize::new(0));
|
||||
let aborted = Arc::new(AtomicBool::new(false));
|
||||
let signal = cancellation.clone();
|
||||
let bytes = bytes_written.clone();
|
||||
let abort = aborted.clone();
|
||||
let layer = MockLayerBuilder::default()
|
||||
.writer_factory(Arc::new(move |_, _, inner| {
|
||||
Box::new(CancellingUploadWriter {
|
||||
inner,
|
||||
cancellation: signal.clone(),
|
||||
bytes_written: bytes.clone(),
|
||||
aborted: abort.clone(),
|
||||
})
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
let remote_store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
|
||||
let cache = env
|
||||
.create_write_cache(local_store.clone(), ReadableSize::mb(10))
|
||||
.await;
|
||||
let key = IndexKey::new(RegionId::new(1024, 1), FileId::random(), FileType::Parquet);
|
||||
let data = vec![42; 1024 * 1024];
|
||||
local_store
|
||||
.write(&cache.file_cache.cache_file_path(key), data.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let result = cache
|
||||
.upload_with_cancellation(
|
||||
key,
|
||||
"cancelled.parquet",
|
||||
&remote_store,
|
||||
UploadOptions {
|
||||
op_type: OperationType::Compact,
|
||||
write_buffer_size: ReadableSize::kb(1),
|
||||
},
|
||||
Some(&cancellation),
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(error::Error::SstWriteCancelled { .. })
|
||||
));
|
||||
assert!(aborted.load(Ordering::Relaxed));
|
||||
let bytes = bytes_written.load(Ordering::Relaxed);
|
||||
assert!(
|
||||
bytes > 0 && bytes < data.len(),
|
||||
"wrote {bytes} bytes before cancelling"
|
||||
);
|
||||
assert!(!remote_store.exists("cancelled.parquet").await.unwrap());
|
||||
}
|
||||
|
||||
struct FailingUploadWriter {
|
||||
inner: object_store::layers::mock::oio::Writer,
|
||||
fail_close: bool,
|
||||
fail_abort: bool,
|
||||
successful_writes: usize,
|
||||
}
|
||||
|
||||
impl object_store::layers::mock::oio::Write for FailingUploadWriter {
|
||||
async fn write(&mut self, bytes: object_store::Buffer) -> object_store::Result<()> {
|
||||
if !self.fail_close && self.successful_writes == 1 {
|
||||
return Err(object_store::Error::new(
|
||||
object_store::ErrorKind::NotFound,
|
||||
"injected write failure",
|
||||
));
|
||||
}
|
||||
self.inner.write(bytes).await?;
|
||||
self.successful_writes += 1;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn close(&mut self) -> object_store::Result<object_store::layers::mock::Metadata> {
|
||||
Err(object_store::Error::new(
|
||||
object_store::ErrorKind::NotFound,
|
||||
"injected close failure",
|
||||
))
|
||||
}
|
||||
|
||||
async fn abort(&mut self) -> object_store::Result<()> {
|
||||
if self.fail_abort {
|
||||
return Err(object_store::Error::new(
|
||||
object_store::ErrorKind::Unexpected,
|
||||
"injected abort failure",
|
||||
));
|
||||
}
|
||||
self.inner.abort().await
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::write(false, false)]
|
||||
#[case::close(true, false)]
|
||||
#[case::write_and_abort(false, true)]
|
||||
#[tokio::test]
|
||||
async fn test_upload_error_preserves_file_context(
|
||||
#[case] fail_close: bool,
|
||||
#[case] fail_abort: bool,
|
||||
) {
|
||||
use object_store::layers::mock::{MockLayerBuilder, ObservedWriter, WriterObservations};
|
||||
|
||||
let env = TestEnv::new().await;
|
||||
let local_store = ObjectStore::new(Memory::default()).unwrap();
|
||||
let observations = Arc::new(std::sync::Mutex::new(WriterObservations::default()));
|
||||
let observed = observations.clone();
|
||||
let layer = MockLayerBuilder::default()
|
||||
.writer_factory(Arc::new(move |_, _, inner| {
|
||||
Box::new(ObservedWriter::new(
|
||||
Box::new(FailingUploadWriter {
|
||||
inner,
|
||||
fail_close,
|
||||
fail_abort,
|
||||
successful_writes: 0,
|
||||
}),
|
||||
observed.clone(),
|
||||
))
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
let remote_store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
|
||||
let cache = env
|
||||
.create_write_cache(local_store.clone(), ReadableSize::mb(10))
|
||||
.await;
|
||||
let key = IndexKey::new(RegionId::new(1024, 1), FileId::random(), FileType::Parquet);
|
||||
local_store
|
||||
.write(&cache.file_cache.cache_file_path(key), vec![42; 4096])
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!cache.file_cache.contains_key(&key));
|
||||
let err = cache
|
||||
.upload(
|
||||
key,
|
||||
"failed.parquet",
|
||||
&remote_store,
|
||||
UploadOptions {
|
||||
op_type: OperationType::Compact,
|
||||
write_buffer_size: ReadableSize::kb(1),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(!err.is_object_not_found());
|
||||
assert!(err.may_have_persisted_manifest_update());
|
||||
match err {
|
||||
error::Error::Upload {
|
||||
region_id,
|
||||
file_id,
|
||||
file_type,
|
||||
error,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(key.region_id, region_id);
|
||||
assert_eq!(key.file_id, file_id);
|
||||
assert_eq!(key.file_type, file_type);
|
||||
assert_eq!(std::io::ErrorKind::NotFound, error.kind());
|
||||
let message = if fail_close {
|
||||
"injected close failure"
|
||||
} else {
|
||||
"injected write failure"
|
||||
};
|
||||
assert!(error.to_string().contains(message));
|
||||
}
|
||||
err => panic!("expected Upload error, got {err:?}"),
|
||||
}
|
||||
assert!(!cache.file_cache.contains_key(&key));
|
||||
assert!(!remote_store.exists("failed.parquet").await.unwrap());
|
||||
let observations = observations.lock().unwrap();
|
||||
// The write failure follows one completed chunk, so partial progress is exercised.
|
||||
assert_eq!(
|
||||
vec![1024; if fail_close { 4 } else { 2 }],
|
||||
observations.write_sizes
|
||||
);
|
||||
assert_eq!(1, observations.abort_calls);
|
||||
assert_eq!(usize::from(fail_close), observations.close_calls);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_upload_local_read_error() {
|
||||
use object_store::layers::mock::MockLayerBuilder;
|
||||
|
||||
let env = TestEnv::new().await;
|
||||
// The unit reader rejects reads after stat and reader creation succeed.
|
||||
let layer = MockLayerBuilder::default()
|
||||
.reader_factory(Arc::new(|_, _, _| Box::new(())))
|
||||
.build()
|
||||
.unwrap();
|
||||
let local_store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
|
||||
let remote_store = ObjectStore::new(Memory::default()).unwrap();
|
||||
let cache = env
|
||||
.create_write_cache(local_store.clone(), ReadableSize::mb(10))
|
||||
.await;
|
||||
let key = IndexKey::new(RegionId::new(1024, 1), FileId::random(), FileType::Parquet);
|
||||
local_store
|
||||
.write(&cache.file_cache.cache_file_path(key), vec![42; 4096])
|
||||
.await
|
||||
.unwrap();
|
||||
let err = cache
|
||||
.upload(
|
||||
key,
|
||||
"failed.parquet",
|
||||
&remote_store,
|
||||
UploadOptions {
|
||||
op_type: OperationType::Compact,
|
||||
write_buffer_size: ReadableSize::kb(1),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, error::Error::OpenDal { .. }));
|
||||
assert!(!remote_store.exists("failed.parquet").await.unwrap());
|
||||
}
|
||||
|
||||
struct RedirectUploadStoreWrapper {
|
||||
target_store: ObjectStore,
|
||||
last_op_type: Mutex<Option<OperationType>>,
|
||||
@@ -776,7 +1075,13 @@ mod tests {
|
||||
// Write to cache and upload sst to mock remote store
|
||||
let mut metrics = Metrics::new(WriteType::Flush);
|
||||
let mut sst_infos = write_cache
|
||||
.write_and_upload_sst(write_request, upload_request, &write_opts, &mut metrics)
|
||||
.write_and_upload_sst_with_context(
|
||||
write_request,
|
||||
upload_request,
|
||||
&write_opts,
|
||||
&mut metrics,
|
||||
SstWriteContext::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let sst_info = sst_infos.remove(0);
|
||||
@@ -890,7 +1195,13 @@ mod tests {
|
||||
|
||||
let mut metrics = Metrics::new(WriteType::Flush);
|
||||
let mut sst_infos = write_cache
|
||||
.write_and_upload_sst(write_request, upload_request, &write_opts, &mut metrics)
|
||||
.write_and_upload_sst_with_context(
|
||||
write_request,
|
||||
upload_request,
|
||||
&write_opts,
|
||||
&mut metrics,
|
||||
SstWriteContext::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let sst_info = sst_infos.remove(0);
|
||||
@@ -983,7 +1294,13 @@ mod tests {
|
||||
|
||||
let mut metrics = Metrics::new(WriteType::Flush);
|
||||
write_cache
|
||||
.write_and_upload_sst(write_request, upload_request, &write_opts, &mut metrics)
|
||||
.write_and_upload_sst_with_context(
|
||||
write_request,
|
||||
upload_request,
|
||||
&write_opts,
|
||||
&mut metrics,
|
||||
SstWriteContext::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
let atomic_write_dir = write_cache_dir.path().join(ATOMIC_WRITE_DIR);
|
||||
|
||||
@@ -17,7 +17,6 @@ use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use common_base::Plugins;
|
||||
use common_base::cancellation::CancellationHandle;
|
||||
use common_meta::key::SchemaMetadataManagerRef;
|
||||
use common_telemetry::{info, warn};
|
||||
use common_time::TimeToLive;
|
||||
@@ -31,6 +30,7 @@ use store_api::ManifestVersion;
|
||||
use store_api::metadata::RegionMetadataRef;
|
||||
use store_api::region_request::PathType;
|
||||
use store_api::storage::{RegionId, SequenceNumber};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::access_layer::{
|
||||
AccessLayer, AccessLayerRef, Metrics, OperationType, SstWriteRequest, WriteType,
|
||||
@@ -61,6 +61,7 @@ use crate::sst::index::puffin_manager::PuffinManagerFactory;
|
||||
use crate::sst::location::region_dir_from_table_dir;
|
||||
use crate::sst::parquet::metadata::extract_primary_key_range;
|
||||
use crate::sst::parquet::parallel_writer::EncodeRuntime;
|
||||
use crate::sst::parquet::writer::SstWriteContext;
|
||||
use crate::sst::parquet::{SstInfo, WriteOptions};
|
||||
use crate::sst::version::{SstVersion, SstVersionRef};
|
||||
|
||||
@@ -555,11 +556,14 @@ pub trait Compactor: Send + Sync + 'static {
|
||||
/// implementations in tests.
|
||||
#[async_trait::async_trait]
|
||||
pub trait SstMerger: Send + Sync + 'static {
|
||||
/// Merges an output, observing cancellation at safe boundaries and draining
|
||||
/// in-flight writers before returning.
|
||||
async fn merge_single_output(
|
||||
&self,
|
||||
context: CompactionContext,
|
||||
output: CompactionOutput,
|
||||
write_opts: WriteOptions,
|
||||
cancellation: CancellationToken,
|
||||
) -> Result<(Vec<FileMeta>, Vec<SstInfo>)>;
|
||||
}
|
||||
|
||||
@@ -618,6 +622,37 @@ impl SstMerger for DefaultSstMerger {
|
||||
context: CompactionContext,
|
||||
output: CompactionOutput,
|
||||
write_opts: WriteOptions,
|
||||
cancellation: CancellationToken,
|
||||
) -> Result<(Vec<FileMeta>, Vec<SstInfo>)> {
|
||||
let uncommitted = UncommittedSsts::new(
|
||||
context.region_id,
|
||||
context.access_layer.clone(),
|
||||
Some(context.cache_manager.clone()),
|
||||
);
|
||||
let result = self
|
||||
.merge_output(context, output, write_opts, &uncommitted, cancellation)
|
||||
.await;
|
||||
if result.is_err() {
|
||||
uncommitted.cleanup().await;
|
||||
} else {
|
||||
// Successful outputs are transferred to the caller for publication.
|
||||
uncommitted.disarm_cleanup();
|
||||
}
|
||||
match result {
|
||||
Err(error::Error::SstWriteCancelled { .. }) => error::CompactionCancelledSnafu.fail(),
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DefaultSstMerger {
|
||||
async fn merge_output(
|
||||
&self,
|
||||
context: CompactionContext,
|
||||
output: CompactionOutput,
|
||||
write_opts: WriteOptions,
|
||||
uncommitted: &UncommittedSsts,
|
||||
cancellation: CancellationToken,
|
||||
) -> Result<(Vec<FileMeta>, Vec<SstInfo>)> {
|
||||
let region_id = context.region_id;
|
||||
validate_region_compaction_output(region_id, &output)?;
|
||||
@@ -652,13 +687,18 @@ impl SstMerger for DefaultSstMerger {
|
||||
merge_mode,
|
||||
read_parallelism: write_opts.encode_parallelism,
|
||||
};
|
||||
let source = spawn_merge(builder.build_flat_sst_reader().await?);
|
||||
let source = tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => return error::CompactionCancelledSnafu.fail(),
|
||||
source = builder.build_flat_sst_reader() => source?,
|
||||
};
|
||||
let source = spawn_merge(source);
|
||||
|
||||
let mut metrics = Metrics::new(WriteType::Compaction);
|
||||
let region_metadata = context.region_metadata.clone();
|
||||
let sst_infos = context
|
||||
.access_layer
|
||||
.write_sst(
|
||||
.write_sst_with_context(
|
||||
SstWriteRequest {
|
||||
index_build_inputs: None,
|
||||
op_type: OperationType::Compact,
|
||||
@@ -683,6 +723,11 @@ impl SstMerger for DefaultSstMerger {
|
||||
},
|
||||
&write_opts,
|
||||
&mut metrics,
|
||||
SstWriteContext {
|
||||
guard: Some(Arc::new(uncommitted.clone())),
|
||||
cancellation: Some(cancellation),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
// Convert partition expression once outside the map
|
||||
@@ -746,7 +791,7 @@ impl SstMerger for DefaultSstMerger {
|
||||
/// implementations in tests.
|
||||
pub struct DefaultCompactor<M = DefaultSstMerger> {
|
||||
merger: M,
|
||||
cancel_handle: Arc<CancellationHandle>,
|
||||
cancel_token: CancellationToken,
|
||||
uncommitted: Option<UncommittedSsts>,
|
||||
}
|
||||
|
||||
@@ -755,37 +800,37 @@ impl<M: SstMerger> DefaultCompactor<M> {
|
||||
pub fn with_merger(merger: M) -> Self {
|
||||
Self {
|
||||
merger,
|
||||
cancel_handle: Arc::new(CancellationHandle::default()),
|
||||
cancel_token: CancellationToken::new(),
|
||||
uncommitted: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DefaultCompactor {
|
||||
/// Creates a new `DefaultCompactor` with the given cancel handle and no
|
||||
/// Creates a new `DefaultCompactor` with the given cancellation token and no
|
||||
/// uncommitted SSTs tracking.
|
||||
///
|
||||
/// This is the public entry point for external crates that want a
|
||||
/// cancellable compactor without opting into uncommitted-SST tracking.
|
||||
///
|
||||
/// Cancellation stops new output batches. Already spawned outputs finish,
|
||||
/// and their successful SSTs remain in the returned [`MergeOutput`]. This
|
||||
/// can delay cancellation until the current batch completes.
|
||||
pub fn new_with_cancel_handle(cancel_handle: Arc<CancellationHandle>) -> Self {
|
||||
/// Cancellation stops new output batches and interrupts active outputs at
|
||||
/// batch and upload-chunk boundaries. Successful complete outputs remain in
|
||||
/// the returned [`MergeOutput`]; incomplete outputs are cleaned up.
|
||||
pub fn new_with_cancel_token(cancel_token: CancellationToken) -> Self {
|
||||
Self {
|
||||
merger: DefaultSstMerger,
|
||||
cancel_handle,
|
||||
cancel_token,
|
||||
uncommitted: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_cancel_handle(
|
||||
cancel_handle: Arc<CancellationHandle>,
|
||||
pub(crate) fn with_cancel_token(
|
||||
cancel_token: CancellationToken,
|
||||
uncommitted: UncommittedSsts,
|
||||
) -> Self {
|
||||
Self {
|
||||
merger: DefaultSstMerger,
|
||||
cancel_handle,
|
||||
cancel_token,
|
||||
uncommitted: Some(uncommitted),
|
||||
}
|
||||
}
|
||||
@@ -804,11 +849,11 @@ impl<M: SstMerger> DefaultCompactor<M> {
|
||||
..Default::default()
|
||||
};
|
||||
for output in &unit.outputs {
|
||||
if self.cancel_handle.is_cancelled() {
|
||||
if self.cancel_token.is_cancelled() {
|
||||
return error::CompactionCancelledSnafu.fail();
|
||||
}
|
||||
// Let the writer finish and register its files before observing cancellation.
|
||||
// Dropping it mid-write can lose ownership of already finalized SSTs.
|
||||
// The merger observes cancellation and drains its writers before returning.
|
||||
// Keep awaiting it so cleanup cannot race with a late finalized SST.
|
||||
let (files, infos) = self
|
||||
.merger
|
||||
.merge_single_output(
|
||||
@@ -828,6 +873,7 @@ impl<M: SstMerger> DefaultCompactor<M> {
|
||||
),
|
||||
encode_runtime: EncodeRuntime::Compact,
|
||||
},
|
||||
self.cancel_token.clone(),
|
||||
)
|
||||
.await?;
|
||||
if let Some(uncommitted) = &self.uncommitted {
|
||||
@@ -836,7 +882,7 @@ impl<M: SstMerger> DefaultCompactor<M> {
|
||||
result.files_to_add.extend(files);
|
||||
result.sst_infos.extend(infos);
|
||||
}
|
||||
if self.cancel_handle.is_cancelled() {
|
||||
if self.cancel_token.is_cancelled() {
|
||||
return error::CompactionCancelledSnafu.fail();
|
||||
}
|
||||
Ok(result)
|
||||
@@ -895,9 +941,10 @@ where
|
||||
let merger = self.merger.clone();
|
||||
let context = context.clone();
|
||||
let uncommitted = self.uncommitted.clone();
|
||||
let cancellation = self.cancel_token.clone();
|
||||
let fut = async move {
|
||||
let result = merger
|
||||
.merge_single_output(context, output, write_opts)
|
||||
.merge_single_output(context, output, write_opts, cancellation)
|
||||
.await;
|
||||
if let (Some(uncommitted), Ok((_, infos))) = (&uncommitted, &result) {
|
||||
uncommitted.track(infos);
|
||||
@@ -915,7 +962,7 @@ where
|
||||
+ picker_output.expired_ssts.len(),
|
||||
);
|
||||
|
||||
while !tasks.is_empty() && !self.cancel_handle.is_cancelled() {
|
||||
while !tasks.is_empty() && !self.cancel_token.is_cancelled() {
|
||||
let mut chunk: Vec<(Vec<FileMeta>, _)> = Vec::with_capacity(internal_parallelism);
|
||||
for _ in 0..internal_parallelism {
|
||||
if let Some(task) = tasks.pop() {
|
||||
@@ -930,9 +977,8 @@ where
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Like merge_unit, cancellation must not drop a writer mid-output:
|
||||
// its blocking encoders would detach and finalized SSTs could lose
|
||||
// their owner. Join the whole batch and retain every successful output.
|
||||
// Each merger cancels cooperatively and owns its incomplete outputs.
|
||||
// Join every sibling before returning so writers and cleanup cannot race.
|
||||
let mut join_error = None;
|
||||
while let Some((inputs, handle)) = spawned.pop() {
|
||||
match handle.await {
|
||||
@@ -943,6 +989,7 @@ where
|
||||
}
|
||||
compacted_inputs.extend(inputs);
|
||||
}
|
||||
Ok(Err(error::Error::CompactionCancelled { .. })) => {}
|
||||
Ok(Err(e)) => {
|
||||
warn!(
|
||||
e; "Failed to merge compaction output for region: {}, inputs: [{}]",
|
||||
@@ -966,7 +1013,7 @@ where
|
||||
return Err(e).context(error::JoinSnafu);
|
||||
}
|
||||
|
||||
if self.cancel_handle.is_cancelled() {
|
||||
if self.cancel_token.is_cancelled() {
|
||||
info!("Compaction merge cancelled for region: {}", region_id);
|
||||
break;
|
||||
}
|
||||
@@ -1006,9 +1053,12 @@ mod tests {
|
||||
use std::time::Duration;
|
||||
|
||||
use datatypes::arrow::record_batch::RecordBatch;
|
||||
use object_store::ObjectStore;
|
||||
use object_store::layers::mock::{self, MockLayerBuilder};
|
||||
use object_store::services::Memory;
|
||||
use parquet::arrow::ArrowWriter;
|
||||
use store_api::storage::{FileId, RegionId};
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
use tokio::sync::{Notify, mpsc, oneshot};
|
||||
use tokio::time::timeout;
|
||||
|
||||
use super::{DefaultCompactor, *};
|
||||
@@ -1032,6 +1082,105 @@ mod tests {
|
||||
FileHandle::new(meta, Arc::new(NoopFilePurger))
|
||||
}
|
||||
|
||||
struct PendingCompactionReader {
|
||||
started: Arc<Notify>,
|
||||
}
|
||||
|
||||
impl mock::oio::Read for PendingCompactionReader {
|
||||
async fn open(
|
||||
&self,
|
||||
_: mock::BytesRange,
|
||||
) -> object_store::Result<(mock::RpRead, Box<dyn mock::oio::ReadStreamDyn>)> {
|
||||
self.started.notify_one();
|
||||
std::future::pending().await
|
||||
}
|
||||
|
||||
async fn read(
|
||||
&self,
|
||||
_: mock::BytesRange,
|
||||
) -> object_store::Result<(mock::RpRead, object_store::Buffer)> {
|
||||
self.started.notify_one();
|
||||
std::future::pending().await
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::unit(true)]
|
||||
#[case::outputs(false)]
|
||||
#[tokio::test]
|
||||
async fn test_default_merger_observes_compactor_cancellation(#[case] merge_unit: bool) {
|
||||
let mut region = new_test_compaction_region().await;
|
||||
let started = Arc::new(Notify::new());
|
||||
let signal = started.clone();
|
||||
let layer = MockLayerBuilder::default()
|
||||
.reader_factory(Arc::new(move |_, _, _| {
|
||||
Box::new(PendingCompactionReader {
|
||||
started: signal.clone(),
|
||||
})
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
let store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
|
||||
let access = ®ion.access_layer;
|
||||
region.access_layer = Arc::new(AccessLayer::new(
|
||||
access.table_dir(),
|
||||
access.path_type(),
|
||||
store.clone(),
|
||||
access.puffin_manager_factory().clone(),
|
||||
access.intermediate_manager().clone(),
|
||||
));
|
||||
let input = new_file_handle(dummy_file_meta());
|
||||
store
|
||||
.write(
|
||||
&input.file_path(
|
||||
region.access_layer.table_dir(),
|
||||
region.access_layer.path_type(),
|
||||
),
|
||||
vec![0; input.meta_ref().file_size as usize],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let picker_output = PickerOutput {
|
||||
outputs: vec![CompactionOutput {
|
||||
output_level: 1,
|
||||
inputs: vec![input],
|
||||
filter_deleted: false,
|
||||
output_time_range: None,
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
let compactor = DefaultCompactor::with_merger(DefaultSstMerger);
|
||||
let context = CompactionContext::from(®ion);
|
||||
let mut merge = Box::pin(async {
|
||||
if merge_unit {
|
||||
let unit = CompactionUnit::from_picker(picker_output)
|
||||
.unwrap()
|
||||
.pop()
|
||||
.unwrap();
|
||||
compactor.merge_unit(&context, &unit).await
|
||||
} else {
|
||||
compactor.merge_ssts(®ion, picker_output).await
|
||||
}
|
||||
});
|
||||
tokio::select! {
|
||||
_ = started.notified() => {},
|
||||
result = &mut merge => panic!("merge returned before cancellation: {result:?}"),
|
||||
_ = tokio::time::sleep(Duration::from_secs(10)) => panic!("merge did not reach the reader"),
|
||||
}
|
||||
compactor.cancel_token.cancel();
|
||||
let result = timeout(Duration::from_secs(10), merge).await.unwrap();
|
||||
if merge_unit {
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(error::Error::CompactionCancelled { .. })
|
||||
));
|
||||
} else {
|
||||
let result = result.unwrap();
|
||||
assert!(result.files_to_add.is_empty());
|
||||
assert!(result.files_to_remove.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_effective_target_trust_and_max_sequence() {
|
||||
let target = RegionId::new(1, 1);
|
||||
@@ -1420,7 +1569,12 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
let (files, _) = DefaultSstMerger
|
||||
.merge_single_output(context, output, WriteOptions::default())
|
||||
.merge_single_output(
|
||||
context,
|
||||
output,
|
||||
WriteOptions::default(),
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(1, files.len());
|
||||
@@ -1481,6 +1635,7 @@ mod tests {
|
||||
_context: CompactionContext,
|
||||
_output: CompactionOutput,
|
||||
_write_opts: WriteOptions,
|
||||
_cancellation: CancellationToken,
|
||||
) -> Result<(Vec<FileMeta>, Vec<SstInfo>)> {
|
||||
let idx = self.call_idx.fetch_add(1, Ordering::SeqCst);
|
||||
match self.results.lock().unwrap().get(idx) {
|
||||
@@ -1713,6 +1868,7 @@ mod tests {
|
||||
_context: CompactionContext,
|
||||
output: CompactionOutput,
|
||||
_write_opts: WriteOptions,
|
||||
_cancellation: CancellationToken,
|
||||
) -> Result<(Vec<FileMeta>, Vec<SstInfo>)> {
|
||||
let input_id = output.inputs[0].meta_ref().file_id;
|
||||
let mut file = dummy_file_meta();
|
||||
@@ -1765,7 +1921,7 @@ mod tests {
|
||||
.collect(),
|
||||
..Default::default()
|
||||
};
|
||||
let cancel_handle = Arc::new(CancellationHandle::default());
|
||||
let cancel_token = CancellationToken::new();
|
||||
let (started_tx, mut started_rx) = mpsc::unbounded_channel();
|
||||
let compactor = DefaultCompactor {
|
||||
merger: GatedEncodingMerger {
|
||||
@@ -1773,7 +1929,7 @@ mod tests {
|
||||
// Outputs are popped from the back, then joined in reverse order.
|
||||
panic_input: panic_merge.then_some(inputs[1].file_id),
|
||||
},
|
||||
cancel_handle: cancel_handle.clone(),
|
||||
cancel_token: cancel_token.clone(),
|
||||
uncommitted: None,
|
||||
};
|
||||
let memory = new_compaction_memory_manager(1024 * 1024);
|
||||
@@ -1792,7 +1948,7 @@ mod tests {
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
cancel_handle.cancel();
|
||||
cancel_token.cancel();
|
||||
assert!(futures::poll!(merge.as_mut()).is_pending());
|
||||
assert!(memory.try_acquire(memory.limit_bytes()).is_none());
|
||||
assert!(
|
||||
@@ -1855,7 +2011,7 @@ mod tests {
|
||||
let merger = MockMerger::new(vec![Ok(vec![dummy_file_meta()])]);
|
||||
let calls = merger.call_idx.clone();
|
||||
let compactor = DefaultCompactor::with_merger(merger);
|
||||
compactor.cancel_handle.cancel();
|
||||
compactor.cancel_token.cancel();
|
||||
let picker_output = PickerOutput {
|
||||
outputs: vec![CompactionOutput {
|
||||
output_level: 1,
|
||||
|
||||
@@ -28,12 +28,12 @@ use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicU64;
|
||||
|
||||
use common_base::cancellation::{CancellableFuture, CancellationHandle};
|
||||
use common_time::Timestamp;
|
||||
use partition::PartitionRuleRef;
|
||||
use snafu::{OptionExt, ensure};
|
||||
use store_api::metadata::RegionMetadataRef;
|
||||
use store_api::storage::RegionId;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::access_layer::AccessLayerRef;
|
||||
use crate::cache::CacheManagerRef;
|
||||
@@ -82,7 +82,7 @@ pub struct MultiRegionCompactionContext {
|
||||
pub cache: CacheManagerRef,
|
||||
pub config: Arc<MitoConfig>,
|
||||
pub protection: Arc<dyn SstWriteGuard>,
|
||||
pub cancellation: Arc<CancellationHandle>,
|
||||
pub cancellation: CancellationToken,
|
||||
}
|
||||
|
||||
/// Source I/O and lossless output accounting, excluding footer and query reads.
|
||||
@@ -137,9 +137,11 @@ pub async fn compact_regions(
|
||||
source::SharedSource::build(&plan, &context, metadata, counters.clone()).await?;
|
||||
worker::execute(plan, context, source, counters).await
|
||||
};
|
||||
CancellableFuture::new(execution, cancellation)
|
||||
.await
|
||||
.map_err(|_| crate::error::CompactionCancelledSnafu.build())?
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => crate::error::CompactionCancelledSnafu.fail(),
|
||||
result = execution => result,
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_plan(plan: &MultiRegionCompactionPlan) -> Result<RegionMetadataRef> {
|
||||
|
||||
@@ -109,7 +109,7 @@ pub(crate) struct SharedSource {
|
||||
window: (Timestamp, Timestamp),
|
||||
pub(crate) limits: MemoryLimits,
|
||||
peak_group_bytes: AtomicU64,
|
||||
cancellation: Arc<common_base::cancellation::CancellationHandle>,
|
||||
cancellation: CancellationToken,
|
||||
stop: CancellationToken,
|
||||
counters: Arc<SourceCounters>,
|
||||
}
|
||||
|
||||
@@ -277,7 +277,7 @@ impl Fixture {
|
||||
cache: Arc::new(CacheManager::default()),
|
||||
config: self.config.clone(),
|
||||
protection: protection.clone(),
|
||||
cancellation: Arc::new(CancellationHandle::default()),
|
||||
cancellation: CancellationToken::new(),
|
||||
},
|
||||
protection,
|
||||
)
|
||||
@@ -2002,7 +2002,7 @@ async fn test_interrupted_indexed_worker_cleans_tantivy_files(#[case] fail_sibli
|
||||
}
|
||||
}
|
||||
|
||||
struct CancelBeforeCreate(Arc<CancellationHandle>);
|
||||
struct CancelBeforeCreate(CancellationToken);
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl SstWriteGuard for CancelBeforeCreate {
|
||||
|
||||
@@ -16,6 +16,7 @@ use common_base::readable_size::ReadableSize;
|
||||
use partition::expr::PartitionExpr;
|
||||
use snafu::{ResultExt, ensure};
|
||||
use store_api::codec::PrimaryKeyEncoding;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::access_layer::{Metrics, OperationType, SstWriteRequest, WriteType};
|
||||
use crate::compaction::multi_region::memory::{self, MemoryLimits};
|
||||
@@ -27,6 +28,7 @@ use crate::sst::FormatType;
|
||||
use crate::sst::file::FileMeta;
|
||||
use crate::sst::parquet::metadata::extract_primary_key_range;
|
||||
use crate::sst::parquet::parallel_writer::EncodeRuntime;
|
||||
use crate::sst::parquet::writer::SstWriteContext;
|
||||
use crate::sst::parquet::{SstInfo, WriteOptions};
|
||||
|
||||
pub(crate) struct TargetWriter<'a> {
|
||||
@@ -71,13 +73,14 @@ impl<'a> TargetWriter<'a> {
|
||||
let infos = self
|
||||
.context
|
||||
.access_layer
|
||||
.write_sst_with_guard(
|
||||
.write_sst_with_context(
|
||||
request,
|
||||
&self.options(),
|
||||
&mut metrics,
|
||||
crate::sst::parquet::writer::SstWriteContext {
|
||||
guard: self.context.protection.clone(),
|
||||
limits: self.limits.writer,
|
||||
SstWriteContext {
|
||||
guard: Some(self.context.protection.clone()),
|
||||
limits: Some(self.limits.writer),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
@@ -207,7 +210,7 @@ impl<'a> TargetWriter<'a> {
|
||||
fn output_projection(
|
||||
target: &MultiRegionTarget,
|
||||
mut source: FlatSource,
|
||||
cancellation: Arc<common_base::cancellation::CancellationHandle>,
|
||||
cancellation: CancellationToken,
|
||||
) -> Result<FlatSource> {
|
||||
let projection: Vec<_> = source
|
||||
.schema()
|
||||
|
||||
@@ -2519,7 +2519,7 @@ async fn test_request_cancel_state_transitions() {
|
||||
let state = status.start_local_task();
|
||||
|
||||
assert_eq!(status.request_cancel(), RequestCancelResult::CancelIssued);
|
||||
assert!(state.cancel_handle().is_cancelled());
|
||||
assert!(state.cancel_token().is_cancelled());
|
||||
assert_eq!(
|
||||
status.request_cancel(),
|
||||
RequestCancelResult::AlreadyCancelling
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
use std::fmt::{Debug, Formatter};
|
||||
use std::sync::Arc;
|
||||
|
||||
use common_base::cancellation::CancellableFuture;
|
||||
use common_memory_manager::OnExhaustedPolicy;
|
||||
use common_telemetry::error;
|
||||
use futures::FutureExt;
|
||||
@@ -91,25 +90,24 @@ impl CompactionTask {
|
||||
None
|
||||
} else {
|
||||
let _wait_timer = crate::metrics::COMPACTION_MEMORY_WAIT.start_timer();
|
||||
let cancellation = self.state.cancel_token();
|
||||
Some(
|
||||
CancellableFuture::new(
|
||||
self.memory_manager
|
||||
.acquire_with_policy(bytes, self.memory_policy),
|
||||
self.state.cancel_handle(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| CompactionCancelledSnafu.build())?
|
||||
.context(CompactionMemoryExhaustedSnafu {
|
||||
region_id,
|
||||
policy: format!("{:?}", self.memory_policy),
|
||||
})?,
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => return CompactionCancelledSnafu.fail(),
|
||||
memory = self.memory_manager.acquire_with_policy(bytes, self.memory_policy) => memory,
|
||||
}
|
||||
.context(CompactionMemoryExhaustedSnafu {
|
||||
region_id,
|
||||
policy: format!("{:?}", self.memory_policy),
|
||||
})?,
|
||||
)
|
||||
};
|
||||
let _timer = COMPACTION_STAGE_ELAPSED
|
||||
.with_label_values(&["merge"])
|
||||
.start_timer();
|
||||
let compactor = DefaultCompactor::with_cancel_handle(
|
||||
self.state.cancel_handle(),
|
||||
let compactor = DefaultCompactor::with_cancel_token(
|
||||
self.state.cancel_token(),
|
||||
self.uncommitted.clone(),
|
||||
);
|
||||
let context = self.compaction_context.clone();
|
||||
|
||||
@@ -1398,6 +1398,9 @@ pub enum Error {
|
||||
#[snafu(display("Compaction is cancelled."))]
|
||||
CompactionCancelled {},
|
||||
|
||||
#[snafu(display("SST write is cancelled."))]
|
||||
SstWriteCancelled {},
|
||||
|
||||
#[snafu(display("Flush is cancelled."))]
|
||||
FlushCancelled {},
|
||||
|
||||
@@ -1818,9 +1821,10 @@ impl ErrorExt for Error {
|
||||
source.status_code()
|
||||
}
|
||||
|
||||
ManualCompactionOverride {} | CompactionCancelled {} | FlushCancelled {} => {
|
||||
StatusCode::Cancelled
|
||||
}
|
||||
ManualCompactionOverride {}
|
||||
| CompactionCancelled {}
|
||||
| FlushCancelled {}
|
||||
| SstWriteCancelled {} => StatusCode::Cancelled,
|
||||
|
||||
// A concurrent manual compaction fails fast instead of being queued;
|
||||
// the conflict is reported to the caller, which decides whether to
|
||||
|
||||
+13
-9
@@ -21,7 +21,6 @@ use std::sync::{Arc, Mutex};
|
||||
use std::time::Instant;
|
||||
|
||||
use bytes::Bytes;
|
||||
use common_base::cancellation::CancellableFuture;
|
||||
use common_telemetry::{debug, error, info};
|
||||
use datatypes::arrow::datatypes::SchemaRef;
|
||||
use partition::expr::PartitionExpr;
|
||||
@@ -537,11 +536,11 @@ impl RegionFlushTask {
|
||||
// that acknowledges appends before their entries are durable can have
|
||||
// handed out `last_entry_id` for an entry that is still in its backlog.
|
||||
// A DDL that cancels the flush must not wait behind that upload.
|
||||
let durable = self.durability_barrier.wait(version_data.last_entry_id);
|
||||
tokio::pin!(durable);
|
||||
match CancellableFuture::new(durable.as_mut(), state.cancel_handle()).await {
|
||||
Ok(result) => result?,
|
||||
Err(_) => return FlushCancelledSnafu.fail(),
|
||||
let cancellation = state.cancel_token();
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => return FlushCancelledSnafu.fail(),
|
||||
result = self.durability_barrier.wait(version_data.last_entry_id) => result?,
|
||||
}
|
||||
|
||||
let edit = RegionEdit {
|
||||
@@ -803,12 +802,17 @@ impl RegionFlushTask {
|
||||
.collect::<Vec<_>>();
|
||||
let join_all = futures::future::join_all(tasks);
|
||||
tokio::pin!(join_all);
|
||||
let results = match CancellableFuture::new(join_all.as_mut(), state.cancel_handle()).await {
|
||||
Ok(results) => results
|
||||
let cancellation = state.cancel_token();
|
||||
let results = match tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => None,
|
||||
results = join_all.as_mut() => Some(results),
|
||||
} {
|
||||
Some(results) => results
|
||||
.into_iter()
|
||||
.map(|result| result.context(JoinSnafu))
|
||||
.collect::<Result<Vec<_>>>()?,
|
||||
Err(_) => {
|
||||
None => {
|
||||
for handle in abort_handles {
|
||||
handle.abort();
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use common_base::cancellation::CancellationHandle;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
pub mod remote_job_scheduler;
|
||||
pub mod scheduler;
|
||||
@@ -23,7 +23,7 @@ pub mod scheduler;
|
||||
/// Shared state for cooperatively cancelling a task until it starts committing.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct CancellableTaskState {
|
||||
cancel_handle: Arc<CancellationHandle>,
|
||||
cancel_token: CancellationToken,
|
||||
commit_started: Arc<Mutex<bool>>,
|
||||
started: Arc<AtomicBool>,
|
||||
}
|
||||
@@ -31,7 +31,7 @@ pub(crate) struct CancellableTaskState {
|
||||
impl CancellableTaskState {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
cancel_handle: Arc::new(CancellationHandle::default()),
|
||||
cancel_token: CancellationToken::new(),
|
||||
commit_started: Arc::new(Mutex::new(false)),
|
||||
started: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
@@ -47,12 +47,12 @@ impl CancellableTaskState {
|
||||
self.started.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub(crate) fn cancel_handle(&self) -> Arc<CancellationHandle> {
|
||||
self.cancel_handle.clone()
|
||||
pub(crate) fn cancel_token(&self) -> CancellationToken {
|
||||
self.cancel_token.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn is_cancelled(&self) -> bool {
|
||||
self.cancel_handle.is_cancelled()
|
||||
self.cancel_token.is_cancelled()
|
||||
}
|
||||
|
||||
/// Returns whether the task has crossed the non-cancellable publication boundary.
|
||||
@@ -65,7 +65,7 @@ impl CancellableTaskState {
|
||||
/// Returns false if cancellation was requested first.
|
||||
pub(crate) fn mark_commit_started(&self) -> bool {
|
||||
let mut commit_started = self.commit_started.lock().unwrap();
|
||||
if self.cancel_handle.is_cancelled() {
|
||||
if self.cancel_token.is_cancelled() {
|
||||
return false;
|
||||
}
|
||||
*commit_started = true;
|
||||
@@ -78,11 +78,11 @@ impl CancellableTaskState {
|
||||
if *commit_started {
|
||||
return RequestCancelResult::TooLateToCancel;
|
||||
}
|
||||
if self.cancel_handle.is_cancelled() {
|
||||
if self.cancel_token.is_cancelled() {
|
||||
return RequestCancelResult::AlreadyCancelling;
|
||||
}
|
||||
|
||||
self.cancel_handle.cancel();
|
||||
self.cancel_token.cancel();
|
||||
RequestCancelResult::CancelIssued
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,6 +50,7 @@ pub mod location;
|
||||
pub mod parquet;
|
||||
pub(crate) mod primary_key;
|
||||
pub mod range_index;
|
||||
pub(crate) mod upload;
|
||||
pub(crate) mod version;
|
||||
|
||||
/// Default write buffer size, it should be greater than the default minimum upload part of S3 (5mb).
|
||||
|
||||
@@ -45,6 +45,7 @@ use crate::sst::file_ref::{FileReferenceManager, FileReferenceManagerRef};
|
||||
use crate::sst::index::fulltext_index::FULLTEXT_BLOOM_VERSION;
|
||||
use crate::sst::location;
|
||||
use crate::sst::parquet::SstInfo;
|
||||
use crate::sst::parquet::writer::SstWriteGuard;
|
||||
use crate::sst::primary_key::PrimaryKeyRangeMapper;
|
||||
|
||||
/// Custom serde functions for Bytes fields serialized as base64 strings.
|
||||
@@ -1042,7 +1043,7 @@ pub async fn delete_files(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Tracks finalized SSTs until their manifest edit is committed.
|
||||
/// Tracks SSTs from creation until their manifest edit is committed.
|
||||
///
|
||||
/// Flush and local compaction jobs use this to remove files that were written successfully but
|
||||
/// abandoned by cancellation or a later failure.
|
||||
@@ -1054,6 +1055,15 @@ pub(crate) struct UncommittedSsts {
|
||||
cache_manager: Option<CacheManagerRef>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl SstWriteGuard for UncommittedSsts {
|
||||
async fn before_create(&self, file_id: RegionFileId) -> crate::error::Result<()> {
|
||||
// An index can be created before its final metadata is available.
|
||||
self.track_before_upload(file_id.file_id(), true);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl UncommittedSsts {
|
||||
pub(crate) fn new(
|
||||
region_id: RegionId,
|
||||
|
||||
@@ -12,15 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use bytes::Bytes;
|
||||
use datatypes::arrow::datatypes::SchemaRef;
|
||||
use datatypes::arrow::record_batch::RecordBatch;
|
||||
use futures::future::BoxFuture;
|
||||
use object_store::{ObjectStore, Writer};
|
||||
use object_store::ObjectStore;
|
||||
use parquet::arrow::AsyncArrowWriter;
|
||||
use parquet::arrow::async_writer::AsyncFileWriter;
|
||||
use parquet::basic::{Compression, Encoding, ZstdLevel};
|
||||
use parquet::errors::ParquetError;
|
||||
use parquet::file::metadata::KeyValue;
|
||||
use parquet::file::properties::WriterProperties;
|
||||
use snafu::{OptionExt, ResultExt};
|
||||
@@ -29,55 +25,9 @@ use crate::access_layer::TempFileCleaner;
|
||||
use crate::error::{OpenDalSnafu, Result, UnexpectedSnafu, WriteParquetSnafu};
|
||||
use crate::sst::{DEFAULT_WRITE_BUFFER_SIZE, DEFAULT_WRITE_CONCURRENCY};
|
||||
|
||||
type ArrowWriter = AsyncArrowWriter<AsyncWriter>;
|
||||
use crate::sst::parquet::writer::ObjectStoreWriter;
|
||||
|
||||
/// Bridges an OpenDAL [`Writer`] with Parquet's [`AsyncFileWriter`] and tracks
|
||||
/// the number of bytes successfully submitted to the object store.
|
||||
struct AsyncWriter {
|
||||
inner: Writer,
|
||||
output_bytes: u64,
|
||||
}
|
||||
|
||||
impl AsyncWriter {
|
||||
fn new(inner: Writer) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
output_bytes: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn output_bytes(&self) -> u64 {
|
||||
self.output_bytes
|
||||
}
|
||||
|
||||
fn into_inner(self) -> Writer {
|
||||
self.inner
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncFileWriter for AsyncWriter {
|
||||
fn write(&mut self, bytes: Bytes) -> BoxFuture<'_, parquet::errors::Result<()>> {
|
||||
Box::pin(async move {
|
||||
let len = bytes.len() as u64;
|
||||
self.inner
|
||||
.write(bytes)
|
||||
.await
|
||||
.map_err(|error| ParquetError::External(Box::new(error)))?;
|
||||
self.output_bytes += len;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn complete(&mut self) -> BoxFuture<'_, parquet::errors::Result<()>> {
|
||||
Box::pin(async move {
|
||||
self.inner
|
||||
.close()
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(|error| ParquetError::External(Box::new(error)))
|
||||
})
|
||||
}
|
||||
}
|
||||
type ArrowWriter = AsyncArrowWriter<ObjectStoreWriter>;
|
||||
|
||||
/// Shared Parquet output and cleanup lifecycle for index writers.
|
||||
pub(crate) struct ParquetIndexWriter {
|
||||
@@ -112,9 +62,12 @@ impl ParquetIndexWriter {
|
||||
.set_statistics_truncate_length(None)
|
||||
.set_key_value_metadata(key_value_metadata)
|
||||
.build();
|
||||
let writer =
|
||||
AsyncArrowWriter::try_new(AsyncWriter::new(output), schema.clone(), Some(properties))
|
||||
.context(WriteParquetSnafu)?;
|
||||
let writer = AsyncArrowWriter::try_new(
|
||||
ObjectStoreWriter::new(output, DEFAULT_WRITE_BUFFER_SIZE.as_bytes() as usize, None),
|
||||
schema.clone(),
|
||||
Some(properties),
|
||||
)
|
||||
.context(WriteParquetSnafu)?;
|
||||
|
||||
Ok(Self {
|
||||
name,
|
||||
|
||||
@@ -19,6 +19,7 @@ use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use bytes::Bytes;
|
||||
use common_base::cancellation::Cancelled;
|
||||
use common_runtime::JoinHandle;
|
||||
use datatypes::arrow::datatypes::SchemaRef;
|
||||
use datatypes::arrow::record_batch::RecordBatch;
|
||||
@@ -31,6 +32,7 @@ use parquet::errors::{ParquetError, Result};
|
||||
use parquet::file::metadata::ParquetMetaData;
|
||||
use parquet::file::properties::WriterProperties;
|
||||
use parquet::file::writer::SerializedFileWriter;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
/// Runtime whose blocking pool encodes the columns of a writer.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
@@ -90,6 +92,7 @@ pub struct ParallelArrowWriter<W> {
|
||||
/// Tasks encoding the previous row group.
|
||||
encoding: Vec<JoinHandle<Result<EncodedFields>>>,
|
||||
num_row_groups: usize,
|
||||
cancellation: Option<CancellationToken>,
|
||||
}
|
||||
|
||||
impl<W: AsyncFileWriter> ParallelArrowWriter<W> {
|
||||
@@ -137,13 +140,20 @@ impl<W: AsyncFileWriter> ParallelArrowWriter<W> {
|
||||
buffered_rows: 0,
|
||||
encoding: Vec::new(),
|
||||
num_row_groups: 0,
|
||||
cancellation: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn with_cancellation(mut self, cancellation: Option<CancellationToken>) -> Self {
|
||||
self.cancellation = cancellation;
|
||||
self
|
||||
}
|
||||
|
||||
/// Buffers `batch`, starting to encode each row group once it is full.
|
||||
pub async fn write(&mut self, batch: &RecordBatch) -> Result<()> {
|
||||
let mut offset = 0;
|
||||
while offset < batch.num_rows() {
|
||||
check_cancelled(self.cancellation.as_ref())?;
|
||||
let len = (self.max_row_group_rows - self.buffered_rows).min(batch.num_rows() - offset);
|
||||
self.buffered.push(batch.slice(offset, len));
|
||||
self.buffered_rows += len;
|
||||
@@ -165,6 +175,11 @@ impl<W: AsyncFileWriter> ParallelArrowWriter<W> {
|
||||
|
||||
/// Writes the remaining rows and the footer, then completes the async writer.
|
||||
pub async fn close(mut self) -> Result<ParquetMetaData> {
|
||||
self.finish().await
|
||||
}
|
||||
|
||||
/// Finalizes the file while retaining the upload for abort if it fails.
|
||||
pub(crate) async fn finish(&mut self) -> Result<ParquetMetaData> {
|
||||
self.flush().await?;
|
||||
let metadata = self.file_writer.finish()?;
|
||||
self.write_buffered_bytes().await?;
|
||||
@@ -173,9 +188,10 @@ impl<W: AsyncFileWriter> ParallelArrowWriter<W> {
|
||||
}
|
||||
|
||||
/// Discards buffered input and waits for encoders without writing a failed SST.
|
||||
pub(crate) async fn abort(mut self) {
|
||||
pub(crate) async fn abort(mut self) -> W {
|
||||
self.buffered.clear();
|
||||
abort_encoding_tasks(&mut self.encoding).await;
|
||||
self.async_writer
|
||||
}
|
||||
|
||||
/// Installs controlled tasks to test SST write-error cleanup independently of codec timing.
|
||||
@@ -191,6 +207,7 @@ impl<W: AsyncFileWriter> ParallelArrowWriter<W> {
|
||||
/// Starts encoding the buffered rows once the previous row group is written.
|
||||
async fn encode_buffered(&mut self) -> Result<()> {
|
||||
self.finish_encoding().await?;
|
||||
check_cancelled(self.cancellation.as_ref())?;
|
||||
|
||||
let batches: Arc<[RecordBatch]> = mem::take(&mut self.buffered).into();
|
||||
self.buffered_rows = 0;
|
||||
@@ -209,8 +226,9 @@ impl<W: AsyncFileWriter> ParallelArrowWriter<W> {
|
||||
.map(|fields| {
|
||||
let batches = batches.clone();
|
||||
let schema = self.arrow_schema.clone();
|
||||
let cancellation = self.cancellation.clone();
|
||||
self.runtime
|
||||
.spawn(move || encode_fields(&schema, &batches, fields))
|
||||
.spawn(move || encode_fields(&schema, &batches, fields, cancellation.as_ref()))
|
||||
})
|
||||
.collect();
|
||||
Ok(())
|
||||
@@ -301,17 +319,26 @@ fn assign_fields(
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn check_cancelled(cancellation: Option<&CancellationToken>) -> Result<()> {
|
||||
if cancellation.is_some_and(CancellationToken::is_cancelled) {
|
||||
return Err(ParquetError::External(Box::new(Cancelled)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Encodes `fields` of all `batches` and closes their column writers.
|
||||
fn encode_fields(
|
||||
schema: &SchemaRef,
|
||||
batches: &[RecordBatch],
|
||||
fields: Vec<FieldWriters>,
|
||||
cancellation: Option<&CancellationToken>,
|
||||
) -> Result<EncodedFields> {
|
||||
let mut chunks = Vec::new();
|
||||
let mut costs = Vec::with_capacity(fields.len());
|
||||
for (field, mut writers) in fields {
|
||||
let start = Instant::now();
|
||||
for batch in batches {
|
||||
check_cancelled(cancellation)?;
|
||||
let leaves = compute_leaves(schema.field(field), batch.column(field))?;
|
||||
if leaves.len() != writers.len() {
|
||||
return Err(ParquetError::General(format!(
|
||||
@@ -322,10 +349,12 @@ fn encode_fields(
|
||||
)));
|
||||
}
|
||||
for (leaf, (_, writer)) in leaves.iter().zip(writers.iter_mut()) {
|
||||
check_cancelled(cancellation)?;
|
||||
writer.write(leaf)?;
|
||||
}
|
||||
}
|
||||
for (leaf, writer) in writers {
|
||||
check_cancelled(cancellation)?;
|
||||
chunks.push((leaf, writer.close()?));
|
||||
}
|
||||
costs.push((field, start.elapsed()));
|
||||
@@ -534,8 +563,12 @@ mod tests {
|
||||
let running = runtime.spawn_blocking(move || {
|
||||
started_tx.send(()).unwrap();
|
||||
release_rx.blocking_recv().unwrap();
|
||||
let _encoded =
|
||||
encode_fields(&schema(), &running_batches, vec![(0, vec![(0, column)])])?;
|
||||
let _encoded = encode_fields(
|
||||
&schema(),
|
||||
&running_batches,
|
||||
vec![(0, vec![(0, column)])],
|
||||
None,
|
||||
)?;
|
||||
Err(ParquetError::General("later encoding failure".to_string()))
|
||||
});
|
||||
// With one blocking thread, this also proves the first task has ended.
|
||||
|
||||
@@ -15,15 +15,15 @@
|
||||
//! Parquet writer.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::error::Error as StdError;
|
||||
use std::future::Future;
|
||||
use std::mem;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Instant;
|
||||
|
||||
use bytes::Bytes;
|
||||
use common_base::cancellation::Cancelled;
|
||||
use common_telemetry::debug;
|
||||
use common_time::Timestamp;
|
||||
use datatypes::arrow::array::{
|
||||
@@ -34,23 +34,24 @@ use datatypes::arrow::compute::{max, min};
|
||||
use datatypes::arrow::datatypes::{DataType, SchemaRef, TimeUnit};
|
||||
use datatypes::arrow::record_batch::RecordBatch;
|
||||
use datatypes::extension::json::is_json2_extension_type;
|
||||
use object_store::{FuturesAsyncWriter, ObjectStore};
|
||||
use futures::future::BoxFuture;
|
||||
use object_store::{ObjectStore, Writer};
|
||||
use parquet::arrow::AsyncArrowWriter;
|
||||
use parquet::arrow::async_writer::AsyncFileWriter;
|
||||
use parquet::basic::{Compression, Encoding, ZstdLevel};
|
||||
use parquet::errors::ParquetError;
|
||||
use parquet::file::metadata::{FileMetaData, KeyValue, ParquetMetaData};
|
||||
use parquet::file::properties::WriterProperties;
|
||||
use smallvec::smallvec;
|
||||
use snafu::{OptionExt, ResultExt, ensure};
|
||||
use snafu::{IntoError, OptionExt, ResultExt, ensure};
|
||||
use store_api::metadata::RegionMetadataRef;
|
||||
use store_api::storage::{FileId, SequenceNumber};
|
||||
use tokio::io::AsyncWrite;
|
||||
use tokio_util::compat::{Compat, FuturesAsyncWriteCompatExt};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::access_layer::{FilePathProvider, Metrics, SstInfoArray, TempFileCleaner};
|
||||
use crate::config::{IndexBuildMode, IndexConfig};
|
||||
use crate::error::{
|
||||
InvalidMetadataSnafu, InvalidRecordBatchSnafu, OpenDalSnafu, Result, UnexpectedSnafu,
|
||||
Error, InvalidMetadataSnafu, InvalidRecordBatchSnafu, OpenDalSnafu, Result, UnexpectedSnafu,
|
||||
WriteParquetSnafu,
|
||||
};
|
||||
use crate::read::FlatSource;
|
||||
@@ -65,6 +66,7 @@ use crate::sst::parquet::{
|
||||
COLUMN_INDEX_TRUNCATE_LENGTH, PARQUET_METADATA_KEY, SstInfo, WriteOptions,
|
||||
apply_column_compression, apply_float_field_encoding, apply_internal_column_encoding,
|
||||
};
|
||||
use crate::sst::upload::{self, UploadError};
|
||||
use crate::sst::{
|
||||
DEFAULT_WRITE_CONCURRENCY, FlatSchemaOptions, SeriesEstimator, maybe_wrap_schema,
|
||||
};
|
||||
@@ -179,16 +181,17 @@ impl<W: AsyncFileWriter> SstArrowWriter<W> {
|
||||
}
|
||||
}
|
||||
|
||||
async fn close(self) -> parquet::errors::Result<ParquetMetaData> {
|
||||
async fn finish(&mut self) -> parquet::errors::Result<ParquetMetaData> {
|
||||
match self {
|
||||
SstArrowWriter::Serial(writer) => writer.close().await,
|
||||
SstArrowWriter::Parallel(writer) => writer.close().await,
|
||||
SstArrowWriter::Serial(writer) => writer.finish().await,
|
||||
SstArrowWriter::Parallel(writer) => writer.finish().await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn abort(self) {
|
||||
if let SstArrowWriter::Parallel(writer) = self {
|
||||
writer.abort().await;
|
||||
async fn abort(self) -> W {
|
||||
match self {
|
||||
SstArrowWriter::Serial(writer) => writer.into_inner(),
|
||||
SstArrowWriter::Parallel(writer) => writer.abort().await,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -248,28 +251,105 @@ pub struct ParquetWriter<'a, F: WriterFactory, I: IndexerBuilder, P: FilePathPro
|
||||
/// Write metrics
|
||||
metrics: &'a mut Metrics,
|
||||
write_guard: Option<Arc<dyn SstWriteGuard>>,
|
||||
cancellation: Option<CancellationToken>,
|
||||
memory: WriteMemory,
|
||||
}
|
||||
|
||||
/// Registers durable ownership before either an SST or its index is published to storage.
|
||||
/// Registers output ownership before either an SST or its index is created in storage.
|
||||
#[async_trait::async_trait]
|
||||
pub trait SstWriteGuard: Send + Sync {
|
||||
async fn before_create(&self, file_id: RegionFileId) -> Result<()>;
|
||||
}
|
||||
|
||||
/// Protection and resource limits travel together for table-compaction output.
|
||||
/// Optional ownership, cancellation, and resource limits for an SST write.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct SstWriteContext {
|
||||
pub(crate) guard: Arc<dyn SstWriteGuard>,
|
||||
pub(crate) limits: SstWriteLimits,
|
||||
/// Registers output ownership before creating files, including cached files.
|
||||
pub(crate) guard: Option<Arc<dyn SstWriteGuard>>,
|
||||
/// Resource bounds enforced by the direct writer, with one upload at a time.
|
||||
/// The write cache does not enforce these limits, so bounded writes bypass it.
|
||||
pub(crate) limits: Option<SstWriteLimits>,
|
||||
/// Signals cancellation; the writer still drains encoders and aborts unfinished output.
|
||||
pub(crate) cancellation: Option<CancellationToken>,
|
||||
}
|
||||
|
||||
pub trait WriterFactory {
|
||||
type Writer: AsyncWrite + Send + Unpin;
|
||||
type Writer: AsyncFileWriter;
|
||||
|
||||
/// Opens output with the same cancellation signal as its encoders.
|
||||
fn create(
|
||||
&mut self,
|
||||
file_path: &str,
|
||||
write_buffer_size: usize,
|
||||
cancellation: Option<&CancellationToken>,
|
||||
) -> impl Future<Output = Result<Self::Writer>>;
|
||||
|
||||
/// Aborts unfinished output after all encoders have stopped using it.
|
||||
fn abort(writer: Self::Writer) -> impl Future<Output = Result<()>>;
|
||||
}
|
||||
|
||||
/// Bridges an OpenDAL [`Writer`] with Parquet's [`AsyncFileWriter`] and tracks
|
||||
/// the number of bytes successfully submitted to the object store.
|
||||
pub struct ObjectStoreWriter {
|
||||
inner: Writer,
|
||||
output_bytes: u64,
|
||||
write_buffer_size: usize,
|
||||
cancellation: Option<CancellationToken>,
|
||||
}
|
||||
|
||||
impl ObjectStoreWriter {
|
||||
pub(crate) fn new(
|
||||
inner: Writer,
|
||||
write_buffer_size: usize,
|
||||
cancellation: Option<CancellationToken>,
|
||||
) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
output_bytes: 0,
|
||||
write_buffer_size,
|
||||
cancellation,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn output_bytes(&self) -> u64 {
|
||||
self.output_bytes
|
||||
}
|
||||
|
||||
pub(crate) fn into_inner(self) -> Writer {
|
||||
self.inner
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncFileWriter for ObjectStoreWriter {
|
||||
fn write(&mut self, bytes: Bytes) -> BoxFuture<'_, parquet::errors::Result<()>> {
|
||||
Box::pin(async move {
|
||||
let written = upload::write(
|
||||
&mut self.inner,
|
||||
bytes.into(),
|
||||
self.write_buffer_size,
|
||||
self.cancellation.as_ref(),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
UploadError::Cancelled => ParquetError::External(Box::new(Cancelled)),
|
||||
UploadError::Read { source } | UploadError::Write { source } => {
|
||||
ParquetError::External(Box::new(source))
|
||||
}
|
||||
})?;
|
||||
self.output_bytes += written;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn complete(&mut self) -> BoxFuture<'_, parquet::errors::Result<()>> {
|
||||
Box::pin(async move {
|
||||
self.inner
|
||||
.close()
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(|error| ParquetError::External(Box::new(error)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ObjectStoreWriterFactory {
|
||||
@@ -278,17 +358,26 @@ pub struct ObjectStoreWriterFactory {
|
||||
}
|
||||
|
||||
impl WriterFactory for ObjectStoreWriterFactory {
|
||||
type Writer = Compat<FuturesAsyncWriter>;
|
||||
type Writer = ObjectStoreWriter;
|
||||
|
||||
async fn create(&mut self, file_path: &str, write_buffer_size: usize) -> Result<Self::Writer> {
|
||||
async fn create(
|
||||
&mut self,
|
||||
file_path: &str,
|
||||
write_buffer_size: usize,
|
||||
cancellation: Option<&CancellationToken>,
|
||||
) -> Result<Self::Writer> {
|
||||
self.object_store
|
||||
.writer_with(file_path)
|
||||
.chunk(write_buffer_size)
|
||||
.concurrent(self.concurrency)
|
||||
.await
|
||||
.map(|v| v.into_futures_async_write().compat_write())
|
||||
.map(|writer| ObjectStoreWriter::new(writer, write_buffer_size, cancellation.cloned()))
|
||||
.context(OpenDalSnafu)
|
||||
}
|
||||
|
||||
async fn abort(writer: Self::Writer) -> Result<()> {
|
||||
writer.into_inner().abort().await.context(OpenDalSnafu)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, I, P> ParquetWriter<'a, ObjectStoreWriterFactory, I, P>
|
||||
@@ -361,9 +450,26 @@ where
|
||||
file_cleaner: None,
|
||||
metrics,
|
||||
write_guard: None,
|
||||
cancellation: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_cancellation(mut self, cancellation: Option<CancellationToken>) -> Self {
|
||||
self.cancellation = cancellation;
|
||||
self
|
||||
}
|
||||
|
||||
fn check_cancelled(&self) -> Result<()> {
|
||||
ensure!(
|
||||
!self
|
||||
.cancellation
|
||||
.as_ref()
|
||||
.is_some_and(|handle| handle.is_cancelled()),
|
||||
crate::error::SstWriteCancelledSnafu
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn with_write_guard(mut self, guard: Option<Arc<dyn SstWriteGuard>>) -> Self {
|
||||
self.write_guard = guard;
|
||||
self
|
||||
@@ -409,8 +515,9 @@ where
|
||||
ssts: &mut SstInfoArray,
|
||||
stats: &mut SourceStats,
|
||||
) -> Result<()> {
|
||||
self.check_cancelled()?;
|
||||
// maybe_init_writer will re-create a new file.
|
||||
if let Some(mut current_writer) = mem::take(&mut self.writer) {
|
||||
if self.writer.is_some() {
|
||||
let mut stats = mem::take(stats);
|
||||
// At least one row has been written.
|
||||
assert!(stats.num_rows > 0);
|
||||
@@ -436,9 +543,28 @@ where
|
||||
);
|
||||
}
|
||||
}
|
||||
current_writer.flush().await.context(WriteParquetSnafu)?;
|
||||
|
||||
let parquet_metadata = current_writer.close().await.context(WriteParquetSnafu)?;
|
||||
self.check_cancelled()?;
|
||||
self.writer
|
||||
.as_mut()
|
||||
.context(UnexpectedSnafu {
|
||||
reason: "Missing SST writer during finalization",
|
||||
})?
|
||||
.flush()
|
||||
.await
|
||||
.map_err(map_write_error)?;
|
||||
self.check_cancelled()?;
|
||||
// Keep the upload owned until finalization succeeds, so an error
|
||||
// still reaches the common abort path.
|
||||
let parquet_metadata = self
|
||||
.writer
|
||||
.as_mut()
|
||||
.context(UnexpectedSnafu {
|
||||
reason: "Missing SST writer during finalization",
|
||||
})?
|
||||
.finish()
|
||||
.await
|
||||
.map_err(map_write_error)?;
|
||||
self.writer.take();
|
||||
let file_size = self.bytes_written.load(Ordering::Relaxed) as u64;
|
||||
if self.memory.limits.is_some() {
|
||||
self.memory.completed = self.memory.completed.saturating_add(
|
||||
@@ -517,6 +643,7 @@ where
|
||||
);
|
||||
let res = self.write_all_flat_inner(source, &converter, opts).await;
|
||||
if res.is_err() {
|
||||
self.abort_current_indexer().await;
|
||||
self.abort_current_file().await;
|
||||
}
|
||||
res
|
||||
@@ -542,6 +669,7 @@ where
|
||||
};
|
||||
let res = self.write_all_flat_inner(source, &converter, opts).await;
|
||||
if res.is_err() {
|
||||
self.abort_current_indexer().await;
|
||||
self.abort_current_file().await;
|
||||
}
|
||||
res
|
||||
@@ -558,7 +686,17 @@ where
|
||||
|
||||
loop {
|
||||
let start = Instant::now();
|
||||
let batch = match source.next_batch().await {
|
||||
self.check_cancelled()?;
|
||||
let next = if let Some(cancellation) = &self.cancellation {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => return crate::error::SstWriteCancelledSnafu.fail(),
|
||||
next = source.next_batch() => next,
|
||||
}
|
||||
} else {
|
||||
source.next_batch().await
|
||||
};
|
||||
let batch = match next {
|
||||
Ok(Some(batch)) => batch,
|
||||
Ok(None) => break,
|
||||
Err(e) => {
|
||||
@@ -622,15 +760,18 @@ where
|
||||
stats: &mut SourceStats,
|
||||
) -> Result<()> {
|
||||
let result = async {
|
||||
self.check_cancelled()?;
|
||||
let arrow_batch = converter.convert_batch(batch)?;
|
||||
self.check_cancelled()?;
|
||||
let start = Instant::now();
|
||||
self.maybe_init_writer(arrow_batch.schema_ref(), opts)
|
||||
.await?
|
||||
.write(&arrow_batch)
|
||||
.await
|
||||
.context(WriteParquetSnafu)?;
|
||||
.map_err(map_write_error)?;
|
||||
self.metrics.write_batch += start.elapsed();
|
||||
|
||||
self.check_cancelled()?;
|
||||
stats.update_flat(batch)?;
|
||||
if matches!(self.index_config.build_mode, IndexBuildMode::Sync) {
|
||||
let start = Instant::now();
|
||||
@@ -664,7 +805,10 @@ where
|
||||
/// Joins encoders before dropping the writer and deleting its failed output.
|
||||
async fn abort_current_file(&mut self) {
|
||||
if let Some(writer) = self.writer.take() {
|
||||
writer.abort().await;
|
||||
let output = writer.abort().await;
|
||||
if let Err(error) = F::abort(output.inner).await {
|
||||
common_telemetry::warn!(error; "Failed to abort SST writer, file: {}", self.current_file);
|
||||
}
|
||||
}
|
||||
if let Some(cleaner) = &self.file_cleaner {
|
||||
cleaner.clean_by_file_id(self.current_file).await;
|
||||
@@ -672,7 +816,7 @@ where
|
||||
}
|
||||
|
||||
async fn abort_current_indexer(&mut self) {
|
||||
if let Some(indexer) = &mut self.current_indexer {
|
||||
if let Some(mut indexer) = self.current_indexer.take() {
|
||||
indexer.abort().await;
|
||||
}
|
||||
}
|
||||
@@ -682,6 +826,7 @@ where
|
||||
schema: &SchemaRef,
|
||||
opts: &WriteOptions,
|
||||
) -> Result<&mut SstArrowWriter<SizeAwareWriter<F::Writer>>> {
|
||||
self.check_cancelled()?;
|
||||
if let Some(ref mut w) = self.writer {
|
||||
Ok(w)
|
||||
} else {
|
||||
@@ -719,7 +864,11 @@ where
|
||||
}
|
||||
let writer = SizeAwareWriter::new(
|
||||
self.writer_factory
|
||||
.create(&sst_file_path, opts.write_buffer_size.as_bytes() as usize)
|
||||
.create(
|
||||
&sst_file_path,
|
||||
opts.write_buffer_size.as_bytes() as usize,
|
||||
self.cancellation.as_ref(),
|
||||
)
|
||||
.await?,
|
||||
self.bytes_written.clone(),
|
||||
);
|
||||
@@ -736,12 +885,13 @@ where
|
||||
opts.encode_parallelism,
|
||||
opts.encode_runtime,
|
||||
)
|
||||
.context(WriteParquetSnafu)?,
|
||||
.map_err(map_write_error)?
|
||||
.with_cancellation(self.cancellation.clone()),
|
||||
)
|
||||
} else {
|
||||
SstArrowWriter::Serial(
|
||||
AsyncArrowWriter::try_new(writer, schema, Some(writer_props))
|
||||
.context(WriteParquetSnafu)?,
|
||||
.map_err(map_write_error)?,
|
||||
)
|
||||
};
|
||||
self.writer = Some(arrow_writer);
|
||||
@@ -762,6 +912,19 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Keeps cancellation inside the SST write error contract, including nested encoder errors.
|
||||
fn map_write_error(error: ParquetError) -> Error {
|
||||
let cancelled = std::iter::successors(Some(&error as &(dyn StdError + 'static)), |source| {
|
||||
(*source).source()
|
||||
})
|
||||
.any(|source| source.is::<Cancelled>());
|
||||
if cancelled {
|
||||
crate::error::SstWriteCancelledSnafu.build()
|
||||
} else {
|
||||
WriteParquetSnafu.into_error(error)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct SourceStats {
|
||||
/// Number of rows fetched.
|
||||
@@ -875,47 +1038,34 @@ impl<W> SizeAwareWriter<W> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<W> AsyncWrite for SizeAwareWriter<W>
|
||||
where
|
||||
W: AsyncWrite + Unpin,
|
||||
{
|
||||
fn poll_write(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &[u8],
|
||||
) -> Poll<std::result::Result<usize, std::io::Error>> {
|
||||
let this = self.as_mut().get_mut();
|
||||
|
||||
match Pin::new(&mut this.inner).poll_write(cx, buf) {
|
||||
Poll::Ready(Ok(bytes_written)) => {
|
||||
this.size.fetch_add(bytes_written, Ordering::Relaxed);
|
||||
Poll::Ready(Ok(bytes_written))
|
||||
}
|
||||
other => other,
|
||||
}
|
||||
impl<W: AsyncFileWriter> AsyncFileWriter for SizeAwareWriter<W> {
|
||||
fn write(&mut self, bytes: Bytes) -> BoxFuture<'_, parquet::errors::Result<()>> {
|
||||
Box::pin(async move {
|
||||
let len = bytes.len();
|
||||
self.inner.write(bytes).await?;
|
||||
self.size.fetch_add(len, Ordering::Relaxed);
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn poll_flush(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
) -> Poll<std::result::Result<(), std::io::Error>> {
|
||||
Pin::new(&mut self.inner).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
) -> Poll<std::result::Result<(), std::io::Error>> {
|
||||
Pin::new(&mut self.inner).poll_shutdown(cx)
|
||||
fn complete(&mut self) -> BoxFuture<'_, parquet::errors::Result<()>> {
|
||||
self.inner.complete()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::time::Duration;
|
||||
|
||||
use common_base::readable_size::ReadableSize;
|
||||
use common_runtime::runtime::{BuilderBuild, RuntimeTrait};
|
||||
use object_store::layers::mock::{
|
||||
Metadata, MockLayerBuilder, ObservedWriter, WriterObservations, oio,
|
||||
};
|
||||
use object_store::services::Memory;
|
||||
use object_store::{Buffer, Error as ObjectStoreError, ErrorKind, Result as ObjectStoreResult};
|
||||
use parquet::arrow::ArrowWriter;
|
||||
use parquet::errors::ParquetError;
|
||||
use store_api::region_request::PathType;
|
||||
@@ -925,20 +1075,445 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::access_layer::{RegionFilePathFactory, WriteType};
|
||||
use crate::compaction::memory_manager::new_compaction_memory_manager;
|
||||
use crate::error::Error;
|
||||
use crate::sst::FormatType;
|
||||
use crate::sst::parquet::tests::NoopIndexBuilder;
|
||||
use crate::test_util::TestEnv;
|
||||
use crate::test_util::sst_util::{new_record_batch_by_range, sst_region_metadata};
|
||||
|
||||
struct GatedObjectStoreWriter {
|
||||
inner: oio::Writer,
|
||||
started: Option<oneshot::Sender<()>>,
|
||||
release: Option<oneshot::Receiver<()>>,
|
||||
completed: Arc<AtomicUsize>,
|
||||
write_error: Option<ObjectStoreError>,
|
||||
}
|
||||
|
||||
impl oio::Write for GatedObjectStoreWriter {
|
||||
async fn write(&mut self, bytes: Buffer) -> ObjectStoreResult<()> {
|
||||
if let Some(started) = self.started.take() {
|
||||
started.send(()).unwrap();
|
||||
self.release.take().unwrap().await.unwrap();
|
||||
}
|
||||
if let Some(error) = self.write_error.take() {
|
||||
return Err(error);
|
||||
}
|
||||
self.inner.write(bytes).await?;
|
||||
self.completed.fetch_add(1, Ordering::Relaxed);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn close(&mut self) -> ObjectStoreResult<Metadata> {
|
||||
self.inner.close().await
|
||||
}
|
||||
|
||||
async fn abort(&mut self) -> ObjectStoreResult<()> {
|
||||
self.inner.abort().await
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[tokio::test]
|
||||
async fn test_object_store_writer_cancellation_between_chunks(
|
||||
#[values(false, true)] observe_cancellation: bool,
|
||||
) {
|
||||
let cancellation = CancellationToken::new();
|
||||
let observations = Arc::new(Mutex::new(WriterObservations::default()));
|
||||
let completed = Arc::new(AtomicUsize::new(0));
|
||||
let (started_tx, started_rx) = oneshot::channel();
|
||||
let (release_tx, release_rx) = oneshot::channel();
|
||||
let gate = Mutex::new(Some((started_tx, release_rx)));
|
||||
let observed = observations.clone();
|
||||
let completed_writes = completed.clone();
|
||||
let layer = MockLayerBuilder::default()
|
||||
.writer_factory(Arc::new(move |_, _, inner| {
|
||||
let (started, release) = gate.lock().unwrap().take().unwrap();
|
||||
Box::new(ObservedWriter::new(
|
||||
Box::new(GatedObjectStoreWriter {
|
||||
inner,
|
||||
started: Some(started),
|
||||
release: Some(release),
|
||||
completed: completed_writes.clone(),
|
||||
write_error: None,
|
||||
}),
|
||||
observed.clone(),
|
||||
))
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
let store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
|
||||
let inner = store
|
||||
.writer_with("output.parquet")
|
||||
.chunk(4)
|
||||
.concurrent(1)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut writer =
|
||||
ObjectStoreWriter::new(inner, 4, observe_cancellation.then(|| cancellation.clone()));
|
||||
let data = Bytes::from(vec![42; 32]);
|
||||
let mut write = Box::pin(writer.write(data.clone()));
|
||||
assert!(futures::poll!(write.as_mut()).is_pending());
|
||||
timeout(Duration::from_secs(5), started_rx)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
cancellation.cancel();
|
||||
// A started storage write retains its owner until it completes.
|
||||
assert!(futures::poll!(write.as_mut()).is_pending());
|
||||
assert_eq!(0, completed.load(Ordering::Relaxed));
|
||||
release_tx.send(()).unwrap();
|
||||
let result = timeout(Duration::from_secs(5), write).await.unwrap();
|
||||
{
|
||||
let observed = observations.lock().unwrap();
|
||||
assert_eq!(0, observed.abort_calls);
|
||||
assert_eq!(0, observed.close_calls);
|
||||
}
|
||||
if observe_cancellation {
|
||||
assert!(
|
||||
matches!(result, Err(ParquetError::External(ref source)) if source.is::<Cancelled>())
|
||||
);
|
||||
assert_eq!(1, completed.load(Ordering::Relaxed));
|
||||
assert_eq!(vec![4], observations.lock().unwrap().write_sizes);
|
||||
// The caller owns abort; the adapter only reports cancellation.
|
||||
writer.into_inner().abort().await.unwrap();
|
||||
assert_eq!(1, observations.lock().unwrap().abort_calls);
|
||||
assert!(!store.exists("output.parquet").await.unwrap());
|
||||
} else {
|
||||
result.unwrap();
|
||||
writer.complete().await.unwrap();
|
||||
assert_eq!(data, store.read("output.parquet").await.unwrap().to_bytes());
|
||||
let observed = observations.lock().unwrap();
|
||||
assert_eq!(1, observed.close_calls);
|
||||
assert_eq!(0, observed.abort_calls);
|
||||
assert_eq!(vec![4; 8], observed.write_sizes);
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[tokio::test]
|
||||
async fn test_parquet_write_error_aborts_upload(
|
||||
#[values(1, 2)] parallelism: usize,
|
||||
#[values(false, true)] cancel_during_write: bool,
|
||||
) {
|
||||
let cancellation = CancellationToken::new();
|
||||
let observations = Arc::new(Mutex::new(WriterObservations::default()));
|
||||
let completed = Arc::new(AtomicUsize::new(0));
|
||||
let (started_tx, started_rx) = oneshot::channel();
|
||||
let (release_tx, release_rx) = oneshot::channel();
|
||||
let gate = Mutex::new(Some((started_tx, release_rx)));
|
||||
let observed = observations.clone();
|
||||
let completed_writes = completed.clone();
|
||||
let layer = MockLayerBuilder::default()
|
||||
.writer_factory(Arc::new(move |_, _, inner| {
|
||||
let (started, release) = gate.lock().unwrap().take().unwrap();
|
||||
Box::new(ObservedWriter::new(
|
||||
Box::new(GatedObjectStoreWriter {
|
||||
inner,
|
||||
started: Some(started),
|
||||
release: Some(release),
|
||||
completed: completed_writes.clone(),
|
||||
write_error: (!cancel_during_write).then(|| {
|
||||
ObjectStoreError::new(
|
||||
ErrorKind::PermissionDenied,
|
||||
"injected upload failure",
|
||||
)
|
||||
}),
|
||||
}),
|
||||
observed.clone(),
|
||||
))
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
let store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
|
||||
let metadata = Arc::new(sst_region_metadata());
|
||||
let mut metrics = Metrics::new(WriteType::Compaction);
|
||||
let mut writer = ParquetWriter::new_with_object_store(
|
||||
store,
|
||||
metadata,
|
||||
IndexConfig::default(),
|
||||
NoopIndexBuilder,
|
||||
RegionFilePathFactory::new(String::new(), PathType::Bare),
|
||||
&mut metrics,
|
||||
)
|
||||
.await
|
||||
.with_write_concurrency(1)
|
||||
.with_cancellation(Some(cancellation.clone()));
|
||||
let batch = new_record_batch_by_range(&["a", "b"], 0, 8);
|
||||
let opts = WriteOptions {
|
||||
// A partial group reaches the upload through finish_current_file.
|
||||
row_group_size: batch.num_rows() + 1,
|
||||
write_buffer_size: ReadableSize(1),
|
||||
encode_parallelism: parallelism,
|
||||
..Default::default()
|
||||
};
|
||||
let source_finished = Arc::new(AtomicBool::new(false));
|
||||
let finished = source_finished.clone();
|
||||
let source = FlatSource::new_stream(
|
||||
batch.schema(),
|
||||
Box::pin(async_stream::try_stream! {
|
||||
yield batch;
|
||||
finished.store(true, Ordering::Relaxed);
|
||||
}),
|
||||
);
|
||||
let mut write = Box::pin(writer.write_all_flat(source, None, &opts));
|
||||
tokio::select! {
|
||||
started = started_rx => started.unwrap(),
|
||||
result = &mut write => panic!("writer returned before the upload gate: {result:?}"),
|
||||
_ = tokio::time::sleep(Duration::from_secs(5)) => panic!("writer did not reach the upload gate"),
|
||||
}
|
||||
assert!(source_finished.load(Ordering::Relaxed));
|
||||
if cancel_during_write {
|
||||
cancellation.cancel();
|
||||
}
|
||||
assert!(futures::poll!(write.as_mut()).is_pending());
|
||||
assert_eq!(0, observations.lock().unwrap().abort_calls);
|
||||
release_tx.send(()).unwrap();
|
||||
let error = timeout(Duration::from_secs(5), write)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap_err();
|
||||
if cancel_during_write {
|
||||
assert!(
|
||||
matches!(error, Error::SstWriteCancelled { .. }),
|
||||
"{error:?}"
|
||||
);
|
||||
} else {
|
||||
assert!(matches!(error, Error::WriteParquet { .. }), "{error:?}");
|
||||
let source =
|
||||
std::iter::successors(Some(&error as &(dyn StdError + 'static)), |error| {
|
||||
(*error).source()
|
||||
})
|
||||
.find_map(|error| error.downcast_ref::<ObjectStoreError>())
|
||||
.unwrap();
|
||||
assert_eq!(ErrorKind::PermissionDenied, source.kind());
|
||||
assert!(source.to_string().contains("injected upload failure"));
|
||||
}
|
||||
assert_eq!(
|
||||
usize::from(cancel_during_write),
|
||||
completed.load(Ordering::Relaxed)
|
||||
);
|
||||
let observed = observations.lock().unwrap();
|
||||
assert_eq!(vec![1], observed.write_sizes);
|
||||
assert_eq!(1, observed.abort_calls);
|
||||
assert_eq!(0, observed.close_calls);
|
||||
assert!(writer.writer.is_none());
|
||||
assert!(writer.current_indexer.is_none());
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::flat(FormatType::Flat)]
|
||||
#[case::primary_key(FormatType::PrimaryKey)]
|
||||
#[tokio::test]
|
||||
async fn test_source_error_waits_for_encoding_cleanup(#[case] format: FormatType) {
|
||||
async fn test_encoding_error_classification(
|
||||
#[case] format: FormatType,
|
||||
#[values(false, true)] finish_file: bool,
|
||||
#[values(false, true)] cancelled: bool,
|
||||
) {
|
||||
let mut env = TestEnv::new().await;
|
||||
let metadata = Arc::new(sst_region_metadata());
|
||||
let mut metrics = Metrics::new(WriteType::Compaction);
|
||||
let mut writer = ParquetWriter::new_with_object_store(
|
||||
env.init_object_store_manager(),
|
||||
metadata.clone(),
|
||||
IndexConfig::default(),
|
||||
NoopIndexBuilder,
|
||||
RegionFilePathFactory::new(String::new(), PathType::Bare),
|
||||
&mut metrics,
|
||||
)
|
||||
.await;
|
||||
let batch = new_record_batch_by_range(&["a", "b"], 0, 4);
|
||||
let converter = match format {
|
||||
FormatType::Flat => FlatBatchConverter::Flat(FlatWriteFormat::new(
|
||||
metadata.clone(),
|
||||
&FlatSchemaOptions::default(),
|
||||
)),
|
||||
FormatType::PrimaryKey => FlatBatchConverter::PrimaryKey {
|
||||
format: PrimaryKeyWriteFormat::new(metadata.clone()),
|
||||
num_fields: metadata.field_columns().count(),
|
||||
},
|
||||
};
|
||||
let schema = converter.convert_batch(&batch).unwrap().schema();
|
||||
let opts = WriteOptions {
|
||||
// A full row group joins the previous task in append_flat_batch;
|
||||
// a partial group joins it in finish_current_file instead.
|
||||
row_group_size: batch.num_rows() + usize::from(finish_file),
|
||||
encode_parallelism: 2,
|
||||
..Default::default()
|
||||
};
|
||||
let SstArrowWriter::Parallel(parallel) =
|
||||
writer.maybe_init_writer(&schema, &opts).await.unwrap()
|
||||
else {
|
||||
panic!("expected parallel encoding");
|
||||
};
|
||||
let task = common_runtime::spawn_blocking_compact(move || {
|
||||
let source: Box<dyn std::error::Error + Send + Sync> = if cancelled {
|
||||
Box::new(Cancelled)
|
||||
} else {
|
||||
Box::new(std::io::Error::other("encoding failed"))
|
||||
};
|
||||
Err(ParquetError::External(source))
|
||||
});
|
||||
// No SST-level cancellation handle: only the encoder's error can report cancellation.
|
||||
parallel.set_encoding_tasks_for_test(vec![task]);
|
||||
let source =
|
||||
FlatSource::new_stream(batch.schema(), Box::pin(futures::stream::iter([Ok(batch)])));
|
||||
let error = match format {
|
||||
FormatType::Flat => writer.write_all_flat(source, None, &opts).await,
|
||||
FormatType::PrimaryKey => {
|
||||
writer
|
||||
.write_all_flat_as_primary_key(source, None, &opts)
|
||||
.await
|
||||
}
|
||||
}
|
||||
.unwrap_err();
|
||||
if cancelled {
|
||||
assert!(matches!(error, Error::SstWriteCancelled { .. }));
|
||||
} else {
|
||||
assert!(
|
||||
matches!(error, Error::WriteParquet { error: ParquetError::External(ref source), .. }
|
||||
if source.downcast_ref::<std::io::Error>().is_some_and(|error| error.to_string() == "encoding failed"))
|
||||
);
|
||||
}
|
||||
assert!(writer.writer.is_none());
|
||||
assert!(writer.current_indexer.is_none());
|
||||
assert_eq!(0, writer.bytes_written.load(Ordering::Relaxed));
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::flat(FormatType::Flat)]
|
||||
#[case::primary_key(FormatType::PrimaryKey)]
|
||||
#[tokio::test]
|
||||
async fn test_cancel_pending_source(
|
||||
#[case] format: FormatType,
|
||||
#[values(1, 2)] parallelism: usize,
|
||||
) {
|
||||
let mut env = TestEnv::new().await;
|
||||
let metadata = Arc::new(sst_region_metadata());
|
||||
let cancellation = CancellationToken::new();
|
||||
let mut metrics = Metrics::new(WriteType::Compaction);
|
||||
let mut writer = ParquetWriter::new_with_object_store(
|
||||
env.init_object_store_manager(),
|
||||
metadata,
|
||||
IndexConfig::default(),
|
||||
NoopIndexBuilder,
|
||||
RegionFilePathFactory::new(String::new(), PathType::Bare),
|
||||
&mut metrics,
|
||||
)
|
||||
.await
|
||||
.with_cancellation(Some(cancellation.clone()));
|
||||
let batch = new_record_batch_by_range(&["a", "b"], 0, 4);
|
||||
let schema = batch.schema();
|
||||
let source = FlatSource::new_stream(
|
||||
schema,
|
||||
Box::pin(async_stream::try_stream! {
|
||||
yield batch;
|
||||
futures::future::pending::<()>().await;
|
||||
}),
|
||||
);
|
||||
let opts = WriteOptions {
|
||||
row_group_size: 8,
|
||||
encode_parallelism: parallelism,
|
||||
..Default::default()
|
||||
};
|
||||
let mut write = Box::pin(async {
|
||||
match format {
|
||||
FormatType::Flat => writer.write_all_flat(source, None, &opts).await,
|
||||
FormatType::PrimaryKey => {
|
||||
writer
|
||||
.write_all_flat_as_primary_key(source, None, &opts)
|
||||
.await
|
||||
}
|
||||
}
|
||||
});
|
||||
assert!(futures::poll!(write.as_mut()).is_pending());
|
||||
cancellation.cancel();
|
||||
let error = timeout(Duration::from_secs(5), write)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, Error::SstWriteCancelled { .. }));
|
||||
assert!(writer.writer.is_none());
|
||||
}
|
||||
|
||||
/// Controlled comparison: identical batches and cancellation point, changing
|
||||
/// only whether the writer observes cancellation.
|
||||
#[rstest::rstest]
|
||||
#[case::flat(FormatType::Flat)]
|
||||
#[case::primary_key(FormatType::PrimaryKey)]
|
||||
#[tokio::test]
|
||||
async fn test_batch_cancellation_ablation(
|
||||
#[case] format: FormatType,
|
||||
#[values(1, 2)] parallelism: usize,
|
||||
) {
|
||||
for enabled in [false, true] {
|
||||
let mut env = TestEnv::new().await;
|
||||
let cancellation = CancellationToken::new();
|
||||
let mut metrics = Metrics::new(WriteType::Compaction);
|
||||
let mut writer = ParquetWriter::new_with_object_store(
|
||||
env.init_object_store_manager(),
|
||||
Arc::new(sst_region_metadata()),
|
||||
IndexConfig::default(),
|
||||
NoopIndexBuilder,
|
||||
RegionFilePathFactory::new(String::new(), PathType::Bare),
|
||||
&mut metrics,
|
||||
)
|
||||
.await
|
||||
.with_cancellation(enabled.then(|| cancellation.clone()));
|
||||
let batch = new_record_batch_by_range(&["a", "b"], 0, 4);
|
||||
let consumed = Arc::new(AtomicUsize::new(0));
|
||||
let observed = consumed.clone();
|
||||
let source = FlatSource::new_stream(
|
||||
batch.schema(),
|
||||
Box::pin(async_stream::try_stream! {
|
||||
for i in 0..32 {
|
||||
if i == 1 { cancellation.cancel(); }
|
||||
observed.fetch_add(1, Ordering::Relaxed);
|
||||
yield new_record_batch_by_range(&["a", "b"], i * 8, i * 8 + 8);
|
||||
}
|
||||
}),
|
||||
);
|
||||
let opts = WriteOptions {
|
||||
row_group_size: 8,
|
||||
encode_parallelism: parallelism,
|
||||
..Default::default()
|
||||
};
|
||||
let result = match format {
|
||||
FormatType::Flat => writer.write_all_flat(source, None, &opts).await,
|
||||
FormatType::PrimaryKey => {
|
||||
writer
|
||||
.write_all_flat_as_primary_key(source, None, &opts)
|
||||
.await
|
||||
}
|
||||
};
|
||||
if enabled {
|
||||
assert!(result.is_err());
|
||||
assert_eq!(2, consumed.load(Ordering::Relaxed));
|
||||
assert!(writer.writer.is_none());
|
||||
} else {
|
||||
assert_eq!(
|
||||
256,
|
||||
result
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|info| info.num_rows)
|
||||
.sum::<usize>()
|
||||
);
|
||||
assert_eq!(32, consumed.load(Ordering::Relaxed));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::flat(FormatType::Flat)]
|
||||
#[case::primary_key(FormatType::PrimaryKey)]
|
||||
#[tokio::test]
|
||||
async fn test_source_error_waits_for_encoding_cleanup(
|
||||
#[case] format: FormatType,
|
||||
#[values(false, true)] cancel_source: bool,
|
||||
) {
|
||||
let mut env = TestEnv::new().await;
|
||||
let metadata = Arc::new(sst_region_metadata());
|
||||
let cancellation = CancellationToken::new();
|
||||
let mut metrics = Metrics::new(WriteType::Compaction);
|
||||
let mut writer = ParquetWriter::new_with_object_store(
|
||||
env.init_object_store_manager(),
|
||||
metadata.clone(),
|
||||
@@ -947,7 +1522,8 @@ mod tests {
|
||||
RegionFilePathFactory::new("".to_string(), PathType::Bare),
|
||||
&mut metrics,
|
||||
)
|
||||
.await;
|
||||
.await
|
||||
.with_cancellation(Some(cancellation.clone()));
|
||||
let opts = WriteOptions {
|
||||
row_group_size: 8,
|
||||
encode_parallelism: 2,
|
||||
@@ -1010,13 +1586,11 @@ mod tests {
|
||||
parallel.set_encoding_tasks_for_test(vec![running, queued]);
|
||||
let source = FlatSource::new_stream(
|
||||
batch.schema(),
|
||||
Box::pin(futures::stream::iter(vec![
|
||||
Ok(batch),
|
||||
InvalidRecordBatchSnafu {
|
||||
reason: "source read failed",
|
||||
}
|
||||
.fail(),
|
||||
])),
|
||||
Box::pin(async_stream::try_stream! {
|
||||
yield batch;
|
||||
if cancel_source { futures::future::pending::<()>().await; }
|
||||
Err(InvalidRecordBatchSnafu { reason: "source read failed" }.build())?;
|
||||
}),
|
||||
);
|
||||
let memory = new_compaction_memory_manager(1024 * 1024);
|
||||
let budget = memory.try_acquire(memory.limit_bytes()).unwrap();
|
||||
@@ -1034,6 +1608,10 @@ mod tests {
|
||||
// The next row group is buffered, then the source fails. No codec timing
|
||||
// or I/O wait can make the old indexer-only cleanup pass this assertion.
|
||||
assert!(futures::poll!(write.as_mut()).is_pending());
|
||||
if cancel_source {
|
||||
cancellation.cancel();
|
||||
assert!(futures::poll!(write.as_mut()).is_pending());
|
||||
}
|
||||
assert!(memory.try_acquire(memory.limit_bytes()).is_none());
|
||||
assert!(retained_field.upgrade().is_some());
|
||||
release_tx.send(()).unwrap();
|
||||
@@ -1041,9 +1619,13 @@ mod tests {
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
matches!(error, Error::InvalidRecordBatch { reason, .. } if reason == "source read failed")
|
||||
);
|
||||
if cancel_source {
|
||||
assert!(matches!(error, Error::SstWriteCancelled { .. }));
|
||||
} else {
|
||||
assert!(
|
||||
matches!(error, Error::InvalidRecordBatch { reason, .. } if reason == "source read failed")
|
||||
);
|
||||
}
|
||||
assert!(!queued_started.load(Ordering::SeqCst));
|
||||
assert!(retained_field.upgrade().is_none());
|
||||
assert!(memory.try_acquire(memory.limit_bytes()).is_some());
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Cooperative byte transfer for SST uploads. Callers own close, abort, and file cleanup.
|
||||
|
||||
use futures::{Stream, StreamExt};
|
||||
use object_store::{Buffer, Error as ObjectStoreError, Result as ObjectStoreResult, Writer};
|
||||
use snafu::Snafu;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
/// Preserves read/write attribution until the caller adds its file context.
|
||||
#[derive(Debug, Snafu)]
|
||||
pub(crate) enum UploadError {
|
||||
#[snafu(display("Upload cancelled"))]
|
||||
Cancelled,
|
||||
#[snafu(display("Failed to read upload source: {source}"))]
|
||||
Read { source: ObjectStoreError },
|
||||
#[snafu(display("Failed to write upload: {source}"))]
|
||||
Write { source: ObjectStoreError },
|
||||
}
|
||||
|
||||
/// Copies chunks without dropping an in-flight write when cancellation arrives.
|
||||
pub(crate) async fn copy(
|
||||
reader: &mut (impl Stream<Item = ObjectStoreResult<Buffer>> + Unpin),
|
||||
writer: &mut Writer,
|
||||
write_buffer_size: usize,
|
||||
cancellation: Option<&CancellationToken>,
|
||||
) -> Result<u64, UploadError> {
|
||||
let mut written = 0;
|
||||
loop {
|
||||
let next = if let Some(cancellation) = cancellation {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => return Err(UploadError::Cancelled),
|
||||
next = reader.next() => next,
|
||||
}
|
||||
} else {
|
||||
reader.next().await
|
||||
};
|
||||
let Some(bytes) = next else {
|
||||
return Ok(written);
|
||||
};
|
||||
let bytes = bytes.map_err(|source| UploadError::Read { source })?;
|
||||
written += write(writer, bytes, write_buffer_size, cancellation).await?;
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes bounded, zero-copy slices, awaiting each started write to completion.
|
||||
pub(crate) async fn write(
|
||||
writer: &mut Writer,
|
||||
mut bytes: Buffer,
|
||||
write_buffer_size: usize,
|
||||
cancellation: Option<&CancellationToken>,
|
||||
) -> Result<u64, UploadError> {
|
||||
let len = bytes.len() as u64;
|
||||
// Keep each slice nonempty even when the configured size is zero.
|
||||
let write_buffer_size = write_buffer_size.max(1);
|
||||
while !bytes.is_empty() {
|
||||
if cancellation.is_some_and(CancellationToken::is_cancelled) {
|
||||
return Err(UploadError::Cancelled);
|
||||
}
|
||||
let chunk = bytes.split_to(bytes.len().min(write_buffer_size));
|
||||
writer
|
||||
.write(chunk)
|
||||
.await
|
||||
.map_err(|source| UploadError::Write { source })?;
|
||||
}
|
||||
if cancellation.is_some_and(CancellationToken::is_cancelled) {
|
||||
return Err(UploadError::Cancelled);
|
||||
}
|
||||
Ok(len)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::stream;
|
||||
use object_store::ObjectStore;
|
||||
use object_store::layers::mock::{
|
||||
Metadata, MockLayerBuilder, ObservedWriter, WriterObservations, oio,
|
||||
};
|
||||
use object_store::services::Memory;
|
||||
use tokio::sync::Notify;
|
||||
use tokio::time::timeout;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_copy_cancels_pending_read() {
|
||||
let store = ObjectStore::new(Memory::default()).unwrap();
|
||||
let mut writer = store.writer("pending").await.unwrap();
|
||||
let mut reader = stream::pending();
|
||||
let cancellation = CancellationToken::new();
|
||||
let mut copy = Box::pin(super::copy(
|
||||
&mut reader,
|
||||
&mut writer,
|
||||
4,
|
||||
Some(&cancellation),
|
||||
));
|
||||
assert!(futures::poll!(copy.as_mut()).is_pending());
|
||||
cancellation.cancel();
|
||||
let error = timeout(Duration::from_secs(5), copy)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, UploadError::Cancelled));
|
||||
writer.abort().await.unwrap();
|
||||
assert!(!store.exists("pending").await.unwrap());
|
||||
}
|
||||
|
||||
struct GatedWriter {
|
||||
inner: oio::Writer,
|
||||
started: Arc<Notify>,
|
||||
release: Arc<Notify>,
|
||||
}
|
||||
|
||||
impl oio::Write for GatedWriter {
|
||||
async fn write(&mut self, bytes: Buffer) -> ObjectStoreResult<()> {
|
||||
self.started.notify_one();
|
||||
self.release.notified().await;
|
||||
self.inner.write(bytes).await
|
||||
}
|
||||
|
||||
async fn close(&mut self) -> ObjectStoreResult<Metadata> {
|
||||
self.inner.close().await
|
||||
}
|
||||
|
||||
async fn abort(&mut self) -> ObjectStoreResult<()> {
|
||||
self.inner.abort().await
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_copy_cancellation_waits_for_started_write() {
|
||||
let started = Arc::new(Notify::new());
|
||||
let release = Arc::new(Notify::new());
|
||||
let observations = Arc::new(Mutex::new(WriterObservations::default()));
|
||||
let observer = observations.clone();
|
||||
let started_write = started.clone();
|
||||
let release_write = release.clone();
|
||||
let layer = MockLayerBuilder::default()
|
||||
.writer_factory(Arc::new(move |_, _, inner| {
|
||||
Box::new(ObservedWriter::new(
|
||||
Box::new(GatedWriter {
|
||||
inner,
|
||||
started: started_write.clone(),
|
||||
release: release_write.clone(),
|
||||
}),
|
||||
observer.clone(),
|
||||
))
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
let store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
|
||||
// Leave the OpenDAL writer unbuffered so the observer sees helper boundaries.
|
||||
let mut writer = store.writer("gated").await.unwrap();
|
||||
let mut reader = stream::iter([Ok(Buffer::from(Bytes::from_static(b"abcdefghijkl")))]);
|
||||
let cancellation = CancellationToken::new();
|
||||
let mut copy = Box::pin(super::copy(
|
||||
&mut reader,
|
||||
&mut writer,
|
||||
4,
|
||||
Some(&cancellation),
|
||||
));
|
||||
assert!(futures::poll!(copy.as_mut()).is_pending());
|
||||
timeout(Duration::from_secs(5), started.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
cancellation.cancel();
|
||||
assert!(futures::poll!(copy.as_mut()).is_pending());
|
||||
release.notify_one();
|
||||
let error = timeout(Duration::from_secs(5), copy)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, UploadError::Cancelled));
|
||||
{
|
||||
let observations = observations.lock().unwrap();
|
||||
assert_eq!(vec![4], observations.write_sizes);
|
||||
assert_eq!(0, observations.close_calls);
|
||||
assert_eq!(0, observations.abort_calls);
|
||||
}
|
||||
writer.abort().await.unwrap();
|
||||
assert_eq!(1, observations.lock().unwrap().abort_calls);
|
||||
assert!(!store.exists("gated").await.unwrap());
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[tokio::test]
|
||||
async fn test_copy_preserves_contents(#[values(false, true)] cancellable: bool) {
|
||||
let observations = Arc::new(Mutex::new(WriterObservations::default()));
|
||||
let observer = observations.clone();
|
||||
let layer = MockLayerBuilder::default()
|
||||
.writer_factory(Arc::new(move |_, _, inner| {
|
||||
Box::new(ObservedWriter::new(inner, observer.clone()))
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
let store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
|
||||
let mut writer = store.writer("complete").await.unwrap();
|
||||
let mut reader = stream::iter([
|
||||
Ok(Buffer::from(Bytes::from_static(b"abcde"))),
|
||||
Ok(Buffer::from(Bytes::from_static(b"fghijkl"))),
|
||||
]);
|
||||
let cancellation = CancellationToken::new();
|
||||
let copied = super::copy(
|
||||
&mut reader,
|
||||
&mut writer,
|
||||
4,
|
||||
cancellable.then_some(&cancellation),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(12, copied);
|
||||
{
|
||||
let observations = observations.lock().unwrap();
|
||||
assert_eq!(vec![4, 1, 4, 3], observations.write_sizes);
|
||||
assert_eq!(0, observations.close_calls);
|
||||
assert_eq!(0, observations.abort_calls);
|
||||
}
|
||||
writer.close().await.unwrap();
|
||||
assert_eq!(1, observations.lock().unwrap().close_calls);
|
||||
assert_eq!(
|
||||
Bytes::from_static(b"abcdefghijkl"),
|
||||
store.read("complete").await.unwrap().to_bytes()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,6 @@ use std::num::NonZeroU64;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use common_base::cancellation::CancellableFuture;
|
||||
use common_telemetry::{error, info, warn};
|
||||
use futures::TryStreamExt;
|
||||
use snafu::ensure;
|
||||
@@ -416,16 +415,22 @@ impl<S: LogStore> TailReplayTask<S> {
|
||||
|
||||
/// Awaits `future` unless the task is cancelled first.
|
||||
async fn unless_cancelled<T>(&self, future: impl Future<Output = Result<T>>) -> Result<T> {
|
||||
CancellableFuture::new(future, self.state.cancel_handle())
|
||||
.await
|
||||
.unwrap_or_else(|_| FlushCancelledSnafu.fail())
|
||||
let cancellation = self.state.cancel_token();
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => FlushCancelledSnafu.fail(),
|
||||
result = future => result,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sleeps for `delay`. Returns false if the task is cancelled first.
|
||||
async fn sleep(&self, delay: Duration) -> bool {
|
||||
CancellableFuture::new(tokio::time::sleep(delay), self.state.cancel_handle())
|
||||
.await
|
||||
.is_ok()
|
||||
self.unless_cancelled(async {
|
||||
tokio::time::sleep(delay).await;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user