rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-10-20 10:52:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: sandbox SQL local filesystem access (#8708)

Browse Source 
* fix: sandbox SQL local filesystem access

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* fix: address local file sandbox review findings

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* fix: support Windows local copy paths

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* fix: improve sandbox path errors

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* refactor: simplify local path error context

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* perf: stream secure filesystem listings

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* style: derive local file access default

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* fix: improve local file access errors

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* fix: address local file access review findings

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* test: simplify local file access coverage

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* fix: harden sandboxed local file backends

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* fix: reject directory copy targets before creation

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* fix: avoid implicit string clone in file table listing

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

---------

Signed-off-by: jeremyhi <fengjiachun@gmail.com>
This commit is contained in:
jeremyhi
2026-07-31 13:23:15 +00:00
committed by GitHub
parent 1aa35716cb
commit 448f973593
64 changed files with 2153 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/operator/src/error.rs
+3 -19
View File
@@ -469,13 +469,6 @@ pub enum Error {
source: table::error::Error,
},
#[snafu(display("Failed to parse data source url"))]
ParseUrl {
#[snafu(implicit)]
location: Location,
source: common_datasource::error::Error,
},
#[snafu(display("Unsupported format: {:?}", format))]
UnsupportedFormat {
#[snafu(implicit)]
@@ -852,13 +845,6 @@ pub enum Error {
location: Location,
},
#[snafu(display("Path not found: {path}"))]
PathNotFound {
path: String,
#[snafu(implicit)]
location: Location,
},
#[snafu(display("Invalid time index type: {}", ty))]
InvalidTimeIndexType {
ty: arrow::datatypes::DataType,
@@ -1040,9 +1026,9 @@ impl ErrorExt for Error {
Error::ReadObject { .. }
| Error::ReadParquetMetadata { .. }
| Error::ReadOrc { .. } => StatusCode::StorageUnavailable,
Error::ListObjects { source, .. }
| Error::ParseUrl { source, .. }
| Error::BuildBackend { source, .. } => source.status_code(),
Error::ListObjects { source, .. } | Error::BuildBackend { source, .. } => {
source.status_code()
}
Error::ExecuteDdl { source, .. } => source.status_code(),
Error::InvalidCopyParameter { .. } | Error::InvalidCopyDatabasePath { .. } => {
StatusCode::InvalidArguments
@@ -1063,7 +1049,6 @@ impl ErrorExt for Error {
}
Error::InvalidProcessId { .. } => StatusCode::InvalidArguments,
Error::ProcessManagerMissing { .. } => StatusCode::Unexpected,
Error::PathNotFound { .. } => StatusCode::InvalidArguments,
Error::TimestampFormatNotSupported { .. } => StatusCode::InvalidArguments,
Error::SqlCommon { source, .. } => source.status_code(),
#[cfg(feature = "enterprise")]
@@ -1096,7 +1081,6 @@ impl ErrorExt for Error {
Error::ParseFileFormat { source, .. }
| Error::InferSchema { source, .. }
| Error::ListObjects { source, .. }
| Error::ParseUrl { source, .. }
| Error::BuildBackend { source, .. }
| Error::ReadOrc { source, .. } => source.retry_hint(),
src/operator/src/expr_helper.rs
+3 -1
View File
@@ -31,6 +31,7 @@ use api::v1::{
UnsetIndex, UnsetIndexes, UnsetInverted, UnsetSkipping, UnsetTableOptions, set_index,
unset_index,
};
use common_datasource::object_store::LocalFileAccess;
use common_error::ext::BoxedError;
use common_grpc_expr::util::ColumnExpr;
use common_time::Timezone;
@@ -140,6 +141,7 @@ pub fn extract_add_columns_expr(
pub(crate) async fn create_external_expr(
create: CreateExternalTable,
query_ctx: &QueryContextRef,
local_file_access: &LocalFileAccess,
) -> Result<CreateTableExpr> {
let (catalog_name, schema_name, table_name) =
table_idents_to_full_name(&create.name, query_ctx)
@@ -148,7 +150,7 @@ pub(crate) async fn create_external_expr(
let mut table_options = create.options.into_map();
let (object_store, files) = prepare_file_table_files(&table_options)
let (object_store, files) = prepare_file_table_files(&table_options, local_file_access)
.await
.context(PrepareFileTableSnafu)?;
src/operator/src/statement.rs
+4
View File
@@ -37,6 +37,7 @@ use catalog::process_manager::ProcessManagerRef;
use client::RecordBatches;
use client::error::{ExternalSnafu as ClientExternalSnafu, Result as ClientResult};
use client::inserter::{InsertOptions, Inserter};
use common_datasource::object_store::LocalFileAccess;
use common_error::ext::BoxedError;
use common_meta::cache_invalidator::CacheInvalidatorRef;
use common_meta::key::flow::{FlowMetadataManager, FlowMetadataManagerRef};
@@ -137,6 +138,7 @@ pub struct StatementExecutor {
inserter: InserterRef,
process_manager: Option<ProcessManagerRef>,
origin_frontend_addr: String,
pub(crate) local_file_access: LocalFileAccess,
#[cfg(feature = "enterprise")]
create_database_handler: Option<CreateDatabaseHandlerRef>,
#[cfg(feature = "enterprise")]
@@ -175,6 +177,7 @@ impl StatementExecutor {
partition_manager: PartitionRuleManagerRef,
process_manager: Option<ProcessManagerRef>,
origin_frontend_addr: String,
local_file_access: LocalFileAccess,
) -> Self {
Self {
catalog_manager,
@@ -188,6 +191,7 @@ impl StatementExecutor {
inserter,
process_manager,
origin_frontend_addr,
local_file_access,
#[cfg(feature = "enterprise")]
create_database_handler: None,
#[cfg(feature = "enterprise")]
src/operator/src/statement/copy_database.rs
+45 -11
View File
@@ -21,7 +21,9 @@ use client::{Output, OutputData, OutputMeta};
use common_catalog::format_full_table_name;
use common_datasource::file_format::Format;
use common_datasource::lister::{Lister, Source};
use common_datasource::object_store::build_backend;
#[cfg(windows)]
use common_datasource::object_store::{FS_SCHEMA, parse_url};
use common_datasource::object_store::{LocalFileAccess, build_backend, build_backend_for_write};
use common_stat::get_total_cpu_cores;
use common_telemetry::{debug, error, info, tracing};
use futures::future::try_join_all;
@@ -43,6 +45,24 @@ pub(crate) const COPY_DATABASE_TIME_END_KEY: &str = "end_time";
pub(crate) const CONTINUE_ON_ERROR_KEY: &str = "continue_on_error";
pub(crate) const PARALLELISM_KEY: &str = "parallelism";
fn is_directory_location(location: &str) -> bool {
if location.ends_with('/') {
return true;
}
#[cfg(windows)]
{
location.ends_with(std::path::MAIN_SEPARATOR)
&& matches!(
parse_url(location),
Ok((schema, _, _)) if schema.eq_ignore_ascii_case(FS_SCHEMA)
)
}
#[cfg(not(windows))]
false
}
/// Get parallelism from options, default to total CPU cores.
fn parse_parallelism_from_option_map(options: &HashMap<String, String>) -> usize {
options
@@ -59,13 +79,16 @@ impl StatementExecutor {
req: CopyDatabaseRequest,
ctx: QueryContextRef,
) -> error::Result<Output> {
// location must end with / so that every table is exported to a file.
// Location must end with a separator so that every table is exported to a file.
ensure!(
req.location.ends_with('/'),
is_directory_location(&req.location),
InvalidCopyDatabasePathSnafu {
value: req.location,
}
);
build_backend_for_write(&req.location, &req.connection, &self.local_file_access)
.await
.context(error::BuildBackendSnafu)?;
let parallelism = parse_parallelism_from_option_map(&req.with);
info!(
@@ -153,9 +176,9 @@ impl StatementExecutor {
req: CopyDatabaseRequest,
ctx: QueryContextRef,
) -> error::Result<Output> {
// location must end with /
// Location must end with a directory separator.
ensure!(
req.location.ends_with('/'),
is_directory_location(&req.location),
InvalidCopyDatabasePathSnafu {
value: req.location,
}
@@ -170,7 +193,7 @@ impl StatementExecutor {
.context(error::ParseFileFormatSnafu)?
.suffix();
let entries = list_files_to_copy(&req, suffix).await?;
let entries = list_files_to_copy(&req, suffix, &self.local_file_access).await?;
let continue_on_error = req
.with
@@ -198,7 +221,7 @@ impl StatementExecutor {
catalog_name: req.catalog_name.clone(),
schema_name: req.schema_name.clone(),
table_name: table_name.clone(),
location: format!("{}/{}", req.location, e.path()),
location: format!("{}{}", req.location, e.path()),
with: req.with.clone(),
connection: req.connection.clone(),
pattern: None,
@@ -255,9 +278,14 @@ fn parse_file_name_to_copy(e: &Entry) -> error::Result<String> {
}
/// Lists all files with expected suffix that can be imported to database.
async fn list_files_to_copy(req: &CopyDatabaseRequest, suffix: &str) -> error::Result<Vec<Entry>> {
let object_store =
build_backend(&req.location, &req.connection).context(error::BuildBackendSnafu)?;
async fn list_files_to_copy(
req: &CopyDatabaseRequest,
suffix: &str,
local_file_access: &LocalFileAccess,
) -> error::Result<Vec<Entry>> {
let object_store = build_backend(&req.location, &req.connection, local_file_access)
.await
.context(error::BuildBackendSnafu)?;
let pattern = Regex::try_from(format!(".*{}", suffix)).context(error::BuildRegexSnafu)?;
let lister = Lister::new(
@@ -273,10 +301,12 @@ async fn list_files_to_copy(req: &CopyDatabaseRequest, suffix: &str) -> error::R
mod tests {
use std::collections::{HashMap, HashSet};
use common_datasource::object_store::LocalFileAccess;
use common_stat::get_total_cpu_cores;
use object_store::ObjectStore;
use object_store::services::Fs;
use object_store::util::normalize_dir;
#[cfg(not(windows))]
use path_slash::PathExt;
use table::requests::CopyDatabaseRequest;
@@ -296,7 +326,10 @@ mod tests {
object_store.write("d", "").await.unwrap();
object_store.write("e.f.parquet", "").await.unwrap();
#[cfg(not(windows))]
let location = normalize_dir(&dir.path().to_slash().unwrap());
#[cfg(windows)]
let location = format!("{}\\", dir.path().display());
let request = CopyDatabaseRequest {
catalog_name: "catalog_0".to_string(),
schema_name: "schema_0".to_string(),
@@ -307,7 +340,8 @@ mod tests {
connection: Default::default(),
time_range: None,
};
let listed = list_files_to_copy(&request, ".parquet")
let local_file_access = LocalFileAccess::sandboxed(dir.path()).unwrap();
let listed = list_files_to_copy(&request, ".parquet", &local_file_access)
.await
.unwrap()
.into_iter()
src/operator/src/statement/copy_table_from.rs
+18 -18
View File
@@ -14,7 +14,6 @@
use std::collections::{BTreeSet, HashMap, HashSet};
use std::future::Future;
use std::path::Path;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
@@ -28,8 +27,7 @@ use common_datasource::file_format::json::JsonFormat;
use common_datasource::file_format::orc::{ReaderAdapter, infer_orc_schema, new_orc_stream_reader};
use common_datasource::file_format::{FileFormat, Format, file_to_stream};
use common_datasource::lister::{Lister, Source};
use common_datasource::object_store::{FS_SCHEMA, build_backend, parse_url};
use common_datasource::util::find_dir_and_filename;
use common_datasource::object_store::build_backend_with_path;
use common_query::{OutputCost, OutputRows};
use common_recordbatch::DfSendableRecordBatchStream;
use common_recordbatch::adapter::RecordBatchStreamTypeAdapter;
@@ -54,7 +52,7 @@ use table::requests::{CopyTableRequest, InsertRequest};
use table::table_reference::TableReference;
use tokio_util::compat::FuturesAsyncReadCompatExt;
use crate::error::{self, IntoVectorsSnafu, PathNotFoundSnafu, Result};
use crate::error::{self, IntoVectorsSnafu, Result};
use crate::statement::StatementExecutor;
const DEFAULT_BATCH_SIZE: usize = 8192;
@@ -99,16 +97,10 @@ impl StatementExecutor {
&self,
req: &CopyTableRequest,
) -> Result<(ObjectStore, Vec<Entry>)> {
let (schema, _host, path) = parse_url(&req.location).context(error::ParseUrlSnafu)?;
if schema.to_uppercase() == FS_SCHEMA {
ensure!(Path::new(&path).exists(), PathNotFoundSnafu { path });
}
let object_store =
build_backend(&req.location, &req.connection).context(error::BuildBackendSnafu)?;
let (dir, filename) = find_dir_and_filename(&path);
let backend =
build_backend_with_path(&req.location, &req.connection, &self.local_file_access)
.await
.context(error::BuildBackendSnafu)?;
let regex = req
.pattern
.as_ref()
@@ -116,17 +108,25 @@ impl StatementExecutor {
.transpose()
.context(error::BuildRegexSnafu)?;
let source = if let Some(filename) = filename {
let source = if let Some(filename) = backend.object_path {
Source::Filename(filename)
} else {
Source::Dir
};
let lister = Lister::new(object_store.clone(), source.clone(), dir.clone(), regex);
let lister = Lister::new(
backend.object_store.clone(),
source.clone(),
req.location.clone(),
regex,
);
let entries = lister.list().await.context(error::ListObjectsSnafu)?;
debug!("Copy from dir: {dir:?}, {source:?}, entries: {entries:?}");
Ok((object_store, entries))
debug!(
"Copy from location: {:?}, {source:?}, entries: {entries:?}",
req.location
);
Ok((backend.object_store, entries))
}
async fn collect_metadata(
src/operator/src/statement/copy_table_to.rs
+8 -8
View File
@@ -21,8 +21,7 @@ use common_datasource::file_format::Format;
use common_datasource::file_format::csv::stream_to_csv;
use common_datasource::file_format::json::stream_to_json;
use common_datasource::file_format::parquet::stream_to_parquet;
use common_datasource::object_store::{build_backend, parse_url};
use common_datasource::util::find_dir_and_filename;
use common_datasource::object_store::build_backend_for_write_with_path;
use common_query::Output;
use common_recordbatch::adapter::DfRecordBatchStreamAdapter;
use common_recordbatch::{
@@ -178,13 +177,14 @@ impl StatementExecutor {
_ => unreachable!(),
};
let (_schema, _host, path) = parse_url(location).context(error::ParseUrlSnafu)?;
let (_, filename) = find_dir_and_filename(&path);
let filename = filename.context(error::UnexpectedSnafu {
violated: format!("Expected filename, path: {path}"),
let backend =
build_backend_for_write_with_path(location, connection, &self.local_file_access)
.await
.context(error::BuildBackendSnafu)?;
let filename = backend.object_path.context(error::UnexpectedSnafu {
violated: format!("Expected filename, path: {location}"),
})?;
let object_store = build_backend(location, connection).context(error::BuildBackendSnafu)?;
self.stream_to_file(stream, format, object_store, &filename)
self.stream_to_file(stream, format, backend.object_store, &filename)
.await
}
}
src/operator/src/statement/ddl.rs
+3 -1
View File
@@ -361,7 +361,9 @@ impl StatementExecutor {
create_expr: CreateExternalTable,
ctx: QueryContextRef,
) -> Result<TableRef> {
let create_expr = &mut expr_helper::create_external_expr(create_expr, &ctx).await?;
let create_expr =
&mut expr_helper::create_external_expr(create_expr, &ctx, &self.local_file_access)
.await?;
self.create_table_inner(create_expr, None, ctx).await
}