fix: sandbox SQL local filesystem access (#8708)

* fix: sandbox SQL local filesystem access Signed-off-by: jeremyhi <fengjiachun@gmail.com> * fix: address local file sandbox review findings Signed-off-by: jeremyhi <fengjiachun@gmail.com> * fix: support Windows local copy paths Signed-off-by: jeremyhi <fengjiachun@gmail.com> * fix: improve sandbox path errors Signed-off-by: jeremyhi <fengjiachun@gmail.com> * refactor: simplify local path error context Signed-off-by: jeremyhi <fengjiachun@gmail.com> * perf: stream secure filesystem listings Signed-off-by: jeremyhi <fengjiachun@gmail.com> * style: derive local file access default Signed-off-by: jeremyhi <fengjiachun@gmail.com> * fix: improve local file access errors Signed-off-by: jeremyhi <fengjiachun@gmail.com> * fix: address local file access review findings Signed-off-by: jeremyhi <fengjiachun@gmail.com> * test: simplify local file access coverage Signed-off-by: jeremyhi <fengjiachun@gmail.com> * fix: harden sandboxed local file backends Signed-off-by: jeremyhi <fengjiachun@gmail.com> * fix: reject directory copy targets before creation Signed-off-by: jeremyhi <fengjiachun@gmail.com> * fix: avoid implicit string clone in file table listing Signed-off-by: jeremyhi <fengjiachun@gmail.com> --------- Signed-off-by: jeremyhi <fengjiachun@gmail.com>
2026-09-12 08:22:16 +00:00 · 2026-07-31 13:23:15 +00:00
parent 1aa35716cb
commit 448f973593
64 changed files with 2153 additions and 140 deletions
@@ -424,7 +424,8 @@ impl ErrorExt for Error {
            | InvalidQueryContextExtension { .. }
            | ConflictingSnapshotSequence { .. } => StatusCode::InvalidArguments,

-            BuildBackend { .. } | ListObjects { .. } => StatusCode::StorageUnavailable,
+            BuildBackend { source, .. } => source.status_code(),
+            ListObjects { .. } => StatusCode::StorageUnavailable,

            TableNotFound { .. } => StatusCode::TableNotFound,

@@ -494,3 +495,23 @@ impl From<Error> for DataFusionError {
        DataFusionError::External(Box::new(e))
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_build_backend_delegates_error_metadata() {
+        let source = common_datasource::error::LocalFileAccessDisabledSnafu {
+            path: "file:///tmp/data.parquet",
+        }
+        .build();
+        let error = Error::BuildBackend {
+            source,
+            location: Location::default(),
+        };
+
+        assert_eq!(error.status_code(), StatusCode::InvalidArguments);
+        assert_eq!(error.retry_hint(), RetryHint::NonRetryable);
+    }
+}
@@ -30,8 +30,7 @@ use common_catalog::consts::{
 use common_catalog::format_full_table_name;
 use common_datasource::file_format::{FileFormat, Format, infer_schemas};
 use common_datasource::lister::{Lister, Source};
-use common_datasource::object_store::build_backend;
-use common_datasource::util::find_dir_and_filename;
+use common_datasource::object_store::{LocalFileAccess, build_backend_with_path};
 use common_meta::SchemaOptions;
 use common_meta::ddl::create_flow::FlowType;
 use common_meta::key::flow::flow_info::FlowInfoValue;
@@ -1149,6 +1148,7 @@ fn describe_column_semantic_types(
 // lists files in the frontend to reduce unnecessary scan requests repeated in each datanode.
 pub async fn prepare_file_table_files(
    options: &HashMap<String, String>,
+    local_file_access: &LocalFileAccess,
 ) -> Result<(ObjectStore, Vec<String>)> {
    let url = options
        .get(FILE_TABLE_LOCATION_KEY)
@@ -1156,19 +1156,20 @@ pub async fn prepare_file_table_files(
            name: FILE_TABLE_LOCATION_KEY,
        })?;

-    let (dir, filename) = find_dir_and_filename(url);
-    let source = if let Some(filename) = filename {
-        Source::Filename(filename)
-    } else {
-        Source::Dir
-    };
    let regex = options
        .get(FILE_TABLE_PATTERN_KEY)
        .map(|x| Regex::new(x))
        .transpose()
        .context(error::BuildRegexSnafu)?;
-    let object_store = build_backend(url, options).context(error::BuildBackendSnafu)?;
-    let lister = Lister::new(object_store.clone(), source, dir, regex);
+    let backend = build_backend_with_path(url, options, local_file_access)
+        .await
+        .context(error::BuildBackendSnafu)?;
+    let source = if let Some(filename) = backend.object_path {
+        Source::Filename(filename)
+    } else {
+        Source::Dir
+    };
+    let lister = Lister::new(backend.object_store.clone(), source, url.clone(), regex);
    // If we scan files in a directory every time the database restarts,
    // then it might lead to a potential undefined behavior:
    // If a user adds a file with an incompatible schema to that directory,
@@ -1186,7 +1187,7 @@ pub async fn prepare_file_table_files(
            }
        })
        .collect::<Vec<_>>();
-    Ok((object_store, files))
+    Ok((backend.object_store, files))
 }

 pub async fn infer_file_table_schema(