Skip to main content

object_store/
secure_fs.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! A capability-based filesystem backend for untrusted object paths.
16
17use std::path::{Component, Path, PathBuf};
18use std::sync::{Arc, Mutex};
19use std::vec::IntoIter;
20use std::{fmt, io};
21
22use cap_std::ambient_authority;
23use cap_std::fs::{Dir, DirEntry, OpenOptions, ReadDir};
24use opendal::raw::*;
25use opendal::{
26    Buffer, Capability, EntryMode, Error, ErrorKind, Metadata, Operator, OperatorBuilder, Result,
27};
28use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
29
30const LIST_BATCH_SIZE: usize = 128;
31
32/// An opened filesystem root that confines all descendant path resolution.
33#[derive(Clone)]
34pub struct SecureFsRoot {
35    dir: Arc<Dir>,
36    path: Arc<PathBuf>,
37}
38
39impl fmt::Debug for SecureFsRoot {
40    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
41        f.debug_struct("SecureFsRoot")
42            .field("path", &self.path)
43            .finish_non_exhaustive()
44    }
45}
46
47impl SecureFsRoot {
48    /// Creates and opens `path` using ambient authority.
49    ///
50    /// Callers must only pass a server-controlled path.
51    pub fn open(path: impl AsRef<Path>) -> io::Result<Self> {
52        let path = path.as_ref();
53        std::fs::create_dir_all(path)?;
54        if std::fs::symlink_metadata(path)?.file_type().is_symlink() {
55            return Err(io::Error::new(
56                io::ErrorKind::InvalidInput,
57                "filesystem sandbox root must not be a symbolic link",
58            ));
59        }
60
61        let path = path.canonicalize()?;
62        let dir = Dir::open_ambient_dir(&path, ambient_authority())?;
63        Ok(Self {
64            dir: Arc::new(dir),
65            path: Arc::new(path),
66        })
67    }
68
69    /// Returns the canonical path used to open this root.
70    pub fn path(&self) -> &Path {
71        &self.path
72    }
73
74    /// Opens a descendant directory without leaving this capability root.
75    pub fn open_subdir(&self, path: impl AsRef<Path>) -> io::Result<Self> {
76        let path = normalize_relative_path(path.as_ref())?;
77        if path.as_os_str().is_empty() {
78            return Ok(self.clone());
79        }
80
81        let dir = self.dir.open_dir(&path)?;
82        Ok(Self {
83            dir: Arc::new(dir),
84            path: Arc::new(self.path.join(path)),
85        })
86    }
87
88    /// Creates and opens a descendant directory without leaving this capability root.
89    pub fn create_subdir(&self, path: impl AsRef<Path>) -> io::Result<Self> {
90        let path = normalize_relative_path(path.as_ref())?;
91        if path.as_os_str().is_empty() {
92            return Ok(self.clone());
93        }
94
95        self.dir.create_dir_all(&path)?;
96        let dir = self.dir.open_dir(&path)?;
97        Ok(Self {
98            dir: Arc::new(dir),
99            path: Arc::new(self.path.join(path)),
100        })
101    }
102
103    /// Builds an OpenDAL operator confined to this root.
104    pub fn build_operator(&self) -> Operator {
105        OperatorBuilder::new(SecureFsBackend::new(self.clone())).finish()
106    }
107}
108
109fn normalize_relative_path(path: &Path) -> io::Result<PathBuf> {
110    let mut normalized = PathBuf::new();
111    for component in path.components() {
112        match component {
113            Component::CurDir => {}
114            Component::Normal(value) => normalized.push(value),
115            Component::ParentDir | Component::RootDir | Component::Prefix(_) => {
116                return Err(io::Error::new(
117                    io::ErrorKind::PermissionDenied,
118                    "path escapes the filesystem sandbox",
119                ));
120            }
121        }
122    }
123    Ok(normalized)
124}
125
126fn backend_path(path: &str) -> io::Result<PathBuf> {
127    let path = path.trim_matches('/');
128    if path.is_empty() {
129        Ok(PathBuf::new())
130    } else {
131        normalize_relative_path(Path::new(path))
132    }
133}
134
135fn parse_write_error(error: io::Error, if_not_exists: bool) -> Error {
136    if if_not_exists && error.kind() == io::ErrorKind::AlreadyExists {
137        Error::new(
138            ErrorKind::ConditionNotMatch,
139            "the file already exists in the filesystem",
140        )
141        .set_source(error)
142    } else {
143        new_std_io_error(error)
144    }
145}
146
147fn metadata_from_fs(metadata: cap_std::fs::Metadata) -> Result<Metadata> {
148    let mode = if metadata.is_dir() {
149        EntryMode::DIR
150    } else if metadata.is_file() {
151        EntryMode::FILE
152    } else {
153        EntryMode::Unknown
154    };
155
156    Ok(Metadata::new(mode)
157        .with_content_length(metadata.len())
158        .with_last_modified(Timestamp::try_from(
159            metadata.modified().map_err(new_std_io_error)?.into_std(),
160        )?))
161}
162
163#[derive(Clone, Debug)]
164struct SecureFsBackend {
165    root: SecureFsRoot,
166    info: Arc<AccessorInfo>,
167}
168
169impl SecureFsBackend {
170    fn new(root: SecureFsRoot) -> Self {
171        let info = AccessorInfo::default();
172        info.set_scheme("fs")
173            .set_root(&root.path().to_string_lossy())
174            .set_native_capability(Capability {
175                stat: true,
176                read: true,
177                write: true,
178                write_can_empty: true,
179                write_can_append: true,
180                write_can_multi: true,
181                write_with_if_not_exists: true,
182                create_dir: true,
183                delete: true,
184                delete_with_recursive: true,
185                list: true,
186                shared: true,
187                ..Default::default()
188            });
189        Self {
190            root,
191            info: info.into(),
192        }
193    }
194}
195
196impl Access for SecureFsBackend {
197    type Reader = SecureFsReader;
198    type Writer = SecureFsWriter;
199    type Lister = Option<SecureFsLister>;
200    type Deleter = oio::OneShotDeleter<SecureFsDeleter>;
201    type Copier = ();
202
203    fn info(&self) -> Arc<AccessorInfo> {
204        self.info.clone()
205    }
206
207    async fn create_dir(&self, path: &str, _: OpCreateDir) -> Result<RpCreateDir> {
208        let path = backend_path(path).map_err(new_std_io_error)?;
209        let root = self.root.clone();
210        common_runtime::spawn_blocking_global(move || root.dir.create_dir_all(path))
211            .await
212            .map_err(new_task_join_error)?
213            .map_err(new_std_io_error)?;
214        Ok(RpCreateDir::default())
215    }
216
217    async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
218        let path = backend_path(path).map_err(new_std_io_error)?;
219        let root = self.root.clone();
220        let metadata = common_runtime::spawn_blocking_global(move || {
221            if path.as_os_str().is_empty() {
222                root.dir.dir_metadata()
223            } else {
224                root.dir.metadata(path)
225            }
226        })
227        .await
228        .map_err(new_task_join_error)?
229        .map_err(new_std_io_error)?;
230        Ok(RpStat::new(metadata_from_fs(metadata)?))
231    }
232
233    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
234        let path = backend_path(path).map_err(new_std_io_error)?;
235        let root = self.root.clone();
236        let file = common_runtime::spawn_blocking_global(move || root.dir.open(path))
237            .await
238            .map_err(new_task_join_error)?
239            .map_err(new_std_io_error)?;
240        let mut file = tokio::fs::File::from_std(file.into_std());
241        if args.range().offset() != 0 {
242            file.seek(io::SeekFrom::Start(args.range().offset()))
243                .await
244                .map_err(new_std_io_error)?;
245        }
246        Ok((
247            RpRead::default(),
248            SecureFsReader {
249                file,
250                remaining: args.range().size().unwrap_or(u64::MAX),
251            },
252        ))
253    }
254
255    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
256        let path = backend_path(path).map_err(new_std_io_error)?;
257        let root = self.root.clone();
258        let if_not_exists = args.if_not_exists();
259        let file = common_runtime::spawn_blocking_global(move || {
260            if let Some(parent) = path.parent()
261                && !parent.as_os_str().is_empty()
262            {
263                root.dir.create_dir_all(parent).map_err(new_std_io_error)?;
264            }
265
266            let mut options = OpenOptions::new();
267            options.write(true);
268            if args.if_not_exists() {
269                options.create_new(true);
270            } else {
271                options.create(true);
272            }
273            if args.append() {
274                options.append(true);
275            } else {
276                options.truncate(true);
277            }
278            root.dir
279                .open_with(path, &options)
280                .map_err(|error| parse_write_error(error, if_not_exists))
281        })
282        .await
283        .map_err(new_task_join_error)??;
284
285        Ok((
286            RpWrite::default(),
287            SecureFsWriter {
288                file: tokio::fs::File::from_std(file.into_std()),
289            },
290        ))
291    }
292
293    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
294        Ok((
295            RpDelete::default(),
296            oio::OneShotDeleter::new(SecureFsDeleter {
297                root: self.root.clone(),
298            }),
299        ))
300    }
301
302    async fn list(&self, path: &str, _: OpList) -> Result<(RpList, Self::Lister)> {
303        let path = backend_path(path).map_err(new_std_io_error)?;
304        let display_prefix = if path.as_os_str().is_empty() {
305            String::new()
306        } else {
307            format!("{}/", path.to_string_lossy().replace('\\', "/"))
308        };
309        let root = self.root.clone();
310        let read_dir = common_runtime::spawn_blocking_global(move || {
311            let result = (|| {
312                let dir = if path.as_os_str().is_empty() {
313                    root.dir.open_dir(".")?
314                } else {
315                    root.dir.open_dir(&path)?
316                };
317                dir.entries()
318            })();
319
320            match result {
321                Ok(read_dir) => Ok(Some(read_dir)),
322                Err(error)
323                    if matches!(
324                        error.kind(),
325                        io::ErrorKind::NotFound | io::ErrorKind::NotADirectory
326                    ) =>
327                {
328                    Ok(None)
329                }
330                Err(error) => Err(error),
331            }
332        })
333        .await
334        .map_err(new_task_join_error)?
335        .map_err(new_std_io_error)?;
336
337        let Some(read_dir) = read_dir else {
338            return Ok((RpList::default(), None));
339        };
340        let current_path = oio::Entry::new(
341            if display_prefix.is_empty() {
342                "/"
343            } else {
344                &display_prefix
345            },
346            Metadata::new(EntryMode::DIR),
347        );
348        Ok((
349            RpList::default(),
350            Some(SecureFsLister {
351                read_dir: Arc::new(Mutex::new(read_dir)),
352                display_prefix,
353                entries: vec![current_path].into_iter(),
354                done: false,
355            }),
356        ))
357    }
358}
359
360struct SecureFsReader {
361    file: tokio::fs::File,
362    remaining: u64,
363}
364
365impl oio::Read for SecureFsReader {
366    async fn read(&mut self) -> Result<Buffer> {
367        if self.remaining == 0 {
368            return Ok(Buffer::new());
369        }
370
371        let size = self.remaining.min(2 * 1024 * 1024) as usize;
372        let mut buffer = vec![0; size];
373        let read = self
374            .file
375            .read(&mut buffer)
376            .await
377            .map_err(new_std_io_error)?;
378        self.remaining = self.remaining.saturating_sub(read as u64);
379        buffer.truncate(read);
380        Ok(Buffer::from(buffer))
381    }
382}
383
384struct SecureFsWriter {
385    file: tokio::fs::File,
386}
387
388impl oio::Write for SecureFsWriter {
389    async fn write(&mut self, buffer: Buffer) -> Result<()> {
390        self.file
391            .write_all(&buffer.to_bytes())
392            .await
393            .map_err(new_std_io_error)
394    }
395
396    async fn close(&mut self) -> Result<Metadata> {
397        self.file.flush().await.map_err(new_std_io_error)?;
398        self.file.sync_all().await.map_err(new_std_io_error)?;
399        let metadata = self.file.metadata().await.map_err(new_std_io_error)?;
400        Ok(Metadata::new(EntryMode::FILE)
401            .with_content_length(metadata.len())
402            .with_last_modified(Timestamp::try_from(
403                metadata.modified().map_err(new_std_io_error)?,
404            )?))
405    }
406
407    async fn abort(&mut self) -> Result<()> {
408        Err(Error::new(
409            ErrorKind::Unsupported,
410            "filesystem writes cannot be aborted without atomic writes",
411        ))
412    }
413}
414
415struct SecureFsLister {
416    read_dir: Arc<Mutex<ReadDir>>,
417    display_prefix: String,
418    entries: IntoIter<oio::Entry>,
419    done: bool,
420}
421
422impl oio::List for SecureFsLister {
423    async fn next(&mut self) -> Result<Option<oio::Entry>> {
424        if let Some(entry) = self.entries.next() {
425            return Ok(Some(entry));
426        }
427        if self.done {
428            return Ok(None);
429        }
430
431        let read_dir = self.read_dir.clone();
432        let display_prefix = self.display_prefix.clone();
433        let (entries, done) = common_runtime::spawn_blocking_global(move || {
434            let mut read_dir = read_dir
435                .lock()
436                .map_err(|_| io::Error::other("filesystem directory iterator lock is poisoned"))?;
437            read_list_batch(&mut read_dir, &display_prefix)
438        })
439        .await
440        .map_err(new_task_join_error)?
441        .map_err(new_std_io_error)?;
442
443        self.entries = entries.into_iter();
444        self.done = done;
445        Ok(self.entries.next())
446    }
447}
448
449fn read_list_batch(
450    read_dir: &mut ReadDir,
451    display_prefix: &str,
452) -> io::Result<(Vec<oio::Entry>, bool)> {
453    let mut entries = Vec::with_capacity(LIST_BATCH_SIZE);
454    while entries.len() < LIST_BATCH_SIZE {
455        let entry = match read_dir.next() {
456            Some(Ok(entry)) => entry,
457            Some(Err(error)) if error.kind() == io::ErrorKind::NotFound => {
458                return Ok((entries, true));
459            }
460            Some(Err(error)) => return Err(error),
461            None => return Ok((entries, true)),
462        };
463
464        if let Some(entry) = read_list_entry(entry, display_prefix)? {
465            entries.push(entry);
466        }
467    }
468    Ok((entries, false))
469}
470
471fn read_list_entry(entry: DirEntry, display_prefix: &str) -> io::Result<Option<oio::Entry>> {
472    let file_type = match entry.file_type() {
473        Ok(file_type) => file_type,
474        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None),
475        Err(error) => return Err(error),
476    };
477    let name = entry.file_name().to_string_lossy().to_string();
478    let (path, mode) = if file_type.is_dir() {
479        (format!("{display_prefix}{name}/"), EntryMode::DIR)
480    } else if file_type.is_file() {
481        (format!("{display_prefix}{name}"), EntryMode::FILE)
482    } else {
483        (format!("{display_prefix}{name}"), EntryMode::Unknown)
484    };
485    let metadata = if mode == EntryMode::Unknown {
486        Metadata::new(mode)
487    } else {
488        match entry.metadata() {
489            Ok(metadata) => match metadata_from_fs(metadata) {
490                Ok(metadata) => metadata,
491                Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None),
492                Err(error) => return Err(io::Error::other(error.to_string())),
493            },
494            Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None),
495            Err(error) => return Err(error),
496        }
497    };
498    Ok(Some(oio::Entry::new(&path, metadata)))
499}
500
501struct SecureFsDeleter {
502    root: SecureFsRoot,
503}
504
505impl oio::OneShotDelete for SecureFsDeleter {
506    async fn delete_once(&self, path: String, args: OpDelete) -> Result<()> {
507        let path = backend_path(&path).map_err(new_std_io_error)?;
508        if path.as_os_str().is_empty() {
509            return Err(Error::new(
510                ErrorKind::Unsupported,
511                "deleting the filesystem sandbox root is not supported",
512            ));
513        }
514        let root = self.root.clone();
515        common_runtime::spawn_blocking_global(move || {
516            let metadata = match root.dir.symlink_metadata(&path) {
517                Ok(metadata) => metadata,
518                Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
519                Err(error) => return Err(error),
520            };
521
522            if metadata.is_dir() {
523                if args.recursive() {
524                    root.dir.remove_dir_all(path)
525                } else {
526                    root.dir.remove_dir(path)
527                }
528            } else {
529                root.dir.remove_file(path)
530            }
531        })
532        .await
533        .map_err(new_task_join_error)?
534        .map_err(new_std_io_error)
535    }
536}
537
538#[cfg(test)]
539mod tests {
540    use bytes::Bytes;
541    use common_test_util::temp_dir::create_temp_dir;
542    use opendal::ErrorKind;
543    use opendal::raw::oio::List;
544    use opendal::raw::{Access, OpList};
545
546    use super::{LIST_BATCH_SIZE, SecureFsBackend, SecureFsRoot, read_list_entry};
547
548    #[tokio::test]
549    async fn test_lister_streams_entries() {
550        let temp_dir = create_temp_dir("secure_fs_lister_streams_entries");
551        for index in 0..129 {
552            std::fs::write(temp_dir.path().join(format!("{index}.parquet")), []).unwrap();
553        }
554
555        let root = SecureFsRoot::open(temp_dir.path()).unwrap();
556        let backend = SecureFsBackend::new(root);
557        let (_, lister) = backend.list("/", OpList::new()).await.unwrap();
558        let mut lister = lister.unwrap();
559
560        assert_eq!(1, lister.entries.len());
561
562        let mut paths = Vec::new();
563        while let Some(entry) = lister.next().await.unwrap() {
564            paths.push(entry.path().to_string());
565            assert!(lister.entries.len() <= LIST_BATCH_SIZE);
566        }
567        assert_eq!(130, paths.len());
568        assert!(paths.iter().any(|path| path == "/"));
569        assert!(paths.iter().any(|path| path == "128.parquet"));
570    }
571
572    #[tokio::test]
573    async fn test_if_not_exists_returns_condition_not_match() {
574        let temp_dir = create_temp_dir("secure_fs_if_not_exists");
575        let operator = SecureFsRoot::open(temp_dir.path())
576            .unwrap()
577            .build_operator();
578        operator
579            .write("existing", Bytes::from_static(b"original"))
580            .await
581            .unwrap();
582
583        let error = operator
584            .write_with("existing", Bytes::from_static(b"replacement"))
585            .if_not_exists(true)
586            .await
587            .unwrap_err();
588
589        assert_eq!(ErrorKind::ConditionNotMatch, error.kind());
590        assert_eq!(
591            Bytes::from_static(b"original"),
592            operator.read("existing").await.unwrap().to_bytes()
593        );
594    }
595
596    #[tokio::test]
597    async fn test_if_not_exists_does_not_remap_parent_directory_error() {
598        let temp_dir = create_temp_dir("secure_fs_if_not_exists_parent_error");
599        std::fs::write(temp_dir.path().join("parent"), []).unwrap();
600        let operator = SecureFsRoot::open(temp_dir.path())
601            .unwrap()
602            .build_operator();
603
604        let error = operator
605            .write_with("parent/file", Bytes::new())
606            .if_not_exists(true)
607            .await
608            .unwrap_err();
609
610        assert_eq!(ErrorKind::AlreadyExists, error.kind());
611    }
612
613    #[tokio::test]
614    async fn test_list_missing_or_non_directory_is_empty() {
615        let temp_dir = create_temp_dir("secure_fs_list_missing_or_non_directory");
616        std::fs::write(temp_dir.path().join("file"), []).unwrap();
617        let operator = SecureFsRoot::open(temp_dir.path())
618            .unwrap()
619            .build_operator();
620
621        assert!(operator.list("missing/").await.unwrap().is_empty());
622        assert!(operator.list("file/").await.unwrap().is_empty());
623    }
624
625    // On Windows, `std::fs::DirEntry` is a snapshot taken by
626    // `FindFirstFileW`: `file_type()` and `metadata()` keep returning the
627    // cached data even after the file is removed, so `read_list_entry`
628    // may return `Some` with the stale entry. On platforms where metadata
629    // is fetched from the live filesystem (e.g. Unix `lstat` returns
630    // `ENOENT` after removal), `read_list_entry` returns `None` instead.
631    // The strong assertion is Unix-specific; the portable part still
632    // verifies the call succeeds and yields the expected path.
633    #[test]
634    fn test_lister_skips_entry_removed_during_iteration() {
635        let temp_dir = create_temp_dir("secure_fs_lister_removed_entry");
636        let path = temp_dir.path().join("removed");
637        std::fs::write(&path, []).unwrap();
638        let root = SecureFsRoot::open(temp_dir.path()).unwrap();
639        let mut read_dir = root.dir.entries().unwrap();
640        let entry = read_dir.next().unwrap().unwrap();
641        std::fs::remove_file(path).unwrap();
642
643        let result = read_list_entry(entry, "").unwrap();
644
645        if let Some(entry) = &result {
646            assert_eq!("removed", entry.path());
647        }
648
649        #[cfg(not(windows))]
650        assert!(result.is_none());
651    }
652
653    #[tokio::test]
654    async fn test_delete_root_is_unsupported() {
655        let temp_dir = create_temp_dir("secure_fs_delete_root");
656        let operator = SecureFsRoot::open(temp_dir.path())
657            .unwrap()
658            .build_operator();
659        operator
660            .write("nested/file", Bytes::from_static(b"data"))
661            .await
662            .unwrap();
663
664        let error = operator.delete_with("/").recursive(true).await.unwrap_err();
665
666        assert_eq!(ErrorKind::Unsupported, error.kind());
667        assert!(temp_dir.path().join("nested/file").exists());
668
669        operator
670            .delete_with("nested/")
671            .recursive(true)
672            .await
673            .unwrap();
674        assert!(!temp_dir.path().join("nested").exists());
675    }
676
677    #[tokio::test]
678    async fn test_writer_abort_is_unsupported_without_atomic_write() {
679        let temp_dir = create_temp_dir("secure_fs_writer_abort");
680        let operator = SecureFsRoot::open(temp_dir.path())
681            .unwrap()
682            .build_operator();
683        let mut writer = operator.writer("partial").await.unwrap();
684        writer.write(Bytes::from_static(b"partial")).await.unwrap();
685
686        let error = writer.abort().await.unwrap_err();
687
688        assert_eq!(ErrorKind::Unsupported, error.kind());
689    }
690}