object_store/
temp_file.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Temporary files of non-append writes, and how to reclaim them.
16
17use std::time::Duration;
18
19use common_telemetry::warn;
20use opendal::raw::Timestamp;
21use uuid::Uuid;
22
23use crate::{ErrorKind, ObjectStore};
24
25/// Prefix of a temporary file name.
26const TEMPORARY_PREFIX: &str = ".";
27
28/// Marker that follows the name of the target in a temporary file name.
29const TEMPORARY_MARKER: &str = ".greptime-";
30
31/// Suffix of a temporary file name.
32const TEMPORARY_SUFFIX: &str = ".tmp";
33
34/// Scheme of the only service that publishes non-append writes with a rename.
35const TEMPORARY_FILE_SCHEME: &str = crate::services::HDFS_NATIVE_SCHEME;
36
37/// Returns true if `store` publishes non-append writes with a rename, which is
38/// what leaves temporary files behind when the writer dies.
39///
40/// Only the HDFS compatibility layer does that; a write to any other service
41/// appears under its final name.
42pub fn writes_temporary_files(store: &ObjectStore) -> bool {
43    store.info().scheme() == TEMPORARY_FILE_SCHEME
44}
45
46/// What a sweep did.
47///
48/// A sweep goes through its directories and files one at a time, and a failure
49/// on either doesn't stop it, so it reports what it reclaimed and what it
50/// couldn't.
51#[derive(Debug, Clone, PartialEq, Eq, Default)]
52pub struct SweepReport {
53    /// Temporary files deleted.
54    pub deleted: usize,
55    /// List and delete operations that failed.
56    pub failed: usize,
57}
58
59impl SweepReport {
60    fn merge(&mut self, other: SweepReport) {
61        self.deleted += other.deleted;
62        self.failed += other.failed;
63    }
64}
65
66/// Deletes the temporary files in `dirs` that are older than `older_than`.
67///
68/// A non-append write goes to a temporary file next to its target and is
69/// renamed on close, so a writer that dies in between leaves the file behind
70/// with nothing to reclaim it: GC can't parse the name, and the startup cleanup
71/// of [`ATOMIC_WRITE_DIR`](crate::ATOMIC_WRITE_DIR) doesn't reach it.
72///
73/// The caller runs this for a region that just became its writable leader, so a
74/// file that is old enough belongs to a writer that is gone. Failures are
75/// logged and don't stop the sweep.
76pub async fn sweep_temporary_files(
77    store: &ObjectStore,
78    dirs: &[String],
79    older_than: Duration,
80) -> SweepReport {
81    let mut report = SweepReport::default();
82    for dir in dirs {
83        report.merge(sweep_dir(store, dir, older_than).await);
84    }
85    report
86}
87
88async fn sweep_dir(store: &ObjectStore, dir: &str, older_than: Duration) -> SweepReport {
89    let entries = match store.list(dir).await {
90        Ok(entries) => entries,
91        // Not every region has every directory, for example one that was never
92        // repartitioned has no staging directory.
93        Err(error) if error.kind() == ErrorKind::NotFound => return SweepReport::default(),
94        Err(error) => {
95            warn!(error; "Failed to list {dir} for temporary files");
96            return SweepReport {
97                failed: 1,
98                ..Default::default()
99            };
100        }
101    };
102
103    let expired_before = Timestamp::now()
104        .into_inner()
105        .as_millisecond()
106        .saturating_sub(i64::try_from(older_than.as_millis()).unwrap_or(i64::MAX));
107
108    let mut report = SweepReport::default();
109    for entry in entries {
110        if !is_temporary_file(entry.name()) {
111            continue;
112        }
113
114        // HDFS advances the modification time of a file under construction only
115        // as its blocks complete, so an old file is not necessarily abandoned.
116        // That is what the threshold covers.
117        let is_expired = entry
118            .metadata()
119            .last_modified()
120            .map(|last_modified| last_modified.into_inner().as_millisecond() < expired_before)
121            // Without a modification time there is no way to tell whether the
122            // writer is gone, so keep the file.
123            .unwrap_or(false);
124        if !is_expired {
125            continue;
126        }
127
128        // Delete one file at a time: the bulk delete stops at the first
129        // failure, and one file that can't be deleted must not keep the rest of
130        // the directory from being swept.
131        let path = entry.path();
132        match store.delete(path).await {
133            Ok(()) => report.deleted += 1,
134            Err(error) => {
135                warn!(error; "Failed to delete temporary file {path}");
136                report.failed += 1;
137            }
138        }
139    }
140
141    report
142}
143
144/// Returns the path a non-append write to `path` goes to before it is renamed.
145///
146/// A writer that dies in between leaves the file behind, which is what
147/// [`sweep_temporary_files`] reclaims.
148pub(crate) fn temporary_path(path: &str) -> String {
149    let suffix = format!("{TEMPORARY_MARKER}{}{TEMPORARY_SUFFIX}", Uuid::new_v4());
150    match path.rsplit_once('/') {
151        Some((parent, name)) => format!("{parent}/{TEMPORARY_PREFIX}{name}{suffix}"),
152        None => format!("{TEMPORARY_PREFIX}{path}{suffix}"),
153    }
154}
155
156fn is_temporary_file(name: &str) -> bool {
157    // `name` is the last segment of a path, so a directory name ends with `/`.
158    name.starts_with(TEMPORARY_PREFIX)
159        && name.contains(TEMPORARY_MARKER)
160        && name.ends_with(TEMPORARY_SUFFIX)
161}
162
163#[cfg(test)]
164mod tests {
165    use std::sync::Arc;
166
167    use opendal::Operator;
168    use opendal::services::Fs;
169    use tempfile::TempDir;
170
171    use super::*;
172    use crate::layers::HdfsCompatibilityLayer;
173    use crate::layers::mock::{MockLayer, MockLayerBuilder, OpDelete, oio};
174    use crate::{Error, Result};
175
176    const EXPIRED: Duration = Duration::from_millis(1);
177
178    fn test_store() -> (TempDir, ObjectStore) {
179        let directory = tempfile::tempdir().unwrap();
180        let store = Operator::new(Fs::default().root(directory.path().to_str().unwrap())).unwrap();
181        (directory, store)
182    }
183
184    fn test_store_of(layer: MockLayer) -> (TempDir, ObjectStore) {
185        let directory = tempfile::tempdir().unwrap();
186        let store = Operator::new(Fs::default().root(directory.path().to_str().unwrap()))
187            .unwrap()
188            .layer(layer);
189        (directory, store)
190    }
191
192    fn test_store_of_failing_list(kind: ErrorKind) -> (TempDir, ObjectStore) {
193        struct FailingLister(ErrorKind);
194
195        impl oio::List for FailingLister {
196            async fn next(&mut self) -> Result<Option<oio::Entry>> {
197                Err(Error::new(self.0, "Failed to list"))
198            }
199        }
200
201        let layer = MockLayerBuilder::default()
202            .lister_factory(Arc::new(move |_, _, _| Box::new(FailingLister(kind))))
203            .build()
204            .unwrap();
205        test_store_of(layer)
206    }
207
208    // A store that fails to delete every path containing `marker`.
209    fn test_store_of_failing_delete(marker: &'static str) -> (TempDir, ObjectStore) {
210        struct FailingDeleter {
211            inner: oio::Deleter,
212            marker: &'static str,
213        }
214
215        impl oio::Delete for FailingDeleter {
216            async fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
217                if path.contains(self.marker) {
218                    return Err(Error::new(ErrorKind::Unexpected, "Failed to delete"));
219                }
220                self.inner.delete(path, args).await
221            }
222
223            async fn close(&mut self) -> Result<()> {
224                self.inner.close().await
225            }
226        }
227
228        let layer = MockLayerBuilder::default()
229            .deleter_factory(Arc::new(move |inner| {
230                Box::new(FailingDeleter { inner, marker })
231            }))
232            .build()
233            .unwrap();
234        test_store_of(layer)
235    }
236
237    async fn wait_until_expired() {
238        // A file written just before this call can carry the same modification
239        // time in milliseconds as the sweep that follows it, which would make the
240        // sweep keep it.
241        tokio::time::sleep(Duration::from_millis(10)).await;
242    }
243
244    #[test]
245    fn test_is_temporary_file() {
246        assert!(is_temporary_file(
247            ".00000000000000000042.json.greptime-53f0f4a6-3f3e-4a2f-9a2f-6fe4b0d5b5b0.tmp"
248        ));
249        assert!(is_temporary_file(".a5f2.parquet.greptime-53f0f4a6.tmp"));
250
251        assert!(!is_temporary_file("00000000000000000042.json"));
252        assert!(!is_temporary_file(".00000000000000000042.json"));
253        assert!(!is_temporary_file(
254            "00000000000000000042.json.greptime-53f0f4a6.tmp"
255        ));
256        assert!(!is_temporary_file(
257            ".00000000000000000042.json.greptime-53f0f4a6"
258        ));
259        assert!(!is_temporary_file("index/"));
260    }
261
262    #[tokio::test]
263    async fn test_sweep_dir_deletes_expired_temporary_file() {
264        let (_directory, store) = test_store();
265        let temporary = "data/.42.parquet.greptime-53f0f4a6.tmp";
266        store.create_dir("data/").await.unwrap();
267        store.write(temporary, "partial").await.unwrap();
268        store.write("data/42.parquet", "data").await.unwrap();
269
270        wait_until_expired().await;
271        assert_eq!(
272            SweepReport {
273                deleted: 1,
274                failed: 0
275            },
276            sweep_dir(&store, "data/", EXPIRED).await
277        );
278
279        assert!(!store.exists(temporary).await.unwrap());
280        assert!(store.exists("data/42.parquet").await.unwrap());
281    }
282
283    #[tokio::test]
284    async fn test_sweep_dir_keeps_young_temporary_file() {
285        let (_directory, store) = test_store();
286        let temporary = "data/.42.parquet.greptime-53f0f4a6.tmp";
287        store.create_dir("data/").await.unwrap();
288        store.write(temporary, "partial").await.unwrap();
289
290        assert_eq!(
291            SweepReport::default(),
292            sweep_dir(&store, "data/", Duration::from_secs(24 * 60 * 60)).await
293        );
294
295        assert!(store.exists(temporary).await.unwrap());
296    }
297
298    #[tokio::test]
299    async fn test_sweep_dir_of_missing_directory() {
300        // Fs lists a directory it doesn't have as an empty one, but HDFS reports
301        // it as not found.
302        let (_directory, store) = test_store_of_failing_list(ErrorKind::NotFound);
303
304        // A region that was never repartitioned has no staging directory, so a
305        // missing directory is an expected answer rather than a failure.
306        assert_eq!(
307            SweepReport::default(),
308            sweep_dir(&store, "data/staging/blob/", Duration::ZERO).await
309        );
310    }
311
312    #[tokio::test]
313    async fn test_sweep_dir_reports_failed_list() {
314        let (_directory, store) = test_store_of_failing_list(ErrorKind::Unexpected);
315
316        assert_eq!(
317            SweepReport {
318                deleted: 0,
319                failed: 1
320            },
321            sweep_dir(&store, "data/", Duration::ZERO).await
322        );
323    }
324
325    #[tokio::test]
326    async fn test_sweep_dir_keeps_going_after_failed_delete() {
327        let (_directory, store) = test_store_of_failing_delete("fail");
328        let failing = "data/.fail-42.parquet.greptime-53f0f4a6.tmp";
329        let sweepable = "data/.ok-42.parquet.greptime-53f0f4a6.tmp";
330        store.create_dir("data/").await.unwrap();
331        store.write(failing, "partial").await.unwrap();
332        store.write(sweepable, "partial").await.unwrap();
333
334        wait_until_expired().await;
335        assert_eq!(
336            SweepReport {
337                deleted: 1,
338                failed: 1
339            },
340            sweep_dir(&store, "data/", EXPIRED).await
341        );
342
343        assert!(store.exists(failing).await.unwrap());
344        assert!(!store.exists(sweepable).await.unwrap());
345    }
346
347    #[tokio::test]
348    async fn test_sweep_temporary_files_on_compatibility_layer() {
349        let (_directory, store) = test_store();
350        let temporary = "data/.42.parquet.greptime-53f0f4a6.tmp";
351        store.create_dir("data/").await.unwrap();
352        // What a writer that died before it renamed its file leaves behind.
353        store.write(temporary, "partial").await.unwrap();
354        // The same layer HDFS writes through, on an Fs store whose scheme stays `fs`.
355        let store = store.layer(HdfsCompatibilityLayer::new_for_test());
356
357        wait_until_expired().await;
358        assert_eq!(
359            SweepReport {
360                deleted: 1,
361                failed: 0
362            },
363            sweep_temporary_files(
364                &store,
365                &["data/".to_string(), "data/staging/blob/".to_string()],
366                EXPIRED,
367            )
368            .await
369        );
370
371        assert!(!store.exists(temporary).await.unwrap());
372    }
373}
object_store/temp_file.rs

object_store/
temp_file.rs
