mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-10 23:12:54 +00:00
a22be5af72
Migrates the remaining crates to edition 2024. We like to stay on the latest edition if possible. There is no functional changes, however some code changes had to be done to accommodate the edition's breaking changes. Like the previous migration PRs, this is comprised of three commits: * the first does the edition update and makes `cargo check`/`cargo clippy` pass. we had to update bindgen to make its output [satisfy the requirements of edition 2024](https://doc.rust-lang.org/edition-guide/rust-2024/unsafe-extern.html) * the second commit does a `cargo fmt` for the new style edition. * the third commit reorders imports as a one-off change. As before, it is entirely optional. Part of #10918
74 lines
2.4 KiB
Rust
74 lines
2.4 KiB
Rust
use std::io::SeekFrom;
|
|
|
|
use anyhow::{Context, Result};
|
|
use async_compression::Level;
|
|
use async_compression::tokio::bufread::ZstdDecoder;
|
|
use async_compression::tokio::write::ZstdEncoder;
|
|
use async_compression::zstd::CParameter;
|
|
use camino::Utf8Path;
|
|
use nix::NixPath;
|
|
use tokio::fs::{File, OpenOptions};
|
|
use tokio::io::{AsyncBufRead, AsyncSeekExt, AsyncWriteExt};
|
|
use tokio_tar::{Archive, Builder, HeaderMode};
|
|
use walkdir::WalkDir;
|
|
|
|
/// Creates a Zstandard tarball.
|
|
pub async fn create_zst_tarball(path: &Utf8Path, tarball: &Utf8Path) -> Result<(File, u64)> {
|
|
let file = OpenOptions::new()
|
|
.create(true)
|
|
.truncate(true)
|
|
.read(true)
|
|
.write(true)
|
|
.open(&tarball)
|
|
.await
|
|
.with_context(|| format!("tempfile creation {tarball}"))?;
|
|
|
|
let mut paths = Vec::new();
|
|
for entry in WalkDir::new(path) {
|
|
let entry = entry?;
|
|
let metadata = entry.metadata().expect("error getting dir entry metadata");
|
|
// Also allow directories so that we also get empty directories
|
|
if !(metadata.is_file() || metadata.is_dir()) {
|
|
continue;
|
|
}
|
|
let path = entry.into_path();
|
|
paths.push(path);
|
|
}
|
|
// Do a sort to get a more consistent listing
|
|
paths.sort_unstable();
|
|
let zstd = ZstdEncoder::with_quality_and_params(
|
|
file,
|
|
Level::Default,
|
|
&[CParameter::enable_long_distance_matching(true)],
|
|
);
|
|
let mut builder = Builder::new(zstd);
|
|
// Use reproducible header mode
|
|
builder.mode(HeaderMode::Deterministic);
|
|
for p in paths {
|
|
let rel_path = p.strip_prefix(path)?;
|
|
if rel_path.is_empty() {
|
|
// The top directory should not be compressed,
|
|
// the tar crate doesn't like that
|
|
continue;
|
|
}
|
|
builder.append_path_with_name(&p, rel_path).await?;
|
|
}
|
|
let mut zstd = builder.into_inner().await?;
|
|
zstd.shutdown().await?;
|
|
let mut compressed = zstd.into_inner();
|
|
let compressed_len = compressed.metadata().await?.len();
|
|
compressed.seek(SeekFrom::Start(0)).await?;
|
|
Ok((compressed, compressed_len))
|
|
}
|
|
|
|
/// Creates a Zstandard tarball.
|
|
pub async fn extract_zst_tarball(
|
|
path: &Utf8Path,
|
|
tarball: impl AsyncBufRead + Unpin,
|
|
) -> Result<()> {
|
|
let decoder = Box::pin(ZstdDecoder::new(tarball));
|
|
let mut archive = Archive::new(decoder);
|
|
archive.unpack(path).await?;
|
|
Ok(())
|
|
}
|