From 32ea0d3a112076fab0190b2d976da66252d7d958 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Mon, 31 Mar 2025 15:58:24 -0500 Subject: [PATCH] Blha --- Cargo.lock | 22 ++- Cargo.toml | 4 +- cicc/Cargo.toml | 12 ++ cicc/extension.schema.json | 22 +++ cicc/src/main.rs | 145 ++++++++++++++++++ compute/extensions/timescaledb.yaml | 20 +++ .../extensions/v14/dependencies/sfcgal.toml | 0 compute/extensions/v14/ip4r.toml | 9 ++ .../extensions/v15/dependencies/sfcgal.toml | 0 compute/extensions/v15/ip4r.toml | 9 ++ .../extensions/v16/dependencies/sfcgal.toml | 0 compute/extensions/v16/ip4r.toml | 9 ++ .../extensions/v17/dependencies/sfcgal.toml | 0 compute/extensions/v17/ip4r.toml | 9 ++ compute/extensions/v17/timescaledb.toml | 14 ++ 15 files changed, 269 insertions(+), 6 deletions(-) create mode 100644 cicc/Cargo.toml create mode 100644 cicc/extension.schema.json create mode 100644 cicc/src/main.rs create mode 100644 compute/extensions/timescaledb.yaml create mode 100644 compute/extensions/v14/dependencies/sfcgal.toml create mode 100644 compute/extensions/v14/ip4r.toml create mode 100644 compute/extensions/v15/dependencies/sfcgal.toml create mode 100644 compute/extensions/v15/ip4r.toml create mode 100644 compute/extensions/v16/dependencies/sfcgal.toml create mode 100644 compute/extensions/v16/ip4r.toml create mode 100644 compute/extensions/v17/dependencies/sfcgal.toml create mode 100644 compute/extensions/v17/ip4r.toml create mode 100644 compute/extensions/v17/timescaledb.toml diff --git a/Cargo.lock b/Cargo.lock index a8e400524e..cd1376a605 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1167,6 +1167,18 @@ dependencies = [ "half", ] +[[package]] +name = "cicc" +version = "0.0.0" +dependencies = [ + "anyhow", + "clap", + "glob", + "serde", + "toml", + "url", +] + [[package]] name = "clang-sys" version = "1.6.1" @@ -2498,9 +2510,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "governor" @@ -4692,7 +4704,7 @@ dependencies = [ [[package]] name = "postgres-protocol" version = "0.6.6" -source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#1f21e7959a96a34dcfbfce1b14b73286cdadffe9" +source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#394851e467755562b4173ff68f9eb0e7f737be13" dependencies = [ "base64 0.22.1", "byteorder", @@ -4726,7 +4738,7 @@ dependencies = [ [[package]] name = "postgres-types" version = "0.2.6" -source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#1f21e7959a96a34dcfbfce1b14b73286cdadffe9" +source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#394851e467755562b4173ff68f9eb0e7f737be13" dependencies = [ "bytes", "chrono", @@ -7170,7 +7182,7 @@ dependencies = [ [[package]] name = "tokio-postgres" version = "0.7.10" -source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#1f21e7959a96a34dcfbfce1b14b73286cdadffe9" +source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#394851e467755562b4173ff68f9eb0e7f737be13" dependencies = [ "async-trait", "byteorder", diff --git a/Cargo.toml b/Cargo.toml index 9bbc5a1a38..5db1c86428 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] resolver = "2" members = [ + "cicc", "compute_tools", "control_plane", "control_plane/storcon_cli", @@ -95,6 +96,7 @@ futures = "0.3" futures-core = "0.3" futures-util = "0.3" git-version = "0.3" +glob = "0.3.2" governor = "0.8" hashbrown = "0.14" hashlink = "0.9.1" @@ -210,7 +212,7 @@ tracing-subscriber = { version = "0.3", default-features = false, features = ["s try-lock = "0.2.5" twox-hash = { version = "1.6.3", default-features = false } typed-json = "0.1" -url = "2.2" +url = { version = "2.2", features = ["serde"] } urlencoding = "2.1" uuid = { version = "1.6.1", features = ["v4", "v7", "serde"] } walkdir = "2.3.2" diff --git a/cicc/Cargo.toml b/cicc/Cargo.toml new file mode 100644 index 0000000000..313cdd8f97 --- /dev/null +++ b/cicc/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "cicc" +edition = "2024" +license.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +glob.workspace = true +serde.workspace = true +toml.workspace = true +url.workspace = true diff --git a/cicc/extension.schema.json b/cicc/extension.schema.json new file mode 100644 index 0000000000..26a7a884e0 --- /dev/null +++ b/cicc/extension.schema.json @@ -0,0 +1,22 @@ +{ + "$id": "extension", + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "build-system": { + "enum": [ + "pgxs" + ] + }, + "tarball": { + "type": "string" + }, + "checksum": { + "type": "string" + } + } +} diff --git a/cicc/src/main.rs b/cicc/src/main.rs new file mode 100644 index 0000000000..ea3b7de2af --- /dev/null +++ b/cicc/src/main.rs @@ -0,0 +1,145 @@ +use std::{io::Write, path::PathBuf}; + +use anyhow::Result; +use clap::Parser; +use glob::glob; +use serde::Deserialize; + +#[derive(Debug, Parser)] +#[command(rename_all = "kebab-case")] +struct Cli { + directory: PathBuf, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +enum BuildSystem { + #[serde(rename = "pgxs")] + PGXS, + #[serde(rename = "cmake-ninja")] + CmakeNinja, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +struct ExtensionManifest { + pub name: String, + pub version: String, + pub build_system: BuildSystem, + #[serde(default)] + pub build_arguments: Vec, + pub tarball: String, + pub checksum: String, + pub trusted: bool, +} + +impl TryFrom for ExtensionManifest { + type Error = anyhow::Error; + + fn try_from(path: PathBuf) -> std::result::Result { + let content = std::fs::read_to_string(path)?; + + let manifest: ExtensionManifest = toml::from_str(&content)?; + + Ok(manifest) + } +} + +impl ExtensionManifest { + pub fn compile(self) { + let mut stdout = std::io::stdout().lock(); + + writeln!( + &mut stdout, + r"FROM build-deps AS {name}-src +ARG PG_VERSION + +WORKDIR /ext-src +RUN wget '{tarball}' -O '{name}.tar.gz' && \ + echo '{checksum} {name}.tar.gz' | sha256sum --check && \ + mkdir '{name}-src' && cd '{name}-src' && tar xzf '../{name}.tar.gz' --strip-components=1 -C . + +FROM pg-build AS {name}-build +COPY --from={name}-src /ext-src/ /ext-src/ +WORKDIR /ext-src/{name}-src +", + name = self.name, + tarball = self.tarball, + checksum = self.checksum + ) + .unwrap(); + + match self.build_system { + BuildSystem::PGXS => writeln!( + &mut stdout, + r#"RUN make -j "$(getconf _NPROCESSORS_ONLN)" install && \"#, + ) + .unwrap(), + BuildSystem::CmakeNinja => writeln!( + &mut stdout, + r"RUN cmake -G Ninja -B build -DCMAKE_BUILD_TYPE=Release {build_args} + ninja -C install && \", + build_args = self.build_arguments.join(" ") + ) + .unwrap(), + } + + if self.trusted { + writeln!( + &mut stdout, + " echo 'trusted = true' >> '/usr/local/pgsql/share/extension/{name}.control", + name = self.name + ) + .unwrap(); + } else { + writeln!(&mut stdout, " true").unwrap(); + } + + write!(&mut stdout, "\n").unwrap(); + } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +struct DependencyManifest {} + +fn main() -> Result<()> { + let cli = Cli::parse(); + + if !cli.directory.exists() { + eprintln!("Directory does not exist"); + + std::process::exit(1); + } + + for entry in glob(cli.directory.join("*.toml").to_str().unwrap()).unwrap() { + let path = match entry { + Ok(entry) => entry, + Err(_) => continue, + }; + + let manifest: ExtensionManifest = match path.clone().try_into() { + Ok(manifest) => manifest, + Err(e) => { + eprintln!("Failed to read {}: {}", path.display(), e); + std::process::exit(1); + } + }; + + manifest.compile(); + } + + Ok(()) +} + +#[cfg(test)] +mod test { + use clap::CommandFactory; + + use super::Cli; + + #[test] + fn verify_cli() { + Cli::command().debug_assert() + } +} diff --git a/compute/extensions/timescaledb.yaml b/compute/extensions/timescaledb.yaml new file mode 100644 index 0000000000..f4f9e0f617 --- /dev/null +++ b/compute/extensions/timescaledb.yaml @@ -0,0 +1,20 @@ +name: ip4r + +build-system: "pgxs" + +trusted: true + +pg14: &pg14 + version: "2.10.1" + tarball: "https://github.com/timescale/timescaledb/archive/refs/tags/{version}.tar.gz" + checksum: 6fca72a6ed0f6d32d2b3523951ede73dc5f9b0077b38450a029a5f411fdb8c73 + +pg15: + <<: *pg14 + +pg16: + <<: *pg14 + + +pg17: + <<: *pg14 diff --git a/compute/extensions/v14/dependencies/sfcgal.toml b/compute/extensions/v14/dependencies/sfcgal.toml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/compute/extensions/v14/ip4r.toml b/compute/extensions/v14/ip4r.toml new file mode 100644 index 0000000000..63c6dfef4f --- /dev/null +++ b/compute/extensions/v14/ip4r.toml @@ -0,0 +1,9 @@ +name = "ip4r" +version = "2.4.2" + +trusted = true + +build-system = "pgxs" + +tarball = "https://github.com/RhodiumToad/ip4r/archive/refs/tags/2.4.2.tar.gz" +checksum = "0f7b1f159974f49a47842a8ab6751aecca1ed1142b6d5e38d81b064b2ead1b4b" diff --git a/compute/extensions/v15/dependencies/sfcgal.toml b/compute/extensions/v15/dependencies/sfcgal.toml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/compute/extensions/v15/ip4r.toml b/compute/extensions/v15/ip4r.toml new file mode 100644 index 0000000000..63c6dfef4f --- /dev/null +++ b/compute/extensions/v15/ip4r.toml @@ -0,0 +1,9 @@ +name = "ip4r" +version = "2.4.2" + +trusted = true + +build-system = "pgxs" + +tarball = "https://github.com/RhodiumToad/ip4r/archive/refs/tags/2.4.2.tar.gz" +checksum = "0f7b1f159974f49a47842a8ab6751aecca1ed1142b6d5e38d81b064b2ead1b4b" diff --git a/compute/extensions/v16/dependencies/sfcgal.toml b/compute/extensions/v16/dependencies/sfcgal.toml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/compute/extensions/v16/ip4r.toml b/compute/extensions/v16/ip4r.toml new file mode 100644 index 0000000000..63c6dfef4f --- /dev/null +++ b/compute/extensions/v16/ip4r.toml @@ -0,0 +1,9 @@ +name = "ip4r" +version = "2.4.2" + +trusted = true + +build-system = "pgxs" + +tarball = "https://github.com/RhodiumToad/ip4r/archive/refs/tags/2.4.2.tar.gz" +checksum = "0f7b1f159974f49a47842a8ab6751aecca1ed1142b6d5e38d81b064b2ead1b4b" diff --git a/compute/extensions/v17/dependencies/sfcgal.toml b/compute/extensions/v17/dependencies/sfcgal.toml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/compute/extensions/v17/ip4r.toml b/compute/extensions/v17/ip4r.toml new file mode 100644 index 0000000000..63c6dfef4f --- /dev/null +++ b/compute/extensions/v17/ip4r.toml @@ -0,0 +1,9 @@ +name = "ip4r" +version = "2.4.2" + +trusted = true + +build-system = "pgxs" + +tarball = "https://github.com/RhodiumToad/ip4r/archive/refs/tags/2.4.2.tar.gz" +checksum = "0f7b1f159974f49a47842a8ab6751aecca1ed1142b6d5e38d81b064b2ead1b4b" diff --git a/compute/extensions/v17/timescaledb.toml b/compute/extensions/v17/timescaledb.toml new file mode 100644 index 0000000000..01bff3065a --- /dev/null +++ b/compute/extensions/v17/timescaledb.toml @@ -0,0 +1,14 @@ +name = "timescaledb" +version = "2.17.1" + +trusted = true + +build-system = "cmake-ninja" +build-arguments = [ + "-DSEND_TELEMETRY_DEFAULT:BOOL=OFF", + "-DUSE_TELEMETRY:BOOL=OFF", + "-DAPACHE_ONLY:BOOL=ON", +] + +tarball = "https://github.com/timescale/timescaledb/archive/refs/tags/2.17.1.tar.gz" +checksum = "6277cf43f5695e23dae1c5cfeba00474d730b66ed53665a84b787a6bb1a57e28"