From 632cde7f136da8e7962153a6570b2903d8e6bd75 Mon Sep 17 00:00:00 2001 From: Suhas Thalanki <54014218+thesuhas@users.noreply.github.com> Date: Mon, 16 Jun 2025 12:30:41 -0700 Subject: [PATCH] schema and github workflow for validation of compute manifest (#12069) Adds a schema to validate the manifest.yaml described in [this RFC](https://github.com/neondatabase/neon/blob/main/docs/rfcs/038-independent-compute-release.md) and a github workflow to test this. --- .github/workflows/build_and_test.yml | 22 +++ compute/.gitignore | 3 + compute/Makefile | 8 + compute/manifest.schema.json | 209 +++++++++++++++++++++++++++ compute/manifest.yaml | 8 +- compute/package-lock.json | 37 +++++ compute/package.json | 7 + 7 files changed, 290 insertions(+), 4 deletions(-) create mode 100644 compute/manifest.schema.json create mode 100644 compute/package-lock.json create mode 100644 compute/package.json diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 9f2fa3d52c..7faaed49c1 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -199,6 +199,28 @@ jobs: build-tools-image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm secrets: inherit + validate-compute-manifest: + runs-on: ubuntu-22.04 + needs: [ meta, check-permissions ] + # We do need to run this in `.*-rc-pr` because of hotfixes. + if: ${{ contains(fromJSON('["pr", "push-main", "storage-rc-pr", "proxy-rc-pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 + with: + egress-policy: audit + + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Set up Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '24' + + - name: Validate manifest against schema + run: | + make -C compute manifest-schema-validation + build-and-test-locally: needs: [ meta, build-build-tools-image ] # We do need to run this in `.*-rc-pr` because of hotfixes. diff --git a/compute/.gitignore b/compute/.gitignore index 70980d335a..aab2afaa4e 100644 --- a/compute/.gitignore +++ b/compute/.gitignore @@ -3,3 +3,6 @@ etc/neon_collector.yml etc/neon_collector_autoscaling.yml etc/sql_exporter.yml etc/sql_exporter_autoscaling.yml + +# Node.js dependencies +node_modules/ diff --git a/compute/Makefile b/compute/Makefile index 0036196160..c53d040887 100644 --- a/compute/Makefile +++ b/compute/Makefile @@ -48,3 +48,11 @@ jsonnetfmt-test: .PHONY: jsonnetfmt-format jsonnetfmt-format: jsonnetfmt --in-place $(jsonnet_files) + +.PHONY: manifest-schema-validation +manifest-schema-validation: node_modules + node_modules/.bin/jsonschema validate -d https://json-schema.org/draft/2020-12/schema manifest.schema.json manifest.yaml + +node_modules: package.json + npm install + touch node_modules diff --git a/compute/manifest.schema.json b/compute/manifest.schema.json new file mode 100644 index 0000000000..a25055b45a --- /dev/null +++ b/compute/manifest.schema.json @@ -0,0 +1,209 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Neon Compute Manifest Schema", + "description": "Schema for Neon compute node configuration manifest", + "type": "object", + "properties": { + "pg_settings": { + "type": "object", + "properties": { + "common": { + "type": "object", + "properties": { + "client_connection_check_interval": { + "type": "string", + "description": "Check for client disconnection interval in milliseconds" + }, + "effective_io_concurrency": { + "type": "string", + "description": "Effective IO concurrency setting" + }, + "fsync": { + "type": "string", + "enum": ["on", "off"], + "description": "Whether to force fsync to disk" + }, + "hot_standby": { + "type": "string", + "enum": ["on", "off"], + "description": "Whether hot standby is enabled" + }, + "idle_in_transaction_session_timeout": { + "type": "string", + "description": "Timeout for idle transactions in milliseconds" + }, + "listen_addresses": { + "type": "string", + "description": "Addresses to listen on" + }, + "log_connections": { + "type": "string", + "enum": ["on", "off"], + "description": "Whether to log connections" + }, + "log_disconnections": { + "type": "string", + "enum": ["on", "off"], + "description": "Whether to log disconnections" + }, + "log_temp_files": { + "type": "string", + "description": "Size threshold for logging temporary files in KB" + }, + "log_error_verbosity": { + "type": "string", + "enum": ["terse", "verbose", "default"], + "description": "Error logging verbosity level" + }, + "log_min_error_statement": { + "type": "string", + "description": "Minimum error level for statement logging" + }, + "maintenance_io_concurrency": { + "type": "string", + "description": "Maintenance IO concurrency setting" + }, + "max_connections": { + "type": "string", + "description": "Maximum number of connections" + }, + "max_replication_flush_lag": { + "type": "string", + "description": "Maximum replication flush lag" + }, + "max_replication_slots": { + "type": "string", + "description": "Maximum number of replication slots" + }, + "max_replication_write_lag": { + "type": "string", + "description": "Maximum replication write lag" + }, + "max_wal_senders": { + "type": "string", + "description": "Maximum number of WAL senders" + }, + "max_wal_size": { + "type": "string", + "description": "Maximum WAL size" + }, + "neon.unstable_extensions": { + "type": "string", + "description": "List of unstable extensions" + }, + "neon.protocol_version": { + "type": "string", + "description": "Neon protocol version" + }, + "password_encryption": { + "type": "string", + "description": "Password encryption method" + }, + "restart_after_crash": { + "type": "string", + "enum": ["on", "off"], + "description": "Whether to restart after crash" + }, + "superuser_reserved_connections": { + "type": "string", + "description": "Number of reserved connections for superuser" + }, + "synchronous_standby_names": { + "type": "string", + "description": "Names of synchronous standby servers" + }, + "wal_keep_size": { + "type": "string", + "description": "WAL keep size" + }, + "wal_level": { + "type": "string", + "description": "WAL level" + }, + "wal_log_hints": { + "type": "string", + "enum": ["on", "off"], + "description": "Whether to log hints in WAL" + }, + "wal_sender_timeout": { + "type": "string", + "description": "WAL sender timeout in milliseconds" + } + }, + "required": [ + "client_connection_check_interval", + "effective_io_concurrency", + "fsync", + "hot_standby", + "idle_in_transaction_session_timeout", + "listen_addresses", + "log_connections", + "log_disconnections", + "log_temp_files", + "log_error_verbosity", + "log_min_error_statement", + "maintenance_io_concurrency", + "max_connections", + "max_replication_flush_lag", + "max_replication_slots", + "max_replication_write_lag", + "max_wal_senders", + "max_wal_size", + "neon.unstable_extensions", + "neon.protocol_version", + "password_encryption", + "restart_after_crash", + "superuser_reserved_connections", + "synchronous_standby_names", + "wal_keep_size", + "wal_level", + "wal_log_hints", + "wal_sender_timeout" + ] + }, + "replica": { + "type": "object", + "properties": { + "hot_standby": { + "type": "string", + "enum": ["on", "off"], + "description": "Whether hot standby is enabled for replicas" + } + }, + "required": ["hot_standby"] + }, + "per_version": { + "type": "object", + "patternProperties": { + "^1[4-7]$": { + "type": "object", + "properties": { + "common": { + "type": "object", + "properties": { + "io_combine_limit": { + "type": "string", + "description": "IO combine limit" + } + } + }, + "replica": { + "type": "object", + "properties": { + "recovery_prefetch": { + "type": "string", + "enum": ["on", "off"], + "description": "Whether to enable recovery prefetch for PostgreSQL replicas" + } + } + } + } + } + } + } + }, + "required": ["common", "replica", "per_version"] + } + }, + "required": ["pg_settings"] +} diff --git a/compute/manifest.yaml b/compute/manifest.yaml index f1cd20c497..4425241d8a 100644 --- a/compute/manifest.yaml +++ b/compute/manifest.yaml @@ -105,17 +105,17 @@ pg_settings: # Neon hot standby ignores pages that are not in the shared_buffers recovery_prefetch: "off" 16: - common: + common: {} replica: # prefetching of blocks referenced in WAL doesn't make sense for us # Neon hot standby ignores pages that are not in the shared_buffers recovery_prefetch: "off" 15: - common: + common: {} replica: # prefetching of blocks referenced in WAL doesn't make sense for us # Neon hot standby ignores pages that are not in the shared_buffers recovery_prefetch: "off" 14: - common: - replica: + common: {} + replica: {} diff --git a/compute/package-lock.json b/compute/package-lock.json new file mode 100644 index 0000000000..693a37cfcb --- /dev/null +++ b/compute/package-lock.json @@ -0,0 +1,37 @@ +{ + "name": "neon-compute", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "neon-compute", + "dependencies": { + "@sourcemeta/jsonschema": "9.3.4" + } + }, + "node_modules/@sourcemeta/jsonschema": { + "version": "9.3.4", + "resolved": "https://registry.npmjs.org/@sourcemeta/jsonschema/-/jsonschema-9.3.4.tgz", + "integrity": "sha512-hkujfkZAIGXUs4U//We9faZW8LZ4/H9LqagRYsFSulH/VLcKPNhZyCTGg7AhORuzm27zqENvKpnX4g2FzudYFw==", + "cpu": [ + "x64", + "arm64" + ], + "license": "AGPL-3.0", + "os": [ + "darwin", + "linux", + "win32" + ], + "bin": { + "jsonschema": "cli.js" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sourcemeta" + } + } + } +} diff --git a/compute/package.json b/compute/package.json new file mode 100644 index 0000000000..581384dc13 --- /dev/null +++ b/compute/package.json @@ -0,0 +1,7 @@ +{ + "name": "neon-compute", + "private": true, + "dependencies": { + "@sourcemeta/jsonschema": "9.3.4" + } +} \ No newline at end of file