diff --git a/control_plane/src/endpoint.rs b/control_plane/src/endpoint.rs index cba364c049..bf267df9b5 100644 --- a/control_plane/src/endpoint.rs +++ b/control_plane/src/endpoint.rs @@ -45,6 +45,7 @@ use std::sync::Arc; use std::time::Duration; use anyhow::{anyhow, bail, Context, Result}; +use compute_api::spec::GenericOption; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; use utils::id::{NodeId, TenantId, TimelineId}; @@ -478,29 +479,192 @@ impl Endpoint { } } - // Create spec file - let spec = ComputeSpec { - skip_pg_catalog_updates: self.skip_pg_catalog_updates, - format_version: 1.0, - operation_uuid: None, - cluster: Cluster { - cluster_id: None, // project ID: not used - name: None, // project name: not used - state: None, - roles: vec![], - databases: vec![], - settings: None, - postgresql_conf: Some(postgresql_conf), + let raw_spec = r#" + { + "cluster": { + "cluster_id": "young-forest-08365916", + "name": "young-forest-08365916", + "settings": [ + { + "name": "listen_addresses", + "value": "*", + "vartype": "string" + }, + { + "name": "wal_level", + "value": "replica", + "vartype": "enum" + }, + { + "name": "max_wal_size", + "value": "1024", + "vartype": "integer" + }, + { + "name": "max_parallel_workers", + "value": "8", + "vartype": "integer" + }, + { + "name": "max_connections", + "value": "112", + "vartype": "integer" + }, + { + "name": "wal_sender_timeout", + "value": "10000", + "vartype": "integer" + }, + { + "name": "synchronous_standby_names", + "value": "walproposer", + "vartype": "string" + }, + { + "name": "effective_io_concurrency", + "value": "100", + "vartype": "integer" + }, + { + "name": "shared_preload_libraries", + "value": "neon", + "vartype": "string" + }, + { + "name": "max_replication_slots", + "value": "10", + "vartype": "integer" + }, + { + "name": "neon.max_cluster_size", + "value": "204800", + "vartype": "integer" + }, + { + "name": "fsync", + "value": "off", + "vartype": "bool" + }, + { + "name": "max_replication_write_lag", + "value": "15", + "vartype": "integer" + }, + { + "name": "wal_log_hints", + "value": "off", + "vartype": "bool" + }, + { + "name": "maintenance_io_concurrency", + "value": "100", + "vartype": "integer" + }, + { + "name": "max_worker_processes", + "value": "8", + "vartype": "integer" + }, + { + "name": "idle_in_transaction_session_timeout", + "value": "300000", + "vartype": "integer" + }, + { + "name": "password_encryption", + "value": "scram-sha-256", + "vartype": "enum" + }, + { + "name": "max_replication_flush_lag", + "value": "10240", + "vartype": "integer" + }, + { + "name": "max_wal_senders", + "value": "10", + "vartype": "integer" + }, + { + "name": "restart_after_crash", + "value": "off", + "vartype": "bool" + }, + { + "name": "shared_buffers", + "value": "65536", + "vartype": "integer" + }, + { + "name": "hot_standby", + "value": "off", + "vartype": "bool" + }, + { + "name": "superuser_reserved_connections", + "value": "4", + "vartype": "integer" + }, + { + "name": "maintenance_work_mem", + "value": "65536", + "vartype": "integer" + } + ], + "roles": [ + { + "name": "arthur", + "encrypted_password": "SCRAM-SHA-256$4096:goIf+DVnIR1HglUPCerpKw==$cCdVfO1J81t5q6Ry54Eu9wJyjtm/Q5gu281MHbV/0r8=:1vyoQkj0Z/rMEzZ96QcoSECrYckSvQNgWWIL7x2Lcyw=", + "options": [] + } + ], + "databases": [ + { + "name": "neondb", + "owner": "arthur", + "options": null + } + ] }, - delta_operations: None, - tenant_id: Some(self.tenant_id), - timeline_id: Some(self.timeline_id), - mode: self.mode, - pageserver_connstring: Some(pageserver_connstring), - safekeeper_connstrings, - storage_auth_token: auth_token.clone(), - remote_extensions: None, - }; + "skip_pg_catalog_updates": false, + "mode": "Primary", + "delta_operations": null, + "format_version": 1, + "operation_uuid": "9eaaac44-1d3b-405f-abd9-3562f633c6b5", + "timestamp": "2023-09-14T16:10:56.936947442Z", + "storage_auth_token": "" + } +"#; + + let mut spec: ComputeSpec = serde_json::from_str(raw_spec)?; + if let Some(ref mut settings) = spec.cluster.settings { + settings.push(GenericOption{ + name: "neon.safekeepers".to_owned(), + value: Some(safekeeper_connstrings.join(",")), + vartype: "string".to_owned(), + }); + settings.push(GenericOption{ + name: "neon.pageserver_connstring".to_owned(), + value: Some(pageserver_connstring), + vartype: "string".to_owned(), + }); + settings.push(GenericOption{ + name: "neon.tenant_id".to_owned(), + value: Some(self.tenant_id.to_string()), + vartype: "string".to_owned(), + }); + settings.push(GenericOption{ + name: "neon.timeline_id".to_owned(), + value: Some(self.timeline_id.to_string()), + vartype: "string".to_owned(), + }); + settings.push(GenericOption{ + name: "port".to_owned(), + value: Some(self.pg_address.port().to_string()), + vartype: "string".to_owned(), + }); + } + let spec_path = self.endpoint_path().join("spec.json"); std::fs::write(spec_path, serde_json::to_string_pretty(&spec)?)?; diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index c7759d9b44..885f33e8bc 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2308,7 +2308,7 @@ class Endpoint(PgProtocol): http_port: int, check_stop_result: bool = True, ): - super().__init__(host="localhost", port=pg_port, user="cloud_admin", dbname="postgres") + super().__init__(host="localhost", port=pg_port, user="arthur", dbname="neondb", password="I2GH6gtFBpCz") self.env = env self.running = False self.branch_name: Optional[str] = None # dubious diff --git a/test_runner/regress/test_tmp.py b/test_runner/regress/test_tmp.py new file mode 100644 index 0000000000..525bce7d55 --- /dev/null +++ b/test_runner/regress/test_tmp.py @@ -0,0 +1,59 @@ +import random +import time +from fixtures.log_helper import log +from fixtures.neon_fixtures import NeonEnvBuilder +from fixtures.types import TimelineId +from fixtures.utils import print_gc_result, query_scalar +import pytest + + +@pytest.mark.parametrize('execution_number', range(10000)) +def test_tmp(neon_env_builder: NeonEnvBuilder, execution_number): + neon_env_builder.num_safekeepers = 3 + env = neon_env_builder.init_start() + try: + endpoint = env.endpoints.create_start("main") + except Exception as e: + log.error(f"failed to create endpoint: {e}") + time.sleep(100) + raise e + + log.info("postgres is running on main branch") + + def exec(sql): + before_ts = time.time() + res = endpoint.safe_psql(sql) + after_ts = time.time() + log.info(f"executed in {after_ts - before_ts:.2f}s, result {res}") + if after_ts - before_ts > 1: + raise Exception(f"executed in {after_ts - before_ts:.2f}s, result {res}") + return res + + time.sleep(random.random() * 4) + + exec("SELECT 1") + exec("CREATE TABLE IF NOT EXISTS activity_v1 (\n\t\tid SERIAL PRIMARY KEY,\n\t\tnonce BIGINT,\n\t\tval FLOAT,\n\t\tcreated_at TIMESTAMP DEFAULT NOW()\n\t )") + exec("INSERT INTO activity_v1(nonce,val) SELECT 5122547621334681000 AS nonce, avg(id) AS val FROM activity_v1 RETURNING *") + + time.sleep(random.random() * 4) + + exec("SELECT 1") + exec("CREATE TABLE IF NOT EXISTS activity_v1 (\n\t\tid SERIAL PRIMARY KEY,\n\t\tnonce BIGINT,\n\t\tval FLOAT,\n\t\tcreated_at TIMESTAMP DEFAULT NOW()\n\t )") + exec("INSERT INTO activity_v1(nonce,val) SELECT 2137073174395130327 AS nonce, avg(id) AS val FROM activity_v1 RETURNING *") + + exec("SELECT 1") + exec("CREATE TABLE IF NOT EXISTS activity_v1 (\n\t\tid SERIAL PRIMARY KEY,\n\t\tnonce BIGINT,\n\t\tval FLOAT,\n\t\tcreated_at TIMESTAMP DEFAULT NOW()\n\t )") + exec("INSERT INTO activity_v1(nonce,val) SELECT (random() * 100)::int AS nonce, avg(id) AS val FROM activity_v1 RETURNING *") + + exec("SELECT 1") + exec("CREATE TABLE IF NOT EXISTS activity_v1 (\n\t\tid SERIAL PRIMARY KEY,\n\t\tnonce BIGINT,\n\t\tval FLOAT,\n\t\tcreated_at TIMESTAMP DEFAULT NOW()\n\t )") + exec("INSERT INTO activity_v1(nonce,val) SELECT (random() * 100)::int AS nonce, avg(id) AS val FROM activity_v1 RETURNING *") + + exec("SELECT 1") + exec("CREATE TABLE IF NOT EXISTS activity_v1 (\n\t\tid SERIAL PRIMARY KEY,\n\t\tnonce BIGINT,\n\t\tval FLOAT,\n\t\tcreated_at TIMESTAMP DEFAULT NOW()\n\t )") + exec("INSERT INTO activity_v1(nonce,val) SELECT (random() * 100)::int AS nonce, avg(id) AS val FROM activity_v1 RETURNING *") + + # ERROR: could not read relation existence of rel 1663/16386/16427.1 from page server at lsn 0/014A0888 + # I2GH6gtFBpCz + # +