diff --git a/scripts/neon_grep.txt b/scripts/neon_grep.txt deleted file mode 100644 index a41cc714c5..0000000000 --- a/scripts/neon_grep.txt +++ /dev/null @@ -1,2235 +0,0 @@ -compute_tools/src/bin/compute_ctl.rs://! `ExecStart` option. It will handle all the `Neon` specifics during compute node -compute_tools/src/bin/compute_ctl.rs://! - Sync safekeepers and get commit LSN. -compute_tools/src/bin/compute_ctl.rs://! `vm-monitor` located in [`neon/libs/vm_monitor`]. For VM compute nodes, -compute_tools/src/bin/compute_ctl.rs: #[arg(long, default_value = "neon-postgres")] -compute_tools/src/bin/compute_ctl.rs: // from the neon control plane. This allows linking together the wider -compute_tools/src/bin/compute_ctl.rs: // /sys/fs/cgroup/neon-postgres. -compute_tools/src/bin/compute_ctl.rs: // Maybe sync safekeepers again, to speed up next startup -compute_tools/src/bin/compute_ctl.rs: info!("syncing safekeepers on shutdown"); -compute_tools/src/bin/compute_ctl.rs: let lsn = compute.sync_safekeepers(storage_auth_token)?; -compute_tools/src/bin/compute_ctl.rs: info!("synced safekeepers at lsn {lsn}"); -compute_tools/src/bin/compute_ctl.rs: // - and our problems with staging https://github.com/neondatabase/cloud/issues/3707#issuecomment-1493983636 -compute_tools/src/bin/compute_ctl.rs:/// When compute_ctl is killed, send also termination signal to sync-safekeepers -compute_tools/src/bin/fast_import.rs://! See cloud.git Fast Imports RFC () -compute_tools/src/bin/fast_import.rs://! is publicly accessible at . -compute_tools/src/bin/fast_import.rs: #[clap(long, env = "NEON_IMPORTER_S3_PREFIX")] -compute_tools/src/bin/fast_import.rs: library_search_path: &pg_lib_dir, // TODO: is this right? Prob works in compute image, not sure about neon_local. -compute_tools/src/bin/fast_import.rs: // Create neondb database in the running postgres -compute_tools/src/bin/fast_import.rs: match client.simple_query("CREATE DATABASE neondb;").await { -compute_tools/src/bin/fast_import.rs: info!("created neondb database"); -compute_tools/src/bin/fast_import.rs: let restore_pg_connstring = restore_pg_connstring.replace("dbname=postgres", "dbname=neondb"); -compute_tools/src/compute.rs: CreateDatabricksMisc, CreateDatabricksRoles, CreateSchemaNeon, CreateSuperUser, -compute_tools/src/compute.rs: DropInvalidDatabases, DropRoles, FinalizeDropLogicalSubscriptions, HandleNeonExtension, -compute_tools/src/compute.rs:use crate::sync_sk::{check_if_synced, ping_safekeeper}; -compute_tools/src/compute.rs:pub static SYNC_SAFEKEEPERS_PID: AtomicU32 = AtomicU32::new(0); -compute_tools/src/compute.rs: pub safekeeper_connstrings: Vec, -compute_tools/src/compute.rs: .or_else(|| spec.cluster.settings.find("neon.pageserver_connstring")) -compute_tools/src/compute.rs: let safekeeper_connstrings = if spec.safekeeper_connstrings.is_empty() { -compute_tools/src/compute.rs: .find("neon.safekeepers") -compute_tools/src/compute.rs: .ok_or("safekeeper connstrings should be provided")? -compute_tools/src/compute.rs: spec.safekeeper_connstrings.clone() -compute_tools/src/compute.rs: .find("neon.tenant_id") -compute_tools/src/compute.rs: .find("neon.timeline_id") -compute_tools/src/compute.rs: safekeeper_connstrings, -compute_tools/src/compute.rs:/// If we are a VM, returns a [`Command`] that will run in the `neon-postgres` -compute_tools/src/compute.rs:/// neon-postgres cgroup if we are a VM. This allows autoscaling to control -compute_tools/src/compute.rs: command.args(["-g", "memory:neon-postgres"]); -compute_tools/src/compute.rs: SELECT FROM pg_catalog.pg_roles WHERE rolname = 'neon_superuser') -compute_tools/src/compute.rs: CREATE ROLE neon_superuser CREATEDB CREATEROLE NOLOGIN BYPASSRLS IN ROLE pg_read_all_data, pg_write_all_data; -compute_tools/src/compute.rs: EXECUTE format('GRANT neon_superuser TO %s', -compute_tools/src/compute.rs: EXECUTE format('GRANT ALL PRIVILEGES ON DATABASE %s TO neon_superuser', -compute_tools/src/compute.rs: ).replace("neon_superuser", "databricks_superuser") -compute_tools/src/compute.rs: let max_attempts = if let Ok(v) = env::var("NEON_COMPUTE_TESTING_BASEBACKUP_RETRIES") { -compute_tools/src/compute.rs: pub async fn check_safekeepers_synced_async( -compute_tools/src/compute.rs: // Construct a connection config for each safekeeper -compute_tools/src/compute.rs: let sk_connstrs: Vec = pspec.safekeeper_connstrings.clone(); -compute_tools/src/compute.rs: // Create task set to query all safekeepers -compute_tools/src/compute.rs: let task = tokio::time::timeout(timeout, ping_safekeeper(id, config)); -compute_tools/src/compute.rs: // be backwards compatible with safekeepers that don't have the -compute_tools/src/compute.rs: "failed sync safekeepers check {:?} {:?} {:?}", -compute_tools/src/compute.rs: // Fast path for sync_safekeepers. If they're already synced we get the lsn -compute_tools/src/compute.rs: // in one roundtrip. If not, we should do a full sync_safekeepers. -compute_tools/src/compute.rs: pub fn check_safekeepers_synced(&self, compute_state: &ComputeState) -> Result> { -compute_tools/src/compute.rs: let result = rt.block_on(self.check_safekeepers_synced_async(compute_state)); -compute_tools/src/compute.rs: // Run `postgres` in a special mode with `--sync-safekeepers` argument -compute_tools/src/compute.rs: pub fn sync_safekeepers(&self, storage_auth_token: Option) -> Result { -compute_tools/src/compute.rs: .args(["--sync-safekeepers"]) -compute_tools/src/compute.rs: vec![("NEON_AUTH_TOKEN", storage_auth_token)] -compute_tools/src/compute.rs: .expect("postgres --sync-safekeepers failed to start"); -compute_tools/src/compute.rs: SYNC_SAFEKEEPERS_PID.store(sync_handle.id(), Ordering::SeqCst); -compute_tools/src/compute.rs: // `postgres --sync-safekeepers` will print all log output to stderr and -compute_tools/src/compute.rs: .expect("postgres --sync-safekeepers failed"); -compute_tools/src/compute.rs: SYNC_SAFEKEEPERS_PID.store(0, Ordering::SeqCst); -compute_tools/src/compute.rs: "postgres --sync-safekeepers exited with non-zero status: {}. stdout: {}", -compute_tools/src/compute.rs: .expect("postgres --sync-safekeepers exited, and stdout is not utf-8"), -compute_tools/src/compute.rs: self.state.lock().unwrap().metrics.sync_safekeepers_ms = Utc::now() -compute_tools/src/compute.rs: /// safekeepers sync, basebackup, etc. -compute_tools/src/compute.rs: // Syncing safekeepers is only safe with primary nodes: if a primary -compute_tools/src/compute.rs: // cannot sync safekeepers. -compute_tools/src/compute.rs: info!("checking if safekeepers are synced"); -compute_tools/src/compute.rs: let lsn = if let Ok(Some(lsn)) = self.check_safekeepers_synced(compute_state) { -compute_tools/src/compute.rs: info!("starting safekeepers syncing"); -compute_tools/src/compute.rs: self.sync_safekeepers(pspec.storage_auth_token.clone()) -compute_tools/src/compute.rs: .with_context(|| "failed to sync safekeepers")? -compute_tools/src/compute.rs: info!("safekeepers synced at LSN {}", lsn); -compute_tools/src/compute.rs: // See https://github.com/neondatabase/autoscaling/issues/800 -compute_tools/src/compute.rs: writeln!(file, "shared_preload_libraries = 'neon'")?; -compute_tools/src/compute.rs: env_vars.extend(storage_auth_token.map(|t| ("NEON_AUTH_TOKEN".to_string(), t))); -compute_tools/src/compute.rs: /// configure the database after applying the compute spec. Currently, it upgrades the neon extension -compute_tools/src/compute.rs: handle_neon_extension_upgrade(&mut client) -compute_tools/src/compute.rs: .context("handle_neon_extension_upgrade")?; -compute_tools/src/compute.rs: // If connection fails, it may be the old node with `zenith_admin` superuser. -compute_tools/src/compute.rs: // In this case we need to connect with old `zenith_admin` name -compute_tools/src/compute.rs: // Connect with zenith_admin if cloud_admin could not authenticate -compute_tools/src/compute.rs: "cannot connect to postgres: {}, retrying with `zenith_admin` username", -compute_tools/src/compute.rs: let mut zenith_admin_conf = postgres::config::Config::from(conf.clone()); -compute_tools/src/compute.rs: zenith_admin_conf.application_name("compute_ctl:apply_config"); -compute_tools/src/compute.rs: zenith_admin_conf.user("zenith_admin"); -compute_tools/src/compute.rs: zenith_admin_conf.connect(NoTls) -compute_tools/src/compute.rs: .context("broken cloud_admin credential: tried connecting with cloud_admin but could not authenticate, and zenith_admin does not work either")?; -compute_tools/src/compute.rs: client.simple_query("SET neon.forward_ddl = false")?; -compute_tools/src/compute.rs: client.simple_query("GRANT zenith_admin TO cloud_admin")?; -compute_tools/src/compute.rs: .simple_query("SET neon.forward_ddl = false") -compute_tools/src/compute.rs: .context("apply_config SET neon.forward_ddl = false")?; -compute_tools/src/compute.rs: // in the neon.drop_subscriptions_done table. -compute_tools/src/compute.rs: CreateSchemaNeon, -compute_tools/src/compute.rs: // We install system objects (neon extensions, health check tables) in the databricks_system database only. -compute_tools/src/compute.rs: // Hadron: In upstream, the HandleNeonExtension phase depends on CreateSchemaNeon. The new CreateSchemaNeon phase -compute_tools/src/compute.rs: // In Hadron deployments, this HandleNeonExtension phase executes only in the databricks_system database. To reduce -compute_tools/src/compute.rs: // the amount of divergent code changes from upstream and to avoid creating any schemas named "neon" in user-visible -compute_tools/src/compute.rs: // databases, we preserved the old code creating the "neon" schema on-demand in the HandleNeonExtension phase. -compute_tools/src/compute.rs: HandleNeonExtension, -compute_tools/src/compute.rs: // This step depends on CreateSchemaNeon -compute_tools/src/compute.rs: config::with_compute_ctl_tmp_override(pgdata_path, "neon.max_cluster_size=-1", || { -compute_tools/src/compute.rs: "neon.max_cluster_size=-1", -compute_tools/src/compute.rs: "neon.disable_logical_replication_subscribers=false", -compute_tools/src/compute.rs: info!("updated postgresql.conf to set neon.disable_logical_replication_subscribers=false"); -compute_tools/src/compute.rs: .filter(|s| *s != "neon" && *s != "databricks_auth" && !s.is_empty()) -compute_tools/src/compute.rs: // that is used in neon_local and python tests -compute_tools/src/compute.rs: .filter(|s| *s != "neon" && *s != "databricks_auth" && !s.is_empty()) -compute_tools/src/compute.rs: let ss_pid = SYNC_SAFEKEEPERS_PID.load(Ordering::SeqCst); -compute_tools/src/config.rs: writeln!(file, "# Neon storage settings")?; -compute_tools/src/config.rs: writeln!(file, "neon.pageserver_connstring={}", escape_conf_value(s))?; -compute_tools/src/config.rs: writeln!(file, "neon.stripe_size={stripe_size}")?; -compute_tools/src/config.rs: if !spec.safekeeper_connstrings.is_empty() { -compute_tools/src/config.rs: "neon.safekeepers={}", -compute_tools/src/config.rs: escape_conf_value(&spec.safekeeper_connstrings.join(",")) -compute_tools/src/config.rs: writeln!(file, "neon.tenant_id={}", escape_conf_value(&s.to_string()))?; -compute_tools/src/config.rs: "neon.timeline_id={}", -compute_tools/src/config.rs: // This is (maybe?) temporary - for more, see https://github.com/neondatabase/cloud/issues/12047. -compute_tools/src/config.rs: writeln!(file, "neon.extension_server_port={}", extension_server_port)?; -compute_tools/src/config.rs: writeln!(file, "neon.disable_logical_replication_subscribers=true")?; -compute_tools/src/config.rs: writeln!(file, "neon.disable_logical_replication_subscribers=false")?; -compute_tools/src/disk_quota.rs:pub const DISK_QUOTA_BIN: &str = "/neonvm/bin/set-disk-quota"; -compute_tools/src/disk_quota.rs: // run `/neonvm/bin/set-disk-quota {size_kb} {mountpoint}` -compute_tools/src/http/openapi_spec.yaml: - sync_safekeepers_ms -compute_tools/src/http/openapi_spec.yaml: sync_safekeepers_ms: -compute_tools/src/http/openapi_spec.yaml: example: "neondb" -compute_tools/src/http/openapi_spec.yaml: example: "neondb" -compute_tools/src/http/openapi_spec.yaml: example: "neon" -compute_tools/src/http/openapi_spec.yaml: example: "neondb" -compute_tools/src/http/openapi_spec.yaml: example: "neon" -compute_tools/src/http/server.rs:/// neon Postgres extension on the other hand does not send one. -compute_tools/src/local_proxy.rs://! Local Proxy is a feature of our BaaS Neon Authorize project. -compute_tools/src/migration.rs: // The neon_migration.migration_id::id column is a bigint, which is equivalent to an i64 -compute_tools/src/migration.rs: /// Get the current value neon_migration.migration_id -compute_tools/src/migration.rs: .query_one("SELECT id FROM neon_migration.migration_id", &[])?; -compute_tools/src/migration.rs: /// Update the neon_migration.migration_id value -compute_tools/src/migration.rs: "UPDATE neon_migration.migration_id SET id = $1", -compute_tools/src/migration.rs: .with_context(|| format!("update neon_migration.migration_id to {migration_id}"))?; -compute_tools/src/migration.rs: .simple_query("CREATE SCHEMA IF NOT EXISTS neon_migration")?; -compute_tools/src/migration.rs: self.client.simple_query("CREATE TABLE IF NOT EXISTS neon_migration.migration_id (key INT NOT NULL PRIMARY KEY, id bigint NOT NULL DEFAULT 0)")?; -compute_tools/src/migration.rs: "INSERT INTO neon_migration.migration_id VALUES (0, 0) ON CONFLICT DO NOTHING", -compute_tools/src/migration.rs: .simple_query("ALTER SCHEMA neon_migration OWNER TO cloud_admin")?; -compute_tools/src/migration.rs: .simple_query("REVOKE ALL ON SCHEMA neon_migration FROM PUBLIC")?; -compute_tools/src/migration.rs: .replace("neon_superuser", "databricks_superuser"); -compute_tools/src/migrations/0001-neon_superuser_bypass_rls.sql:ALTER ROLE neon_superuser BYPASSRLS; -compute_tools/src/migrations/0002-alter_roles.sql: FOR role_name IN SELECT rolname FROM pg_roles WHERE pg_has_role(rolname, 'neon_superuser', 'member') -compute_tools/src/migrations/0002-alter_roles.sql: NOT pg_has_role(rolname, 'neon_superuser', 'member') AND NOT starts_with(rolname, 'pg_') -compute_tools/src/migrations/0003-grant_pg_create_subscription_to_neon_superuser.sql: EXECUTE 'GRANT pg_create_subscription TO neon_superuser'; -compute_tools/src/migrations/0004-grant_pg_monitor_to_neon_superuser.sql:GRANT pg_monitor TO neon_superuser WITH ADMIN OPTION; -compute_tools/src/migrations/0005-grant_all_on_tables_to_neon_superuser.sql:-- interacted with by neon_superuser without permission issues. -compute_tools/src/migrations/0005-grant_all_on_tables_to_neon_superuser.sql:ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO neon_superuser; -compute_tools/src/migrations/0006-grant_all_on_sequences_to_neon_superuser.sql:-- interacted with by neon_superuser without permission issues. -compute_tools/src/migrations/0006-grant_all_on_sequences_to_neon_superuser.sql:ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO neon_superuser; -compute_tools/src/migrations/0007-grant_all_on_tables_to_neon_superuser_with_grant_option.sql:ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO neon_superuser WITH GRANT OPTION; -compute_tools/src/migrations/0008-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql:ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO neon_superuser WITH GRANT OPTION; -compute_tools/src/migrations/0010-grant_snapshot_synchronization_funcs_to_neon_superuser.sql: EXECUTE 'GRANT EXECUTE ON FUNCTION pg_export_snapshot TO neon_superuser'; -compute_tools/src/migrations/0010-grant_snapshot_synchronization_funcs_to_neon_superuser.sql: EXECUTE 'GRANT EXECUTE ON FUNCTION pg_log_standby_snapshot TO neon_superuser'; -compute_tools/src/migrations/0011-grant_pg_show_replication_origin_status_to_neon_superuser.sql:GRANT EXECUTE ON FUNCTION pg_show_replication_origin_status TO neon_superuser; -compute_tools/src/migrations/tests/0001-neon_superuser_bypass_rls.sql: SELECT rolbypassrls INTO bypassrls FROM pg_roles WHERE rolname = 'neon_superuser'; -compute_tools/src/migrations/tests/0001-neon_superuser_bypass_rls.sql: RAISE EXCEPTION 'neon_superuser cannot bypass RLS'; -compute_tools/src/migrations/tests/0002-alter_roles.sql: WHERE pg_has_role(rolname, 'neon_superuser', 'member') -compute_tools/src/migrations/tests/0002-alter_roles.sql: WHERE NOT pg_has_role(rolname, 'neon_superuser', 'member') -compute_tools/src/migrations/tests/0003-grant_pg_create_subscription_to_neon_superuser.sql: IF NOT (SELECT pg_has_role('neon_superuser', 'pg_create_subscription', 'member')) THEN -compute_tools/src/migrations/tests/0003-grant_pg_create_subscription_to_neon_superuser.sql: RAISE EXCEPTION 'neon_superuser cannot execute pg_create_subscription'; -compute_tools/src/migrations/tests/0004-grant_pg_monitor_to_neon_superuser.sql: SELECT pg_has_role('neon_superuser', 'pg_monitor', 'member') AS member, -compute_tools/src/migrations/tests/0004-grant_pg_monitor_to_neon_superuser.sql: RAISE EXCEPTION 'neon_superuser is not a member of pg_monitor'; -compute_tools/src/migrations/tests/0004-grant_pg_monitor_to_neon_superuser.sql: RAISE EXCEPTION 'neon_superuser cannot grant pg_monitor'; -compute_tools/src/migrations/tests/0010-grant_snapshot_synchronization_funcs_to_neon_superuser.sql: SELECT bool_and(has_function_privilege('neon_superuser', oid, 'execute')) -compute_tools/src/migrations/tests/0010-grant_snapshot_synchronization_funcs_to_neon_superuser.sql: RAISE EXCEPTION 'neon_superuser cannot execute both pg_export_snapshot and pg_log_standby_snapshot'; -compute_tools/src/migrations/tests/0011-grant_pg_show_replication_origin_status_to_neon_superuser.sql: SELECT has_function_privilege('neon_superuser', oid, 'execute') -compute_tools/src/migrations/tests/0011-grant_pg_show_replication_origin_status_to_neon_superuser.sql: RAISE EXCEPTION 'neon_superuser cannot execute pg_show_replication_origin_status'; -compute_tools/src/monitor.rs: // to start PG (e.g., if we can't pull the spec, can't sync safekeepers, or can't get the basebackup). -compute_tools/src/spec.rs: // https://github.com/neondatabase/cloud/issues/2353 -compute_tools/src/spec.rs:/// Request spec from the control-plane by compute_id. If `NEON_CONTROL_PLANE_TOKEN` -compute_tools/src/spec.rs: let jwt: String = match std::env::var("NEON_CONTROL_PLANE_TOKEN") { -compute_tools/src/spec.rs: // Update pg_hba to contains databricks specfic settings before adding neon settings -compute_tools/src/spec.rs: // our rules before the default ones from neon. -compute_tools/src/spec.rs:pub fn handle_neon_extension_upgrade(client: &mut Client) -> Result<()> { -compute_tools/src/spec.rs: info!("handle neon extension upgrade"); -compute_tools/src/spec.rs: let query = "ALTER EXTENSION neon UPDATE"; -compute_tools/src/spec.rs: info!("update neon extension version with query: {}", query); -compute_tools/src/spec.rs: include_str!("./migrations/0001-neon_superuser_bypass_rls.sql"), -compute_tools/src/spec.rs: include_str!("./migrations/0003-grant_pg_create_subscription_to_neon_superuser.sql"), -compute_tools/src/spec.rs: include_str!("./migrations/0004-grant_pg_monitor_to_neon_superuser.sql"), -compute_tools/src/spec.rs: include_str!("./migrations/0005-grant_all_on_tables_to_neon_superuser.sql"), -compute_tools/src/spec.rs: include_str!("./migrations/0006-grant_all_on_sequences_to_neon_superuser.sql"), -compute_tools/src/spec.rs: "./migrations/0007-grant_all_on_tables_to_neon_superuser_with_grant_option.sql" -compute_tools/src/spec.rs: "./migrations/0008-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql" -compute_tools/src/spec.rs: "./migrations/0010-grant_snapshot_synchronization_funcs_to_neon_superuser.sql" -compute_tools/src/spec.rs: "./migrations/0011-grant_pg_show_replication_origin_status_to_neon_superuser.sql" -compute_tools/src/spec_apply.rs: CreateSchemaNeon, -compute_tools/src/spec_apply.rs: HandleNeonExtension, -compute_tools/src/spec_apply.rs: // Postgres Neon extension is done the way, that db is de-registered -compute_tools/src/spec_apply.rs: ApplySpecPhase::CreateSchemaNeon => Ok(Box::new(once(Operation { -compute_tools/src/spec_apply.rs: // Revoke some potentially blocking privileges (Neon-specific currently) -compute_tools/src/spec_apply.rs: .replace("neon_superuser", "databricks_superuser"), -compute_tools/src/spec_apply.rs: .replace("neon_superuser", "databricks_superuser"), -compute_tools/src/spec_apply.rs: ApplySpecPhase::HandleNeonExtension => { -compute_tools/src/spec_apply.rs: query: String::from("CREATE SCHEMA IF NOT EXISTS neon"), -compute_tools/src/spec_apply.rs: query: String::from("CREATE EXTENSION IF NOT EXISTS neon WITH SCHEMA neon"), -compute_tools/src/spec_apply.rs: "UPDATE pg_extension SET extrelocatable = true WHERE extname = 'neon'", -compute_tools/src/spec_apply.rs: comment: Some(String::from("compat/fix: make neon relocatable")), -compute_tools/src/spec_apply.rs: query: String::from("ALTER EXTENSION neon SET SCHEMA neon"), -compute_tools/src/spec_apply.rs: comment: Some(String::from("compat/fix: alter neon extension schema")), -compute_tools/src/spec_apply.rs: query: String::from("ALTER EXTENSION neon UPDATE"), -compute_tools/src/spec_apply.rs: comment: Some(String::from("compat/update: update neon extension version")), -compute_tools/src/spec_apply.rs: query: String::from("GRANT USAGE ON SCHEMA neon TO databricks_monitor"), -compute_tools/src/spec_apply.rs: comment: Some(String::from("Permissions needed to execute neon.* functions (in the databricks_system database)")), -compute_tools/src/sql/default_grants.sql: ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO neon_superuser WITH GRANT OPTION; -compute_tools/src/sql/default_grants.sql: ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO neon_superuser WITH GRANT OPTION; -compute_tools/src/sql/finalize_drop_subscriptions.sql: INSERT INTO __db_system.drop_subscriptions_done VALUES (1, current_setting('neon.timeline_id')) -compute_tools/src/sql/finalize_drop_subscriptions.sql: SET timeline_id = current_setting('neon.timeline_id'); -compute_tools/src/sql/pre_drop_role_revoke_privileges.sql:SET SESSION ROLE neon_superuser; -compute_tools/src/sql/pre_drop_role_revoke_privileges.sql: -- See https://github.com/neondatabase/cloud/issues/13582 for the context. -compute_tools/src/sql/pre_drop_role_revoke_privileges.sql: 'REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA %I FROM {role_name} GRANTED BY neon_superuser;', -compute_tools/src/sql/set_public_schema_owner.sql: IF schema_owner = 'cloud_admin' OR schema_owner = 'zenith_admin' -compute_tools/src/swap.rs:pub const RESIZE_SWAP_BIN: &str = "/neonvm/bin/resize-swap"; -compute_tools/src/swap.rs: // run `/neonvm/bin/resize-swap --once {size_bytes}` -compute_tools/src/sync_sk.rs:// Utils for running sync_safekeepers -compute_tools/src/sync_sk.rs:/// Get a safekeeper's metadata for our timeline. The id is only used for logging -compute_tools/src/sync_sk.rs:pub async fn ping_safekeeper( -compute_tools/src/sync_sk.rs:/// Given a quorum of responses, check if safekeepers are synced at some Lsn -pgxn/neon/Makefile:# pgxs/neon/Makefile -pgxn/neon/Makefile:MODULE_big = neon -pgxn/neon/Makefile: neon.o \ -pgxn/neon/Makefile: neon_pgversioncompat.o \ -pgxn/neon/Makefile: neon_perf_counters.o \ -pgxn/neon/Makefile: neon_utils.o \ -pgxn/neon/Makefile: neon_walreader.o \ -pgxn/neon/Makefile:EXTENSION = neon -pgxn/neon/Makefile: neon--1.0.sql \ -pgxn/neon/Makefile: neon--1.0--1.1.sql \ -pgxn/neon/Makefile: neon--1.1--1.2.sql \ -pgxn/neon/Makefile: neon--1.2--1.3.sql \ -pgxn/neon/Makefile: neon--1.3--1.4.sql \ -pgxn/neon/Makefile: neon--1.4--1.5.sql \ -pgxn/neon/Makefile: neon--1.5--1.4.sql \ -pgxn/neon/Makefile: neon--1.4--1.3.sql \ -pgxn/neon/Makefile: neon--1.3--1.2.sql \ -pgxn/neon/Makefile: neon--1.2--1.1.sql \ -pgxn/neon/Makefile: neon--1.1--1.0.sql -pgxn/neon/Makefile:PGFILEDESC = "neon - cloud storage for PostgreSQL" -pgxn/neon/Makefile: neon_utils.o \ -pgxn/neon/Makefile: $(FIND_TYPEDEF) . > neon.typedefs -pgxn/neon/Makefile: INDENT=$(INDENT) $(PGINDENT_SCRIPT) --typedefs neon.typedefs $(srcdir)/*.c $(srcdir)/*.h -pgxn/neon/README.md:neon extension consists of several parts: -pgxn/neon/README.md:### shared preload library `neon.so` -pgxn/neon/README.md:- walproposer: implements broadcast protocol between postgres and WAL safekeepers. -pgxn/neon/README.md:- relsize_cache: Relation size cache for better neon performance. -pgxn/neon/README.md:### SQL functions in `neon--*.sql` -pgxn/neon/README.md:Utility functions to expose neon specific information to user and metrics collection. -pgxn/neon/bitmap.h:#ifndef NEON_BITMAP_H -pgxn/neon/bitmap.h:#define NEON_BITMAP_H -pgxn/neon/bitmap.h:#endif //NEON_BITMAP_H -pgxn/neon/control_plane_connector.c: * via HTTP to the URL specified by the GUC neon.console_url when the -pgxn/neon/control_plane_connector.c: * setting neon.forward_ddl to false. -pgxn/neon/control_plane_connector.c: * contrib/neon/control_plane_connector.c -pgxn/neon/control_plane_connector.c:#include "neon_utils.h" -pgxn/neon/control_plane_connector.c: * Neon forwards all DDL from PG to neon control plane. During -pgxn/neon/control_plane_connector.c:NeonSubXactCallback( -pgxn/neon/control_plane_connector.c:NeonXactCallback(XactEvent event, void *arg) -pgxn/neon/control_plane_connector.c:RoleIsNeonSuperuser(const char *role_name) -pgxn/neon/control_plane_connector.c: if (RoleIsNeonSuperuser(owner_name)) -pgxn/neon/control_plane_connector.c: if (RoleIsNeonSuperuser(new_owner)) -pgxn/neon/control_plane_connector.c: if (RoleIsNeonSuperuser(role_name) && !superuser()) -pgxn/neon/control_plane_connector.c:NeonProcessUtility( -pgxn/neon/control_plane_connector.c: ProcessUtility_hook = NeonProcessUtility; -pgxn/neon/control_plane_connector.c: RegisterXactCallback(NeonXactCallback, NULL); -pgxn/neon/control_plane_connector.c: RegisterSubXactCallback(NeonSubXactCallback, NULL); -pgxn/neon/control_plane_connector.c: "neon.console_url", -pgxn/neon/control_plane_connector.c: "URL of the Neon Console, which will be forwarded changes to dbs and roles", -pgxn/neon/control_plane_connector.c: "neon.forward_ddl", -pgxn/neon/control_plane_connector.c: "neon.regress_test_mode", -pgxn/neon/control_plane_connector.c: jwt_token = getenv("NEON_CONTROL_PLANE_TOKEN"); -pgxn/neon/extension_server.c: * contrib/neon/extension_server.c -pgxn/neon/extension_server.c:#include "neon_utils.h" -pgxn/neon/extension_server.c:neon_download_extension_file_http(const char *filename, bool is_library) -pgxn/neon/extension_server.c: DefineCustomIntVariable("neon.extension_server_port", -pgxn/neon/extension_server.c: download_extension_file_hook = neon_download_extension_file_http; -pgxn/neon/extension_server.h: * contrib/neon/extension_server.h -pgxn/neon/file_cache.c: * pgxn/neon/file_cache.c -pgxn/neon/file_cache.c:#include "neon_pgversioncompat.h" -pgxn/neon/file_cache.c:#include "neon.h" -pgxn/neon/file_cache.c:#include "neon_perf_counters.h" -pgxn/neon/file_cache.c: * at server startup (neon.max_file_cache_size). After server startup, we -pgxn/neon/file_cache.c: * (neon.file_cache_size_limit). If the soft limit is later reduced, we shrink -pgxn/neon/file_cache.c: * Local file cache is optional and Neon can work without it. -pgxn/neon/file_cache.c: pgstat_report_wait_start(WAIT_EVENT_NEON_LFC_TRUNCATE); -pgxn/neon/file_cache.c: neon_log(LOG, "Detected COPY FROM, disabling file cache writes."); -pgxn/neon/file_cache.c: neon_log(LOG, "Detected COPY FROM, but not disabling file cache writes."); -pgxn/neon/file_cache.c: neon_log(LOG, "Failed to punch hole in file: %m"); -pgxn/neon/file_cache.c: neon_log(DEBUG1, "set local file cache limit to %d", new_size); -pgxn/neon/file_cache.c: * Hook for neon.file_cache_max_outstanding_ios. -pgxn/neon/file_cache.c: neon_log(ERROR, "databricks_storage module should be loaded via shared_preload_libraries"); -pgxn/neon/file_cache.c: DefineCustomIntVariable("neon.max_file_cache_size", -pgxn/neon/file_cache.c: "Maximal size of Neon local file cache", -pgxn/neon/file_cache.c: DefineCustomIntVariable("neon.file_cache_size_limit", -pgxn/neon/file_cache.c: "Current limit for size of Neon local file cache", -pgxn/neon/file_cache.c: DefineCustomStringVariable("neon.file_cache_path", -pgxn/neon/file_cache.c: * Notes on neon.file_cache_max_outstanding_ios and -pgxn/neon/file_cache.c: * neon.file_cache_bypass_for_copy: -pgxn/neon/file_cache.c: DefineCustomIntVariable("neon.file_cache_max_outstanding_ios", -pgxn/neon/file_cache.c: DefineCustomBoolVariable("neon.file_cache_bypass_for_copy", -pgxn/neon/file_cache.c: pgstat_report_wait_start(WAIT_EVENT_NEON_LFC_READ); -pgxn/neon/file_cache.c: neon_log(DEBUG2, "Swap file cache page"); -pgxn/neon/file_cache.c: pgstat_report_wait_start(WAIT_EVENT_NEON_LFC_WRITE); -pgxn/neon/file_cache.c:} NeonGetStatsCtx; -pgxn/neon/file_cache.c:#define NUM_NEON_GET_STATS_COLS 2 -pgxn/neon/file_cache.c:PG_FUNCTION_INFO_V1(neon_get_lfc_stats); -pgxn/neon/file_cache.c:neon_get_lfc_stats(PG_FUNCTION_ARGS) -pgxn/neon/file_cache.c: NeonGetStatsCtx *fctx; -pgxn/neon/file_cache.c: Datum values[NUM_NEON_GET_STATS_COLS]; -pgxn/neon/file_cache.c: bool nulls[NUM_NEON_GET_STATS_COLS]; -pgxn/neon/file_cache.c: fctx = (NeonGetStatsCtx *) palloc(sizeof(NeonGetStatsCtx)); -pgxn/neon/file_cache.c: tupledesc = CreateTemplateTupleDesc(NUM_NEON_GET_STATS_COLS); -pgxn/neon/file_cache.c: fctx = (NeonGetStatsCtx *) funcctx->user_fctx; -pgxn/neon/file_cache.c: neon_log(ERROR, "return type must be a row type"); -pgxn/neon/file_cache.c: neon_log(ERROR, "incorrect number of output arguments"); -pgxn/neon/libpagestore.c: * contrib/neon/libpqpagestore.c -pgxn/neon/libpagestore.c:#include "neon.h" -pgxn/neon/libpagestore.c:#include "neon_perf_counters.h" -pgxn/neon/libpagestore.c:#include "neon_utils.h" -pgxn/neon/libpagestore.c:char *neon_timeline; -pgxn/neon/libpagestore.c:char *neon_tenant; -pgxn/neon/libpagestore.c:char *neon_auth_token; -pgxn/neon/libpagestore.c:int neon_protocol_version = 2; -pgxn/neon/libpagestore.c: * The "neon.pageserver_connstring" GUC is marked with the PGC_SIGHUP option, -pgxn/neon/libpagestore.c: neon_log(LOG, "Too many shards"); -pgxn/neon/libpagestore.c: neon_log(LOG, "Connection string too long"); -pgxn/neon/libpagestore.c: // e.g., sync-safekeepers on shutdown. -pgxn/neon/libpagestore.c: neon_log(LOG, "get_num_shards(): pagestore_shared is NULL. Return 0 shards."); -pgxn/neon/libpagestore.c: neon_log(ERROR, "Shard %d is greater or equal than number of shards %d", -pgxn/neon/libpagestore.c: MyNeonCounters->pageserver_disconnects_total++; -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, DEBUG5, "Connection state: Disconnected"); -pgxn/neon/libpagestore.c: * cf. https://github.com/neondatabase/neon/issues/7897 -pgxn/neon/libpagestore.c: * neon.pageserver_connstring GUC. If the NEON_AUTH_TOKEN environment -pgxn/neon/libpagestore.c: if (neon_auth_token) -pgxn/neon/libpagestore.c: values[1] = neon_auth_token; -pgxn/neon/libpagestore.c: errmsg(NEON_TAG "[shard %d] could not establish connection to pageserver", shard_no), -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, DEBUG5, "Connection state: Connecting_Startup"); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, DEBUG5, "POLLING_FAILED"); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, elevel, -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, elevel, -pgxn/neon/libpagestore.c: WAIT_EVENT_NEON_PS_STARTING); -pgxn/neon/libpagestore.c: WAIT_EVENT_NEON_PS_STARTING); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, DEBUG5, "POLLING_OK"); -pgxn/neon/libpagestore.c: switch (neon_protocol_version) -pgxn/neon/libpagestore.c: pagestream_query = psprintf("pagestream_v3 %s %s", neon_tenant, neon_timeline); -pgxn/neon/libpagestore.c: pagestream_query = psprintf("pagestream_v2 %s %s", neon_tenant, neon_timeline); -pgxn/neon/libpagestore.c: elog(ERROR, "unexpected databricks_protocol_version %d", neon_protocol_version); -pgxn/neon/libpagestore.c: errmsg(NEON_TAG "[shard %d] could not establish connection to pageserver", shard_no), -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, elevel, "could not send pagestream command to pageserver"); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, DEBUG5, "Connection state: Connecting_PageStream"); -pgxn/neon/libpagestore.c: errmsg(NEON_TAG "[shard %d] could not establish connection to pageserver", shard_no), -pgxn/neon/libpagestore.c: WAIT_EVENT_NEON_PS_CONFIGURING); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, elevel, "could not complete handshake with pageserver: %s", -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, DEBUG5, "Connection state: Connected"); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "libpagestore: connected to '%s' with protocol version %d", connstr, neon_protocol_version); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, ERROR, "libpagestore: invalid connection state %d", shard->state); -pgxn/neon/libpagestore.c: WAIT_EVENT_NEON_PS_READ); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "could not get response from pageserver: %s", msg); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "no response received from pageserver for %0.3f s, still waiting (sent " UINT64_FORMAT " requests, received " UINT64_FORMAT " responses) (socket sndbuf=%d recvbuf=%d)", -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "received response from pageserver after %0.3f s", -pgxn/neon/libpagestore.c:pageserver_send(shardno_t shard_no, NeonRequest *request) -pgxn/neon/libpagestore.c: MyNeonCounters->pageserver_requests_sent_total++; -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "pageserver_send disconnect bad connection"); -pgxn/neon/libpagestore.c: * https://github.com/neondatabase/neon/issues/1138 So try to reestablish -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, ERROR, "request failed too many times, cancelling query"); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "pageserver_send disconnected: failed to send page request (try to reconnect): %s", msg); -pgxn/neon/libpagestore.c: char *msg = nm_to_string((NeonMessage *) request); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, PageStoreTrace, "sent request: %s", msg); -pgxn/neon/libpagestore.c:static NeonResponse * -pgxn/neon/libpagestore.c: NeonResponse *resp; -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "pageserver_receive: disconnect due to failure while parsing response"); -pgxn/neon/libpagestore.c: char *msg = nm_to_string((NeonMessage *) resp); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, PageStoreTrace, "got response: %s", msg); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "pageserver_receive disconnect: psql end of copy data: %s", pchomp(PQerrorMessage(pageserver_conn))); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, ERROR, "pageserver_receive disconnect: could not read COPY data: %s", msg); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, ERROR, "pageserver_receive disconnect: unexpected PQgetCopyData return value: %d", rc); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, ERROR, "refresh_configuration request failed, cancelling query"); -pgxn/neon/libpagestore.c: return (NeonResponse *) resp; -pgxn/neon/libpagestore.c:static NeonResponse * -pgxn/neon/libpagestore.c: NeonResponse *resp; -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "pageserver_receive: disconnect due to failure while parsing response"); -pgxn/neon/libpagestore.c: char *msg = nm_to_string((NeonMessage *) resp); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, PageStoreTrace, "got response: %s", msg); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "pageserver_receive disconnect: psql end of copy data: %s", pchomp(PQerrorMessage(pageserver_conn))); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, ERROR, "pageserver_receive disconnect: could not read COPY data: %s", msg); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, ERROR, "pageserver_receive disconnect: unexpected PQgetCopyData return value: %d", rc); -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, ERROR, "refresh_configuration request failed, cancelling query"); -pgxn/neon/libpagestore.c: return (NeonResponse *) resp; -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, WARNING, "Tried to flush while disconnected"); -pgxn/neon/libpagestore.c: MyNeonCounters->pageserver_send_flushes_total++; -pgxn/neon/libpagestore.c: neon_shard_log(shard_no, LOG, "pageserver_flush disconnect because failed to flush page requests: %s", msg); -pgxn/neon/libpagestore.c:check_neon_id(char **newval, void **extra, GucSource source) -pgxn/neon/libpagestore.c: size = add_size(size, NeonPerfCountersShmemSize()); -pgxn/neon/libpagestore.c: NeonPerfCountersShmemInit(); -pgxn/neon/libpagestore.c: DefineCustomStringVariable("neon.pageserver_connstring", -pgxn/neon/libpagestore.c: DefineCustomStringVariable("neon.timeline_id", -pgxn/neon/libpagestore.c: "Neon timeline_id the server is running on", -pgxn/neon/libpagestore.c: &neon_timeline, -pgxn/neon/libpagestore.c: check_neon_id, NULL, NULL); -pgxn/neon/libpagestore.c: DefineCustomStringVariable("neon.tenant_id", -pgxn/neon/libpagestore.c: "Neon tenant_id the server is running on", -pgxn/neon/libpagestore.c: &neon_tenant, -pgxn/neon/libpagestore.c: check_neon_id, NULL, NULL); -pgxn/neon/libpagestore.c: DefineCustomIntVariable("neon.stripe_size", -pgxn/neon/libpagestore.c: DefineCustomIntVariable("neon.max_cluster_size", -pgxn/neon/libpagestore.c: DefineCustomIntVariable("neon.flush_output_after", -pgxn/neon/libpagestore.c: DefineCustomIntVariable("neon.max_reconnect_attempts", -pgxn/neon/libpagestore.c: DefineCustomIntVariable("neon.readahead_buffer_size", -pgxn/neon/libpagestore.c: DefineCustomIntVariable("neon.protocol_version", -pgxn/neon/libpagestore.c: &neon_protocol_version, -pgxn/neon/libpagestore.c: neon_log(ERROR, "libpagestore already loaded"); -pgxn/neon/libpagestore.c: neon_log(PageStoreTrace, "libpagestore already loaded"); -pgxn/neon/libpagestore.c: * safekeepers -pgxn/neon/libpagestore.c: neon_auth_token = getenv("NEON_AUTH_TOKEN"); -pgxn/neon/libpagestore.c: if (neon_auth_token) -pgxn/neon/libpagestore.c: neon_log(LOG, "using storage auth token from DATABRICKS_AUTH_TOKEN environment variable"); -pgxn/neon/libpagestore.c: neon_log(PageStoreTrace, "set databricks_smgr hook"); -pgxn/neon/libpagestore.c: smgr_hook = smgr_neon; -pgxn/neon/libpagestore.c: smgr_init_hook = smgr_init_neon; -pgxn/neon/libpagestore.c: dbsize_hook = neon_dbsize; -pgxn/neon/libpqwalproposer.h: * Interface to set of libpq wrappers walproposer and neon_walreader need. -pgxn/neon/libpqwalproposer.h: * provide an error message indicating which safekeeper messed up. -pgxn/neon/logical_replication_monitor.c: "neon.logical_replication_max_snap_files", -pgxn/neon/logical_replication_monitor.c: "neon.logical_replication_max_logicalsnapdir_size", -pgxn/neon/logical_replication_monitor.c: snprintf(bgw.bgw_library_name, BGW_MAXLEN, "neon"); -pgxn/neon/logical_replication_monitor.h:#ifndef __NEON_LOGICAL_REPLICATION_MONITOR_H__ -pgxn/neon/logical_replication_monitor.h:#define __NEON_LOGICAL_REPLICATION_MONITOR_H__ -pgxn/neon/neon--1.0--1.1.sql:\echo Use "ALTER EXTENSION neon UPDATE TO '1.1'" to load this file. \quit -pgxn/neon/neon--1.0--1.1.sql:CREATE FUNCTION neon_get_lfc_stats() -pgxn/neon/neon--1.0--1.1.sql:AS 'MODULE_PATHNAME', 'neon_get_lfc_stats' -pgxn/neon/neon--1.0--1.1.sql:CREATE VIEW neon_lfc_stats AS -pgxn/neon/neon--1.0--1.1.sql: SELECT P.* FROM neon_get_lfc_stats() AS P (lfc_key text, lfc_value bigint); -pgxn/neon/neon--1.0.sql:\echo Use "CREATE EXTENSION neon" to load this file. \quit -pgxn/neon/neon--1.1--1.0.sql:DROP VIEW IF EXISTS neon_lfc_stats CASCADE; -pgxn/neon/neon--1.1--1.0.sql:DROP FUNCTION IF EXISTS neon_get_lfc_stats CASCADE; -pgxn/neon/neon--1.1--1.2.sql:\echo Use "ALTER EXTENSION neon UPDATE TO '1.2'" to load this file. \quit -pgxn/neon/neon--1.1--1.2.sql:CREATE OR REPLACE VIEW NEON_STAT_FILE_CACHE AS -pgxn/neon/neon--1.1--1.2.sql: FROM neon_get_lfc_stats() AS t(stat_name text, count bigint) -pgxn/neon/neon--1.1--1.2.sql:GRANT SELECT ON NEON_STAT_FILE_CACHE TO PG_MONITOR; -pgxn/neon/neon--1.2--1.1.sql:DROP VIEW IF EXISTS NEON_STAT_FILE_CACHE CASCADE; -pgxn/neon/neon--1.2--1.3.sql:\echo Use "ALTER EXTENSION neon UPDATE TO '1.3'" to load this file. \quit -pgxn/neon/neon--1.3--1.4.sql:\echo Use "ALTER EXTENSION neon UPDATE TO '1.4'" to load this file. \quit -pgxn/neon/neon--1.4--1.5.sql:\echo Use "ALTER EXTENSION neon UPDATE TO '1.5'" to load this file. \quit -pgxn/neon/neon--1.4--1.5.sql:AS 'MODULE_PATHNAME', 'neon_get_backend_perf_counters' -pgxn/neon/neon--1.4--1.5.sql:AS 'MODULE_PATHNAME', 'neon_get_perf_counters' -pgxn/neon/neon--1.4--1.5.sql:CREATE VIEW neon_backend_perf_counters AS -pgxn/neon/neon--1.4--1.5.sql:-- an aggregate query over neon_backend_perf_counters view.) -pgxn/neon/neon--1.4--1.5.sql:CREATE VIEW neon_perf_counters AS -pgxn/neon/neon--1.5--1.4.sql:DROP VIEW IF EXISTS neon_perf_counters; -pgxn/neon/neon--1.5--1.4.sql:DROP VIEW IF EXISTS neon_backend_perf_counters; -pgxn/neon/neon.c: * neon.c -pgxn/neon/neon.c: * Utility functions to expose neon specific information to user -pgxn/neon/neon.c: * contrib/neon/neon.c -pgxn/neon/neon.c:#include "neon.h" -pgxn/neon/neon.c:#include "neon_perf_counters.h" -pgxn/neon/neon.c:static void neon_shmem_startup_hook(void); -pgxn/neon/neon.c:uint32 WAIT_EVENT_NEON_LFC_MAINTENANCE; -pgxn/neon/neon.c:uint32 WAIT_EVENT_NEON_LFC_READ; -pgxn/neon/neon.c:uint32 WAIT_EVENT_NEON_LFC_TRUNCATE; -pgxn/neon/neon.c:uint32 WAIT_EVENT_NEON_LFC_WRITE; -pgxn/neon/neon.c:uint32 WAIT_EVENT_NEON_PS_STARTING; -pgxn/neon/neon.c:uint32 WAIT_EVENT_NEON_PS_CONFIGURING; -pgxn/neon/neon.c:uint32 WAIT_EVENT_NEON_PS_SEND; -pgxn/neon/neon.c:uint32 WAIT_EVENT_NEON_PS_READ; -pgxn/neon/neon.c:uint32 WAIT_EVENT_NEON_WAL_DL; -pgxn/neon/neon.c: * Neon: A standby in Neon doesn't need to start WAL replay from a checkpoint -pgxn/neon/neon.c: * Also load 'neon_rmgr'. This makes it unnecessary to list both 'neon' -pgxn/neon/neon.c: * and 'neon_rmgr' in shared_preload_libraries. -pgxn/neon/neon.c: load_file("$libdir/neon_rmgr", false); -pgxn/neon/neon.c: shmem_startup_hook = neon_shmem_startup_hook; -pgxn/neon/neon.c: Custom_XLogReaderRoutines = NeonOnDemandXLogReaderRoutines; -pgxn/neon/neon.c: "neon.disable_logical_replication_subscribers", -pgxn/neon/neon.c: "neon.allow_replica_misconfig", -pgxn/neon/neon.c: "neon.running_xacts_overflow_policy", -pgxn/neon/neon.c: EmitWarningsOnPlaceholders("neon"); -pgxn/neon/neon.c: size = GetNeonCurrentClusterSize(); -pgxn/neon/neon.c:neon_shmem_startup_hook(void) -pgxn/neon/neon.c: WAIT_EVENT_NEON_LFC_MAINTENANCE = WaitEventExtensionNew("Neon/FileCache_Maintenance"); -pgxn/neon/neon.c: WAIT_EVENT_NEON_LFC_READ = WaitEventExtensionNew("Neon/FileCache_Read"); -pgxn/neon/neon.c: WAIT_EVENT_NEON_LFC_TRUNCATE = WaitEventExtensionNew("Neon/FileCache_Truncate"); -pgxn/neon/neon.c: WAIT_EVENT_NEON_LFC_WRITE = WaitEventExtensionNew("Neon/FileCache_Write"); -pgxn/neon/neon.c: WAIT_EVENT_NEON_PS_STARTING = WaitEventExtensionNew("Neon/PS_Starting"); -pgxn/neon/neon.c: WAIT_EVENT_NEON_PS_CONFIGURING = WaitEventExtensionNew("Neon/PS_Configuring"); -pgxn/neon/neon.c: WAIT_EVENT_NEON_PS_SEND = WaitEventExtensionNew("Neon/PS_SendIO"); -pgxn/neon/neon.c: WAIT_EVENT_NEON_PS_READ = WaitEventExtensionNew("Neon/PS_ReadIO"); -pgxn/neon/neon.c: WAIT_EVENT_NEON_WAL_DL = WaitEventExtensionNew("Neon/WAL_Download"); -pgxn/neon/neon.control:# neon extension -pgxn/neon/neon.control:module_pathname = '$libdir/neon' -pgxn/neon/neon.h: * neon.h -pgxn/neon/neon.h: * contrib/neon/neon.h -pgxn/neon/neon.h:#ifndef NEON_H -pgxn/neon/neon.h:#define NEON_H -pgxn/neon/neon.h:extern char *neon_auth_token; -pgxn/neon/neon.h:extern char *neon_timeline; -pgxn/neon/neon.h:extern char *neon_tenant; -pgxn/neon/neon.h:extern uint32 WAIT_EVENT_NEON_LFC_MAINTENANCE; -pgxn/neon/neon.h:extern uint32 WAIT_EVENT_NEON_LFC_READ; -pgxn/neon/neon.h:extern uint32 WAIT_EVENT_NEON_LFC_TRUNCATE; -pgxn/neon/neon.h:extern uint32 WAIT_EVENT_NEON_LFC_WRITE; -pgxn/neon/neon.h:extern uint32 WAIT_EVENT_NEON_PS_STARTING; -pgxn/neon/neon.h:extern uint32 WAIT_EVENT_NEON_PS_CONFIGURING; -pgxn/neon/neon.h:extern uint32 WAIT_EVENT_NEON_PS_SEND; -pgxn/neon/neon.h:extern uint32 WAIT_EVENT_NEON_PS_READ; -pgxn/neon/neon.h:extern uint32 WAIT_EVENT_NEON_WAL_DL; -pgxn/neon/neon.h:#define WAIT_EVENT_NEON_LFC_MAINTENANCE PG_WAIT_EXTENSION -pgxn/neon/neon.h:#define WAIT_EVENT_NEON_LFC_READ WAIT_EVENT_BUFFILE_READ -pgxn/neon/neon.h:#define WAIT_EVENT_NEON_LFC_TRUNCATE WAIT_EVENT_BUFFILE_TRUNCATE -pgxn/neon/neon.h:#define WAIT_EVENT_NEON_LFC_WRITE WAIT_EVENT_BUFFILE_WRITE -pgxn/neon/neon.h:#define WAIT_EVENT_NEON_PS_STARTING PG_WAIT_EXTENSION -pgxn/neon/neon.h:#define WAIT_EVENT_NEON_PS_CONFIGURING PG_WAIT_EXTENSION -pgxn/neon/neon.h:#define WAIT_EVENT_NEON_PS_SEND PG_WAIT_EXTENSION -pgxn/neon/neon.h:#define WAIT_EVENT_NEON_PS_READ PG_WAIT_EXTENSION -pgxn/neon/neon.h:#define WAIT_EVENT_NEON_WAL_DL WAIT_EVENT_WAL_READ -pgxn/neon/neon.h:extern void SetNeonCurrentClusterSize(uint64 size); -pgxn/neon/neon.h:extern uint64 GetNeonCurrentClusterSize(void); -pgxn/neon/neon.h:#endif /* NEON_H */ -pgxn/neon/neon_perf_counters.c: * neon_perf_counters.c -pgxn/neon/neon_perf_counters.c: * Collect statistics about Neon I/O -pgxn/neon/neon_perf_counters.c:#include "neon_perf_counters.h" -pgxn/neon/neon_perf_counters.c:#include "neon_pgversioncompat.h" -pgxn/neon/neon_perf_counters.c:neon_per_backend_counters *neon_per_backend_counters_shared; -pgxn/neon/neon_perf_counters.c:NeonPerfCountersShmemSize(void) -pgxn/neon/neon_perf_counters.c: size = add_size(size, mul_size(NUM_NEON_PERF_COUNTER_SLOTS, -pgxn/neon/neon_perf_counters.c: sizeof(neon_per_backend_counters))); -pgxn/neon/neon_perf_counters.c:NeonPerfCountersShmemInit(void) -pgxn/neon/neon_perf_counters.c: neon_per_backend_counters_shared = -pgxn/neon/neon_perf_counters.c: mul_size(NUM_NEON_PERF_COUNTER_SLOTS, -pgxn/neon/neon_perf_counters.c: sizeof(neon_per_backend_counters)), -pgxn/neon/neon_perf_counters.c: inc_iohist(&MyNeonCounters->getpage_hist, latency); -pgxn/neon/neon_perf_counters.c: inc_iohist(&MyNeonCounters->file_cache_read_hist, latency); -pgxn/neon/neon_perf_counters.c: inc_iohist(&MyNeonCounters->file_cache_write_hist, latency); -pgxn/neon/neon_perf_counters.c: * Support functions for the views, neon_backend_perf_counters and -pgxn/neon/neon_perf_counters.c: * neon_perf_counters. -pgxn/neon/neon_perf_counters.c:neon_perf_counters_to_metrics(neon_per_backend_counters *counters) -pgxn/neon/neon_perf_counters.c:PG_FUNCTION_INFO_V1(neon_get_backend_perf_counters); -pgxn/neon/neon_perf_counters.c:neon_get_backend_perf_counters(PG_FUNCTION_ARGS) -pgxn/neon/neon_perf_counters.c: for (int procno = 0; procno < NUM_NEON_PERF_COUNTER_SLOTS; procno++) -pgxn/neon/neon_perf_counters.c: neon_per_backend_counters *counters = &neon_per_backend_counters_shared[procno]; -pgxn/neon/neon_perf_counters.c: metric_t *metrics = neon_perf_counters_to_metrics(counters); -pgxn/neon/neon_perf_counters.c:PG_FUNCTION_INFO_V1(neon_get_perf_counters); -pgxn/neon/neon_perf_counters.c:neon_get_perf_counters(PG_FUNCTION_ARGS) -pgxn/neon/neon_perf_counters.c: neon_per_backend_counters totals = {0}; -pgxn/neon/neon_perf_counters.c: uint32 num_safekeepers; -pgxn/neon/neon_perf_counters.c: uint32 num_active_safekeepers; -pgxn/neon/neon_perf_counters.c: XLogRecPtr max_active_safekeeper_commit_lag; -pgxn/neon/neon_perf_counters.c: for (int procno = 0; procno < NUM_NEON_PERF_COUNTER_SLOTS; procno++) -pgxn/neon/neon_perf_counters.c: neon_per_backend_counters *counters = &neon_per_backend_counters_shared[procno]; -pgxn/neon/neon_perf_counters.c: metrics = neon_perf_counters_to_metrics(&totals); -pgxn/neon/neon_perf_counters.c: // Not ideal but piggyback our databricks counters into the neon perf counters view -pgxn/neon/neon_perf_counters.c: // so that we don't need to introduce neon--1.x+1.sql to add a new view. -pgxn/neon/neon_perf_counters.c: // Read safekeeper status from wal proposer shared memory first. -pgxn/neon/neon_perf_counters.c: // Note that we are taking a mutex when reading from walproposer shared memory so that the total safekeeper count is -pgxn/neon/neon_perf_counters.c: num_safekeepers = wp_shmem->num_safekeepers; -pgxn/neon/neon_perf_counters.c: num_active_safekeepers = 0; -pgxn/neon/neon_perf_counters.c: for (int i = 0; i < num_safekeepers; i++) { -pgxn/neon/neon_perf_counters.c: if (wp_shmem->safekeeper_status[i] == 1) { -pgxn/neon/neon_perf_counters.c: num_active_safekeepers++; -pgxn/neon/neon_perf_counters.c: // Only track the commit LSN lag among active safekeepers. -pgxn/neon/neon_perf_counters.c: // If there are inactive safekeepers we will raise another alert so this lag value -pgxn/neon/neon_perf_counters.c: lsn = wp_shmem->safekeeper_commit_lsn[i]; -pgxn/neon/neon_perf_counters.c: // Calculate max commit LSN lag across active safekeepers -pgxn/neon/neon_perf_counters.c: max_active_safekeeper_commit_lag = (XLogRecPtrIsInvalid(min_commit_lsn) ? 0 : max_commit_lsn - min_commit_lsn); -pgxn/neon/neon_perf_counters.c: {"num_active_safekeepers", false, 0.0, (double) num_active_safekeepers}, -pgxn/neon/neon_perf_counters.c: {"num_configured_safekeepers", false, 0.0, (double) num_safekeepers}, -pgxn/neon/neon_perf_counters.c: {"max_active_safekeeper_commit_lag", false, 0.0, (double) max_active_safekeeper_commit_lag}, -pgxn/neon/neon_perf_counters.h: * neon_perf_counters.h -pgxn/neon/neon_perf_counters.h: * Performance counters for neon storage requests -pgxn/neon/neon_perf_counters.h:#ifndef NEON_PERF_COUNTERS_H -pgxn/neon/neon_perf_counters.h:#define NEON_PERF_COUNTERS_H -pgxn/neon/neon_perf_counters.h: * the backend, but the 'neon_backend_perf_counters' view will convert -pgxn/neon/neon_perf_counters.h:} neon_per_backend_counters; -pgxn/neon/neon_perf_counters.h:/* Pointer to the shared memory array of neon_per_backend_counters structs */ -pgxn/neon/neon_perf_counters.h:extern neon_per_backend_counters *neon_per_backend_counters_shared; -pgxn/neon/neon_perf_counters.h:#define NUM_NEON_PERF_COUNTER_SLOTS (MaxBackends + NUM_AUXILIARY_PROCS) -pgxn/neon/neon_perf_counters.h:#define MyNeonCounters (&neon_per_backend_counters_shared[MyProcNumber]) -pgxn/neon/neon_perf_counters.h:#define MyNeonCounters (&neon_per_backend_counters_shared[MyProc->pgprocno]) -pgxn/neon/neon_perf_counters.h:extern Size NeonPerfCountersShmemSize(void); -pgxn/neon/neon_perf_counters.h:extern void NeonPerfCountersShmemInit(void); -pgxn/neon/neon_perf_counters.h:#endif /* NEON_PERF_COUNTERS_H */ -pgxn/neon/neon_pgversioncompat.c: * Support functions for the compatibility macros in neon_pgversioncompat.h -pgxn/neon/neon_pgversioncompat.c:#include "neon_pgversioncompat.h" -pgxn/neon/neon_pgversioncompat.h:#ifndef NEON_PGVERSIONCOMPAT_H -pgxn/neon/neon_pgversioncompat.h:#define NEON_PGVERSIONCOMPAT_H -pgxn/neon/neon_pgversioncompat.h:#endif /* NEON_PGVERSIONCOMPAT_H */ -pgxn/neon/neon_utils.c:#include "neon_utils.h" -pgxn/neon/neon_utils.c: * place to call curl_global_init() would be _PG_init(), but Neon has to be -pgxn/neon/neon_utils.h:#ifndef __NEON_UTILS_H__ -pgxn/neon/neon_utils.h:#define __NEON_UTILS_H__ -pgxn/neon/neon_utils.h:#endif /* __NEON_UTILS_H__ */ -pgxn/neon/neon_walreader.c: * ERROR asynchronously tries to fetch it from the most advanced safekeeper. -pgxn/neon/neon_walreader.c:#include "neon_walreader.h" -pgxn/neon/neon_walreader.c:#define NEON_WALREADER_ERR_MSG_LEN 512 -pgxn/neon/neon_walreader.c: * Can be called where NeonWALReader *state is available in the context, adds log_prefix. -pgxn/neon/neon_walreader.c:static NeonWALReadResult NeonWALReadRemote(NeonWALReader *state, char *buf, XLogRecPtr startptr, Size count, TimeLineID tli); -pgxn/neon/neon_walreader.c:static NeonWALReadResult NeonWALReaderReadMsg(NeonWALReader *state); -pgxn/neon/neon_walreader.c:static bool NeonWALReadLocal(NeonWALReader *state, char *buf, XLogRecPtr startptr, Size count, TimeLineID tli); -pgxn/neon/neon_walreader.c: * State of connection to donor safekeeper. -pgxn/neon/neon_walreader.c:} NeonWALReaderRemoteState; -pgxn/neon/neon_walreader.c:struct NeonWALReader -pgxn/neon/neon_walreader.c: char err_msg[NEON_WALREADER_ERR_MSG_LEN]; -pgxn/neon/neon_walreader.c: char donor_name[64]; /* saved donor safekeeper name for logging */ -pgxn/neon/neon_walreader.c: /* state of connection to safekeeper */ -pgxn/neon/neon_walreader.c: NeonWALReaderRemoteState rem_state; -pgxn/neon/neon_walreader.c: /* prepended to lines logged by neon_walreader, if provided */ -pgxn/neon/neon_walreader.c:/* palloc and initialize NeonWALReader */ -pgxn/neon/neon_walreader.c:NeonWALReader * -pgxn/neon/neon_walreader.c:NeonWALReaderAllocate(int wal_segment_size, XLogRecPtr available_lsn, char *log_prefix) -pgxn/neon/neon_walreader.c: NeonWALReader *reader; -pgxn/neon/neon_walreader.c: reader = (NeonWALReader *) -pgxn/neon/neon_walreader.c: MemoryContextAllocZero(TopMemoryContext, sizeof(NeonWALReader)); -pgxn/neon/neon_walreader.c:NeonWALReaderFree(NeonWALReader *state) -pgxn/neon/neon_walreader.c: neon_wal_segment_close(state); -pgxn/neon/neon_walreader.c: * advanced safekeeper. -pgxn/neon/neon_walreader.c: * Returns NEON_WALREAD_SUCCESS if succeeded, NEON_WALREAD_ERROR if an error -pgxn/neon/neon_walreader.c: * NEON_WALREAD_WOULDBLOCK means caller should obtain socket to wait for with -pgxn/neon/neon_walreader.c: * NeonWALReaderSocket and call NeonWALRead again with exactly the same -pgxn/neon/neon_walreader.c: * arguments when NeonWALReaderEvents happen on the socket. Note that per libpq -pgxn/neon/neon_walreader.c: * should remove subscription to socket then by checking NeonWALReaderEvents -pgxn/neon/neon_walreader.c: * NEON_WALREAD_WOULDBLOCK might be always returned. -pgxn/neon/neon_walreader.c:NeonWALReadResult -pgxn/neon/neon_walreader.c:NeonWALRead(NeonWALReader *state, char *buf, XLogRecPtr startptr, Size count, TimeLineID tli) -pgxn/neon/neon_walreader.c: return NeonWALReadRemote(state, buf, startptr, count, tli); -pgxn/neon/neon_walreader.c: if (NeonWALReadLocal(state, buf, startptr, count, tli)) -pgxn/neon/neon_walreader.c: return NEON_WALREAD_SUCCESS; -pgxn/neon/neon_walreader.c: return NeonWALReadRemote(state, buf, startptr, count, tli); -pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; -pgxn/neon/neon_walreader.c:/* Do the read from remote safekeeper. */ -pgxn/neon/neon_walreader.c:static NeonWALReadResult -pgxn/neon/neon_walreader.c:NeonWALReadRemote(NeonWALReader *state, char *buf, XLogRecPtr startptr, Size count, TimeLineID tli) -pgxn/neon/neon_walreader.c: if (!NeonWALReaderUpdateDonor(state)) -pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; -pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); -pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; -pgxn/neon/neon_walreader.c: return NEON_WALREAD_WOULDBLOCK; -pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); -pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; -pgxn/neon/neon_walreader.c: return NEON_WALREAD_WOULDBLOCK; -pgxn/neon/neon_walreader.c: return NEON_WALREAD_WOULDBLOCK; -pgxn/neon/neon_walreader.c: * might stream WAL not yet committed by safekeepers. It -pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); -pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; -pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; /* keep the compiler quiet */ -pgxn/neon/neon_walreader.c: return NEON_WALREAD_WOULDBLOCK; -pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); -pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; -pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); -pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; -pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); -pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; -pgxn/neon/neon_walreader.c: NeonWALReadResult read_msg_res = NeonWALReaderReadMsg(state); -pgxn/neon/neon_walreader.c: if (read_msg_res != NEON_WALREAD_SUCCESS) -pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); -pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; -pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); -pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); -pgxn/neon/neon_walreader.c: return NEON_WALREAD_SUCCESS; -pgxn/neon/neon_walreader.c:static NeonWALReadResult -pgxn/neon/neon_walreader.c:NeonWALReaderReadMsg(NeonWALReader *state) -pgxn/neon/neon_walreader.c: return NEON_WALREAD_WOULDBLOCK; -pgxn/neon/neon_walreader.c: return NEON_WALREAD_SUCCESS; -pgxn/neon/neon_walreader.c: NeonWALReaderResetRemote(state); -pgxn/neon/neon_walreader.c: return NEON_WALREAD_ERROR; -pgxn/neon/neon_walreader.c:NeonWALReaderResetRemote(NeonWALReader *state) -pgxn/neon/neon_walreader.c: * connection exists (NeonWALReaderEvents returns non zero). -pgxn/neon/neon_walreader.c:NeonWALReaderSocket(NeonWALReader *state) -pgxn/neon/neon_walreader.c:NeonWALReaderIsRemConnEstablished(NeonWALReader *state) -pgxn/neon/neon_walreader.c:NeonWALReaderEvents(NeonWALReader *state) -pgxn/neon/neon_walreader.c:NeonWALReadLocal(NeonWALReader *state, char *buf, XLogRecPtr startptr, Size count, TimeLineID tli) -pgxn/neon/neon_walreader.c: neon_wal_segment_close(state); -pgxn/neon/neon_walreader.c: if (!neon_wal_segment_open(state, nextSegNo, &tli)) -pgxn/neon/neon_walreader.c:NeonWALReaderGetRemLsn(NeonWALReader *state) -pgxn/neon/neon_walreader.c:NeonWALReaderGetSegment(NeonWALReader *state) -pgxn/neon/neon_walreader.c:neon_wal_segment_open(NeonWALReader *state, XLogSegNo nextSegNo, -pgxn/neon/neon_walreader.c:/* copy of vanilla wal_segment_close with NeonWALReader */ -pgxn/neon/neon_walreader.c:neon_wal_segment_close(NeonWALReader *state) -pgxn/neon/neon_walreader.c:NeonWALReaderErrMsg(NeonWALReader *state) -pgxn/neon/neon_walreader.c:NeonWALReaderUpdateDonor(NeonWALReader *state) -pgxn/neon/neon_walreader.h:#ifndef __NEON_WALREADER_H__ -pgxn/neon/neon_walreader.h:#define __NEON_WALREADER_H__ -pgxn/neon/neon_walreader.h:struct NeonWALReader; -pgxn/neon/neon_walreader.h:typedef struct NeonWALReader NeonWALReader; -pgxn/neon/neon_walreader.h:/* NeonWALRead return value */ -pgxn/neon/neon_walreader.h: NEON_WALREAD_SUCCESS, -pgxn/neon/neon_walreader.h: NEON_WALREAD_WOULDBLOCK, -pgxn/neon/neon_walreader.h: NEON_WALREAD_ERROR, -pgxn/neon/neon_walreader.h:} NeonWALReadResult; -pgxn/neon/neon_walreader.h:extern NeonWALReader *NeonWALReaderAllocate(int wal_segment_size, XLogRecPtr available_lsn, char *log_prefix); -pgxn/neon/neon_walreader.h:extern void NeonWALReaderFree(NeonWALReader *state); -pgxn/neon/neon_walreader.h:extern void NeonWALReaderResetRemote(NeonWALReader *state); -pgxn/neon/neon_walreader.h:extern NeonWALReadResult NeonWALRead(NeonWALReader *state, char *buf, XLogRecPtr startptr, Size count, TimeLineID tli); -pgxn/neon/neon_walreader.h:extern pgsocket NeonWALReaderSocket(NeonWALReader *state); -pgxn/neon/neon_walreader.h:extern uint32 NeonWALReaderEvents(NeonWALReader *state); -pgxn/neon/neon_walreader.h:extern bool NeonWALReaderIsRemConnEstablished(NeonWALReader *state); -pgxn/neon/neon_walreader.h:extern char *NeonWALReaderErrMsg(NeonWALReader *state); -pgxn/neon/neon_walreader.h:extern XLogRecPtr NeonWALReaderGetRemLsn(NeonWALReader *state); -pgxn/neon/neon_walreader.h:extern const WALOpenSegment *NeonWALReaderGetSegment(NeonWALReader *state); -pgxn/neon/neon_walreader.h:extern bool neon_wal_segment_open(NeonWALReader *state, XLogSegNo nextSegNo, TimeLineID *tli_p); -pgxn/neon/neon_walreader.h:extern void neon_wal_segment_close(NeonWALReader *state); -pgxn/neon/neon_walreader.h:extern bool NeonWALReaderUpdateDonor(NeonWALReader *state); -pgxn/neon/neon_walreader.h:#endif /* __NEON_WALREADER_H__ */ -pgxn/neon/pagestore_client.h:#include "neon_pgversioncompat.h" -pgxn/neon/pagestore_client.h: T_NeonExistsRequest = 0, -pgxn/neon/pagestore_client.h: T_NeonNblocksRequest, -pgxn/neon/pagestore_client.h: T_NeonGetPageRequest, -pgxn/neon/pagestore_client.h: T_NeonDbSizeRequest, -pgxn/neon/pagestore_client.h: T_NeonGetSlruSegmentRequest, -pgxn/neon/pagestore_client.h: T_NeonTestRequest = 99, /* only in cfg(feature = "testing") */ -pgxn/neon/pagestore_client.h: T_NeonExistsResponse = 100, -pgxn/neon/pagestore_client.h: T_NeonNblocksResponse, -pgxn/neon/pagestore_client.h: T_NeonGetPageResponse, -pgxn/neon/pagestore_client.h: T_NeonErrorResponse, -pgxn/neon/pagestore_client.h: T_NeonDbSizeResponse, -pgxn/neon/pagestore_client.h: T_NeonGetSlruSegmentResponse, -pgxn/neon/pagestore_client.h: T_NeonTestResponse = 199, /* only in cfg(feature = "testing") */ -pgxn/neon/pagestore_client.h:} NeonMessageTag; -pgxn/neon/pagestore_client.h:typedef uint64 NeonRequestId; -pgxn/neon/pagestore_client.h: NeonMessageTag tag; -pgxn/neon/pagestore_client.h: NeonRequestId reqid; -pgxn/neon/pagestore_client.h:} NeonMessage; -pgxn/neon/pagestore_client.h:#define messageTag(m) (((const NeonMessage *)(m))->tag) -pgxn/neon/pagestore_client.h:#define NEON_TAG "[DATABRICKS_SMGR] " -pgxn/neon/pagestore_client.h:#define neon_log(tag, fmt, ...) ereport(tag, \ -pgxn/neon/pagestore_client.h: (errmsg(NEON_TAG fmt, ##__VA_ARGS__), \ -pgxn/neon/pagestore_client.h:#define neon_shard_log(shard_no, tag, fmt, ...) ereport(tag, \ -pgxn/neon/pagestore_client.h: (errmsg(NEON_TAG "[shard %d] " fmt, shard_no, ##__VA_ARGS__), \ -pgxn/neon/pagestore_client.h: * supertype of all the Neon*Request structs below. -pgxn/neon/pagestore_client.h:typedef NeonMessage NeonRequest; -pgxn/neon/pagestore_client.h: NeonRequest hdr; -pgxn/neon/pagestore_client.h:} NeonExistsRequest; -pgxn/neon/pagestore_client.h: NeonRequest hdr; -pgxn/neon/pagestore_client.h:} NeonNblocksRequest; -pgxn/neon/pagestore_client.h: NeonRequest hdr; -pgxn/neon/pagestore_client.h:} NeonDbSizeRequest; -pgxn/neon/pagestore_client.h: NeonRequest hdr; -pgxn/neon/pagestore_client.h:} NeonGetPageRequest; -pgxn/neon/pagestore_client.h: NeonRequest hdr; -pgxn/neon/pagestore_client.h:} NeonGetSlruSegmentRequest; -pgxn/neon/pagestore_client.h:/* supertype of all the Neon*Response structs below */ -pgxn/neon/pagestore_client.h:typedef NeonMessage NeonResponse; -pgxn/neon/pagestore_client.h: NeonExistsRequest req; -pgxn/neon/pagestore_client.h:} NeonExistsResponse; -pgxn/neon/pagestore_client.h: NeonNblocksRequest req; -pgxn/neon/pagestore_client.h:} NeonNblocksResponse; -pgxn/neon/pagestore_client.h: NeonGetPageRequest req; -pgxn/neon/pagestore_client.h:} NeonGetPageResponse; -pgxn/neon/pagestore_client.h:#define PS_GETPAGERESPONSE_SIZE (MAXALIGN(offsetof(NeonGetPageResponse, page) + BLCKSZ)) -pgxn/neon/pagestore_client.h: NeonDbSizeRequest req; -pgxn/neon/pagestore_client.h:} NeonDbSizeResponse; -pgxn/neon/pagestore_client.h: NeonResponse req; -pgxn/neon/pagestore_client.h:} NeonErrorResponse; -pgxn/neon/pagestore_client.h: NeonGetSlruSegmentRequest req; -pgxn/neon/pagestore_client.h:} NeonGetSlruSegmentResponse; -pgxn/neon/pagestore_client.h:extern StringInfoData nm_pack_request(NeonRequest *msg); -pgxn/neon/pagestore_client.h:extern NeonResponse *nm_unpack_response(StringInfo s); -pgxn/neon/pagestore_client.h:extern char *nm_to_string(NeonMessage *msg); -pgxn/neon/pagestore_client.h: bool (*send) (shardno_t shard_no, NeonRequest * request); -pgxn/neon/pagestore_client.h: NeonResponse *(*receive) (shardno_t shard_no); -pgxn/neon/pagestore_client.h: NeonResponse *(*try_receive) (shardno_t shard_no); -pgxn/neon/pagestore_client.h:extern char *neon_timeline; -pgxn/neon/pagestore_client.h:extern char *neon_tenant; -pgxn/neon/pagestore_client.h:extern int neon_protocol_version; -pgxn/neon/pagestore_client.h:extern const f_smgr *smgr_neon(ProcNumber backend, NRelFileInfo rinfo); -pgxn/neon/pagestore_client.h:extern void smgr_init_neon(void); -pgxn/neon/pagestore_client.h:} neon_request_lsns; -pgxn/neon/pagestore_client.h:extern PGDLLEXPORT void neon_read_at_lsn(NRelFileInfo rnode, ForkNumber forkNum, BlockNumber blkno, -pgxn/neon/pagestore_client.h: neon_request_lsns request_lsns, char *buffer); -pgxn/neon/pagestore_client.h:extern PGDLLEXPORT void neon_read_at_lsn(NRelFileInfo rnode, ForkNumber forkNum, BlockNumber blkno, -pgxn/neon/pagestore_client.h: neon_request_lsns request_lsns, void *buffer); -pgxn/neon/pagestore_client.h:extern int64 neon_dbsize(Oid dbNode); -pgxn/neon/pagestore_client.h:/* utils for neon relsize cache */ -pgxn/neon/pagestore_smgr.c: * contrib/neon/pagestore_smgr.c -pgxn/neon/pagestore_smgr.c:#include "neon_perf_counters.h" -pgxn/neon/pagestore_smgr.c:#define NEON_PANIC_CONNECTION_STATE(shard_no, elvl, message, ...) \ -pgxn/neon/pagestore_smgr.c: neon_shard_log(shard_no, elvl, "Broken connection state: " message, \ -pgxn/neon/pagestore_smgr.c:static bool neon_redo_read_buffer_filter(XLogReaderState *record, uint8 block_id); -pgxn/neon/pagestore_smgr.c:static BlockNumber neon_nblocks(SMgrRelation reln, ForkNumber forknum); -pgxn/neon/pagestore_smgr.c:#define GENERATE_REQUEST_ID() (((NeonRequestId)MyProcPid << 32) | ++local_request_counter) -pgxn/neon/pagestore_smgr.c: * flushes sent requests on manual flush, or every neon.flush_output_after -pgxn/neon/pagestore_smgr.c: neon_request_lsns request_lsns; -pgxn/neon/pagestore_smgr.c: NeonRequestId reqid; -pgxn/neon/pagestore_smgr.c: NeonResponse *response; /* may be null */ -pgxn/neon/pagestore_smgr.c:#include "neon.h" -pgxn/neon/pagestore_smgr.c:static void prefetch_do_request(PrefetchRequest *slot, neon_request_lsns *force_request_lsns); -pgxn/neon/pagestore_smgr.c:neon_get_request_lsns(NRelFileInfo rinfo, ForkNumber forknum, -pgxn/neon/pagestore_smgr.c: BlockNumber blkno, neon_request_lsns *output, -pgxn/neon/pagestore_smgr.c:static bool neon_prefetch_response_usable(neon_request_lsns *request_lsns, -pgxn/neon/pagestore_smgr.c: source_slot->request_lsns = (neon_request_lsns) { -pgxn/neon/pagestore_smgr.c: NeonResponse *response; -pgxn/neon/pagestore_smgr.c: neon_shard_log(slot->shard_no, ERROR, -pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetches_buffered = -pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetches_buffered = -pgxn/neon/pagestore_smgr.c: MyNeonCounters->pageserver_open_requests = -pgxn/neon/pagestore_smgr.c: NeonResponse *response; -pgxn/neon/pagestore_smgr.c: neon_shard_log(slot->shard_no, ERROR, -pgxn/neon/pagestore_smgr.c: response = (NeonResponse *) page_server->receive(shard_no); -pgxn/neon/pagestore_smgr.c: neon_shard_log(shard_no, ERROR, -pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetches_buffered = -pgxn/neon/pagestore_smgr.c: neon_shard_log(shard_no, LOG, -pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetch_discards_total += 1; -pgxn/neon/pagestore_smgr.c: MyNeonCounters->pageserver_open_requests = -pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetches_buffered = -pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetches_buffered = -pgxn/neon/pagestore_smgr.c:prefetch_do_request(PrefetchRequest *slot, neon_request_lsns *force_request_lsns) -pgxn/neon/pagestore_smgr.c: NeonGetPageRequest request = { -pgxn/neon/pagestore_smgr.c: .hdr.tag = T_NeonGetPageRequest, -pgxn/neon/pagestore_smgr.c: neon_get_request_lsns(BufTagGetNRelFileInfo(slot->buftag), -pgxn/neon/pagestore_smgr.c: while (!page_server->send(slot->shard_no, (NeonRequest *) &request)) -pgxn/neon/pagestore_smgr.c:prefetch_register_bufferv(BufferTag tag, neon_request_lsns *frlsns, -pgxn/neon/pagestore_smgr.c: MyNeonCounters->pageserver_open_requests = -pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetches_buffered = -pgxn/neon/pagestore_smgr.c: neon_request_lsns *lsns; -pgxn/neon/pagestore_smgr.c: if (!neon_prefetch_response_usable(lsns, slot)) -pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetch_discards_total += 1; -pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetch_misses_total++; -pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetch_discards_total += 1; -pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetch_discards_total += 1; -pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetch_requests_total++; -pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_sync_requests_total++; -pgxn/neon/pagestore_smgr.c: MyNeonCounters->pageserver_open_requests = -pgxn/neon/pagestore_smgr.c:equal_requests(NeonRequest* a, NeonRequest* b) -pgxn/neon/pagestore_smgr.c:static NeonResponse * -pgxn/neon/pagestore_smgr.c: NeonResponse *resp; -pgxn/neon/pagestore_smgr.c: case T_NeonExistsRequest: -pgxn/neon/pagestore_smgr.c: CopyNRelFileInfoToBufTag(tag, ((NeonExistsRequest *) req)->rinfo); -pgxn/neon/pagestore_smgr.c: case T_NeonNblocksRequest: -pgxn/neon/pagestore_smgr.c: CopyNRelFileInfoToBufTag(tag, ((NeonNblocksRequest *) req)->rinfo); -pgxn/neon/pagestore_smgr.c: case T_NeonDbSizeRequest: -pgxn/neon/pagestore_smgr.c: NInfoGetDbOid(BufTagGetNRelFileInfo(tag)) = ((NeonDbSizeRequest *) req)->dbNode; -pgxn/neon/pagestore_smgr.c: case T_NeonGetPageRequest: -pgxn/neon/pagestore_smgr.c: CopyNRelFileInfoToBufTag(tag, ((NeonGetPageRequest *) req)->rinfo); -pgxn/neon/pagestore_smgr.c: tag.blockNum = ((NeonGetPageRequest *) req)->blkno; -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "Unexpected request tag: %d", messageTag(req)); -pgxn/neon/pagestore_smgr.c: if (((NeonRequest *) req)->tag != T_NeonGetPageRequest) -pgxn/neon/pagestore_smgr.c: while (!page_server->send(shard_no, (NeonRequest *) req) -pgxn/neon/pagestore_smgr.c: MyNeonCounters->pageserver_open_requests++; -pgxn/neon/pagestore_smgr.c: MyNeonCounters->pageserver_open_requests--; -pgxn/neon/pagestore_smgr.c: MyNeonCounters->pageserver_open_requests = 0; -pgxn/neon/pagestore_smgr.c:nm_pack_request(NeonRequest *msg) -pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) -pgxn/neon/pagestore_smgr.c: case T_NeonExistsRequest: -pgxn/neon/pagestore_smgr.c: NeonExistsRequest *msg_req = (NeonExistsRequest *) msg; -pgxn/neon/pagestore_smgr.c: case T_NeonNblocksRequest: -pgxn/neon/pagestore_smgr.c: NeonNblocksRequest *msg_req = (NeonNblocksRequest *) msg; -pgxn/neon/pagestore_smgr.c: case T_NeonDbSizeRequest: -pgxn/neon/pagestore_smgr.c: NeonDbSizeRequest *msg_req = (NeonDbSizeRequest *) msg; -pgxn/neon/pagestore_smgr.c: case T_NeonGetPageRequest: -pgxn/neon/pagestore_smgr.c: NeonGetPageRequest *msg_req = (NeonGetPageRequest *) msg; -pgxn/neon/pagestore_smgr.c: case T_NeonGetSlruSegmentRequest: -pgxn/neon/pagestore_smgr.c: NeonGetSlruSegmentRequest *msg_req = (NeonGetSlruSegmentRequest *) msg; -pgxn/neon/pagestore_smgr.c: case T_NeonExistsResponse: -pgxn/neon/pagestore_smgr.c: case T_NeonNblocksResponse: -pgxn/neon/pagestore_smgr.c: case T_NeonGetPageResponse: -pgxn/neon/pagestore_smgr.c: case T_NeonErrorResponse: -pgxn/neon/pagestore_smgr.c: case T_NeonDbSizeResponse: -pgxn/neon/pagestore_smgr.c: case T_NeonGetSlruSegmentResponse: -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unexpected databricks message tag 0x%02x", msg->tag); -pgxn/neon/pagestore_smgr.c:NeonResponse * -pgxn/neon/pagestore_smgr.c: NeonMessageTag tag = pq_getmsgbyte(s); -pgxn/neon/pagestore_smgr.c: NeonResponse resp_hdr = {0}; /* make valgrind happy */ -pgxn/neon/pagestore_smgr.c: NeonResponse *resp = NULL; -pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) -pgxn/neon/pagestore_smgr.c: case T_NeonExistsResponse: -pgxn/neon/pagestore_smgr.c: NeonExistsResponse *msg_resp = palloc0(sizeof(NeonExistsResponse)); -pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) -pgxn/neon/pagestore_smgr.c: resp = (NeonResponse *) msg_resp; -pgxn/neon/pagestore_smgr.c: case T_NeonNblocksResponse: -pgxn/neon/pagestore_smgr.c: NeonNblocksResponse *msg_resp = palloc0(sizeof(NeonNblocksResponse)); -pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) -pgxn/neon/pagestore_smgr.c: resp = (NeonResponse *) msg_resp; -pgxn/neon/pagestore_smgr.c: case T_NeonGetPageResponse: -pgxn/neon/pagestore_smgr.c: NeonGetPageResponse *msg_resp; -pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) -pgxn/neon/pagestore_smgr.c: Assert(msg_resp->req.hdr.tag == T_NeonGetPageResponse); -pgxn/neon/pagestore_smgr.c: resp = (NeonResponse *) msg_resp; -pgxn/neon/pagestore_smgr.c: case T_NeonDbSizeResponse: -pgxn/neon/pagestore_smgr.c: NeonDbSizeResponse *msg_resp = palloc0(sizeof(NeonDbSizeResponse)); -pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) -pgxn/neon/pagestore_smgr.c: resp = (NeonResponse *) msg_resp; -pgxn/neon/pagestore_smgr.c: case T_NeonErrorResponse: -pgxn/neon/pagestore_smgr.c: NeonErrorResponse *msg_resp; -pgxn/neon/pagestore_smgr.c: msg_resp = palloc0(sizeof(NeonErrorResponse) + msglen + 1); -pgxn/neon/pagestore_smgr.c: resp = (NeonResponse *) msg_resp; -pgxn/neon/pagestore_smgr.c: case T_NeonGetSlruSegmentResponse: -pgxn/neon/pagestore_smgr.c: NeonGetSlruSegmentResponse *msg_resp; -pgxn/neon/pagestore_smgr.c: msg_resp = palloc0(sizeof(NeonGetSlruSegmentResponse)); -pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) -pgxn/neon/pagestore_smgr.c: resp = (NeonResponse *) msg_resp; -pgxn/neon/pagestore_smgr.c: case T_NeonExistsRequest: -pgxn/neon/pagestore_smgr.c: case T_NeonNblocksRequest: -pgxn/neon/pagestore_smgr.c: case T_NeonGetPageRequest: -pgxn/neon/pagestore_smgr.c: case T_NeonDbSizeRequest: -pgxn/neon/pagestore_smgr.c: case T_NeonGetSlruSegmentRequest: -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unexpected databricks message tag 0x%02x", tag); -pgxn/neon/pagestore_smgr.c:nm_to_string(NeonMessage *msg) -pgxn/neon/pagestore_smgr.c: case T_NeonExistsRequest: -pgxn/neon/pagestore_smgr.c: NeonExistsRequest *msg_req = (NeonExistsRequest *) msg; -pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonExistsRequest\""); -pgxn/neon/pagestore_smgr.c: case T_NeonNblocksRequest: -pgxn/neon/pagestore_smgr.c: NeonNblocksRequest *msg_req = (NeonNblocksRequest *) msg; -pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonNblocksRequest\""); -pgxn/neon/pagestore_smgr.c: case T_NeonGetPageRequest: -pgxn/neon/pagestore_smgr.c: NeonGetPageRequest *msg_req = (NeonGetPageRequest *) msg; -pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonGetPageRequest\""); -pgxn/neon/pagestore_smgr.c: case T_NeonDbSizeRequest: -pgxn/neon/pagestore_smgr.c: NeonDbSizeRequest *msg_req = (NeonDbSizeRequest *) msg; -pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonDbSizeRequest\""); -pgxn/neon/pagestore_smgr.c: case T_NeonGetSlruSegmentRequest: -pgxn/neon/pagestore_smgr.c: NeonGetSlruSegmentRequest *msg_req = (NeonGetSlruSegmentRequest *) msg; -pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonGetSlruSegmentRequest\""); -pgxn/neon/pagestore_smgr.c: case T_NeonExistsResponse: -pgxn/neon/pagestore_smgr.c: NeonExistsResponse *msg_resp = (NeonExistsResponse *) msg; -pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonExistsResponse\""); -pgxn/neon/pagestore_smgr.c: case T_NeonNblocksResponse: -pgxn/neon/pagestore_smgr.c: NeonNblocksResponse *msg_resp = (NeonNblocksResponse *) msg; -pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonNblocksResponse\""); -pgxn/neon/pagestore_smgr.c: case T_NeonGetPageResponse: -pgxn/neon/pagestore_smgr.c: NeonGetPageResponse *msg_resp = (NeonGetPageResponse *) msg; -pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonGetPageResponse\""); -pgxn/neon/pagestore_smgr.c: case T_NeonErrorResponse: -pgxn/neon/pagestore_smgr.c: NeonErrorResponse *msg_resp = (NeonErrorResponse *) msg; -pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonErrorResponse\""); -pgxn/neon/pagestore_smgr.c: case T_NeonDbSizeResponse: -pgxn/neon/pagestore_smgr.c: NeonDbSizeResponse *msg_resp = (NeonDbSizeResponse *) msg; -pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonDbSizeResponse\""); -pgxn/neon/pagestore_smgr.c: case T_NeonGetSlruSegmentResponse: -pgxn/neon/pagestore_smgr.c: NeonGetSlruSegmentResponse *msg_resp = (NeonGetSlruSegmentResponse *) msg; -pgxn/neon/pagestore_smgr.c: appendStringInfoString(&s, "{\"type\": \"NeonGetSlruSegmentResponse\""); -pgxn/neon/pagestore_smgr.c: // Replace "Neon" with "Dbrx" -pgxn/neon/pagestore_smgr.c: while ((pos = strcasestr(pos, "neon")) != NULL) { -pgxn/neon/pagestore_smgr.c:neon_wallog_pagev(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, -pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Page %u through %u of relation %u/%u/%u.%u " -pgxn/neon/pagestore_smgr.c: * all-zeros pages, and we can just ignore that in Neon. We do need to -pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Page %u of relation %u/%u/%u.%u is all-zeros", -pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Page %u of relation %u/%u/%u.%u is an empty heap page with no LSN", -pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Page %u of relation %u/%u/%u.%u is evicted with zero LSN", -pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Evicting page %u of relation %u/%u/%u.%u with lsn=%X/%X", -pgxn/neon/pagestore_smgr.c:neon_wallog_page(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *buffer, bool force) -pgxn/neon/pagestore_smgr.c:neon_wallog_page(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, const char *buffer, bool force) -pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Page %u of relation %u/%u/%u.%u was force logged. Evicted at lsn=%X/%X", -pgxn/neon/pagestore_smgr.c: * all-zeros pages, and we can just ignore that in Neon. We do need to -pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Page %u of relation %u/%u/%u.%u is all-zeros", -pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Page %u of relation %u/%u/%u.%u is an empty heap page with no LSN", -pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Page %u of relation %u/%u/%u.%u is evicted with zero LSN", -pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "Evicting page %u of relation %u/%u/%u.%u with lsn=%X/%X", -pgxn/neon/pagestore_smgr.c: * neon_init() -- Initialize private state -pgxn/neon/pagestore_smgr.c:neon_init(void) -pgxn/neon/pagestore_smgr.c: if (MyNeonCounters >= &neon_per_backend_counters_shared[NUM_NEON_PERF_COUNTER_SLOTS]) -pgxn/neon/pagestore_smgr.c: elog(ERROR, "MyNeonCounters points past end of array"); -pgxn/neon/pagestore_smgr.c: redo_read_buffer_filter = neon_redo_read_buffer_filter; -pgxn/neon/pagestore_smgr.c:neon_get_request_lsns(NRelFileInfo rinfo, ForkNumber forknum, BlockNumber blkno, -pgxn/neon/pagestore_smgr.c: neon_request_lsns *output, BlockNumber nblocks, -pgxn/neon/pagestore_smgr.c: * page cache (see neon_redo_read_buffer_filter below). -pgxn/neon/pagestore_smgr.c: * neon_redo_read_buffer_filter below). Normally, in a normal -pgxn/neon/pagestore_smgr.c: neon_request_lsns *result = &output[i]; -pgxn/neon/pagestore_smgr.c: * neon_read_at_lsn() call later will wait for the WAL record to be -pgxn/neon/pagestore_smgr.c: neon_log(DEBUG1, "databricks_get_request_lsns request lsn %X/%X, not_modified_since %X/%X", -pgxn/neon/pagestore_smgr.c: neon_request_lsns *result = &output[i]; -pgxn/neon/pagestore_smgr.c: neon_log(DEBUG1, "databricks_get_request_lsns GetLastWrittenLSN lsn %X/%X", -pgxn/neon/pagestore_smgr.c: neon_log(DEBUG5, "last-written LSN %X/%X is ahead of last flushed LSN %X/%X", -pgxn/neon/pagestore_smgr.c: * neon_prefetch_response_usable -- Can a new request be satisfied by old one? -pgxn/neon/pagestore_smgr.c:neon_prefetch_response_usable(neon_request_lsns *request_lsns, -pgxn/neon/pagestore_smgr.c: * interface in neon_test_utils to fetch pages at arbitary LSNs, which -pgxn/neon/pagestore_smgr.c: errmsg(NEON_TAG "request with unexpected LSN after prefetch"), -pgxn/neon/pagestore_smgr.c: * neon_exists() -- Does the physical file exist? -pgxn/neon/pagestore_smgr.c:neon_exists(SMgrRelation reln, ForkNumber forkNum) -pgxn/neon/pagestore_smgr.c: NeonResponse *resp; -pgxn/neon/pagestore_smgr.c: neon_request_lsns request_lsns; -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); -pgxn/neon/pagestore_smgr.c: neon_get_request_lsns(InfoFromSMgrRel(reln), forkNum, -pgxn/neon/pagestore_smgr.c: NeonExistsRequest request = { -pgxn/neon/pagestore_smgr.c: .hdr.tag = T_NeonExistsRequest, -pgxn/neon/pagestore_smgr.c: case T_NeonExistsResponse: -pgxn/neon/pagestore_smgr.c: NeonExistsResponse* exists_resp = (NeonExistsResponse *) resp; -pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) -pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(-1, PANIC, -pgxn/neon/pagestore_smgr.c: case T_NeonErrorResponse: -pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) -pgxn/neon/pagestore_smgr.c: elog(WARNING, NEON_TAG "Error message {reqid=%lx,lsn=%X/%08X, since=%X/%08X} doesn't match exists request {reqid=%lx,lsn=%X/%08X, since=%X/%08X}", -pgxn/neon/pagestore_smgr.c: errmsg(NEON_TAG "[reqid %lx] could not read relation existence of rel %u/%u/%u.%u from page server at lsn %X/%08X", -pgxn/neon/pagestore_smgr.c: ((NeonErrorResponse *) resp)->message))); -pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(-1, PANIC, -pgxn/neon/pagestore_smgr.c: T_NeonExistsResponse, T_NeonErrorResponse, resp->tag); -pgxn/neon/pagestore_smgr.c: * neon_create() -- Create a new relation on neond storage -pgxn/neon/pagestore_smgr.c:neon_create(SMgrRelation reln, ForkNumber forkNum, bool isRedo) -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgrcreate() on rel with unknown persistence"); -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); -pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "Create relation %u/%u/%u.%u", -pgxn/neon/pagestore_smgr.c: * neon_unlink() -- Unlink a relation. -pgxn/neon/pagestore_smgr.c:neon_unlink(NRelFileInfoBackend rinfo, ForkNumber forkNum, bool isRedo) -pgxn/neon/pagestore_smgr.c: * neon_extend() -- Add a block to the specified relation. -pgxn/neon/pagestore_smgr.c:neon_extend(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, -pgxn/neon/pagestore_smgr.c:neon_extend(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgrextend() on rel with unknown persistence"); -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); -pgxn/neon/pagestore_smgr.c: uint64 current_size = GetNeonCurrentClusterSize(); -pgxn/neon/pagestore_smgr.c: n_blocks = neon_nblocks(reln, forkNum); -pgxn/neon/pagestore_smgr.c: neon_wallog_page(reln, forkNum, n_blocks++, buffer, true); -pgxn/neon/pagestore_smgr.c: neon_wallog_page(reln, forkNum, blkno, buffer, false); -pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "smgrextend called for %u/%u/%u.%u blk %u, page LSN: %X/%08X", -pgxn/neon/pagestore_smgr.c:neon_zeroextend(SMgrRelation reln, ForkNumber forkNum, BlockNumber blocknum, -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgrextend() on rel with unknown persistence"); -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); -pgxn/neon/pagestore_smgr.c: uint64 current_size = GetNeonCurrentClusterSize(); -pgxn/neon/pagestore_smgr.c: errmsg(NEON_TAG "cannot extend file \"%s\" beyond %u blocks", -pgxn/neon/pagestore_smgr.c: * neon_open() -- Initialize newly-opened relation. -pgxn/neon/pagestore_smgr.c:neon_open(SMgrRelation reln) -pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "open noop"); -pgxn/neon/pagestore_smgr.c: * neon_close() -- Close the specified relation, if it isn't closed already. -pgxn/neon/pagestore_smgr.c:neon_close(SMgrRelation reln, ForkNumber forknum) -pgxn/neon/pagestore_smgr.c: * neon_prefetch() -- Initiate asynchronous read of the specified block of a relation -pgxn/neon/pagestore_smgr.c:neon_prefetch(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); -pgxn/neon/pagestore_smgr.c: * neon_prefetch() -- Initiate asynchronous read of the specified block of a relation -pgxn/neon/pagestore_smgr.c:neon_prefetch(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum) -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); -pgxn/neon/pagestore_smgr.c: * neon_writeback() -- Tell the kernel to write pages back to storage. -pgxn/neon/pagestore_smgr.c:neon_writeback(SMgrRelation reln, ForkNumber forknum, -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); -pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "writeback noop"); -pgxn/neon/pagestore_smgr.c:neon_read_at_lsnv(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber base_blockno, neon_request_lsns *request_lsns, -pgxn/neon/pagestore_smgr.c:neon_read_at_lsnv(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber base_blockno, neon_request_lsns *request_lsns, -pgxn/neon/pagestore_smgr.c: NeonResponse *resp; -pgxn/neon/pagestore_smgr.c: * See also the description on neon_redo_read_buffer_filter below. -pgxn/neon/pagestore_smgr.c: neon_request_lsns *reqlsns = &request_lsns[i]; -pgxn/neon/pagestore_smgr.c: if (neon_prefetch_response_usable(reqlsns, slot)) -pgxn/neon/pagestore_smgr.c: MyNeonCounters->getpage_prefetch_discards_total++; -pgxn/neon/pagestore_smgr.c: case T_NeonGetPageResponse: -pgxn/neon/pagestore_smgr.c: NeonGetPageResponse* getpage_resp = (NeonGetPageResponse *) resp; -pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) -pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(-1, PANIC, -pgxn/neon/pagestore_smgr.c: case T_NeonErrorResponse: -pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) -pgxn/neon/pagestore_smgr.c: elog(WARNING, NEON_TAG "Error message {reqid=%lx,lsn=%X/%08X, since=%X/%08X} doesn't match get relsize request {reqid=%lx,lsn=%X/%08X, since=%X/%08X}", -pgxn/neon/pagestore_smgr.c: errmsg(NEON_TAG "[shard %d, reqid %lx] could not read block %u in rel %u/%u/%u.%u from page server at lsn %X/%08X", -pgxn/neon/pagestore_smgr.c: ((NeonErrorResponse *) resp)->message))); -pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(slot->shard_no, PANIC, -pgxn/neon/pagestore_smgr.c: T_NeonGetPageResponse, T_NeonErrorResponse, resp->tag); -pgxn/neon/pagestore_smgr.c: * While function is defined in the neon extension it's used within neon_test_utils directly. -pgxn/neon/pagestore_smgr.c:neon_read_at_lsn(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, -pgxn/neon/pagestore_smgr.c: neon_request_lsns request_lsns, char *buffer) -pgxn/neon/pagestore_smgr.c:neon_read_at_lsn(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, -pgxn/neon/pagestore_smgr.c: neon_request_lsns request_lsns, void *buffer) -pgxn/neon/pagestore_smgr.c: neon_read_at_lsnv(rinfo, forkNum, blkno, &request_lsns, &buffer, 1, NULL); -pgxn/neon/pagestore_smgr.c: * neon_read() -- Read the specified block from a relation. -pgxn/neon/pagestore_smgr.c:neon_read(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, char *buffer) -pgxn/neon/pagestore_smgr.c:neon_read(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, void *buffer) -pgxn/neon/pagestore_smgr.c: neon_request_lsns request_lsns; -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgrread() on rel with unknown persistence"); -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); -pgxn/neon/pagestore_smgr.c: MyNeonCounters->file_cache_hits_total++; -pgxn/neon/pagestore_smgr.c: neon_get_request_lsns(InfoFromSMgrRel(reln), forkNum, blkno, &request_lsns, 1, NULL); -pgxn/neon/pagestore_smgr.c: neon_read_at_lsn(InfoFromSMgrRel(reln), forkNum, blkno, request_lsns, buffer); -pgxn/neon/pagestore_smgr.c: neon_log(PANIC, "page is new in MD but not in Page Server at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n%s\n", -pgxn/neon/pagestore_smgr.c: neon_log(PANIC, "page is new in Page Server but not in MD at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n%s\n", -pgxn/neon/pagestore_smgr.c: neon_log(PANIC, "heap buffers differ at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n------ MD ------\n%s\n------ Page Server ------\n%s\n", -pgxn/neon/pagestore_smgr.c: neon_log(PANIC, "btree buffers differ at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n------ MD ------\n%s\n------ Page Server ------\n%s\n", -pgxn/neon/pagestore_smgr.c:neon_readv(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, -pgxn/neon/pagestore_smgr.c: neon_request_lsns request_lsns[PG_IOV_MAX]; -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgrread() on rel with unknown persistence"); -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "Read request too large: %d is larger than max %d", -pgxn/neon/pagestore_smgr.c: MyNeonCounters->file_cache_hits_total += lfc_result; -pgxn/neon/pagestore_smgr.c: neon_get_request_lsns(InfoFromSMgrRel(reln), forknum, blocknum, -pgxn/neon/pagestore_smgr.c: neon_read_at_lsnv(InfoFromSMgrRel(reln), forknum, blocknum, request_lsns, -pgxn/neon/pagestore_smgr.c: neon_log(PANIC, "page is new in MD but not in Page Server at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n%s\n", -pgxn/neon/pagestore_smgr.c: neon_log(PANIC, "page is new in Page Server but not in MD at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n%s\n", -pgxn/neon/pagestore_smgr.c: neon_log(PANIC, "heap buffers differ at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n------ MD ------\n%s\n------ Page Server ------\n%s\n", -pgxn/neon/pagestore_smgr.c: neon_log(PANIC, "btree buffers differ at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n------ MD ------\n%s\n------ Page Server ------\n%s\n", -pgxn/neon/pagestore_smgr.c: * neon_write() -- Write the supplied block at the appropriate location. -pgxn/neon/pagestore_smgr.c:neon_write(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *buffer, bool skipFsync) -pgxn/neon/pagestore_smgr.c:neon_write(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, const void *buffer, bool skipFsync) -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); -pgxn/neon/pagestore_smgr.c: neon_wallog_page(reln, forknum, blocknum, buffer, false); -pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "smgrwrite called for %u/%u/%u.%u blk %u, page LSN: %X/%08X", -pgxn/neon/pagestore_smgr.c:neon_writev(SMgrRelation reln, ForkNumber forknum, BlockNumber blkno, -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); -pgxn/neon/pagestore_smgr.c: neon_wallog_pagev(reln, forknum, blkno, nblocks, (const char **) buffers, false); -pgxn/neon/pagestore_smgr.c: * neon_nblocks() -- Get the number of blocks stored in a relation. -pgxn/neon/pagestore_smgr.c:neon_nblocks(SMgrRelation reln, ForkNumber forknum) -pgxn/neon/pagestore_smgr.c: NeonResponse *resp; -pgxn/neon/pagestore_smgr.c: neon_request_lsns request_lsns; -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgrnblocks() on rel with unknown persistence"); -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); -pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "cached nblocks for %u/%u/%u.%u: %u blocks", -pgxn/neon/pagestore_smgr.c: neon_get_request_lsns(InfoFromSMgrRel(reln), forknum, -pgxn/neon/pagestore_smgr.c: NeonNblocksRequest request = { -pgxn/neon/pagestore_smgr.c: .hdr.tag = T_NeonNblocksRequest, -pgxn/neon/pagestore_smgr.c: case T_NeonNblocksResponse: -pgxn/neon/pagestore_smgr.c: NeonNblocksResponse * relsize_resp = (NeonNblocksResponse *) resp; -pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) -pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(-1, PANIC, -pgxn/neon/pagestore_smgr.c: case T_NeonErrorResponse: -pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) -pgxn/neon/pagestore_smgr.c: elog(WARNING, NEON_TAG "Error message {reqid=%lx,lsn=%X/%08X, since=%X/%08X} doesn't match get relsize request {reqid=%lx,lsn=%X/%08X, since=%X/%08X}", -pgxn/neon/pagestore_smgr.c: errmsg(NEON_TAG "[reqid %lx] could not read relation size of rel %u/%u/%u.%u from page server at lsn %X/%08X", -pgxn/neon/pagestore_smgr.c: ((NeonErrorResponse *) resp)->message))); -pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(-1, PANIC, -pgxn/neon/pagestore_smgr.c: T_NeonNblocksResponse, T_NeonErrorResponse, resp->tag); -pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "databricks_nblocks: rel %u/%u/%u fork %u (request LSN %X/%08X): %u blocks", -pgxn/neon/pagestore_smgr.c: * neon_db_size() -- Get the size of the database in bytes. -pgxn/neon/pagestore_smgr.c:neon_dbsize(Oid dbNode) -pgxn/neon/pagestore_smgr.c: NeonResponse *resp; -pgxn/neon/pagestore_smgr.c: neon_request_lsns request_lsns; -pgxn/neon/pagestore_smgr.c: neon_get_request_lsns(dummy_node, MAIN_FORKNUM, -pgxn/neon/pagestore_smgr.c: NeonDbSizeRequest request = { -pgxn/neon/pagestore_smgr.c: .hdr.tag = T_NeonDbSizeRequest, -pgxn/neon/pagestore_smgr.c: case T_NeonDbSizeResponse: -pgxn/neon/pagestore_smgr.c: NeonDbSizeResponse* dbsize_resp = (NeonDbSizeResponse *) resp; -pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) -pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(-1, PANIC, -pgxn/neon/pagestore_smgr.c: case T_NeonErrorResponse: -pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) -pgxn/neon/pagestore_smgr.c: elog(WARNING, NEON_TAG "Error message {reqid=%lx,lsn=%X/%08X, since=%X/%08X} doesn't match get DB size request {reqid=%lx,lsn=%X/%08X, since=%X/%08X}", -pgxn/neon/pagestore_smgr.c: errmsg(NEON_TAG "[reqid %lx] could not read db size of db %u from page server at lsn %X/%08X", -pgxn/neon/pagestore_smgr.c: ((NeonErrorResponse *) resp)->message))); -pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(-1, PANIC, -pgxn/neon/pagestore_smgr.c: T_NeonDbSizeResponse, T_NeonErrorResponse, resp->tag); -pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "databricks_dbsize: db %u (request LSN %X/%08X): %ld bytes", -pgxn/neon/pagestore_smgr.c: * neon_truncate() -- Truncate relation to specified number of blocks. -pgxn/neon/pagestore_smgr.c:neon_truncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks) -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgrtruncate() on rel with unknown persistence"); -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); -pgxn/neon/pagestore_smgr.c: * neon_immedsync() -- Immediately sync a relation to stable storage. -pgxn/neon/pagestore_smgr.c:neon_immedsync(SMgrRelation reln, ForkNumber forknum) -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgrimmedsync() on rel with unknown persistence"); -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); -pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "[DATABRICKS_SMGR] immedsync noop"); -pgxn/neon/pagestore_smgr.c:neon_registersync(SMgrRelation reln, ForkNumber forknum) -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgrregistersync() on rel with unknown persistence"); -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); -pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, NEON_TAG "registersync noop"); -pgxn/neon/pagestore_smgr.c: * neon_start_unlogged_build() -- Starting build operation on a rel. -pgxn/neon/pagestore_smgr.c: * and WAL-logging the whole relation after it's done. Neon relies on the -pgxn/neon/pagestore_smgr.c:neon_start_unlogged_build(SMgrRelation reln) -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unlogged relation build is already in progress"); -pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "starting unlogged build of relation %u/%u/%u", -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot call smgr_start_unlogged_build() on rel with unknown persistence"); -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); -pgxn/neon/pagestore_smgr.c: neon_log(ERROR, "cannot perform unlogged index build, index is not empty "); -pgxn/neon/pagestore_smgr.c: * neon_finish_unlogged_build_phase_1() -pgxn/neon/pagestore_smgr.c:neon_finish_unlogged_build_phase_1(SMgrRelation reln) -pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "finishing phase 1 of unlogged build of relation %u/%u/%u", -pgxn/neon/pagestore_smgr.c: * neon_end_unlogged_build() -- Finish an unlogged rel build. -pgxn/neon/pagestore_smgr.c:neon_end_unlogged_build(SMgrRelation reln) -pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "ending unlogged build of relation %u/%u/%u", -pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "forgetting cached relsize for %u/%u/%u.%u", -pgxn/neon/pagestore_smgr.c:neon_read_slru_segment(SMgrRelation reln, const char* path, int segno, void* buffer) -pgxn/neon/pagestore_smgr.c: NeonResponse *resp; -pgxn/neon/pagestore_smgr.c: NeonGetSlruSegmentRequest request; -pgxn/neon/pagestore_smgr.c: * Compute a request LSN to use, similar to neon_get_request_lsns() but the -pgxn/neon/pagestore_smgr.c: * This happens in neon startup, we start up without replaying any -pgxn/neon/pagestore_smgr.c: request = (NeonGetSlruSegmentRequest) { -pgxn/neon/pagestore_smgr.c: .hdr.tag = T_NeonGetSlruSegmentRequest, -pgxn/neon/pagestore_smgr.c: case T_NeonGetSlruSegmentResponse: -pgxn/neon/pagestore_smgr.c: NeonGetSlruSegmentResponse* slru_resp = (NeonGetSlruSegmentResponse *) resp; -pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) -pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(-1, PANIC, -pgxn/neon/pagestore_smgr.c: case T_NeonErrorResponse: -pgxn/neon/pagestore_smgr.c: if (neon_protocol_version >= 3) -pgxn/neon/pagestore_smgr.c: elog(WARNING, NEON_TAG "Error message {reqid=%lx,lsn=%X/%08X, since=%X/%08X} doesn't match get SLRU segment request {reqid=%lx,lsn=%X/%08X, since=%X/%08X}", -pgxn/neon/pagestore_smgr.c: errmsg(NEON_TAG "[reqid %lx] could not read SLRU %d segment %d at lsn %X/%08X", -pgxn/neon/pagestore_smgr.c: ((NeonErrorResponse *) resp)->message))); -pgxn/neon/pagestore_smgr.c: NEON_PANIC_CONNECTION_STATE(-1, PANIC, -pgxn/neon/pagestore_smgr.c: T_NeonGetSlruSegmentResponse, T_NeonErrorResponse, resp->tag); -pgxn/neon/pagestore_smgr.c:AtEOXact_neon(XactEvent event, void *arg) -pgxn/neon/pagestore_smgr.c: (errmsg(NEON_TAG "unlogged index build was not properly finished")))); -pgxn/neon/pagestore_smgr.c:static const struct f_smgr neon_smgr = -pgxn/neon/pagestore_smgr.c: .smgr_init = neon_init, -pgxn/neon/pagestore_smgr.c: .smgr_open = neon_open, -pgxn/neon/pagestore_smgr.c: .smgr_close = neon_close, -pgxn/neon/pagestore_smgr.c: .smgr_create = neon_create, -pgxn/neon/pagestore_smgr.c: .smgr_exists = neon_exists, -pgxn/neon/pagestore_smgr.c: .smgr_unlink = neon_unlink, -pgxn/neon/pagestore_smgr.c: .smgr_extend = neon_extend, -pgxn/neon/pagestore_smgr.c: .smgr_zeroextend = neon_zeroextend, -pgxn/neon/pagestore_smgr.c: .smgr_prefetch = neon_prefetch, -pgxn/neon/pagestore_smgr.c: .smgr_readv = neon_readv, -pgxn/neon/pagestore_smgr.c: .smgr_writev = neon_writev, -pgxn/neon/pagestore_smgr.c: .smgr_prefetch = neon_prefetch, -pgxn/neon/pagestore_smgr.c: .smgr_read = neon_read, -pgxn/neon/pagestore_smgr.c: .smgr_write = neon_write, -pgxn/neon/pagestore_smgr.c: .smgr_writeback = neon_writeback, -pgxn/neon/pagestore_smgr.c: .smgr_nblocks = neon_nblocks, -pgxn/neon/pagestore_smgr.c: .smgr_truncate = neon_truncate, -pgxn/neon/pagestore_smgr.c: .smgr_immedsync = neon_immedsync, -pgxn/neon/pagestore_smgr.c: .smgr_registersync = neon_registersync, -pgxn/neon/pagestore_smgr.c: .smgr_start_unlogged_build = neon_start_unlogged_build, -pgxn/neon/pagestore_smgr.c: .smgr_finish_unlogged_build_phase_1 = neon_finish_unlogged_build_phase_1, -pgxn/neon/pagestore_smgr.c: .smgr_end_unlogged_build = neon_end_unlogged_build, -pgxn/neon/pagestore_smgr.c: .smgr_read_slru_segment = neon_read_slru_segment, -pgxn/neon/pagestore_smgr.c:smgr_neon(ProcNumber backend, NRelFileInfo rinfo) -pgxn/neon/pagestore_smgr.c: return &neon_smgr; -pgxn/neon/pagestore_smgr.c:smgr_init_neon(void) -pgxn/neon/pagestore_smgr.c: RegisterXactCallback(AtEOXact_neon, NULL); -pgxn/neon/pagestore_smgr.c: neon_init(); -pgxn/neon/pagestore_smgr.c:neon_extend_rel_size(NRelFileInfo rinfo, ForkNumber forknum, BlockNumber blkno, XLogRecPtr end_recptr) -pgxn/neon/pagestore_smgr.c: NeonResponse *response; -pgxn/neon/pagestore_smgr.c: NeonNblocksResponse *nbresponse; -pgxn/neon/pagestore_smgr.c: NeonNblocksRequest request = { -pgxn/neon/pagestore_smgr.c: .hdr = (NeonRequest) { -pgxn/neon/pagestore_smgr.c: .tag = T_NeonNblocksRequest, -pgxn/neon/pagestore_smgr.c: Assert(response->tag == T_NeonNblocksResponse); -pgxn/neon/pagestore_smgr.c: nbresponse = (NeonNblocksResponse *) response; -pgxn/neon/pagestore_smgr.c: neon_log(SmgrTrace, "Set length to %d", relsize); -pgxn/neon/pagestore_smgr.c:neon_redo_read_buffer_filter(XLogReaderState *record, uint8 block_id) -pgxn/neon/pagestore_smgr.c: neon_log(PANIC, "failed to locate backup block with ID %d", block_id); -pgxn/neon/pagestore_smgr.c: neon_extend_rel_size(rinfo, forknum, blkno, end_recptr); -pgxn/neon/pagestore_smgr.c: neon_extend_rel_size(rinfo, FSM_FORKNUM, get_fsm_physical_block(blkno), end_recptr); -pgxn/neon/relsize_cache.c: * contrib/neon/relsize_cache.c -pgxn/neon/relsize_cache.c:#include "neon_pgversioncompat.h" -pgxn/neon/relsize_cache.c:neon_smgr_shmem_startup(void) -pgxn/neon/relsize_cache.c: DefineCustomIntVariable("neon.relsize_hash_size", -pgxn/neon/relsize_cache.c: "Sets the maximum number of cached relation sizes for neon", -pgxn/neon/relsize_cache.c: shmem_startup_hook = neon_smgr_shmem_startup; -pgxn/neon/relsize_cache.c: * attach to the shared resources in neon_smgr_shmem_startup(). -pgxn/neon/unstable_extensions.c:#include "neon_pgversioncompat.h" -pgxn/neon/unstable_extensions.c: "neon.allow_unstable_extensions", -pgxn/neon/unstable_extensions.c: "neon.unstable_extensions", -pgxn/neon/unstable_extensions.h:#ifndef __NEON_UNSTABLE_EXTENSIONS_H__ -pgxn/neon/unstable_extensions.h:#define __NEON_UNSTABLE_EXTENSIONS_H__ -pgxn/neon/walproposer.c: * and WAL safekeepers. -pgxn/neon/walproposer.c: * will immediately broadcast it to alive safekeepers. -pgxn/neon/walproposer.c: * 2. As a standalone utility by running `postgres --sync-safekeepers`. That -pgxn/neon/walproposer.c: * it and make sure there is a safekeeper who knows this LSN is -pgxn/neon/walproposer.c: * safekeepers, learn start LSN of future epoch and run basebackup' -pgxn/neon/walproposer.c:#include "neon.h" -pgxn/neon/walproposer.c:#include "neon_utils.h" -pgxn/neon/walproposer.c:static void ShutdownConnection(Safekeeper *sk); -pgxn/neon/walproposer.c:static void ResetConnection(Safekeeper *sk); -pgxn/neon/walproposer.c:static void ReconnectSafekeepers(WalProposer *wp); -pgxn/neon/walproposer.c:static void AdvancePollState(Safekeeper *sk, uint32 events); -pgxn/neon/walproposer.c:static void HandleConnectionEvent(Safekeeper *sk); -pgxn/neon/walproposer.c:static void SendStartWALPush(Safekeeper *sk); -pgxn/neon/walproposer.c:static void RecvStartWALPushResult(Safekeeper *sk); -pgxn/neon/walproposer.c:static void SendProposerGreeting(Safekeeper *sk); -pgxn/neon/walproposer.c:static void RecvAcceptorGreeting(Safekeeper *sk); -pgxn/neon/walproposer.c:static void SendVoteRequest(Safekeeper *sk); -pgxn/neon/walproposer.c:static void RecvVoteResponse(Safekeeper *sk); -pgxn/neon/walproposer.c:static term_t GetEpoch(Safekeeper *sk); -pgxn/neon/walproposer.c:static void SendProposerElected(Safekeeper *sk); -pgxn/neon/walproposer.c:static void StartStreaming(Safekeeper *sk); -pgxn/neon/walproposer.c:static void SendMessageToNode(Safekeeper *sk); -pgxn/neon/walproposer.c:static void HandleActiveState(Safekeeper *sk, uint32 events); -pgxn/neon/walproposer.c:static bool SendAppendRequests(Safekeeper *sk); -pgxn/neon/walproposer.c:static bool RecvAppendResponses(Safekeeper *sk); -pgxn/neon/walproposer.c:static void HandleSafekeeperResponse(WalProposer *wp, Safekeeper *sk); -pgxn/neon/walproposer.c:static bool AsyncRead(Safekeeper *sk, char **buf, int *buf_size); -pgxn/neon/walproposer.c:static bool AsyncReadMessage(Safekeeper *sk, AcceptorProposerMessage *anymsg); -pgxn/neon/walproposer.c:static bool BlockingWrite(Safekeeper *sk, void *msg, size_t msg_size, SafekeeperState success_state); -pgxn/neon/walproposer.c:static bool AsyncWrite(Safekeeper *sk, void *msg, size_t msg_size, SafekeeperState flush_state); -pgxn/neon/walproposer.c:static bool AsyncFlush(Safekeeper *sk); -pgxn/neon/walproposer.c:static char *FormatSafekeeperState(Safekeeper *sk); -pgxn/neon/walproposer.c:static void AssertEventsOkForState(uint32 events, Safekeeper *sk); -pgxn/neon/walproposer.c: for (host = wp->config->safekeepers_list; host != NULL && *host != '\0'; host = sep) -pgxn/neon/walproposer.c: if (wp->n_safekeepers + 1 >= MAX_SAFEKEEPERS) -pgxn/neon/walproposer.c: wp_log(FATAL, "too many safekeepers"); -pgxn/neon/walproposer.c: wp->safekeeper[wp->n_safekeepers].host = host; -pgxn/neon/walproposer.c: wp->safekeeper[wp->n_safekeepers].port = port; -pgxn/neon/walproposer.c: wp->safekeeper[wp->n_safekeepers].state = SS_OFFLINE; -pgxn/neon/walproposer.c: wp->safekeeper[wp->n_safekeepers].active_state = SS_ACTIVE_SEND; -pgxn/neon/walproposer.c: wp->safekeeper[wp->n_safekeepers].wp = wp; -pgxn/neon/walproposer.c: wp->safekeeper[wp->n_safekeepers].index = wp->n_safekeepers; -pgxn/neon/walproposer.c: Safekeeper *sk = &wp->safekeeper[wp->n_safekeepers]; -pgxn/neon/walproposer.c: sk->host, sk->port, wp->config->neon_timeline, wp->config->neon_tenant); -pgxn/neon/walproposer.c: wp_log(FATAL, "could not create connection string for safekeeper %s:%s", sk->host, sk->port); -pgxn/neon/walproposer.c: initStringInfo(&wp->safekeeper[wp->n_safekeepers].outbuf); -pgxn/neon/walproposer.c: wp->safekeeper[wp->n_safekeepers].startStreamingAt = InvalidXLogRecPtr; -pgxn/neon/walproposer.c: wp->safekeeper[wp->n_safekeepers].streamingAt = InvalidXLogRecPtr; -pgxn/neon/walproposer.c: wp->n_safekeepers += 1; -pgxn/neon/walproposer.c: if (wp->n_safekeepers < 1) -pgxn/neon/walproposer.c: wp_log(FATAL, "safekeepers addresses are not specified"); -pgxn/neon/walproposer.c: wp->quorum = wp->n_safekeepers / 2 + 1; -pgxn/neon/walproposer.c: wp->api.reset_safekeeper_statuses_for_metrics(wp, wp->n_safekeepers); -pgxn/neon/walproposer.c: if (!wp->config->neon_timeline) -pgxn/neon/walproposer.c: if (*wp->config->neon_timeline != '\0' && -pgxn/neon/walproposer.c: !HexDecodeString(wp->greetRequest.timeline_id, wp->config->neon_timeline, 16)) -pgxn/neon/walproposer.c: wp_log(FATAL, "could not parse timeline_id, %s", wp->config->neon_timeline); -pgxn/neon/walproposer.c: if (!wp->config->neon_tenant) -pgxn/neon/walproposer.c: if (*wp->config->neon_tenant != '\0' && -pgxn/neon/walproposer.c: !HexDecodeString(wp->greetRequest.tenant_id, wp->config->neon_tenant, 16)) -pgxn/neon/walproposer.c: wp_log(FATAL, "could not parse tenant_id, %s", wp->config->neon_tenant); -pgxn/neon/walproposer.c: for (int i = 0; i < wp->n_safekeepers; i++) -pgxn/neon/walproposer.c: Safekeeper *sk = &wp->safekeeper[i]; -pgxn/neon/walproposer.c: Safekeeper *sk = NULL; -pgxn/neon/walproposer.c: * If the event contains something that one of our safekeeper states -pgxn/neon/walproposer.c: * If the timeout expired, attempt to reconnect to any safekeepers -pgxn/neon/walproposer.c: ReconnectSafekeepers(wp); -pgxn/neon/walproposer.c: if (!wp->config->syncSafekeepers) -pgxn/neon/walproposer.c: for (int i = 0; i < wp->n_safekeepers; i++) -pgxn/neon/walproposer.c: sk = &wp->safekeeper[i]; -pgxn/neon/walproposer.c: wp->config->safekeeper_connection_timeout)) -pgxn/neon/walproposer.c: sk->host, sk->port, FormatSafekeeperState(sk), wp->config->safekeeper_connection_timeout); -pgxn/neon/walproposer.c: /* Initiate connections to all safekeeper nodes */ -pgxn/neon/walproposer.c: for (int i = 0; i < wp->n_safekeepers; i++) -pgxn/neon/walproposer.c: ResetConnection(&wp->safekeeper[i]); -pgxn/neon/walproposer.c:/* Shuts down and cleans up the connection for a safekeeper. Sets its state to SS_OFFLINE */ -pgxn/neon/walproposer.c:ShutdownConnection(Safekeeper *sk) -pgxn/neon/walproposer.c: sk->wp->api.update_safekeeper_status_for_metrics(sk->wp, sk->index, 0); -pgxn/neon/walproposer.c: sk->wp->api.rm_safekeeper_event_set(sk); -pgxn/neon/walproposer.c:ResetConnection(Safekeeper *sk) -pgxn/neon/walproposer.c: wp->api.add_safekeeper_event_set(sk, WL_SOCKET_WRITEABLE); -pgxn/neon/walproposer.c: * safekeepers? Returns 0 if it is already high time, -1 if we never reconnect -pgxn/neon/walproposer.c: if (wp->config->safekeeper_reconnect_timeout <= 0) -pgxn/neon/walproposer.c: till_reconnect = wp->config->safekeeper_reconnect_timeout * 1000 - passed; -pgxn/neon/walproposer.c:/* If the timeout has expired, attempt to reconnect to all offline safekeepers */ -pgxn/neon/walproposer.c:ReconnectSafekeepers(WalProposer *wp) -pgxn/neon/walproposer.c: for (int i = 0; i < wp->n_safekeepers; i++) -pgxn/neon/walproposer.c: if (wp->safekeeper[i].state == SS_OFFLINE) -pgxn/neon/walproposer.c: ResetConnection(&wp->safekeeper[i]); -pgxn/neon/walproposer.c: * Performs the logic for advancing the state machine of the specified safekeeper, -pgxn/neon/walproposer.c:AdvancePollState(Safekeeper *sk, uint32 events) -pgxn/neon/walproposer.c: * safekeepers are only taken out of SS_OFFLINE by calls to -pgxn/neon/walproposer.c: * safekeeper. -pgxn/neon/walproposer.c: sk->port, FormatSafekeeperState(sk)); -pgxn/neon/walproposer.c: /* Read the safekeeper response for our candidate */ -pgxn/neon/walproposer.c: sk->port, FormatSafekeeperState(sk)); -pgxn/neon/walproposer.c:HandleConnectionEvent(Safekeeper *sk) -pgxn/neon/walproposer.c: * ReconnectSafekeepers. -pgxn/neon/walproposer.c: wp->api.rm_safekeeper_event_set(sk); -pgxn/neon/walproposer.c: wp->api.add_safekeeper_event_set(sk, new_events); -pgxn/neon/walproposer.c: * Send "START_WAL_PUSH" message as an empty query to the safekeeper. Performs -pgxn/neon/walproposer.c:SendStartWALPush(Safekeeper *sk) -pgxn/neon/walproposer.c:RecvStartWALPushResult(Safekeeper *sk) -pgxn/neon/walproposer.c: * safekeeper. After sending, we wait on SS_HANDSHAKE_RECV for -pgxn/neon/walproposer.c:SendProposerGreeting(Safekeeper *sk) -pgxn/neon/walproposer.c:RecvAcceptorGreeting(Safekeeper *sk) -pgxn/neon/walproposer.c: * Note: it would be better to track the counter on per safekeeper basis, -pgxn/neon/walproposer.c: * Check if we have quorum. If there aren't enough safekeepers, wait and -pgxn/neon/walproposer.c: for (int j = 0; j < wp->n_safekeepers; j++) -pgxn/neon/walproposer.c: * Remember: SS_VOTING indicates that the safekeeper is -pgxn/neon/walproposer.c: if (wp->safekeeper[j].state == SS_VOTING) -pgxn/neon/walproposer.c: SendVoteRequest(&wp->safekeeper[j]); -pgxn/neon/walproposer.c:SendVoteRequest(Safekeeper *sk) -pgxn/neon/walproposer.c:RecvVoteResponse(Safekeeper *sk) -pgxn/neon/walproposer.c: * Synchronously download WAL from the most advanced safekeeper. We do -pgxn/neon/walproposer.c: * neon_walreader is a todo.) -pgxn/neon/walproposer.c: if (!wp->api.recovery_download(wp, &wp->safekeeper[wp->donor])) -pgxn/neon/walproposer.c: * Zero propEpochStartLsn means majority of safekeepers doesn't have any -pgxn/neon/walproposer.c: * otherwise we must be sync-safekeepers and we have nothing to do then. -pgxn/neon/walproposer.c: * safekeepers term history starting with 0/0. These hacks will go away once -pgxn/neon/walproposer.c: * we disable implicit timeline creation on safekeepers and create it with -pgxn/neon/walproposer.c: Assert(wp->config->syncSafekeepers); -pgxn/neon/walproposer.c: wp->api.finish_sync_safekeepers(wp, wp->propEpochStartLsn); -pgxn/neon/walproposer.c: if (wp->truncateLsn == wp->propEpochStartLsn && wp->config->syncSafekeepers) -pgxn/neon/walproposer.c: wp->api.finish_sync_safekeepers(wp, wp->propEpochStartLsn); -pgxn/neon/walproposer.c: for (int i = 0; i < wp->n_safekeepers; i++) -pgxn/neon/walproposer.c: if (wp->safekeeper[i].state == SS_IDLE) -pgxn/neon/walproposer.c: SendProposerElected(&wp->safekeeper[i]); -pgxn/neon/walproposer.c: * after this point. There will be no safekeeper with state SS_IDLE also, -pgxn/neon/walproposer.c: if (wp->config->syncSafekeepers) -pgxn/neon/walproposer.c: /* keep polling until all safekeepers are synced */ -pgxn/neon/walproposer.c:/* safekeeper's epoch is the term of the highest entry in the log */ -pgxn/neon/walproposer.c:GetEpoch(Safekeeper *sk) -pgxn/neon/walproposer.c: * advanced safekeeper (who will be the donor) and epochStartLsn -- LSN since -pgxn/neon/walproposer.c: for (int i = 0; i < wp->n_safekeepers; i++) -pgxn/neon/walproposer.c: if (wp->safekeeper[i].state == SS_IDLE) -pgxn/neon/walproposer.c: if (GetEpoch(&wp->safekeeper[i]) > wp->donorEpoch || -pgxn/neon/walproposer.c: (GetEpoch(&wp->safekeeper[i]) == wp->donorEpoch && -pgxn/neon/walproposer.c: wp->safekeeper[i].voteResponse.flushLsn > wp->propEpochStartLsn)) -pgxn/neon/walproposer.c: wp->donorEpoch = GetEpoch(&wp->safekeeper[i]); -pgxn/neon/walproposer.c: wp->propEpochStartLsn = wp->safekeeper[i].voteResponse.flushLsn; -pgxn/neon/walproposer.c: wp->truncateLsn = Max(wp->safekeeper[i].voteResponse.truncateLsn, wp->truncateLsn); -pgxn/neon/walproposer.c: if (wp->safekeeper[i].voteResponse.timelineStartLsn != InvalidXLogRecPtr) -pgxn/neon/walproposer.c: wp->timelineStartLsn != wp->safekeeper[i].voteResponse.timelineStartLsn) -pgxn/neon/walproposer.c: LSN_FORMAT_ARGS(wp->safekeeper[i].voteResponse.timelineStartLsn)); -pgxn/neon/walproposer.c: wp->timelineStartLsn = wp->safekeeper[i].voteResponse.timelineStartLsn; -pgxn/neon/walproposer.c: * This is a rare case that can be triggered if safekeeper has voted -pgxn/neon/walproposer.c: if (wp->propEpochStartLsn == InvalidXLogRecPtr && !wp->config->syncSafekeepers) -pgxn/neon/walproposer.c: * Safekeepers are setting truncateLsn after timelineStartLsn is known, so -pgxn/neon/walproposer.c: * timelineStartLsn can be zero only on the first syncSafekeepers run. -pgxn/neon/walproposer.c: (wp->config->syncSafekeepers && wp->truncateLsn == wp->timelineStartLsn)); -pgxn/neon/walproposer.c: dth = &wp->safekeeper[wp->donor].voteResponse.termHistory; -pgxn/neon/walproposer.c: wp->safekeeper[wp->donor].host, wp->safekeeper[wp->donor].port, -pgxn/neon/walproposer.c: if (!wp->config->syncSafekeepers) -pgxn/neon/walproposer.c: * Safekeepers don't skip header as they need continious stream of -pgxn/neon/walproposer.c: * safekeeper is synced, being important for sync-safekeepers) -pgxn/neon/walproposer.c: * 2) Communicating starting streaming point -- safekeeper must truncate its WAL -pgxn/neon/walproposer.c:SendProposerElected(Safekeeper *sk) -pgxn/neon/walproposer.c: * Determine start LSN by comparing safekeeper's log term switch history -pgxn/neon/walproposer.c: * there is some WAL on safekeeper, if immediately after bootstrap compute -pgxn/neon/walproposer.c: /* safekeeper is empty or no common point, start from the beginning */ -pgxn/neon/walproposer.c: * one; there it is flush_lsn in case of safekeeper or, in case of -pgxn/neon/walproposer.c: * safekeeper pos as it obviously can't be higher. -pgxn/neon/walproposer.c: * Start streaming to safekeeper sk, always updates state to SS_ACTIVE and sets -pgxn/neon/walproposer.c:StartStreaming(Safekeeper *sk) -pgxn/neon/walproposer.c: sk->wp->api.update_safekeeper_status_for_metrics(sk->wp, sk->index, 1); -pgxn/neon/walproposer.c: * Can be used only for safekeepers in SS_ACTIVE state. State can be changed -pgxn/neon/walproposer.c:SendMessageToNode(Safekeeper *sk) -pgxn/neon/walproposer.c: * Note: we always send everything to the safekeeper until WOULDBLOCK or -pgxn/neon/walproposer.c: * Broadcast new message to all caught-up safekeepers -pgxn/neon/walproposer.c: for (int i = 0; i < wp->n_safekeepers; i++) -pgxn/neon/walproposer.c: if (wp->safekeeper[i].state == SS_ACTIVE) -pgxn/neon/walproposer.c: SendMessageToNode(&wp->safekeeper[i]); -pgxn/neon/walproposer.c:HandleActiveState(Safekeeper *sk, uint32 events) -pgxn/neon/walproposer.c: * socket or neon_walreader blocks, whichever comes first; active_state is -pgxn/neon/walproposer.c:SendAppendRequests(Safekeeper *sk) -pgxn/neon/walproposer.c: case NEON_WALREAD_SUCCESS: -pgxn/neon/walproposer.c: case NEON_WALREAD_WOULDBLOCK: -pgxn/neon/walproposer.c: case NEON_WALREAD_ERROR: -pgxn/neon/walproposer.c: sk->host, sk->port, FormatSafekeeperState(sk), -pgxn/neon/walproposer.c:RecvAppendResponses(Safekeeper *sk) -pgxn/neon/walproposer.c: * on safekeepers for migration purposes, in this case we do want -pgxn/neon/walproposer.c: HandleSafekeeperResponse(wp, sk); -pgxn/neon/walproposer.c: * Get minimum of flushed LSNs of all safekeepers, which is the LSN of the -pgxn/neon/walproposer.c: XLogRecPtr lsn = wp->n_safekeepers > 0 -pgxn/neon/walproposer.c: ? wp->safekeeper[0].appendResponse.flushLsn -pgxn/neon/walproposer.c: for (int i = 1; i < wp->n_safekeepers; i++) -pgxn/neon/walproposer.c: lsn = Min(lsn, wp->safekeeper[i].appendResponse.flushLsn); -pgxn/neon/walproposer.c: XLogRecPtr responses[MAX_SAFEKEEPERS]; -pgxn/neon/walproposer.c: for (int i = 0; i < wp->n_safekeepers; i++) -pgxn/neon/walproposer.c: responses[i] = wp->safekeeper[i].appendResponse.flushLsn >= wp->propEpochStartLsn ? wp->safekeeper[i].appendResponse.flushLsn : 0; -pgxn/neon/walproposer.c: qsort(responses, wp->n_safekeepers, sizeof(XLogRecPtr), CompareLsn); -pgxn/neon/walproposer.c: return responses[wp->n_safekeepers - wp->quorum]; -pgxn/neon/walproposer.c: * Return safekeeper with active connection from which WAL can be downloaded, or -pgxn/neon/walproposer.c: Safekeeper *donor = NULL; -pgxn/neon/walproposer.c: if (wp->safekeeper[wp->donor].state >= SS_IDLE) -pgxn/neon/walproposer.c: donor = &wp->safekeeper[wp->donor]; -pgxn/neon/walproposer.c: for (i = 0; i < wp->n_safekeepers; i++) -pgxn/neon/walproposer.c: Safekeeper *sk = &wp->safekeeper[i]; -pgxn/neon/walproposer.c: * Process AppendResponse message from safekeeper. -pgxn/neon/walproposer.c:HandleSafekeeperResponse(WalProposer *wp, Safekeeper *fromsk) -pgxn/neon/walproposer.c: /* Send new value to all safekeepers. */ -pgxn/neon/walproposer.c: * called to notify safekeepers about the new commitLsn. -pgxn/neon/walproposer.c: wp->api.process_safekeeper_feedback(wp, fromsk); -pgxn/neon/walproposer.c: * safekeepers. -pgxn/neon/walproposer.c: * from safekeeper who is still in the previous epoch (similar to 'leader -pgxn/neon/walproposer.c: * is alive, there will be at least one safekeeper who is able to stream -pgxn/neon/walproposer.c: * most advanced safekeeper who should push the wal into pageserver and -pgxn/neon/walproposer.c: * (due to pageserver connecting to not-synced-safekeeper) we currently -pgxn/neon/walproposer.c: * wait for all seemingly alive safekeepers to get synced. -pgxn/neon/walproposer.c: if (wp->config->syncSafekeepers) -pgxn/neon/walproposer.c: for (int i = 0; i < wp->n_safekeepers; i++) -pgxn/neon/walproposer.c: Safekeeper *sk = &wp->safekeeper[i]; -pgxn/neon/walproposer.c: /* alive safekeeper which is not synced yet; wait for it */ -pgxn/neon/walproposer.c: /* A quorum of safekeepers has been synced! */ -pgxn/neon/walproposer.c: * safekeepers. This helps to finish next sync-safekeepers -pgxn/neon/walproposer.c: * safekeepers in case of network working properly. -pgxn/neon/walproposer.c: wp->api.finish_sync_safekeepers(wp, wp->propEpochStartLsn); -pgxn/neon/walproposer.c: * Try to read CopyData message from i'th safekeeper, resetting connection on -pgxn/neon/walproposer.c:AsyncRead(Safekeeper *sk, char **buf, int *buf_size) -pgxn/neon/walproposer.c: sk->port, FormatSafekeeperState(sk), -pgxn/neon/walproposer.c: * block from the safekeeper's postgres connection, returning whether the read -pgxn/neon/walproposer.c:AsyncReadMessage(Safekeeper *sk, AcceptorProposerMessage *anymsg) -pgxn/neon/walproposer.c: sk->port, FormatSafekeeperState(sk)); -pgxn/neon/walproposer.c:BlockingWrite(Safekeeper *sk, void *msg, size_t msg_size, SafekeeperState success_state) -pgxn/neon/walproposer.c: sk->host, sk->port, FormatSafekeeperState(sk), -pgxn/neon/walproposer.c: SafekeeperStateDesiredEvents(sk, &sk_events, &nwr_events); -pgxn/neon/walproposer.c: * Starts a write into the 'i'th safekeeper's postgres connection, moving to -pgxn/neon/walproposer.c:AsyncWrite(Safekeeper *sk, void *msg, size_t msg_size, SafekeeperState flush_state) -pgxn/neon/walproposer.c: sk->host, sk->port, FormatSafekeeperState(sk), -pgxn/neon/walproposer.c:AsyncFlush(Safekeeper *sk) -pgxn/neon/walproposer.c: sk->host, sk->port, FormatSafekeeperState(sk), -pgxn/neon/walproposer.c:/* Returns a human-readable string corresonding to the SafekeeperState -pgxn/neon/walproposer.c: * wp_log(LOG, "currently in %s state", FormatSafekeeperState(sk)); -pgxn/neon/walproposer.c: * wp_log(LOG, "currently in state [%s]", FormatSafekeeperState(sk)); -pgxn/neon/walproposer.c:FormatSafekeeperState(Safekeeper *sk) -pgxn/neon/walproposer.c:/* Asserts that the provided events are expected for given safekeeper's state */ -pgxn/neon/walproposer.c:AssertEventsOkForState(uint32 events, Safekeeper *sk) -pgxn/neon/walproposer.c: SafekeeperStateDesiredEvents(sk, &sk_events, &nwr_events); -pgxn/neon/walproposer.c: FormatEvents(wp, events), sk->host, sk->port, FormatSafekeeperState(sk)); -pgxn/neon/walproposer.c:/* Returns the set of events for both safekeeper (sk_events) and neon_walreader -pgxn/neon/walproposer.c: * (nwr_events) sockets a safekeeper in this state should be waiting on. -pgxn/neon/walproposer.c:SafekeeperStateDesiredEvents(Safekeeper *sk, uint32 *sk_events, uint32 *nwr_events) -pgxn/neon/walproposer.c: * safekeeper in one wakeup (unless it blocks). Otherwise -pgxn/neon/walproposer.c: * Waiting for neon_walreader socket, but we still read -pgxn/neon/walproposer.c: * Need to flush the sk socket, so ignore neon_walreader -pgxn/neon/walproposer.h:#ifndef __NEON_WALPROPOSER_H__ -pgxn/neon/walproposer.h:#define __NEON_WALPROPOSER_H__ -pgxn/neon/walproposer.h:#include "neon_walreader.h" -pgxn/neon/walproposer.h:#define MAX_SAFEKEEPERS 32 -pgxn/neon/walproposer.h: * WAL safekeeper state, which is used to wait for some event. -pgxn/neon/walproposer.h: * After we get a successful result, sends handshake to safekeeper. -pgxn/neon/walproposer.h:} SafekeeperState; -pgxn/neon/walproposer.h: * Polling neon_walreader to receive chunk of WAL (probably remotely) to -pgxn/neon/walproposer.h: * send to this safekeeper. -pgxn/neon/walproposer.h: * Also, while in this state we don't touch safekeeper socket, so in -pgxn/neon/walproposer.h:} SafekeeperActiveState; -pgxn/neon/walproposer.h:/* neon storage node id */ -pgxn/neon/walproposer.h: uint32 protocolVersion; /* proposer-safekeeper protocol version */ -pgxn/neon/walproposer.h: uint8 timeline_id[16]; /* Neon timeline id */ -pgxn/neon/walproposer.h:/* Vote itself, sent from safekeeper to proposer */ -pgxn/neon/walproposer.h: * Safekeeper flush_lsn (end of WAL) + history of term switches allow -pgxn/neon/walproposer.h: * recovery of some safekeeper */ -pgxn/neon/walproposer.h: * Header of request with WAL message sent from proposer to safekeeper. -pgxn/neon/walproposer.h: XLogRecPtr commitLsn; /* LSN committed by quorum of safekeepers */ -pgxn/neon/walproposer.h: * minimal LSN which may be needed for recovery of some safekeeper (end -pgxn/neon/walproposer.h: /* standby_status_update fields that safekeeper received from pageserver */ -pgxn/neon/walproposer.h: /* Number of safekeepers in the config */ -pgxn/neon/walproposer.h: uint32 num_safekeepers; -pgxn/neon/walproposer.h: /* Per-safekeeper status flags: 0=inactive, 1=active */ -pgxn/neon/walproposer.h: uint8 safekeeper_status[MAX_SAFEKEEPERS]; -pgxn/neon/walproposer.h: /* Per-safekeeper commit LSN for metrics */ -pgxn/neon/walproposer.h: XLogRecPtr safekeeper_commit_lsn[MAX_SAFEKEEPERS]; -pgxn/neon/walproposer.h: * Report safekeeper state to proposer -pgxn/neon/walproposer.h: * Current term of the safekeeper; if it is higher than proposer's, the -pgxn/neon/walproposer.h: /* Safekeeper reports back his awareness about which WAL is committed, as */ -pgxn/neon/walproposer.h: /* and custom neon feedback. */ -pgxn/neon/walproposer.h: * Descriptor of safekeeper -pgxn/neon/walproposer.h:typedef struct Safekeeper -pgxn/neon/walproposer.h: /* index of this safekeeper in the WalProposer array */ -pgxn/neon/walproposer.h: * Temporary buffer for the message being sent to the safekeeper. -pgxn/neon/walproposer.h: AppendRequestHeader appendRequest; /* request for sending to safekeeper */ -pgxn/neon/walproposer.h: SafekeeperState state; /* safekeeper state machine state */ -pgxn/neon/walproposer.h: SafekeeperActiveState active_state; -pgxn/neon/walproposer.h: * WAL reader, allocated for each safekeeper. -pgxn/neon/walproposer.h: NeonWALReader *xlogreader; -pgxn/neon/walproposer.h: * Neon WAL reader position in wait event set, or -1 if no socket. Note -pgxn/neon/walproposer.h:} Safekeeper; -pgxn/neon/walproposer.h: void (*update_donor) (WalProposer *wp, Safekeeper *donor, XLogRecPtr donor_lsn); -pgxn/neon/walproposer.h: char *(*conn_error_message) (Safekeeper *sk); -pgxn/neon/walproposer.h: WalProposerConnStatusType (*conn_status) (Safekeeper *sk); -pgxn/neon/walproposer.h: void (*conn_connect_start) (Safekeeper *sk); -pgxn/neon/walproposer.h: WalProposerConnectPollStatusType (*conn_connect_poll) (Safekeeper *sk); -pgxn/neon/walproposer.h: bool (*conn_send_query) (Safekeeper *sk, char *query); -pgxn/neon/walproposer.h: WalProposerExecStatusType (*conn_get_query_result) (Safekeeper *sk); -pgxn/neon/walproposer.h: int (*conn_flush) (Safekeeper *sk); -pgxn/neon/walproposer.h: void (*conn_finish) (Safekeeper *sk); -pgxn/neon/walproposer.h: * Try to read CopyData message from the safekeeper, aka PQgetCopyData. -pgxn/neon/walproposer.h: PGAsyncReadResult (*conn_async_read) (Safekeeper *sk, char **buf, int *amount); -pgxn/neon/walproposer.h: PGAsyncWriteResult (*conn_async_write) (Safekeeper *sk, void const *buf, size_t size); -pgxn/neon/walproposer.h: bool (*conn_blocking_write) (Safekeeper *sk, void const *buf, size_t size); -pgxn/neon/walproposer.h: bool (*recovery_download) (WalProposer *wp, Safekeeper *sk); -pgxn/neon/walproposer.h: void (*wal_reader_allocate) (Safekeeper *sk); -pgxn/neon/walproposer.h: NeonWALReadResult (*wal_read) (Safekeeper *sk, char *buf, XLogRecPtr startptr, Size count, char **errmsg); -pgxn/neon/walproposer.h: uint32 (*wal_reader_events) (Safekeeper *sk); -pgxn/neon/walproposer.h: /* Update events for an existing safekeeper connection. */ -pgxn/neon/walproposer.h: void (*update_event_set) (Safekeeper *sk, uint32 events); -pgxn/neon/walproposer.h: void (*active_state_update_event_set) (Safekeeper *sk); -pgxn/neon/walproposer.h: /* Add a new safekeeper connection to the event set. */ -pgxn/neon/walproposer.h: void (*add_safekeeper_event_set) (Safekeeper *sk, uint32 events); -pgxn/neon/walproposer.h: /* Remove safekeeper connection from event set */ -pgxn/neon/walproposer.h: void (*rm_safekeeper_event_set) (Safekeeper *sk); -pgxn/neon/walproposer.h: * safekeeper connection - new WAL is available -pgxn/neon/walproposer.h: * events mask to indicate events and sets sk to the safekeeper which has -pgxn/neon/walproposer.h: int (*wait_event_set) (WalProposer *wp, long timeout, Safekeeper **sk, uint32 *events); -pgxn/neon/walproposer.h: * LSN on the safekeepers. -pgxn/neon/walproposer.h: * Finish sync safekeepers with the given LSN. This function should not -pgxn/neon/walproposer.h: void (*finish_sync_safekeepers) (WalProposer *wp, XLogRecPtr lsn); -pgxn/neon/walproposer.h: * Called after every AppendResponse from the safekeeper. Used to -pgxn/neon/walproposer.h: * been commited on the quorum of safekeepers). -pgxn/neon/walproposer.h: void (*process_safekeeper_feedback) (WalProposer *wp, Safekeeper *sk); -pgxn/neon/walproposer.h: * APIs manipulating shared memory state used for Safekeeper quorum health metrics. -pgxn/neon/walproposer.h: * Reset the safekeeper statuses in shared memory for metric purposes. -pgxn/neon/walproposer.h: void (*reset_safekeeper_statuses_for_metrics) (WalProposer *wp, uint32 num_safekeepers); -pgxn/neon/walproposer.h: * Update the safekeeper status in shared memory for metric purposes. -pgxn/neon/walproposer.h: void (*update_safekeeper_status_for_metrics) (WalProposer *wp, uint32 sk_index, uint8 status); -pgxn/neon/walproposer.h: char *neon_tenant; -pgxn/neon/walproposer.h: char *neon_timeline; -pgxn/neon/walproposer.h: * Comma-separated list of safekeepers, in the following format: -pgxn/neon/walproposer.h: char *safekeepers_list; -pgxn/neon/walproposer.h: * WalProposer reconnects to offline safekeepers once in this interval. -pgxn/neon/walproposer.h: int safekeeper_reconnect_timeout; -pgxn/neon/walproposer.h: * from the safekeeper in this interval. Time is in milliseconds. -pgxn/neon/walproposer.h: int safekeeper_connection_timeout; -pgxn/neon/walproposer.h: * WAL segment size. Will be passed to safekeepers in greet request. Also -pgxn/neon/walproposer.h: * If safekeeper was started in sync mode, walproposer will not subscribe -pgxn/neon/walproposer.h: * for new WAL and will exit when quorum of safekeepers will be synced to -pgxn/neon/walproposer.h: bool syncSafekeepers; -pgxn/neon/walproposer.h: /* Will be passed to safekeepers in greet request. */ -pgxn/neon/walproposer.h: /* Will be passed to safekeepers in greet request. */ -pgxn/neon/walproposer.h: int n_safekeepers; -pgxn/neon/walproposer.h: /* (n_safekeepers / 2) + 1 */ -pgxn/neon/walproposer.h: Safekeeper safekeeper[MAX_SAFEKEEPERS]; -pgxn/neon/walproposer.h: /* Vote request for safekeeper */ -pgxn/neon/walproposer.h: * Minimal LSN which may be needed for recovery of some safekeeper, -pgxn/neon/walproposer.h: * collect terms from safekeepers quorum, choose max and +1. After that -pgxn/neon/walproposer.h: * safekeeper has higher term, it means that we have another running -pgxn/neon/walproposer.h: /* number of votes collected from safekeepers */ -pgxn/neon/walproposer.h: * config->safekeeper_reconnect_timeout -pgxn/neon/walproposer.h:extern void SafekeeperStateDesiredEvents(Safekeeper *sk, uint32 *sk_events, uint32 *nwr_events); -pgxn/neon/walproposer.h:#endif /* __NEON_WALPROPOSER_H__ */ -pgxn/neon/walproposer_pg.c:#include "neon.h" -pgxn/neon/walproposer_pg.c:#include "neon_perf_counters.h" -pgxn/neon/walproposer_pg.c:#include "neon_walreader.h" -pgxn/neon/walproposer_pg.c:static void assign_neon_safekeepers(const char *newval, void *extra); -pgxn/neon/walproposer_pg.c:static void walprop_pg_init_standalone_sync_safekeepers(void); -pgxn/neon/walproposer_pg.c:static void WalproposerShmemInit_SyncSafekeeper(void); -pgxn/neon/walproposer_pg.c:static void add_nwr_event_set(Safekeeper *sk, uint32 events); -pgxn/neon/walproposer_pg.c:static void update_nwr_event_set(Safekeeper *sk, uint32 events); -pgxn/neon/walproposer_pg.c:static void rm_safekeeper_event_set(Safekeeper *to_remove, bool is_sk); -pgxn/neon/walproposer_pg.c:init_walprop_config(bool syncSafekeepers) -pgxn/neon/walproposer_pg.c: walprop_config.neon_tenant = neon_tenant; -pgxn/neon/walproposer_pg.c: walprop_config.neon_timeline = neon_timeline; -pgxn/neon/walproposer_pg.c: walprop_config.safekeepers_list = pstrdup(wal_acceptors_list); -pgxn/neon/walproposer_pg.c: walprop_config.safekeeper_reconnect_timeout = wal_acceptor_reconnect_timeout; -pgxn/neon/walproposer_pg.c: walprop_config.safekeeper_connection_timeout = wal_acceptor_connection_timeout; -pgxn/neon/walproposer_pg.c: walprop_config.syncSafekeepers = syncSafekeepers; -pgxn/neon/walproposer_pg.c: if (!syncSafekeepers) -pgxn/neon/walproposer_pg.c: * Entry point for `postgres --sync-safekeepers`. -pgxn/neon/walproposer_pg.c: WalproposerShmemInit_SyncSafekeeper(); -pgxn/neon/walproposer_pg.c: walprop_pg_init_standalone_sync_safekeepers(); -pgxn/neon/walproposer_pg.c: "neon.safekeepers", -pgxn/neon/walproposer_pg.c: "List of Neon WAL acceptors (host:port)", -pgxn/neon/walproposer_pg.c: NULL, assign_neon_safekeepers, NULL); -pgxn/neon/walproposer_pg.c: "neon.safekeeper_reconnect_timeout", -pgxn/neon/walproposer_pg.c: "Walproposer reconnects to offline safekeepers once in this interval.", -pgxn/neon/walproposer_pg.c: "neon.safekeeper_connect_timeout", -pgxn/neon/walproposer_pg.c: "Connection or connection attempt to safekeeper is terminated if no message is received (or connection attempt doesn't finish) within this period.", -pgxn/neon/walproposer_pg.c:split_safekeepers_list(char *safekeepers_list, char *safekeepers[]) -pgxn/neon/walproposer_pg.c: int n_safekeepers = 0; -pgxn/neon/walproposer_pg.c: char *curr_sk = safekeepers_list; -pgxn/neon/walproposer_pg.c: for (char *coma = safekeepers_list; coma != NULL && *coma != '\0'; curr_sk = coma) -pgxn/neon/walproposer_pg.c: if (++n_safekeepers >= MAX_SAFEKEEPERS) { -pgxn/neon/walproposer_pg.c: safekeepers[n_safekeepers-1] = curr_sk; -pgxn/neon/walproposer_pg.c: return n_safekeepers; -pgxn/neon/walproposer_pg.c: * Accept two coma-separated strings with list of safekeeper host:port addresses. -pgxn/neon/walproposer_pg.c:safekeepers_cmp(char *old, char *new) -pgxn/neon/walproposer_pg.c: char *safekeepers_old[MAX_SAFEKEEPERS]; -pgxn/neon/walproposer_pg.c: char *safekeepers_new[MAX_SAFEKEEPERS]; -pgxn/neon/walproposer_pg.c: len_old = split_safekeepers_list(old, safekeepers_old); -pgxn/neon/walproposer_pg.c: len_new = split_safekeepers_list(new, safekeepers_new); -pgxn/neon/walproposer_pg.c: qsort(&safekeepers_old, len_old, sizeof(char *), pg_qsort_strcmp); -pgxn/neon/walproposer_pg.c: qsort(&safekeepers_new, len_new, sizeof(char *), pg_qsort_strcmp); -pgxn/neon/walproposer_pg.c: if (strcmp(safekeepers_old[i], safekeepers_new[i]) != 0) -pgxn/neon/walproposer_pg.c: * GUC assign_hook for neon.safekeepers. Restarts walproposer through FATAL if -pgxn/neon/walproposer_pg.c:assign_neon_safekeepers(const char *newval, void *extra) -pgxn/neon/walproposer_pg.c: wpg_log(FATAL, "neon.safekeepers is empty"); -pgxn/neon/walproposer_pg.c: /* Copy values because we will modify them in split_safekeepers_list() */ -pgxn/neon/walproposer_pg.c: * XXX: If you change anything here, sync with test_safekeepers_reconfigure_reorder. -pgxn/neon/walproposer_pg.c: if (!safekeepers_cmp(oldval, newval_copy)) -pgxn/neon/walproposer_pg.c: wpg_log(FATAL, "restarting walproposer to change safekeeper list from %s to %s", -pgxn/neon/walproposer_pg.c:WalproposerShmemInit_SyncSafekeeper(void) -pgxn/neon/walproposer_pg.c: snprintf(bgw.bgw_library_name, BGW_MAXLEN, "neon"); -pgxn/neon/walproposer_pg.c:walprop_pg_init_standalone_sync_safekeepers(void) -pgxn/neon/walproposer_pg.c: * reaches durable storage (in safekeepers), before the server shuts down -pgxn/neon/walproposer_pg.c:walprop_pg_update_donor(WalProposer *wp, Safekeeper *donor, XLogRecPtr donor_lsn) -pgxn/neon/walproposer_pg.c:walprop_error_message(Safekeeper *sk) -pgxn/neon/walproposer_pg.c:walprop_status(Safekeeper *sk) -pgxn/neon/walproposer_pg.c: char *password = neon_auth_token; -pgxn/neon/walproposer_pg.c: * Connect using the given connection string. If the NEON_AUTH_TOKEN -pgxn/neon/walproposer_pg.c:walprop_connect_start(Safekeeper *sk) -pgxn/neon/walproposer_pg.c:walprop_connect_poll(Safekeeper *sk) -pgxn/neon/walproposer_pg.c:walprop_send_query(Safekeeper *sk, char *query) -pgxn/neon/walproposer_pg.c:walprop_get_query_result(Safekeeper *sk) -pgxn/neon/walproposer_pg.c:walprop_socket(Safekeeper *sk) -pgxn/neon/walproposer_pg.c:walprop_flush(Safekeeper *sk) -pgxn/neon/walproposer_pg.c: * The protocol we use between walproposer and safekeeper means that we -pgxn/neon/walproposer_pg.c: * safekeeper won't normally send a CopyDone message. -pgxn/neon/walproposer_pg.c: * Receive a message from the safekeeper. -pgxn/neon/walproposer_pg.c:walprop_async_read(Safekeeper *sk, char **buf, int *amount) -pgxn/neon/walproposer_pg.c:walprop_async_write(Safekeeper *sk, void const *buf, size_t size) -pgxn/neon/walproposer_pg.c:walprop_blocking_write(Safekeeper *sk, void const *buf, size_t size) -pgxn/neon/walproposer_pg.c:walprop_finish(Safekeeper *sk) -pgxn/neon/walproposer_pg.c: NeonWALReaderFree(sk->xlogreader); -pgxn/neon/walproposer_pg.c: rm_safekeeper_event_set(sk, false); -pgxn/neon/walproposer_pg.c: * Subscribe for new WAL and stream it in the loop to safekeepers. -pgxn/neon/walproposer_pg.c: * Neon doesn't currently use PG Timelines, but it may in the future, so -pgxn/neon/walproposer_pg.c: longer used, replaced by neon_walreader; but callback still exists because -pgxn/neon/walproposer_pg.c:WalProposerRecovery(WalProposer *wp, Safekeeper *sk) -pgxn/neon/walproposer_pg.c:walprop_pg_wal_reader_allocate(Safekeeper *sk) -pgxn/neon/walproposer_pg.c: sk->xlogreader = NeonWALReaderAllocate(wal_segment_size, sk->wp->propEpochStartLsn, log_prefix); -pgxn/neon/walproposer_pg.c:static NeonWALReadResult -pgxn/neon/walproposer_pg.c:walprop_pg_wal_read(Safekeeper *sk, char *buf, XLogRecPtr startptr, Size count, char **errmsg) -pgxn/neon/walproposer_pg.c: NeonWALReadResult res; -pgxn/neon/walproposer_pg.c: res = NeonWALRead(sk->xlogreader, -pgxn/neon/walproposer_pg.c: if (res == NEON_WALREAD_SUCCESS) -pgxn/neon/walproposer_pg.c: if (NeonWALReaderEvents(sk->xlogreader) == 0) -pgxn/neon/walproposer_pg.c: rm_safekeeper_event_set(sk, false); -pgxn/neon/walproposer_pg.c: else if (res == NEON_WALREAD_ERROR) -pgxn/neon/walproposer_pg.c: *errmsg = NeonWALReaderErrMsg(sk->xlogreader); -pgxn/neon/walproposer_pg.c:walprop_pg_wal_reader_events(Safekeeper *sk) -pgxn/neon/walproposer_pg.c: return NeonWALReaderEvents(sk->xlogreader); -pgxn/neon/walproposer_pg.c: for (int i = 0; i < wp->n_safekeepers; i++) -pgxn/neon/walproposer_pg.c: wp->safekeeper[i].eventPos = -1; -pgxn/neon/walproposer_pg.c: wp->safekeeper[i].nwrEventPos = -1; -pgxn/neon/walproposer_pg.c: wp->safekeeper[i].nwrConnEstablished = false; -pgxn/neon/walproposer_pg.c: /* for each sk, we have socket plus potentially socket for neon walreader */ -pgxn/neon/walproposer_pg.c: waitEvents = CreateWaitEventSet(NULL, 2 + 2 * wp->n_safekeepers); -pgxn/neon/walproposer_pg.c: waitEvents = CreateWaitEventSet(TopMemoryContext, 2 + 2 * wp->n_safekeepers); -pgxn/neon/walproposer_pg.c: for (int i = 0; i < wp->n_safekeepers; i++) -pgxn/neon/walproposer_pg.c: wp->safekeeper[i].eventPos = -1; -pgxn/neon/walproposer_pg.c: wp->safekeeper[i].nwrEventPos = -1; -pgxn/neon/walproposer_pg.c: wp->safekeeper[i].nwrConnEstablished = false; -pgxn/neon/walproposer_pg.c:/* add safekeeper socket to wait event set */ -pgxn/neon/walproposer_pg.c:walprop_pg_add_safekeeper_event_set(Safekeeper *sk, uint32 events) -pgxn/neon/walproposer_pg.c:/* add neon wal reader socket to wait event set */ -pgxn/neon/walproposer_pg.c:add_nwr_event_set(Safekeeper *sk, uint32 events) -pgxn/neon/walproposer_pg.c: sk->nwrEventPos = AddWaitEventToSet(waitEvents, events, NeonWALReaderSocket(sk->xlogreader), NULL, sk); -pgxn/neon/walproposer_pg.c: sk->nwrConnEstablished = NeonWALReaderIsRemConnEstablished(sk->xlogreader); -pgxn/neon/walproposer_pg.c:walprop_pg_update_event_set(Safekeeper *sk, uint32 events) -pgxn/neon/walproposer_pg.c: * Update neon_walreader event. -pgxn/neon/walproposer_pg.c:update_nwr_event_set(Safekeeper *sk, uint32 events) -pgxn/neon/walproposer_pg.c:walprop_pg_active_state_update_event_set(Safekeeper *sk) -pgxn/neon/walproposer_pg.c: SafekeeperStateDesiredEvents(sk, &sk_events, &nwr_events); -pgxn/neon/walproposer_pg.c: * If we need to wait for neon_walreader, ensure we have up to date socket -pgxn/neon/walproposer_pg.c: rm_safekeeper_event_set(sk, false); -pgxn/neon/walproposer_pg.c: * reconstructing the whole set, SafekeeperStateDesiredEvents instead -pgxn/neon/walproposer_pg.c: rm_safekeeper_event_set(sk, false); -pgxn/neon/walproposer_pg.c:walprop_pg_rm_safekeeper_event_set(Safekeeper *to_remove) -pgxn/neon/walproposer_pg.c: rm_safekeeper_event_set(to_remove, true); -pgxn/neon/walproposer_pg.c: * If is_sk is true, socket of connection to safekeeper is removed; otherwise -pgxn/neon/walproposer_pg.c: * socket of neon_walreader. -pgxn/neon/walproposer_pg.c:rm_safekeeper_event_set(Safekeeper *to_remove, bool is_sk) -pgxn/neon/walproposer_pg.c: * function with safekeeper socket not existing, but do that with neon -pgxn/neon/walproposer_pg.c: /* Re-initialize it without adding any safekeeper events */ -pgxn/neon/walproposer_pg.c: * loop through the existing safekeepers. If they aren't the one we're -pgxn/neon/walproposer_pg.c: for (int i = 0; i < wp->n_safekeepers; i++) -pgxn/neon/walproposer_pg.c: Safekeeper *sk = &wp->safekeeper[i]; -pgxn/neon/walproposer_pg.c: * If this safekeeper isn't offline, add events for it, except for the -pgxn/neon/walproposer_pg.c: SafekeeperStateDesiredEvents(sk, &sk_events, &nwr_events); -pgxn/neon/walproposer_pg.c: wp->api.add_safekeeper_event_set(sk, sk_events); -pgxn/neon/walproposer_pg.c:walprop_pg_wait_event_set(WalProposer *wp, long timeout, Safekeeper **sk, uint32 *events) -pgxn/neon/walproposer_pg.c: * Do that only when we're interested in new WAL: without sync-safekeepers -pgxn/neon/walproposer_pg.c: if (!wp->config->syncSafekeepers && wp->availableLsn != InvalidXLogRecPtr && GetFlushRecPtr(NULL) > wp->availableLsn) -pgxn/neon/walproposer_pg.c: * Wait for a wait event to happen, or timeout: - Safekeeper socket can -pgxn/neon/walproposer_pg.c: * Process config if requested. This restarts walproposer if safekeepers -pgxn/neon/walproposer_pg.c: * list changed. Don't do that for sync-safekeepers because quite probably -pgxn/neon/walproposer_pg.c: * sync-safekeepers should be quick to finish anyway. -pgxn/neon/walproposer_pg.c: if (!wp->config->syncSafekeepers && ConfigReloadPending) -pgxn/neon/walproposer_pg.c: * event from a safekeeper socket. -pgxn/neon/walproposer_pg.c: *sk = (Safekeeper *) event.user_data; -pgxn/neon/walproposer_pg.c:walprop_pg_finish_sync_safekeepers(WalProposer *wp, XLogRecPtr lsn) -pgxn/neon/walproposer_pg.c: * Note that unlike sync-safekeepers waiting here is not reliable: we -pgxn/neon/walproposer_pg.c: * don't check that majority of safekeepers received and persisted -pgxn/neon/walproposer_pg.c: * sync-safekeepers which must bump commit_lsn or basebackup will fail, -pgxn/neon/walproposer_pg.c: * this catchup is important only for tests where safekeepers/network -pgxn/neon/walproposer_pg.c: * Combine hot standby feedbacks from all safekeepers. -pgxn/neon/walproposer_pg.c: for (int i = 0; i < wp->n_safekeepers; i++) -pgxn/neon/walproposer_pg.c: if (wp->safekeeper[i].state == SS_ACTIVE) -pgxn/neon/walproposer_pg.c: HotStandbyFeedback *skhs = &wp->safekeeper[i].appendResponse.hs; -pgxn/neon/walproposer_pg.c: * Based on commitLsn and safekeeper responses including pageserver feedback, -pgxn/neon/walproposer_pg.c: * None of that is functional in sync-safekeepers. -pgxn/neon/walproposer_pg.c:walprop_pg_process_safekeeper_feedback(WalProposer *wp, Safekeeper *sk) -pgxn/neon/walproposer_pg.c: if (wp->config->syncSafekeepers) -pgxn/neon/walproposer_pg.c: // Record safekeeper commit LSN in shared memory for lag monitoring -pgxn/neon/walproposer_pg.c: Assert(sk->index < MAX_SAFEKEEPERS); -pgxn/neon/walproposer_pg.c: shmem->safekeeper_commit_lsn[sk->index] = sk->appendResponse.commitLsn; -pgxn/neon/walproposer_pg.c: SetNeonCurrentClusterSize(sk->appendResponse.ps_feedback.currentClusterSize); -pgxn/neon/walproposer_pg.c: * hardened and will be fetched from one of safekeepers by -pgxn/neon/walproposer_pg.c: * neon_walreader if needed. -pgxn/neon/walproposer_pg.c: /* write_lsn - This is what durably stored in safekeepers quorum. */ -pgxn/neon/walproposer_pg.c: /* flush_lsn - This is what durably stored in safekeepers quorum. */ -pgxn/neon/walproposer_pg.c:SetNeonCurrentClusterSize(uint64 size) -pgxn/neon/walproposer_pg.c:GetNeonCurrentClusterSize(void) -pgxn/neon/walproposer_pg.c:uint64 GetNeonCurrentClusterSize(void); -pgxn/neon/walproposer_pg.c:walprop_pg_reset_safekeeper_statuses_for_metrics(WalProposer *wp, uint32 num_safekeepers) -pgxn/neon/walproposer_pg.c: shmem->num_safekeepers = num_safekeepers; -pgxn/neon/walproposer_pg.c: memset(shmem->safekeeper_status, 0, sizeof(shmem->safekeeper_status)); -pgxn/neon/walproposer_pg.c: memset(shmem->safekeeper_commit_lsn, 0, sizeof(shmem->safekeeper_commit_lsn)); -pgxn/neon/walproposer_pg.c:walprop_pg_update_safekeeper_status_for_metrics(WalProposer *wp, uint32 sk_index, uint8 status) -pgxn/neon/walproposer_pg.c: Assert(sk_index < MAX_SAFEKEEPERS); -pgxn/neon/walproposer_pg.c: shmem->safekeeper_status[sk_index] = status; -pgxn/neon/walproposer_pg.c: .add_safekeeper_event_set = walprop_pg_add_safekeeper_event_set, -pgxn/neon/walproposer_pg.c: .rm_safekeeper_event_set = walprop_pg_rm_safekeeper_event_set, -pgxn/neon/walproposer_pg.c: .finish_sync_safekeepers = walprop_pg_finish_sync_safekeepers, -pgxn/neon/walproposer_pg.c: .process_safekeeper_feedback = walprop_pg_process_safekeeper_feedback, -pgxn/neon/walproposer_pg.c: .reset_safekeeper_statuses_for_metrics = walprop_pg_reset_safekeeper_statuses_for_metrics, -pgxn/neon/walproposer_pg.c: .update_safekeeper_status_for_metrics = walprop_pg_update_safekeeper_status_for_metrics, -pgxn/neon/walsender_hooks.c: * Implements XLogReaderRoutine in terms of NeonWALReader. Allows for -pgxn/neon/walsender_hooks.c: * fetching WAL from safekeepers, which normal xlogreader can't do. -pgxn/neon/walsender_hooks.c:#include "neon.h" -pgxn/neon/walsender_hooks.c:#include "neon_walreader.h" -pgxn/neon/walsender_hooks.c:static NeonWALReader *wal_reader = NULL; -pgxn/neon/walsender_hooks.c:NeonWALReadWaitForWAL(XLogRecPtr loc) -pgxn/neon/walsender_hooks.c: while (!NeonWALReaderUpdateDonor(wal_reader)) -pgxn/neon/walsender_hooks.c:NeonWALPageRead( -pgxn/neon/walsender_hooks.c: XLogRecPtr flushptr = NeonWALReadWaitForWAL(targetPagePtr + reqLen); -pgxn/neon/walsender_hooks.c: rem_lsn = NeonWALReaderGetRemLsn(wal_reader); -pgxn/neon/walsender_hooks.c: NeonWALReaderResetRemote(wal_reader); -pgxn/neon/walsender_hooks.c: NeonWALReadResult res = NeonWALRead( -pgxn/neon/walsender_hooks.c: if (res == NEON_WALREAD_SUCCESS) -pgxn/neon/walsender_hooks.c: xlogreader->seg.ws_tli = NeonWALReaderGetSegment(wal_reader)->ws_tli; -pgxn/neon/walsender_hooks.c: xlogreader->seg.ws_segno = NeonWALReaderGetSegment(wal_reader)->ws_segno; -pgxn/neon/walsender_hooks.c: if (res == NEON_WALREAD_ERROR) -pgxn/neon/walsender_hooks.c: NeonWALReaderErrMsg(wal_reader)); -pgxn/neon/walsender_hooks.c: pgsocket sock = NeonWALReaderSocket(wal_reader); -pgxn/neon/walsender_hooks.c: uint32_t reader_events = NeonWALReaderEvents(wal_reader); -pgxn/neon/walsender_hooks.c: WAIT_EVENT_NEON_WAL_DL); -pgxn/neon/walsender_hooks.c:NeonWALReadSegmentOpen(XLogReaderState *xlogreader, XLogSegNo nextSegNo, TimeLineID *tli_p) -pgxn/neon/walsender_hooks.c: neon_wal_segment_open(wal_reader, nextSegNo, tli_p); -pgxn/neon/walsender_hooks.c: xlogreader->seg.ws_file = NeonWALReaderGetSegment(wal_reader)->ws_file; -pgxn/neon/walsender_hooks.c:NeonWALReadSegmentClose(XLogReaderState *xlogreader) -pgxn/neon/walsender_hooks.c: neon_wal_segment_close(wal_reader); -pgxn/neon/walsender_hooks.c: xlogreader->seg.ws_file = NeonWALReaderGetSegment(wal_reader)->ws_file; -pgxn/neon/walsender_hooks.c:NeonOnDemandXLogReaderRoutines(XLogReaderRoutine *xlr) -pgxn/neon/walsender_hooks.c: * If safekeepers are not configured, assume we don't need neon_walreader, -pgxn/neon/walsender_hooks.c: * i.e. running neon fork locally. -pgxn/neon/walsender_hooks.c: wal_reader = NeonWALReaderAllocate(wal_segment_size, basebackupLsn, "[walsender] "); -pgxn/neon/walsender_hooks.c: xlr->page_read = NeonWALPageRead; -pgxn/neon/walsender_hooks.c: xlr->segment_open = NeonWALReadSegmentOpen; -pgxn/neon/walsender_hooks.c: xlr->segment_close = NeonWALReadSegmentClose; -pgxn/neon/walsender_hooks.h:void NeonOnDemandXLogReaderRoutines(struct XLogReaderRoutine *xlr); -pgxn/neon_rmgr/Makefile:# pgxs/neon/Makefile -pgxn/neon_rmgr/Makefile:MODULE_big = neon_rmgr -pgxn/neon_rmgr/Makefile: neon_rmgr.o \ -pgxn/neon_rmgr/Makefile: neon_rmgr_decode.o \ -pgxn/neon_rmgr/Makefile: neon_rmgr_desc.o -pgxn/neon_rmgr/Makefile:EXTENSION = neon_rmgr -pgxn/neon_rmgr/Makefile:PGFILEDESC = "Neon WAL Resource Manager - custom WAL records used to make Neon work (since PG 16)" -pgxn/neon_rmgr/neon_rmgr.c:#include "access/neon_xlog.h" -pgxn/neon_rmgr/neon_rmgr.c:#include "neon_rmgr.h" -pgxn/neon_rmgr/neon_rmgr.c:static void neon_rm_redo(XLogReaderState *record); -pgxn/neon_rmgr/neon_rmgr.c:static void neon_rm_startup(void); -pgxn/neon_rmgr/neon_rmgr.c:static void neon_rm_cleanup(void); -pgxn/neon_rmgr/neon_rmgr.c:static void neon_rm_mask(char *pagedata, BlockNumber blkno); -pgxn/neon_rmgr/neon_rmgr.c:static void redo_neon_heap_insert(XLogReaderState *record); -pgxn/neon_rmgr/neon_rmgr.c:static void redo_neon_heap_delete(XLogReaderState *record); -pgxn/neon_rmgr/neon_rmgr.c:static void redo_neon_heap_update(XLogReaderState *record, bool hot_update); -pgxn/neon_rmgr/neon_rmgr.c:static void redo_neon_heap_lock(XLogReaderState *record); -pgxn/neon_rmgr/neon_rmgr.c:static void redo_neon_heap_multi_insert(XLogReaderState *record); -pgxn/neon_rmgr/neon_rmgr.c:const static RmgrData NeonRmgr = { -pgxn/neon_rmgr/neon_rmgr.c: .rm_redo = neon_rm_redo, -pgxn/neon_rmgr/neon_rmgr.c: .rm_desc = neon_rm_desc, -pgxn/neon_rmgr/neon_rmgr.c: .rm_identify = neon_rm_identify, -pgxn/neon_rmgr/neon_rmgr.c: .rm_startup = neon_rm_startup, -pgxn/neon_rmgr/neon_rmgr.c: .rm_cleanup = neon_rm_cleanup, -pgxn/neon_rmgr/neon_rmgr.c: .rm_mask = neon_rm_mask, -pgxn/neon_rmgr/neon_rmgr.c: .rm_decode = neon_rm_decode, -pgxn/neon_rmgr/neon_rmgr.c: RegisterCustomRmgr(RM_NEON_ID, &NeonRmgr); -pgxn/neon_rmgr/neon_rmgr.c:neon_rm_redo(XLogReaderState *record) -pgxn/neon_rmgr/neon_rmgr.c: switch (info & XLOG_NEON_OPMASK) -pgxn/neon_rmgr/neon_rmgr.c: case XLOG_NEON_HEAP_INSERT: -pgxn/neon_rmgr/neon_rmgr.c: redo_neon_heap_insert(record); -pgxn/neon_rmgr/neon_rmgr.c: case XLOG_NEON_HEAP_DELETE: -pgxn/neon_rmgr/neon_rmgr.c: redo_neon_heap_delete(record); -pgxn/neon_rmgr/neon_rmgr.c: case XLOG_NEON_HEAP_UPDATE: -pgxn/neon_rmgr/neon_rmgr.c: redo_neon_heap_update(record, false); -pgxn/neon_rmgr/neon_rmgr.c: case XLOG_NEON_HEAP_HOT_UPDATE: -pgxn/neon_rmgr/neon_rmgr.c: redo_neon_heap_update(record, true); -pgxn/neon_rmgr/neon_rmgr.c: case XLOG_NEON_HEAP_LOCK: -pgxn/neon_rmgr/neon_rmgr.c: redo_neon_heap_lock(record); -pgxn/neon_rmgr/neon_rmgr.c: case XLOG_NEON_HEAP_MULTI_INSERT: -pgxn/neon_rmgr/neon_rmgr.c: redo_neon_heap_multi_insert(record); -pgxn/neon_rmgr/neon_rmgr.c:neon_rm_startup(void) -pgxn/neon_rmgr/neon_rmgr.c:neon_rm_cleanup(void) -pgxn/neon_rmgr/neon_rmgr.c:neon_rm_mask(char *pagedata, BlockNumber blkno) -pgxn/neon_rmgr/neon_rmgr.c:redo_neon_heap_insert(XLogReaderState *record) -pgxn/neon_rmgr/neon_rmgr.c: xl_neon_heap_insert *xlrec = (xl_neon_heap_insert *) XLogRecGetData(record); -pgxn/neon_rmgr/neon_rmgr.c: xl_neon_heap_header xlhdr; -pgxn/neon_rmgr/neon_rmgr.c: newlen = datalen - SizeOfNeonHeapHeader; -pgxn/neon_rmgr/neon_rmgr.c: Assert(datalen > SizeOfNeonHeapHeader && newlen <= MaxHeapTupleSize); -pgxn/neon_rmgr/neon_rmgr.c: memcpy((char *) &xlhdr, data, SizeOfNeonHeapHeader); -pgxn/neon_rmgr/neon_rmgr.c: data += SizeOfNeonHeapHeader; -pgxn/neon_rmgr/neon_rmgr.c:redo_neon_heap_delete(XLogReaderState *record) -pgxn/neon_rmgr/neon_rmgr.c: xl_neon_heap_delete *xlrec = (xl_neon_heap_delete *) XLogRecGetData(record); -pgxn/neon_rmgr/neon_rmgr.c:redo_neon_heap_update(XLogReaderState *record, bool hot_update) -pgxn/neon_rmgr/neon_rmgr.c: xl_neon_heap_update *xlrec = (xl_neon_heap_update *) XLogRecGetData(record); -pgxn/neon_rmgr/neon_rmgr.c: xl_neon_heap_header xlhdr; -pgxn/neon_rmgr/neon_rmgr.c: memcpy((char *) &xlhdr, recdata, SizeOfNeonHeapHeader); -pgxn/neon_rmgr/neon_rmgr.c: recdata += SizeOfNeonHeapHeader; -pgxn/neon_rmgr/neon_rmgr.c:redo_neon_heap_lock(XLogReaderState *record) -pgxn/neon_rmgr/neon_rmgr.c: xl_neon_heap_lock *xlrec = (xl_neon_heap_lock *) XLogRecGetData(record); -pgxn/neon_rmgr/neon_rmgr.c:redo_neon_heap_multi_insert(XLogReaderState *record) -pgxn/neon_rmgr/neon_rmgr.c: xl_neon_heap_multi_insert *xlrec; -pgxn/neon_rmgr/neon_rmgr.c: xlrec = (xl_neon_heap_multi_insert *) XLogRecGetData(record); -pgxn/neon_rmgr/neon_rmgr.c: xl_neon_multi_insert_tuple *xlhdr; -pgxn/neon_rmgr/neon_rmgr.c: xlhdr = (xl_neon_multi_insert_tuple *) SHORTALIGN(tupdata); -pgxn/neon_rmgr/neon_rmgr.c: tupdata = ((char *) xlhdr) + SizeOfNeonMultiInsertTuple; -pgxn/neon_rmgr/neon_rmgr.control:# neon_rmgr extension -pgxn/neon_rmgr/neon_rmgr.control:comment = 'Neon WAL Resource Manager - custom WAL records used to make Neon work (since PG 16)' -pgxn/neon_rmgr/neon_rmgr.control:module_pathname = '$libdir/neon_rmgr' -pgxn/neon_rmgr/neon_rmgr.h:#ifndef NEON_RMGR_H -pgxn/neon_rmgr/neon_rmgr.h:#define NEON_RMGR_H -pgxn/neon_rmgr/neon_rmgr.h:extern void neon_rm_desc(StringInfo buf, XLogReaderState *record); -pgxn/neon_rmgr/neon_rmgr.h:extern void neon_rm_decode(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); -pgxn/neon_rmgr/neon_rmgr.h:extern const char *neon_rm_identify(uint8 info); -pgxn/neon_rmgr/neon_rmgr.h:#endif //NEON_RMGR_H -pgxn/neon_rmgr/neon_rmgr_decode.c:#include "access/neon_xlog.h" -pgxn/neon_rmgr/neon_rmgr_decode.c:#include "neon_rmgr.h" -pgxn/neon_rmgr/neon_rmgr_decode.c:static void DecodeNeonInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); -pgxn/neon_rmgr/neon_rmgr_decode.c:static void DecodeNeonUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); -pgxn/neon_rmgr/neon_rmgr_decode.c:static void DecodeNeonDelete(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); -pgxn/neon_rmgr/neon_rmgr_decode.c:static void DecodeNeonMultiInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); -pgxn/neon_rmgr/neon_rmgr_decode.c:neon_rm_decode(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) -pgxn/neon_rmgr/neon_rmgr_decode.c: uint8 info = XLogRecGetInfo(buf->record) & XLOG_NEON_OPMASK; -pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_INSERT: -pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeNeonInsert(ctx, buf); -pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_DELETE: -pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeNeonDelete(ctx, buf); -pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_UPDATE: -pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_HOT_UPDATE: -pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeNeonUpdate(ctx, buf); -pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_LOCK: -pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_MULTI_INSERT: -pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeNeonMultiInsert(ctx, buf); -pgxn/neon_rmgr/neon_rmgr_decode.c:DecodeNeonInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) -pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_insert *xlrec; -pgxn/neon_rmgr/neon_rmgr_decode.c: xlrec = (xl_neon_heap_insert *) XLogRecGetData(r); -pgxn/neon_rmgr/neon_rmgr_decode.c: tuplelen = datalen - SizeOfNeonHeapHeader; -pgxn/neon_rmgr/neon_rmgr_decode.c:DecodeNeonDelete(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) -pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_delete *xlrec; -pgxn/neon_rmgr/neon_rmgr_decode.c: xlrec = (xl_neon_heap_delete *) XLogRecGetData(r); -pgxn/neon_rmgr/neon_rmgr_decode.c: Size datalen = XLogRecGetDataLen(r) - SizeOfNeonHeapHeader; -pgxn/neon_rmgr/neon_rmgr_decode.c: Size tuplelen = datalen - SizeOfNeonHeapHeader; -pgxn/neon_rmgr/neon_rmgr_decode.c: Assert(XLogRecGetDataLen(r) > (SizeOfNeonHeapDelete + SizeOfNeonHeapHeader)); -pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeXLogTuple((char *) xlrec + SizeOfNeonHeapDelete, -pgxn/neon_rmgr/neon_rmgr_decode.c:DecodeNeonUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) -pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_update *xlrec; -pgxn/neon_rmgr/neon_rmgr_decode.c: xlrec = (xl_neon_heap_update *) XLogRecGetData(r); -pgxn/neon_rmgr/neon_rmgr_decode.c: tuplelen = datalen - SizeOfNeonHeapHeader; -pgxn/neon_rmgr/neon_rmgr_decode.c: data = XLogRecGetData(r) + SizeOfNeonHeapUpdate; -pgxn/neon_rmgr/neon_rmgr_decode.c: datalen = XLogRecGetDataLen(r) - SizeOfNeonHeapUpdate; -pgxn/neon_rmgr/neon_rmgr_decode.c: tuplelen = datalen - SizeOfNeonHeapHeader; -pgxn/neon_rmgr/neon_rmgr_decode.c:DecodeNeonMultiInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) -pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_multi_insert *xlrec; -pgxn/neon_rmgr/neon_rmgr_decode.c: xlrec = (xl_neon_heap_multi_insert *) XLogRecGetData(r); -pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_multi_insert_tuple *xlhdr; -pgxn/neon_rmgr/neon_rmgr_decode.c: xlhdr = (xl_neon_multi_insert_tuple *) SHORTALIGN(data); -pgxn/neon_rmgr/neon_rmgr_decode.c: data = ((char *) xlhdr) + SizeOfNeonMultiInsertTuple; -pgxn/neon_rmgr/neon_rmgr_decode.c: /* move to the next xl_neon_multi_insert_tuple entry */ -pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_header xlhdr; -pgxn/neon_rmgr/neon_rmgr_decode.c: int datalen = len - SizeOfNeonHeapHeader; -pgxn/neon_rmgr/neon_rmgr_decode.c: SizeOfNeonHeapHeader); -pgxn/neon_rmgr/neon_rmgr_decode.c: data + SizeOfNeonHeapHeader, -pgxn/neon_rmgr/neon_rmgr_decode.c:static void DecodeNeonInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); -pgxn/neon_rmgr/neon_rmgr_decode.c:static void DecodeNeonUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); -pgxn/neon_rmgr/neon_rmgr_decode.c:static void DecodeNeonDelete(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); -pgxn/neon_rmgr/neon_rmgr_decode.c:static void DecodeNeonMultiInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); -pgxn/neon_rmgr/neon_rmgr_decode.c:neon_rm_decode(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) -pgxn/neon_rmgr/neon_rmgr_decode.c: uint8 info = XLogRecGetInfo(buf->record) & XLOG_NEON_OPMASK; -pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_INSERT: -pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeNeonInsert(ctx, buf); -pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_DELETE: -pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeNeonDelete(ctx, buf); -pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_UPDATE: -pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_HOT_UPDATE: -pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeNeonUpdate(ctx, buf); -pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_LOCK: -pgxn/neon_rmgr/neon_rmgr_decode.c: case XLOG_NEON_HEAP_MULTI_INSERT: -pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeNeonMultiInsert(ctx, buf); -pgxn/neon_rmgr/neon_rmgr_decode.c:DecodeNeonInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) -pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_insert *xlrec; -pgxn/neon_rmgr/neon_rmgr_decode.c: xlrec = (xl_neon_heap_insert *) XLogRecGetData(r); -pgxn/neon_rmgr/neon_rmgr_decode.c:DecodeNeonDelete(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) -pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_delete *xlrec; -pgxn/neon_rmgr/neon_rmgr_decode.c: xlrec = (xl_neon_heap_delete *) XLogRecGetData(r); -pgxn/neon_rmgr/neon_rmgr_decode.c: Size datalen = XLogRecGetDataLen(r) - SizeOfNeonHeapHeader; -pgxn/neon_rmgr/neon_rmgr_decode.c: Size tuplelen = datalen - SizeOfNeonHeapHeader; -pgxn/neon_rmgr/neon_rmgr_decode.c: Assert(XLogRecGetDataLen(r) > (SizeOfNeonHeapDelete + SizeOfNeonHeapHeader)); -pgxn/neon_rmgr/neon_rmgr_decode.c: DecodeXLogTuple((char *) xlrec + SizeOfNeonHeapDelete, -pgxn/neon_rmgr/neon_rmgr_decode.c:DecodeNeonUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) -pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_update *xlrec; -pgxn/neon_rmgr/neon_rmgr_decode.c: xlrec = (xl_neon_heap_update *) XLogRecGetData(r); -pgxn/neon_rmgr/neon_rmgr_decode.c: tuplelen = datalen - SizeOfNeonHeapHeader; -pgxn/neon_rmgr/neon_rmgr_decode.c: data = XLogRecGetData(r) + SizeOfNeonHeapUpdate; -pgxn/neon_rmgr/neon_rmgr_decode.c: datalen = XLogRecGetDataLen(r) - SizeOfNeonHeapUpdate; -pgxn/neon_rmgr/neon_rmgr_decode.c: tuplelen = datalen - SizeOfNeonHeapHeader; -pgxn/neon_rmgr/neon_rmgr_decode.c:DecodeNeonMultiInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) -pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_multi_insert *xlrec; -pgxn/neon_rmgr/neon_rmgr_decode.c: xlrec = (xl_neon_heap_multi_insert *) XLogRecGetData(r); -pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_multi_insert_tuple *xlhdr; -pgxn/neon_rmgr/neon_rmgr_decode.c: xlhdr = (xl_neon_multi_insert_tuple *) SHORTALIGN(data); -pgxn/neon_rmgr/neon_rmgr_decode.c: data = ((char *) xlhdr) + SizeOfNeonMultiInsertTuple; -pgxn/neon_rmgr/neon_rmgr_decode.c: /* move to the next xl_neon_multi_insert_tuple entry */ -pgxn/neon_rmgr/neon_rmgr_decode.c: xl_neon_heap_header xlhdr; -pgxn/neon_rmgr/neon_rmgr_decode.c: int datalen = len - SizeOfNeonHeapHeader; -pgxn/neon_rmgr/neon_rmgr_decode.c: SizeOfNeonHeapHeader); -pgxn/neon_rmgr/neon_rmgr_decode.c: data + SizeOfNeonHeapHeader, -pgxn/neon_rmgr/neon_rmgr_desc.c:#include "access/neon_xlog.h" -pgxn/neon_rmgr/neon_rmgr_desc.c:#include "neon_rmgr.h" -pgxn/neon_rmgr/neon_rmgr_desc.c:neon_rm_desc(StringInfo buf, XLogReaderState *record) -pgxn/neon_rmgr/neon_rmgr_desc.c: info &= XLOG_NEON_OPMASK; -pgxn/neon_rmgr/neon_rmgr_desc.c: if (info == XLOG_NEON_HEAP_INSERT) -pgxn/neon_rmgr/neon_rmgr_desc.c: xl_neon_heap_insert *xlrec = (xl_neon_heap_insert *) rec; -pgxn/neon_rmgr/neon_rmgr_desc.c: else if (info == XLOG_NEON_HEAP_DELETE) -pgxn/neon_rmgr/neon_rmgr_desc.c: xl_neon_heap_delete *xlrec = (xl_neon_heap_delete *) rec; -pgxn/neon_rmgr/neon_rmgr_desc.c: else if (info == XLOG_NEON_HEAP_UPDATE) -pgxn/neon_rmgr/neon_rmgr_desc.c: xl_neon_heap_update *xlrec = (xl_neon_heap_update *) rec; -pgxn/neon_rmgr/neon_rmgr_desc.c: else if (info == XLOG_NEON_HEAP_HOT_UPDATE) -pgxn/neon_rmgr/neon_rmgr_desc.c: xl_neon_heap_update *xlrec = (xl_neon_heap_update *) rec; -pgxn/neon_rmgr/neon_rmgr_desc.c: else if (info == XLOG_NEON_HEAP_LOCK) -pgxn/neon_rmgr/neon_rmgr_desc.c: xl_neon_heap_lock *xlrec = (xl_neon_heap_lock *) rec; -pgxn/neon_rmgr/neon_rmgr_desc.c: else if (info == XLOG_NEON_HEAP_MULTI_INSERT) -pgxn/neon_rmgr/neon_rmgr_desc.c: xl_neon_heap_multi_insert *xlrec = (xl_neon_heap_multi_insert *) rec; -pgxn/neon_rmgr/neon_rmgr_desc.c: bool isinit = (XLogRecGetInfo(record) & XLOG_NEON_INIT_PAGE) != 0; -pgxn/neon_rmgr/neon_rmgr_desc.c:neon_rm_identify(uint8 info) -pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_INSERT: -pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_INSERT | XLOG_NEON_INIT_PAGE: -pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_DELETE: -pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_UPDATE: -pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_UPDATE | XLOG_NEON_INIT_PAGE: -pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_HOT_UPDATE: -pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_HOT_UPDATE | XLOG_HEAP_INIT_PAGE: -pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_LOCK: -pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_MULTI_INSERT: -pgxn/neon_rmgr/neon_rmgr_desc.c: case XLOG_NEON_HEAP_MULTI_INSERT | XLOG_NEON_INIT_PAGE: -contrib/pg_trgm/data/trgm2.data:Riad Zenith Marrakech -doc/src/sgml/release-16.sgml:2022-08-29 [82739d4a8] Use ARM Advanced SIMD (NEON) intrinsics where available -doc/src/sgml/release-16.sgml: Instruction Multiple Data) (NEON) instructions -src/backend/access/brin/brin_xlog.c: //ZENITH XXX Don't use BufferGetBlockNumber because wal-redo doesn't pin buffer. -src/backend/access/gin/ginxlog.c: /* NEON: we do not not apply WAL record if target page is absent at replica */ -src/backend/access/heap/heapam.c:#include "access/neon_xlog.h" -src/backend/access/heap/heapam.c: xl_neon_heap_insert xlrec; -src/backend/access/heap/heapam.c: xl_neon_heap_header xlhdr; -src/backend/access/heap/heapam.c: uint8 info = XLOG_NEON_HEAP_INSERT; -src/backend/access/heap/heapam.c: info |= XLOG_NEON_INIT_PAGE; -src/backend/access/heap/heapam.c: XLogRegisterData((char *) &xlrec, SizeOfNeonHeapInsert); -src/backend/access/heap/heapam.c: XLogRegisterBufData(0, (char *) &xlhdr, SizeOfNeonHeapHeader); -src/backend/access/heap/heapam.c: recptr = XLogInsert(RM_NEON_ID, info); -src/backend/access/heap/heapam.c: * NEON: speculative token is not stored in WAL, so if the page is evicted -src/backend/access/heap/heapam.c: xl_neon_heap_multi_insert *xlrec; -src/backend/access/heap/heapam.c: uint8 info = XLOG_NEON_HEAP_MULTI_INSERT; -src/backend/access/heap/heapam.c: /* allocate xl_neon_heap_multi_insert struct from the scratch area */ -src/backend/access/heap/heapam.c: xlrec = (xl_neon_heap_multi_insert *) scratchptr; -src/backend/access/heap/heapam.c: scratchptr += SizeOfNeonHeapMultiInsert; -src/backend/access/heap/heapam.c: xl_neon_multi_insert_tuple *tuphdr; -src/backend/access/heap/heapam.c: tuphdr = (xl_neon_multi_insert_tuple *) SHORTALIGN(scratchptr); -src/backend/access/heap/heapam.c: scratchptr = ((char *) tuphdr) + SizeOfNeonMultiInsertTuple; -src/backend/access/heap/heapam.c: info |= XLOG_NEON_INIT_PAGE; -src/backend/access/heap/heapam.c: recptr = XLogInsert(RM_NEON_ID, info); -src/backend/access/heap/heapam.c: xl_neon_heap_delete xlrec; -src/backend/access/heap/heapam.c: xl_neon_heap_header xlhdr; -src/backend/access/heap/heapam.c: XLogRegisterData((char *) &xlrec, SizeOfNeonHeapDelete); -src/backend/access/heap/heapam.c: XLogRegisterData((char *) &xlhdr, SizeOfNeonHeapHeader); -src/backend/access/heap/heapam.c: recptr = XLogInsert(RM_NEON_ID, XLOG_NEON_HEAP_DELETE); -src/backend/access/heap/heapam.c: xl_neon_heap_lock xlrec; -src/backend/access/heap/heapam.c: XLogRegisterData((char *) &xlrec, SizeOfNeonHeapLock); -src/backend/access/heap/heapam.c: recptr = XLogInsert(RM_NEON_ID, XLOG_NEON_HEAP_LOCK); -src/backend/access/heap/heapam.c: xl_neon_heap_lock xlrec; -src/backend/access/heap/heapam.c: XLogRegisterData((char *) &xlrec, SizeOfNeonHeapLock); -src/backend/access/heap/heapam.c: recptr = XLogInsert(RM_NEON_ID, XLOG_NEON_HEAP_LOCK); -src/backend/access/heap/heapam.c: ReleaseBuffer(buffer); /* NEON: release buffer pinned by heap_insert */ -src/backend/access/heap/heapam.c: * NEON: release buffer pinned by heap_insert -src/backend/access/heap/heapam.c: xl_neon_heap_delete xlrec; -src/backend/access/heap/heapam.c: XLogRegisterData((char *) &xlrec, SizeOfNeonHeapDelete); -src/backend/access/heap/heapam.c: recptr = XLogInsert(RM_NEON_ID, XLOG_NEON_HEAP_DELETE); -src/backend/access/heap/heapam.c: xl_neon_heap_update xlrec; -src/backend/access/heap/heapam.c: xl_neon_heap_header xlhdr; -src/backend/access/heap/heapam.c: xl_neon_heap_header xlhdr_idx; -src/backend/access/heap/heapam.c: info = XLOG_NEON_HEAP_HOT_UPDATE; -src/backend/access/heap/heapam.c: info = XLOG_NEON_HEAP_UPDATE; -src/backend/access/heap/heapam.c: info |= XLOG_NEON_INIT_PAGE; -src/backend/access/heap/heapam.c: XLogRegisterData((char *) &xlrec, SizeOfNeonHeapUpdate); -src/backend/access/heap/heapam.c: XLogRegisterBufData(0, (char *) &xlhdr, SizeOfNeonHeapHeader); -src/backend/access/heap/heapam.c: XLogRegisterData((char *) &xlhdr_idx, SizeOfNeonHeapHeader); -src/backend/access/heap/heapam.c: recptr = XLogInsert(RM_NEON_ID, info); -src/backend/access/heap/heapam.c: * NEON: despite to the comment above we need to update page LSN here. -src/backend/access/heap/heapam.c: * For Neon this assignment is critical because otherwise last written LSN tracked at compute doesn't -src/backend/access/heap/heapam.c: * It is fixed in upstream in https://github.com/neondatabase/postgres/commit/7bf713dd2d0739fbcd4103971ed69c17ebe677ea -src/backend/access/heap/rewriteheap.c: * NEON: we need to persist mapping file in WAL -src/backend/access/heap/rewriteheap.c: snprintf(prefix, sizeof(prefix), "neon-file:%s", path); -src/backend/access/heap/visibilitymap.c: /* NEON: we have to update page LSN even if wal_log_hints=off -src/backend/access/nbtree/nbtinsert.c: /* NEON: store the page's former cycle ID for FPI check later */ -src/backend/access/nbtree/nbtinsert.c: * NEON: If we split to earlier pages during a btree vacuum cycle, -src/backend/access/nbtree/nbtsearch.c: /* Neon: initialize prefetch */ -src/backend/access/nbtree/nbtsearch.c: * Neon prefetch is efficient only if prefetched blocks are accessed by the same worker -src/backend/access/nbtree/nbtsearch.c: /* Neon: prefetch referenced heap pages. -src/backend/access/nbtree/nbtxlog.c: * NEON: If the original page was supposed to be recovered from FPI, -src/backend/access/transam/slru.c: * NEON: we do not want to include large pg_xact/multixact files in basebackup and prefer -src/backend/access/transam/xlog.c:/* NEON: Hook to allow the neon extension to restore running-xacts from CLOG at replica startup */ -src/backend/access/transam/xlog.c: /* neon: copy of startup's RedoStartLSN for walproposer's use */ -src/backend/access/transam/xlog.c: * Variables read from 'zenith.signal' file. -src/backend/access/transam/xlog.c:bool ZenithRecoveryRequested = false; -src/backend/access/transam/xlog.c:XLogRecPtr zenithLastRec = InvalidXLogRecPtr; -src/backend/access/transam/xlog.c:bool zenithWriteOk = false; -src/backend/access/transam/xlog.c: /* Neon: in case of sync-safekeepers shared memory is not inialized */ -src/backend/access/transam/xlog.c:readZenithSignalFile(void) -src/backend/access/transam/xlog.c: fd = BasicOpenFile(ZENITH_SIGNAL_FILE, O_RDONLY | PG_BINARY); -src/backend/access/transam/xlog.c: if (stat(ZENITH_SIGNAL_FILE, &statbuf) != 0) -src/backend/access/transam/xlog.c: zenithLastRec = InvalidXLogRecPtr; -src/backend/access/transam/xlog.c: zenithWriteOk = false; -src/backend/access/transam/xlog.c: zenithLastRec = InvalidXLogRecPtr; -src/backend/access/transam/xlog.c: zenithWriteOk = true; -src/backend/access/transam/xlog.c: zenithLastRec = ((uint64) hi) << 32 | lo; -src/backend/access/transam/xlog.c: if (zenithLastRec == InvalidXLogRecPtr) -src/backend/access/transam/xlog.c: zenithWriteOk = true; -src/backend/access/transam/xlog.c: ZenithRecoveryRequested = true; -src/backend/access/transam/xlog.c: LSN_FORMAT_ARGS(zenithLastRec)); -src/backend/access/transam/xlog.c: * Read zenith.signal before anything else. -src/backend/access/transam/xlog.c: readZenithSignalFile(); -src/backend/access/transam/xlog.c: if (!XRecOffIsValid(ControlFile->checkPoint) && !ZenithRecoveryRequested) -src/backend/access/transam/xlog.c: * Neon: We also use a similar mechanism to start up sooner at -src/backend/access/transam/xlog.c: /* Neon: called StandbyRecoverPreparedTransactions() above already */ -src/backend/access/transam/xlog.c: * But GetLastWrittenLSN(InvalidOid) is used only by zenith_dbsize which is not performance critical. -src/backend/access/transam/xlog.c: * NEON: perform checkpiont action requiring write to the WAL before we determine the REDO pointer. -src/backend/access/transam/xlog.c: * NEON: we use logical records to persist information of about slots, origins, relation map... -src/backend/access/transam/xlog.c: * NEON: In the special WAL redo process, blocks that are being -src/backend/access/transam/xlogrecovery.c:bool allowReplicaMisconfig = true; /* NEON: GUC is defined in neon extension */ -src/backend/access/transam/xlogrecovery.c: else if (ZenithRecoveryRequested) -src/backend/access/transam/xlogrecovery.c: else if (ZenithRecoveryRequested) -src/backend/access/transam/xlogrecovery.c: * Zenith hacks to spawn compute node without WAL. Pretend that we -src/backend/access/transam/xlogrecovery.c: * just finished reading the record that started at 'zenithLastRec' -src/backend/access/transam/xlogrecovery.c: LSN_FORMAT_ARGS(zenithLastRec)); -src/backend/access/transam/xlogrecovery.c: CheckPointLoc = zenithLastRec; -src/backend/access/transam/xlogrecovery.c: * When a primary Neon compute node is started, we pretend that it -src/backend/access/transam/xlogrecovery.c: if (checkPoint.redo > CheckPointLoc && !ZenithRecoveryRequested) -src/backend/access/transam/xlogrecovery.c: if (!ZenithRecoveryRequested) -src/backend/access/transam/xlogrecovery.c: * When starting from a zenith base backup, we don't have WAL. Initialize -src/backend/access/transam/xlogrecovery.c: if (ZenithRecoveryRequested) -src/backend/access/transam/xlogrecovery.c: if (!zenithWriteOk) -src/backend/access/transam/xlogrecovery.c: // FIXME: should we unlink zenith.signal? -src/backend/access/transam/xlogrecovery.c: if (!ZenithRecoveryRequested) -src/backend/access/transam/xlogrecovery.c: if (!ZenithRecoveryRequested) -src/backend/access/transam/xlogrecovery.c: if (ZenithRecoveryRequested) -src/backend/commands/extension.c: /* Hide neon* extensions. */ -src/backend/commands/extension.c: if (strncasecmp(extname, "neon", strlen("neon")) == 0 && !superuser()) -src/backend/commands/extension.c: /* Hide neon* extensions. */ -src/backend/commands/extension.c: if (strncasecmp(extname, "neon", strlen("neon")) == 0 && !superuser()) -src/backend/commands/publicationcmds.c: if (stmt->for_all_tables && !superuser() && !is_neon_superuser()) -src/backend/commands/publicationcmds.c: if (schemaidlist != NIL && !superuser() && !is_neon_superuser()) -src/backend/commands/sequence.c:/* NEON XXX: to ensure sequence order of sequence in Zenith we need to WAL log each sequence update. */ -src/backend/executor/nodeBitmapHeapscan.c: /* NEON: we are not using prefetch iterator for parallel plan so no need to adjust it */ -src/backend/main/main.c: * neon extension sets PGC_POSTMASTER gucs requiring this. -src/backend/main/main.c: CallExtMain("neon_walredo", "WalRedoMain", argc, argv, false); -src/backend/main/main.c: else if (argc > 1 && strcmp(argv[1], "--sync-safekeepers") == 0) -src/backend/main/main.c: CallExtMain("neon", "WalProposerSync", argc, argv, true); -src/backend/postmaster/postmaster.c: /* Neon: Also allow walproposer background worker to be treated like a WAL sender, so that it's shut down last */ -src/backend/postmaster/postmaster.c: /* Neon: Also allow walproposer background worker to be treated like a WAL sender, so that it's shut down last */ -src/backend/replication/logical/launcher.c:bool disable_logical_replication_subscribers = false; /* NEON: GUC is defined in neon extension */ -src/backend/replication/logical/launcher.c: /* NEON GUC that allows to control logical replication without postgres restart */ -src/backend/replication/logical/logical.c: * NEON: override page_read/segment_open/segment_close functions to support on-demand WAL download -src/backend/replication/logical/logical.c: if (strncmp(prefix, "neon-file", strlen("neon-file")) == 0) -src/backend/replication/logical/logical.c: if (strncmp(prefix, "neon-file", strlen("neon-file")) == 0) -src/backend/replication/logical/snapbuild.c: /* NEON specific: persist snapshot in storage using logical message */ -src/backend/replication/logical/snapbuild.c: snprintf(prefix, sizeof(prefix), "neon-file:%s", path); -src/backend/replication/logical/snapbuild.c: /* NEON specific: delete file from storage using logical message */ -src/backend/replication/logical/snapbuild.c: snprintf(prefix, sizeof(prefix), "neon-file:%s", path); -src/backend/replication/logical/worker.c: * This is particularly important in neon where WAL is hardened only after -src/backend/replication/logical/worker.c: * it is flushed on majority of safekeepers. -src/backend/replication/slot.c: /* NEON specific: delete slot from storage using logical message */ -src/backend/replication/slot.c: snprintf(prefix, sizeof(prefix), "neon-file:%s/state", path); -src/backend/replication/slot.c: /* NEON specific: persist slot in storage using logical message */ -src/backend/replication/slot.c: snprintf(prefix, sizeof(prefix), "neon-file:%s", path); -src/backend/replication/walsender.c: * neon: in vanilla this doesn't happen because walsenders register in -src/backend/replication/walsender.c: * However, in neon walproposer occupies walsender slot but doesn't register -src/backend/replication/walsender.c: // XXX Zenith uses flush_lsn to pass extra payload, so use write_lsn here -src/backend/storage/buffer/bufmgr.c:bool zenith_test_evict = false; -src/backend/storage/buffer/bufmgr.c: * NEON: In earlier Neon PostgreSQL versions, we zeroed the pages -src/backend/storage/buffer/bufmgr.c: if (zenith_test_evict && !InRecovery) -src/backend/storage/buffer/localbuf.c:/* NEON: prevent eviction of the buffer of target page */ -src/backend/storage/buffer/localbuf.c: /* ZENITH: Prevent eviction of the buffer with target wal redo page */ -src/backend/storage/smgr/smgr.c:// * TODO: NEON v15- REMOVED this? -src/backend/storage/smgr/smgr.c: * Neon-added functions to mark the phases of an unlogged index build. -src/backend/storage/smgr/smgr.c: * NEON: we do not want to include large pg_xact/multixact files in basebackup and prefer -src/backend/utils/adt/acl.c:is_neon_superuser(void) -src/backend/utils/adt/acl.c: return is_neon_superuser_arg(GetUserId()); -src/backend/utils/adt/acl.c:is_neon_superuser_arg(Oid roleid) -src/backend/utils/adt/acl.c: Oid neon_superuser_oid = get_role_oid("databricks_superuser", true /*missing_ok*/); -src/backend/utils/adt/acl.c: return neon_superuser_oid != InvalidOid && has_privs_of_role(roleid, neon_superuser_oid); -src/backend/utils/error/elog.c: /* Don't include source file information if it would leak Neon. */ -src/backend/utils/error/elog.c: strcasestr(edata->filename, "neon") == NULL && -src/backend/utils/error/elog.c: strcasestr(edata->filename, "zenith") == NULL) -src/backend/utils/error/elog.c: strcasestr(edata->funcname, "neon") == NULL && -src/backend/utils/error/elog.c: strcasestr(edata->funcname, "zenith") == NULL) -src/backend/utils/fmgr/dfmgr.c:static void neon_try_load(const char *name); -src/backend/utils/fmgr/dfmgr.c: neon_try_load(fullname); -src/backend/utils/fmgr/dfmgr.c:neon_try_load(const char *name) -src/backend/utils/fmgr/dfmgr.c: neon_try_load(fullname); -src/backend/utils/misc/guc_funcs.c: /* Hide neon.* options. */ -src/backend/utils/misc/guc_funcs.c: strncasecmp(conf->name, "neon.", strlen("neon.")) == 0) -src/backend/utils/misc/guc_funcs.c: bool replace_neon; -src/backend/utils/misc/guc_funcs.c: boot_val and reset_val, so 'neon' leaks through there. -src/backend/utils/misc/guc_funcs.c: replace_neon = strcasecmp(conf->name, "shared_preload_libraries") == 0; -src/backend/utils/misc/guc_funcs.c: values[12] = replace_neon ? -src/backend/utils/misc/guc_funcs.c: filter_neon(lconf->boot_val) : pstrdup(lconf->boot_val); -src/backend/utils/misc/guc_funcs.c: values[13] = replace_neon ? -src/backend/utils/misc/guc_funcs.c: filter_neon(lconf->reset_val) : strdup(lconf->reset_val); -src/backend/utils/misc/guc_funcs.c: strncasecmp(conf->name, "neon.", strlen("neon.")) == 0)) -src/backend/utils/misc/guc_tables.c: * Replace 'neon' with 'dbrx'. -src/backend/utils/misc/guc_tables.c:filter_neon(const char *input) -src/backend/utils/misc/guc_tables.c: while ((pos = strcasestr(pos, "neon")) != NULL) { -src/backend/utils/misc/guc_tables.c: * GUC show_hook for shared_preload_libraries to replace 'neon' with 'dbrx'. -src/backend/utils/misc/guc_tables.c: return filter_neon(shared_preload_libraries_string); -src/backend/utils/misc/guc_tables.c: {"neon_test_evict", PGC_POSTMASTER, UNGROUPED, -src/backend/utils/misc/guc_tables.c: &zenith_test_evict, -src/backend/utils/misc/guc_tables.c: gettext_noop("Size of last written LSN cache used by Neon."), -src/include/access/nbtree.h: /* Neon: prefetch state */ -src/include/access/neon_xlog.h:#ifndef NEON_XLOG_H -src/include/access/neon_xlog.h:#define NEON_XLOG_H -src/include/access/neon_xlog.h: * The RMGR id of the Neon RMGR -src/include/access/neon_xlog.h:#define RM_NEON_ID 134 -src/include/access/neon_xlog.h:#define XLOG_NEON_INIT_PAGE 0x80 -src/include/access/neon_xlog.h:#define XLOG_NEON_OPMASK ((~XLOG_NEON_INIT_PAGE) & XLR_RMGR_INFO_MASK) -src/include/access/neon_xlog.h:#define XLOG_NEON_HEAP_INSERT 0x00 -src/include/access/neon_xlog.h:#define XLOG_NEON_HEAP_DELETE 0x10 -src/include/access/neon_xlog.h:#define XLOG_NEON_HEAP_UPDATE 0x20 -src/include/access/neon_xlog.h:#define XLOG_NEON_HEAP_HOT_UPDATE 0x30 -src/include/access/neon_xlog.h:#define XLOG_NEON_HEAP_LOCK 0x40 -src/include/access/neon_xlog.h:#define XLOG_NEON_HEAP_MULTI_INSERT 0x50 -src/include/access/neon_xlog.h:typedef struct xl_neon_heap_header { -src/include/access/neon_xlog.h:} xl_neon_heap_header; -src/include/access/neon_xlog.h:#define SizeOfNeonHeapHeader (offsetof(xl_neon_heap_header, t_hoff) + sizeof(uint8)) -src/include/access/neon_xlog.h:typedef struct xl_neon_heap_insert -src/include/access/neon_xlog.h: /* xl_neon_heap_header & TUPLE DATA in backup block 0 */ -src/include/access/neon_xlog.h:} xl_neon_heap_insert; -src/include/access/neon_xlog.h:#define SizeOfNeonHeapInsert (offsetof(xl_neon_heap_insert, flags) + sizeof(uint8)) -src/include/access/neon_xlog.h:typedef struct xl_neon_heap_delete -src/include/access/neon_xlog.h:} xl_neon_heap_delete; -src/include/access/neon_xlog.h:#define SizeOfNeonHeapDelete (offsetof(xl_neon_heap_delete, t_cid) + sizeof(uint32)) -src/include/access/neon_xlog.h: * After that, xl_neon_heap_header and new tuple data follow. The new tuple -src/include/access/neon_xlog.h:typedef struct xl_neon_heap_update -src/include/access/neon_xlog.h: * are set, xl_neon_heap_header and tuple data for the old tuple follow. -src/include/access/neon_xlog.h:} xl_neon_heap_update; -src/include/access/neon_xlog.h:#define SizeOfNeonHeapUpdate (offsetof(xl_neon_heap_update, new_offnum) + sizeof(OffsetNumber)) -src/include/access/neon_xlog.h:typedef struct xl_neon_heap_lock -src/include/access/neon_xlog.h:} xl_neon_heap_lock; -src/include/access/neon_xlog.h:#define SizeOfNeonHeapLock (offsetof(xl_neon_heap_lock, flags) + sizeof(uint8)) -src/include/access/neon_xlog.h: * The main data of the record consists of this xl_neon_heap_multi_insert header. -src/include/access/neon_xlog.h: * In block 0's data portion, there is an xl_neon_multi_insert_tuple struct, -src/include/access/neon_xlog.h: * each xl_neon_multi_insert_tuple struct. -src/include/access/neon_xlog.h:typedef struct xl_neon_heap_multi_insert -src/include/access/neon_xlog.h:} xl_neon_heap_multi_insert; -src/include/access/neon_xlog.h:#define SizeOfNeonHeapMultiInsert offsetof(xl_neon_heap_multi_insert, offsets) -src/include/access/neon_xlog.h:typedef struct xl_neon_multi_insert_tuple -src/include/access/neon_xlog.h:} xl_neon_multi_insert_tuple; -src/include/access/neon_xlog.h:#define SizeOfNeonMultiInsertTuple (offsetof(xl_neon_multi_insert_tuple, t_hoff) + sizeof(uint8)) -src/include/access/neon_xlog.h:#endif //NEON_XLOG_H -src/include/access/xlog.h:extern bool ZenithRecoveryRequested; -src/include/access/xlog.h:extern XLogRecPtr zenithLastRec; -src/include/access/xlog.h:extern bool zenithWriteOk; -src/include/access/xlog.h:/* neon specifics */ -src/include/access/xlog.h:/* NEON: Hook to allow the neon extension to restore running-xacts from CLOG at replica startup */ -src/include/access/xlog.h:#define ZENITH_SIGNAL_FILE "zenith.signal" -src/include/miscadmin.h:extern bool is_neon_superuser(void); /* current user is neon_superuser */ -src/include/miscadmin.h:extern bool is_neon_superuser_arg(Oid roleid); /* given user is neon_superuser */ -src/include/port/simd.h:#elif defined(__aarch64__) && defined(__ARM_NEON) -src/include/port/simd.h: * We use the Neon instructions if the compiler provides access to them (as -src/include/port/simd.h: * indicated by __ARM_NEON) and we are on aarch64. While Neon support is -src/include/port/simd.h: * hardware does have it. Neon exists in some 32-bit hardware too, but we -src/include/port/simd.h:#include -src/include/port/simd.h:#define USE_NEON -src/include/port/simd.h:#elif defined(USE_NEON) -src/include/port/simd.h:#elif defined(USE_NEON) -src/include/port/simd.h:#elif defined(USE_NEON) -src/include/port/simd.h:#elif defined(USE_NEON) -src/include/port/simd.h:#elif defined(USE_NEON) -src/include/port/simd.h:#if defined(USE_NEON) -src/include/port/simd.h:#elif defined(USE_NEON) -src/include/port/simd.h:#elif defined(USE_NEON) -src/include/port/simd.h:#elif defined(USE_NEON) -src/include/port/simd.h:#elif defined(USE_NEON) -src/include/port/simd.h:#elif defined(USE_NEON) -src/include/replication/walsender.h:/* expose these so that they can be reused by the neon walproposer extension */ -src/include/storage/bufmgr.h:extern bool zenith_test_evict; -src/include/storage/checksum_impl.h: * available on x86 SSE4.1 extensions (pmulld) and ARM NEON (vmul.i32). -src/include/storage/smgr.h: * Neon: extended SMGR API. -src/include/storage/smgr.h: * This define can be used by extensions to determine that them are built for Neon. -src/include/storage/smgr.h:#define NEON_SMGR 1 -src/include/storage/smgr.h:/* Neon: Change relpersistence for unlogged index builds */ -src/include/utils/guc_tables.h:const char * filter_neon(const char *input);