rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2025-12-28 00:23:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

storcon: make pageserver AZ id mandatory (#8856)

Browse Source 
## Problem
https://github.com/neondatabase/neon/pull/8852 introduced a new nullable
column for the `nodes` table: `availability_zone_id`

## Summary of changes
* Make neon local and the test suite always provide an az id
* Make the az id field in the ps registration request mandatory
* Migrate the column to non-nullable and adjust in memory state
accordingly
* Remove the code that was used to populate the az id for pre-existing
nodes
This commit is contained in:
Vlad Lazar
2024-09-05 19:14:21 +01:00
committed by GitHub
parent fd12dd942f
commit 04f99a87bf
12 changed files with 41 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Dockerfile
View File

@@ -87,6 +87,7 @@ RUN mkdir -p /data/.neon/ && \
"pg_distrib_dir='/usr/local/'\n" \
"listen_pg_addr='0.0.0.0:6400'\n" \
"listen_http_addr='0.0.0.0:9898'\n" \
"availability_zone='local'\n" \
> /data/.neon/pageserver.toml && \
chown -R neon:neon /data/.neon

2
control_plane/storcon_cli/src/main.rs
View File

@@ -336,7 +336,7 @@ async fn main() -> anyhow::Result<()> {
listen_pg_port,
listen_http_addr,
listen_http_port,
availability_zone_id: Some(availability_zone_id),
availability_zone_id,
}),
)
.await?;

2
libs/pageserver_api/src/controller_api.rs
View File

@@ -57,7 +57,7 @@ pub struct NodeRegisterRequest {
pub listen_http_addr: String,
pub listen_http_port: u16,
pub availability_zone_id: Option<String>,
pub availability_zone_id: String,
}
#[derive(Serialize, Deserialize)]

24
pageserver/src/control_plane_client.rs
View File

@@ -141,10 +141,24 @@ impl ControlPlaneGenerationsApi for ControlPlaneClient {
m.other
);
let az_id = m
.other
.get("availability_zone_id")
.and_then(|jv| jv.as_str().map(|str| str.to_owned()));
let az_id = {
let az_id_from_metadata = m
.other
.get("availability_zone_id")
.and_then(|jv| jv.as_str().map(|str| str.to_owned()));
match az_id_from_metadata {
Some(az_id) => Some(az_id),
None => {
tracing::warn!("metadata.json does not contain an 'availability_zone_id' field");
conf.availability_zone.clone()
}
}
};
if az_id.is_none() {
panic!("Availablity zone id could not be inferred from metadata.json or pageserver config");
}
Some(NodeRegisterRequest {
node_id: conf.id,
@@ -152,7 +166,7 @@ impl ControlPlaneGenerationsApi for ControlPlaneClient {
listen_pg_port: m.postgres_port,
listen_http_addr: m.http_host,
listen_http_port: m.http_port,
availability_zone_id: az_id,
availability_zone_id: az_id.expect("Checked above"),
})
}
Err(e) => {

1
storage_controller/migrations/2024-08-28-150530_pageserver_az_not_null/down.sql Normal file
View File

@@ -0,0 +1 @@
ALTER TABLE nodes ALTER availability_zone_id DROP NOT NULL;

1
storage_controller/migrations/2024-08-28-150530_pageserver_az_not_null/up.sql Normal file
View File

@@ -0,0 +1 @@
ALTER TABLE nodes ALTER availability_zone_id SET NOT NULL;

23
storage_controller/src/node.rs
View File

@@ -36,7 +36,7 @@ pub(crate) struct Node {
listen_pg_addr: String,
listen_pg_port: u16,
availability_zone_id: Option<String>,
availability_zone_id: String,
// This cancellation token means "stop any RPCs in flight to this node, and don't start
// any more". It is not related to process shutdown.
@@ -63,8 +63,9 @@ impl Node {
self.id
}
pub(crate) fn get_availability_zone_id(&self) -> Option<&str> {
self.availability_zone_id.as_deref()
#[allow(unused)]
pub(crate) fn get_availability_zone_id(&self) -> &str {
self.availability_zone_id.as_str()
}
pub(crate) fn get_scheduling(&self) -> NodeSchedulingPolicy {
@@ -78,22 +79,12 @@ impl Node {
/// Does this registration request match `self`? This is used when deciding whether a registration
/// request should be allowed to update an existing record with the same node ID.
pub(crate) fn registration_match(&self, register_req: &NodeRegisterRequest) -> bool {
let az_ids_match = {
match (
self.availability_zone_id.as_deref(),
register_req.availability_zone_id.as_deref(),
) {
(Some(current_az), Some(register_req_az)) => current_az == register_req_az,
_ => true,
}
};
az_ids_match
&& self.id == register_req.node_id
self.id == register_req.node_id
&& self.listen_http_addr == register_req.listen_http_addr
&& self.listen_http_port == register_req.listen_http_port
&& self.listen_pg_addr == register_req.listen_pg_addr
&& self.listen_pg_port == register_req.listen_pg_port
&& self.availability_zone_id == register_req.availability_zone_id
}
/// For a shard located on this node, populate a response object
@@ -190,7 +181,7 @@ impl Node {
listen_http_port: u16,
listen_pg_addr: String,
listen_pg_port: u16,
availability_zone_id: Option<String>,
availability_zone_id: String,
) -> Self {
Self {
id,

28
storage_controller/src/persistence.rs
View File

@@ -105,7 +105,6 @@ pub(crate) enum DatabaseOperation {
ListMetadataHealthOutdated,
GetLeader,
UpdateLeader,
SetNodeAzId,
}
#[must_use]
@@ -325,31 +324,6 @@ impl Persistence {
}
}
pub(crate) async fn set_node_availability_zone_id(
&self,
input_node_id: NodeId,
input_az_id: String,
) -> DatabaseResult<()> {
use crate::schema::nodes::dsl::*;
let updated = self
.with_measured_conn(DatabaseOperation::SetNodeAzId, move |conn| {
let updated = diesel::update(nodes)
.filter(node_id.eq(input_node_id.0 as i64))
.set((availability_zone_id.eq(input_az_id.clone()),))
.execute(conn)?;
Ok(updated)
})
.await?;
if updated != 1 {
Err(DatabaseError::Logical(format!(
"Node {node_id:?} not found for setting az id",
)))
} else {
Ok(())
}
}
/// At startup, load the high level state for shards, such as their config + policy. This will
/// be enriched at runtime with state discovered on pageservers.
pub(crate) async fn list_tenant_shards(&self) -> DatabaseResult<Vec<TenantShardPersistence>> {
@@ -1110,7 +1084,7 @@ pub(crate) struct NodePersistence {
pub(crate) listen_http_port: i32,
pub(crate) listen_pg_addr: String,
pub(crate) listen_pg_port: i32,
pub(crate) availability_zone_id: Option<String>,
pub(crate) availability_zone_id: String,
}
/// Tenant metadata health status that are stored durably.

2
storage_controller/src/scheduler.rs
View File

@@ -528,7 +528,7 @@ pub(crate) mod test_utils {
80 + i as u16,
format!("pghost-{i}"),
5432 + i as u16,
None,
"test-az".to_string(),
);
node.set_availability(NodeAvailability::Active(test_utilization::simple(0, 0)));
assert!(node.is_available());

2
storage_controller/src/schema.rs
View File

@@ -25,7 +25,7 @@ diesel::table! {
listen_http_port -> Int4,
listen_pg_addr -> Varchar,
listen_pg_port -> Int4,
availability_zone_id -> Nullable<Varchar>,
availability_zone_id -> Varchar,
}
}

44
storage_controller/src/service.rs
View File

@@ -1264,7 +1264,7 @@ impl Service {
123,
"".to_string(),
123,
None,
"test_az".to_string(),
);
scheduler.node_upsert(&node);
@@ -4825,15 +4825,8 @@ impl Service {
)
.await;
if register_req.availability_zone_id.is_none() {
tracing::warn!(
"Node {} registering without specific availability zone id",
register_req.node_id
);
}
enum RegistrationStatus {
Matched(Node),
Matched,
Mismatched,
New,
}
@@ -4842,7 +4835,7 @@ impl Service {
let locked = self.inner.read().unwrap();
if let Some(node) = locked.nodes.get(&register_req.node_id) {
if node.registration_match(&register_req) {
RegistrationStatus::Matched(node.clone())
RegistrationStatus::Matched
} else {
RegistrationStatus::Mismatched
}
@@ -4852,41 +4845,12 @@ impl Service {
};
match registration_status {
RegistrationStatus::Matched(node) => {
RegistrationStatus::Matched => {
tracing::info!(
"Node {} re-registered with matching address",
register_req.node_id
);
if node.get_availability_zone_id().is_none() {
if let Some(az_id) = register_req.availability_zone_id.clone() {
tracing::info!("Extracting availability zone id from registration request for node {}: {}",
register_req.node_id, az_id);
// Persist to the database and update in memory state. See comment below
// on ordering.
self.persistence
.set_node_availability_zone_id(register_req.node_id, az_id)
.await?;
let node_with_az = Node::new(
register_req.node_id,
register_req.listen_http_addr,
register_req.listen_http_port,
register_req.listen_pg_addr,
register_req.listen_pg_port,
register_req.availability_zone_id,
);
let mut locked = self.inner.write().unwrap();
let mut new_nodes = (*locked.nodes).clone();
locked.scheduler.node_upsert(&node_with_az);
new_nodes.insert(register_req.node_id, node_with_az);
locked.nodes = Arc::new(new_nodes);
}
}
return Ok(());
}
RegistrationStatus::Mismatched => {

3
test_runner/fixtures/neon_fixtures.py
View File

@@ -758,6 +758,9 @@ class NeonEnvBuilder:
patch_script = ""
for ps in self.env.pageservers:
patch_script += f"UPDATE nodes SET listen_http_port={ps.service_port.http}, listen_pg_port={ps.service_port.pg} WHERE node_id = '{ps.id}';"
# This is a temporary to get the backward compat test happy
# since the compat snapshot was generated with an older version of neon local
patch_script += f"UPDATE nodes SET availability_zone_id='{ps.az_id}' WHERE node_id = '{ps.id}' AND availability_zone_id IS NULL;"
patch_script_path.write_text(patch_script)
# Update the config with info about tenants and timelines