rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-06-04 14:00:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add optional generation input during create & attach

Browse Source
This commit is contained in:
John Spray
2023-08-24 17:48:00 +01:00
parent 50da1b7983
commit f3a9c2d788
6 changed files with 55 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
control_plane/src/bin/neon_local.rs
View File

@@ -278,6 +278,14 @@ fn parse_tenant_id(sub_match: &ArgMatches) -> anyhow::Result<Option<TenantId>> {
.context("Failed to parse tenant id from the argument string")
}
fn parse_generation(sub_match: &ArgMatches) -> anyhow::Result<Option<u32>> {
sub_match
.get_one::<String>("generation")
.map(|tenant_id| u32::from_str(tenant_id))
.transpose()
.context("Failed to parse generation rom the argument string")
}
fn parse_timeline_id(sub_match: &ArgMatches) -> anyhow::Result<Option<TimelineId>> {
sub_match
.get_one::<String>("timeline-id")
@@ -343,11 +351,13 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an
}
Some(("create", create_match)) => {
let initial_tenant_id = parse_tenant_id(create_match)?;
let generation = parse_generation(create_match)?;
let tenant_conf: HashMap<_, _> = create_match
.get_many::<String>("config")
.map(|vals| vals.flat_map(|c| c.split_once(':')).collect())
.unwrap_or_default();
let new_tenant_id = pageserver.tenant_create(initial_tenant_id, tenant_conf)?;
let new_tenant_id =
pageserver.tenant_create(initial_tenant_id, generation, tenant_conf)?;
println!("tenant {new_tenant_id} successfully created on the pageserver");
// Create an initial timeline for the new tenant

2
control_plane/src/pageserver.rs
View File

@@ -317,6 +317,7 @@ impl PageServerNode {
pub fn tenant_create(
&self,
new_tenant_id: Option<TenantId>,
generation: Option<u32>,
settings: HashMap<&str, &str>,
) -> anyhow::Result<TenantId> {
let mut settings = settings.clone();
@@ -387,6 +388,7 @@ impl PageServerNode {
let request = models::TenantCreateRequest {
new_tenant_id,
generation,
config,
};
if !settings.is_empty() {

16
libs/pageserver_api/src/models.rs
View File

@@ -194,6 +194,9 @@ pub struct TimelineCreateRequest {
pub struct TenantCreateRequest {
#[serde_as(as = "DisplayFromStr")]
pub new_tenant_id: TenantId,
#[serde(default)]
#[serde(skip_serializing_if = "Option::is_none")]
pub generation: Option<u32>,
#[serde(flatten)]
pub config: TenantConfig, // as we have a flattened field, we should reject all unknown fields in it
}
@@ -241,15 +244,6 @@ pub struct StatusResponse {
pub id: NodeId,
}
impl TenantCreateRequest {
pub fn new(new_tenant_id: TenantId) -> TenantCreateRequest {
TenantCreateRequest {
new_tenant_id,
config: TenantConfig::default(),
}
}
}
#[serde_as]
#[derive(Serialize, Deserialize, Debug)]
#[serde(deny_unknown_fields)]
@@ -293,9 +287,11 @@ impl TenantConfigRequest {
}
}
#[derive(Debug, Serialize, Deserialize)]
#[derive(Debug, Deserialize)]
pub struct TenantAttachRequest {
pub config: TenantAttachConfig,
#[serde(default)]
pub generation: Option<u32>,
}
/// Newtype to enforce deny_unknown_fields on TenantConfig for

4
pageserver/src/http/openapi_spec.yml
View File

@@ -383,7 +383,6 @@ paths:
schema:
type: string
format: hex
post:
description: |
Schedules attach operation to happen in the background for the given tenant.
@@ -1020,6 +1019,9 @@ components:
properties:
config:
$ref: '#/components/schemas/TenantConfig'
generation:
type: integer
description: Attachment generation number.
TenantConfigRequest:
allOf:
- $ref: '#/components/schemas/TenantConfig'

22
pageserver/src/http/routes.rs
View File

@@ -32,11 +32,13 @@ use crate::tenant::mgr::{
};
use crate::tenant::size::ModelInputs;
use crate::tenant::storage_layer::LayerAccessStatsReset;
use crate::tenant::{LogicalSizeCalculationCause, PageReconstructError, Timeline};
use crate::tenant::timeline::Timeline;
use crate::tenant::{LogicalSizeCalculationCause, PageReconstructError};
use crate::{config::PageServerConf, tenant::mgr};
use crate::{disk_usage_eviction_task, tenant};
use utils::{
auth::JwtAuth,
generation::Generation,
http::{
endpoint::{self, attach_openapi_ui, auth_middleware, check_permission_with},
error::{ApiError, HttpErrorBody},
@@ -472,11 +474,19 @@ async fn tenant_attach_handler(
check_permission(&request, Some(tenant_id))?;
let maybe_body: Option<TenantAttachRequest> = json_request_or_empty_body(&mut request).await?;
let tenant_conf = match maybe_body {
let tenant_conf = match &maybe_body {
Some(request) => TenantConfOpt::try_from(&*request.config).map_err(ApiError::BadRequest)?,
None => TenantConfOpt::default(),
};
// TODO: make generation mandatory here once control plane supports it
let generation = maybe_body
.as_ref()
.map(|tar| tar.generation)
.flatten()
.map(|g| Generation::new(g))
.unwrap_or(Generation::none());
let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn);
info!("Handling tenant attach {tenant_id}");
@@ -487,6 +497,7 @@ async fn tenant_attach_handler(
mgr::attach_tenant(
state.conf,
tenant_id,
generation,
tenant_conf,
state.broker_client.clone(),
remote_storage.clone(),
@@ -867,6 +878,12 @@ async fn tenant_create_handler(
let tenant_conf =
TenantConfOpt::try_from(&request_data.config).map_err(ApiError::BadRequest)?;
// TODO: make generation mandatory here once control plane supports it.
let generation = request_data
.generation
.map(|g| Generation::new(g))
.unwrap_or(Generation::none());
let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn);
let state = get_state(&request);
@@ -875,6 +892,7 @@ async fn tenant_create_handler(
state.conf,
tenant_conf,
target_tenant_id,
generation,
state.broker_client.clone(),
state.remote_storage.clone(),
&ctx,

17
pageserver/src/tenant/mgr.rs
View File

@@ -126,6 +126,9 @@ pub async fn init_tenant_mgr(
match schedule_local_tenant_processing(
conf,
&tenant_dir_path,
// TODO: call into control plane, don't start Tenants
// purely because they are on disk
Generation::none(),
resources.clone(),
Some(init_order.clone()),
&TENANTS,
@@ -162,6 +165,7 @@ pub async fn init_tenant_mgr(
pub(crate) fn schedule_local_tenant_processing(
conf: &'static PageServerConf,
tenant_path: &Path,
generation: Generation,
resources: TenantSharedResources,
init_order: Option<InitializationOrder>,
tenants: &'static tokio::sync::RwLock<TenantsMap>,
@@ -203,7 +207,7 @@ pub(crate) fn schedule_local_tenant_processing(
match Tenant::spawn_attach(
conf,
tenant_id,
Generation::none(),
generation,
resources.broker_client,
tenants,
remote_storage,
@@ -357,6 +361,7 @@ pub async fn create_tenant(
conf: &'static PageServerConf,
tenant_conf: TenantConfOpt,
tenant_id: TenantId,
generation: Generation,
broker_client: storage_broker::BrokerClientChannel,
remote_storage: Option<GenericRemoteStorage>,
ctx: &RequestContext,
@@ -374,7 +379,8 @@ pub async fn create_tenant(
remote_storage,
};
let created_tenant =
schedule_local_tenant_processing(conf, &tenant_directory, tenant_resources, None, &TENANTS, ctx)?;
schedule_local_tenant_processing(conf, &tenant_directory,
generation, tenant_resources, None, &TENANTS, ctx)?;
// TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here.
// See https://github.com/neondatabase/neon/issues/4233
@@ -537,7 +543,9 @@ pub async fn load_tenant(
broker_client,
remote_storage,
};
let new_tenant = schedule_local_tenant_processing(conf, &tenant_path, resources, None, &TENANTS, ctx)
// TODO: remove the `/load` API once generation support is complete:
// it becomes equivalent to attaching.
let new_tenant = schedule_local_tenant_processing(conf, &tenant_path, Generation::none(), resources, None, &TENANTS, ctx)
.with_context(|| {
format!("Failed to schedule tenant processing in path {tenant_path:?}")
})?;
@@ -601,6 +609,7 @@ pub async fn list_tenants() -> Result<Vec<(TenantId, TenantState)>, TenantMapLis
pub async fn attach_tenant(
conf: &'static PageServerConf,
tenant_id: TenantId,
generation: Generation,
tenant_conf: TenantConfOpt,
broker_client: storage_broker::BrokerClientChannel,
remote_storage: GenericRemoteStorage,
@@ -622,7 +631,7 @@ pub async fn attach_tenant(
broker_client,
remote_storage: Some(remote_storage),
};
let attached_tenant = schedule_local_tenant_processing(conf, &tenant_dir, resources, None, &TENANTS, ctx)?;
let attached_tenant = schedule_local_tenant_processing(conf, &tenant_dir, generation, resources, None, &TENANTS, ctx)?;
// TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here.
// See https://github.com/neondatabase/neon/issues/4233