rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-14 00:42:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement archival_config timeline endpoint in the storage controller (#8680)

Browse Source 
Implement the timeline specific `archival_config` endpoint also in the
storage controller.

It's mostly a copy-paste of the detach handler: the task is the same: do
the same operation on all shards.

Part of #8088.
This commit is contained in:
Arpad Müller
2024-09-02 13:51:45 +02:00
committed by GitHub
parent 516ac0591e
commit 9746b6ea31
5 changed files with 174 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
pageserver/client/src/mgmt_api.rs
View File

@@ -419,6 +419,24 @@ impl Client {
}
}
pub async fn timeline_archival_config(
&self,
tenant_shard_id: TenantShardId,
timeline_id: TimelineId,
req: &TimelineArchivalConfigRequest,
) -> Result<()> {
let uri = format!(
"{}/v1/tenant/{tenant_shard_id}/timeline/{timeline_id}/archival_config",
self.mgmt_api_endpoint
);
self.request(Method::POST, &uri, req)
.await?
.json()
.await
.map_err(Error::ReceiveBody)
}
pub async fn timeline_detach_ancestor(
&self,
tenant_shard_id: TenantShardId,

30
storage_controller/src/http.rs
View File

@@ -17,7 +17,7 @@ use pageserver_api::controller_api::{
};
use pageserver_api::models::{
TenantConfigRequest, TenantLocationConfigRequest, TenantShardSplitRequest,
TenantTimeTravelRequest, TimelineCreateRequest,
TenantTimeTravelRequest, TimelineArchivalConfigRequest, TimelineCreateRequest,
};
use pageserver_api::shard::TenantShardId;
use pageserver_client::mgmt_api;
@@ -334,6 +334,24 @@ async fn handle_tenant_timeline_delete(
.await
}
async fn handle_tenant_timeline_archival_config(
service: Arc<Service>,
mut req: Request<Body>,
) -> Result<Response<Body>, ApiError> {
let tenant_id: TenantId = parse_request_param(&req, "tenant_id")?;
check_permissions(&req, Scope::PageServerApi)?;
let timeline_id: TimelineId = parse_request_param(&req, "timeline_id")?;
let create_req = json_request::<TimelineArchivalConfigRequest>(&mut req).await?;
service
.tenant_timeline_archival_config(tenant_id, timeline_id, create_req)
.await?;
json_response(StatusCode::OK, ())
}
async fn handle_tenant_timeline_detach_ancestor(
service: Arc<Service>,
req: Request<Body>,
@@ -1160,6 +1178,16 @@ pub fn make_router(
RequestName("v1_tenant_timeline"),
)
})
.post(
"/v1/tenant/:tenant_id/timeline/:timeline_id/archival_config",
|r| {
tenant_service_handler(
r,
handle_tenant_timeline_archival_config,
RequestName("v1_tenant_timeline_archival_config"),
)
},
)
.put(
"/v1/tenant/:tenant_id/timeline/:timeline_id/detach_ancestor",
|r| {

20
storage_controller/src/pageserver_client.rs
View File

@@ -2,8 +2,8 @@ use pageserver_api::{
models::{
detach_ancestor::AncestorDetached, LocationConfig, LocationConfigListResponse,
PageserverUtilization, SecondaryProgress, TenantScanRemoteStorageResponse,
TenantShardSplitRequest, TenantShardSplitResponse, TimelineCreateRequest, TimelineInfo,
TopTenantShardsRequest, TopTenantShardsResponse,
TenantShardSplitRequest, TenantShardSplitResponse, TimelineArchivalConfigRequest,
TimelineCreateRequest, TimelineInfo, TopTenantShardsRequest, TopTenantShardsResponse,
},
shard::TenantShardId,
};
@@ -227,6 +227,22 @@ impl PageserverClient {
)
}
pub(crate) async fn timeline_archival_config(
&self,
tenant_shard_id: TenantShardId,
timeline_id: TimelineId,
req: &TimelineArchivalConfigRequest,
) -> Result<()> {
measured_request!(
"timeline_archival_config",
crate::metrics::Method::Post,
&self.node_id_label,
self.inner
.timeline_archival_config(tenant_shard_id, timeline_id, req)
.await
)
}
pub(crate) async fn timeline_detach_ancestor(
&self,
tenant_shard_id: TenantShardId,

73
storage_controller/src/service.rs
View File

@@ -46,7 +46,10 @@ use pageserver_api::{
TenantDescribeResponseShard, TenantLocateResponse, TenantPolicyRequest,
TenantShardMigrateRequest, TenantShardMigrateResponse,
},
models::{SecondaryProgress, TenantConfigRequest, TopTenantShardsRequest},
models::{
SecondaryProgress, TenantConfigRequest, TimelineArchivalConfigRequest,
TopTenantShardsRequest,
},
};
use reqwest::StatusCode;
use tracing::{instrument, Instrument};
@@ -131,6 +134,7 @@ enum TenantOperations {
TimelineCreate,
TimelineDelete,
AttachHook,
TimelineArchivalConfig,
TimelineDetachAncestor,
}
@@ -2918,6 +2922,73 @@ impl Service {
.await?
}
pub(crate) async fn tenant_timeline_archival_config(
&self,
tenant_id: TenantId,
timeline_id: TimelineId,
req: TimelineArchivalConfigRequest,
) -> Result<(), ApiError> {
tracing::info!(
"Setting archival config of timeline {tenant_id}/{timeline_id} to '{:?}'",
req.state
);
let _tenant_lock = trace_shared_lock(
&self.tenant_op_locks,
tenant_id,
TenantOperations::TimelineArchivalConfig,
)
.await;
self.tenant_remote_mutation(tenant_id, move |targets| async move {
if targets.is_empty() {
return Err(ApiError::NotFound(
anyhow::anyhow!("Tenant not found").into(),
));
}
async fn config_one(
tenant_shard_id: TenantShardId,
timeline_id: TimelineId,
node: Node,
jwt: Option<String>,
req: TimelineArchivalConfigRequest,
) -> Result<(), ApiError> {
tracing::info!(
"Setting archival config of timeline on shard {tenant_shard_id}/{timeline_id}, attached to node {node}",
);
let client = PageserverClient::new(node.get_id(), node.base_url(), jwt.as_deref());
client
.timeline_archival_config(tenant_shard_id, timeline_id, &req)
.await
.map_err(|e| match e {
mgmt_api::Error::ApiError(StatusCode::PRECONDITION_FAILED, msg) => {
ApiError::PreconditionFailed(msg.into_boxed_str())
}
_ => passthrough_api_error(&node, e),
})
}
// no shard needs to go first/last; the operation should be idempotent
// TODO: it would be great to ensure that all shards return the same error
let results = self
.tenant_for_shards(targets, |tenant_shard_id, node| {
futures::FutureExt::boxed(config_one(
tenant_shard_id,
timeline_id,
node,
self.config.jwt_token.clone(),
req.clone(),
))
})
.await?;
assert!(!results.is_empty(), "must have at least one result");
Ok(())
}).await?
}
pub(crate) async fn tenant_timeline_detach_ancestor(
&self,
tenant_id: TenantId,

83
test_runner/regress/test_timeline_archive.py
View File

@@ -1,97 +1,90 @@
import pytest
from fixtures.common_types import TenantId, TimelineArchivalState, TimelineId
from fixtures.neon_fixtures import (
NeonEnv,
NeonEnvBuilder,
)
from fixtures.pageserver.http import PageserverApiException
def test_timeline_archive(neon_simple_env: NeonEnv):
env = neon_simple_env
@pytest.mark.parametrize("shard_count", [0, 4])
def test_timeline_archive(neon_env_builder: NeonEnvBuilder, shard_count: int):
unsharded = shard_count == 0
if unsharded:
env = neon_env_builder.init_start()
# If we run the unsharded version, talk to the pageserver directly
ps_http = env.pageserver.http_client()
else:
neon_env_builder.num_pageservers = shard_count
env = neon_env_builder.init_start(initial_tenant_shard_count=shard_count)
# If we run the unsharded version, talk to the storage controller
ps_http = env.storage_controller.pageserver_api()
env.pageserver.allowed_errors.extend(
[
".*Timeline .* was not found.*",
".*timeline not found.*",
".*Cannot archive timeline which has unarchived child timelines.*",
".*Precondition failed: Requested tenant is missing.*",
]
)
ps_http = env.pageserver.http_client()
# first try to archive non existing timeline
# for existing tenant:
# first try to archive a non existing timeline for an existing tenant:
invalid_timeline_id = TimelineId.generate()
with pytest.raises(PageserverApiException, match="timeline not found") as exc:
ps_http.timeline_archival_config(
tenant_id=env.initial_tenant,
timeline_id=invalid_timeline_id,
env.initial_tenant,
invalid_timeline_id,
state=TimelineArchivalState.ARCHIVED,
)
assert exc.value.status_code == 404
# for non existing tenant:
# for a non existing tenant:
invalid_tenant_id = TenantId.generate()
with pytest.raises(
PageserverApiException,
match=f"NotFound: tenant {invalid_tenant_id}",
match="NotFound: [tT]enant",
) as exc:
ps_http.timeline_archival_config(
tenant_id=invalid_tenant_id,
timeline_id=invalid_timeline_id,
invalid_tenant_id,
invalid_timeline_id,
state=TimelineArchivalState.ARCHIVED,
)
assert exc.value.status_code == 404
# construct pair of branches to validate that pageserver prohibits
# construct a pair of branches to validate that pageserver prohibits
# archival of ancestor timelines when they have non-archived child branches
parent_timeline_id = env.neon_cli.create_branch("test_ancestor_branch_archive_parent", "empty")
parent_timeline_id = env.neon_cli.create_branch("test_ancestor_branch_archive_parent")
leaf_timeline_id = env.neon_cli.create_branch(
"test_ancestor_branch_archive_branch1", "test_ancestor_branch_archive_parent"
)
timeline_path = env.pageserver.timeline_dir(env.initial_tenant, parent_timeline_id)
with pytest.raises(
PageserverApiException,
match="Cannot archive timeline which has non-archived child timelines",
) as exc:
assert timeline_path.exists()
ps_http.timeline_archival_config(
tenant_id=env.initial_tenant,
timeline_id=parent_timeline_id,
env.initial_tenant,
parent_timeline_id,
state=TimelineArchivalState.ARCHIVED,
)
assert exc.value.status_code == 412
# Test timeline_detail
leaf_detail = ps_http.timeline_detail(
tenant_id=env.initial_tenant,
env.initial_tenant,
timeline_id=leaf_timeline_id,
)
assert leaf_detail["is_archived"] is False
# Test that archiving the leaf timeline and then the parent works
ps_http.timeline_archival_config(
tenant_id=env.initial_tenant,
timeline_id=leaf_timeline_id,
env.initial_tenant,
leaf_timeline_id,
state=TimelineArchivalState.ARCHIVED,
)
leaf_detail = ps_http.timeline_detail(
tenant_id=env.initial_tenant,
timeline_id=leaf_timeline_id,
env.initial_tenant,
leaf_timeline_id,
)
assert leaf_detail["is_archived"] is True
ps_http.timeline_archival_config(
tenant_id=env.initial_tenant,
timeline_id=parent_timeline_id,
env.initial_tenant,
parent_timeline_id,
state=TimelineArchivalState.ARCHIVED,
)
@@ -100,23 +93,21 @@ def test_timeline_archive(neon_simple_env: NeonEnv):
PageserverApiException,
match="ancestor is archived",
) as exc:
assert timeline_path.exists()
ps_http.timeline_archival_config(
tenant_id=env.initial_tenant,
timeline_id=leaf_timeline_id,
env.initial_tenant,
leaf_timeline_id,
state=TimelineArchivalState.UNARCHIVED,
)
# Unarchive works for the leaf if the parent gets unarchived first
ps_http.timeline_archival_config(
tenant_id=env.initial_tenant,
timeline_id=parent_timeline_id,
env.initial_tenant,
parent_timeline_id,
state=TimelineArchivalState.UNARCHIVED,
)
ps_http.timeline_archival_config(
tenant_id=env.initial_tenant,
timeline_id=leaf_timeline_id,
env.initial_tenant,
leaf_timeline_id,
state=TimelineArchivalState.UNARCHIVED,
)