rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-08-12 09:19:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(storcon): passthrough mark invisible and add tests (#11401)

Browse Source 
## Problem

close https://github.com/neondatabase/neon/issues/11279

## Summary of changes

* Allow passthrough of other methods in tenant timeline shard0
passthrough of storcon.
* Passthrough mark invisible API in storcon.

---------

Signed-off-by: Alex Chi Z <chi@neon.tech>
This commit is contained in:
Alex Chi Z.
2025-04-02 13:11:49 -04:00
committed by GitHub
parent cb19e4e05d
commit dd1299f337
6 changed files with 80 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pageserver/client/src/mgmt_api.rs
+5 -5
View File
@@ -86,17 +86,17 @@ impl Client {
resp.json().await.map_err(Error::ReceiveBody)
}
/// Get an arbitrary path and returning a streaming Response. This function is suitable
/// for pass-through/proxy use cases where we don't care what the response content looks
/// like.
/// Send an HTTP request to an arbitrary path with a desired HTTP method and returning a streaming
/// Response. This function is suitable for pass-through/proxy use cases where we don't care
/// what the response content looks like.
///
/// Use/add one of the properly typed methods below if you know aren't proxying, and
/// know what kind of response you expect.
pub async fn get_raw(&self, path: String) -> Result<reqwest::Response> {
pub async fn op_raw(&self, method: Method, path: String) -> Result<reqwest::Response> {
debug_assert!(path.starts_with('/'));
let uri = format!("{}{}", self.mgmt_api_endpoint, path);
let mut req = self.client.request(Method::GET, uri);
let mut req = self.client.request(method, uri);
if let Some(value) = &self.authorization_header {
req = req.header(reqwest::header::AUTHORIZATION, value);
}
pageserver/src/http/routes.rs
+14 -4
View File
@@ -74,8 +74,8 @@ use crate::tenant::size::ModelInputs;
use crate::tenant::storage_layer::{IoConcurrency, LayerAccessStatsReset, LayerName};
use crate::tenant::timeline::offload::{OffloadError, offload_timeline};
use crate::tenant::timeline::{
CompactFlags, CompactOptions, CompactRequest, CompactionError, Timeline, WaitLsnTimeout,
WaitLsnWaiter, import_pgdata,
CompactFlags, CompactOptions, CompactRequest, CompactionError, MarkInvisibleRequest, Timeline,
WaitLsnTimeout, WaitLsnWaiter, import_pgdata,
};
use crate::tenant::{
GetTimelineError, LogicalSizeCalculationCause, OffloadedTimeline, PageReconstructError,
@@ -2337,21 +2337,31 @@ async fn timeline_compact_handler(
}
async fn timeline_mark_invisible_handler(
request: Request<Body>,
mut request: Request<Body>,
_cancel: CancellationToken,
) -> Result<Response<Body>, ApiError> {
let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
check_permission(&request, Some(tenant_shard_id.tenant_id))?;
let compact_request = json_request_maybe::<Option<MarkInvisibleRequest>>(&mut request).await?;
let state = get_state(&request);
let visibility = match compact_request {
Some(req) => match req.is_visible {
Some(true) => TimelineVisibilityState::Visible,
Some(false) | None => TimelineVisibilityState::Invisible,
},
None => TimelineVisibilityState::Invisible,
};
async {
let tenant = state
.tenant_manager
.get_attached_tenant_shard(tenant_shard_id)?;
let timeline = tenant.get_timeline(timeline_id, true)?;
timeline.remote_client.schedule_index_upload_for_timeline_invisible_state(TimelineVisibilityState::Invisible).map_err(ApiError::InternalServerError)?;
timeline.remote_client.schedule_index_upload_for_timeline_invisible_state(visibility).map_err(ApiError::InternalServerError)?;
json_response(StatusCode::OK, ())
}
.instrument(info_span!("manual_timeline_mark_invisible", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id))
pageserver/src/tenant/timeline.rs
+6
View File
@@ -895,6 +895,12 @@ pub(crate) struct CompactRequest {
pub sub_compaction_max_job_size_mb: Option<u64>,
}
#[derive(Debug, Clone, serde::Deserialize)]
pub(crate) struct MarkInvisibleRequest {
#[serde(default)]
pub is_visible: Option<bool>,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct CompactOptions {
pub flags: EnumSet<CompactFlags>,
storage_controller/src/http.rs
+21 -1
View File
@@ -639,6 +639,15 @@ async fn handle_tenant_timeline_passthrough(
return Err(ApiError::BadRequest(anyhow::anyhow!("Missing path")));
};
let method = match *req.method() {
hyper::Method::GET => reqwest::Method::GET,
hyper::Method::POST => reqwest::Method::POST,
hyper::Method::PUT => reqwest::Method::PUT,
hyper::Method::DELETE => reqwest::Method::DELETE,
hyper::Method::PATCH => reqwest::Method::PATCH,
_ => return Err(ApiError::BadRequest(anyhow::anyhow!("Unsupported method"))),
};
tracing::info!(
"Proxying request for tenant {} ({})",
tenant_or_shard_id.tenant_id,
@@ -686,7 +695,7 @@ async fn handle_tenant_timeline_passthrough(
node.base_url(),
service.get_config().pageserver_jwt_token.as_deref(),
);
let resp = client.get_raw(path).await.map_err(|e|
let resp = client.op_raw(method, path).await.map_err(|e|
// We return 503 here because if we can't successfully send a request to the pageserver,
// either we aren't available or the pageserver is unavailable.
ApiError::ResourceUnavailable(format!("Error sending pageserver API request to {node}: {e}").into()))?;
@@ -2247,6 +2256,17 @@ pub fn make_router(
RequestName("v1_tenant_passthrough"),
)
})
// Tenant timeline mark_invisible passthrough to shard zero
.put(
"/v1/tenant/:tenant_id/timeline/:timeline_id/mark_invisible",
|r| {
tenant_service_handler(
r,
handle_tenant_timeline_passthrough,
RequestName("v1_tenant_timeline_mark_invisible_passthrough"),
)
},
)
}
#[cfg(test)]
test_runner/fixtures/pageserver/http.py
+19
View File
@@ -853,6 +853,25 @@ class PageserverHttpClient(requests.Session, MetricsGetter):
res_json = res.json()
return res_json
def timeline_mark_invisible(
self,
tenant_id: TenantId | TenantShardId,
timeline_id: TimelineId,
is_visible: bool | None = None,
):
data = {
"is_visible": is_visible,
}
log.info(
f"Requesting marking timeline invisible for {is_visible=}, {tenant_id=}, {timeline_id=}"
)
res = self.put(
f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/mark_invisible",
json=data,
)
self.verbose_error(res)
def timeline_get_timestamp_of_lsn(
self, tenant_id: TenantId | TenantShardId, timeline_id: TimelineId, lsn: Lsn
):
test_runner/regress/test_tenant_size.py
+15
View File
@@ -782,6 +782,21 @@ def test_lsn_lease_storcon(neon_env_builder: NeonEnvBuilder):
)
def test_mark_invisible_storcon(neon_env_builder: NeonEnvBuilder):
conf = {
"pitr_interval": "0s",
"gc_period": "0s",
"compaction_period": "0s",
}
env = neon_env_builder.init_start(initial_tenant_conf=conf)
env.storage_controller.pageserver_api().timeline_mark_invisible(
env.initial_tenant, env.initial_timeline
)
env.storage_controller.pageserver_api().timeline_mark_invisible(
env.initial_tenant, env.initial_timeline, True
)
def insert_with_action(
env: NeonEnv,
tenant: TenantId,