From 7f9d2a7d05b4a3e743509af248a8b0234694afa1 Mon Sep 17 00:00:00 2001 From: anastasia Date: Tue, 19 Oct 2021 11:11:25 +0300 Subject: [PATCH] Change 'zenith tenant list' API to return tenant state added in 0dc7a3fc --- control_plane/src/storage.rs | 3 ++- pageserver/src/branches.rs | 14 +---------- pageserver/src/http/openapi_spec.yml | 14 +++++++++-- pageserver/src/http/routes.rs | 2 +- pageserver/src/page_service.rs | 2 +- pageserver/src/tenant_mgr.rs | 25 +++++++++++++++++-- .../batch_others/test_pageserver_api.py | 8 +++--- test_runner/batch_others/test_zenith_cli.py | 6 ++--- test_runner/fixtures/zenith_fixtures.py | 4 +-- zenith/src/main.rs | 4 +-- 10 files changed, 51 insertions(+), 31 deletions(-) diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index 3d331ca2a7..6e1f46b2e5 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -20,6 +20,7 @@ use zenith_utils::zid::ZTenantId; use crate::local_env::LocalEnv; use crate::read_pidfile; use pageserver::branches::BranchInfo; +use pageserver::tenant_mgr::TenantInfo; use zenith_utils::connstring::connection_address; #[derive(Error, Debug)] @@ -269,7 +270,7 @@ impl PageServerNode { Ok(()) } - pub fn tenant_list(&self) -> Result> { + pub fn tenant_list(&self) -> Result> { Ok(self .http_request(Method::GET, format!("{}/{}", self.http_base_url, "tenant")) .send()? diff --git a/pageserver/src/branches.rs b/pageserver/src/branches.rs index 15e56b18e5..ba922acdb7 100644 --- a/pageserver/src/branches.rs +++ b/pageserver/src/branches.rs @@ -4,7 +4,7 @@ // TODO: move all paths construction to conf impl // -use anyhow::{bail, ensure, Context, Result}; +use anyhow::{bail, Context, Result}; use postgres_ffi::ControlFileData; use serde::{Deserialize, Serialize}; use std::{ @@ -248,18 +248,6 @@ fn bootstrap_timeline( Ok(()) } -pub(crate) fn get_tenants(conf: &PageServerConf) -> Result> { - let tenants_dir = conf.tenants_path(); - - std::fs::read_dir(&tenants_dir)? - .map(|dir_entry_res| { - let dir_entry = dir_entry_res?; - ensure!(dir_entry.file_type()?.is_dir()); - Ok(dir_entry.file_name().to_str().unwrap().to_owned()) - }) - .collect() -} - pub(crate) fn get_branches(conf: &PageServerConf, tenantid: &ZTenantId) -> Result> { let repo = tenant_mgr::get_repository_for_tenant(*tenantid)?; diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index b173862146..d3c017d832 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -164,13 +164,13 @@ paths: description: Get tenants list responses: "200": - description: OK + description: TenantInfo content: application/json: schema: type: array items: - type: string + $ref: "#/components/schemas/TenantInfo" "401": description: Unauthorized Error content: @@ -243,6 +243,16 @@ components: scheme: bearer bearerFormat: JWT schemas: + TenantInfo: + type: object + required: + - id + - state + properties: + id: + type: string + state: + type: string BranchInfo: type: object required: diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index cacb98ec84..0ff39b1253 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -124,7 +124,7 @@ async fn tenant_list_handler(request: Request) -> Result, A let response_data = tokio::task::spawn_blocking(move || { let _enter = info_span!("tenant_list").entered(); - crate::branches::get_tenants(get_config(&request)) + crate::tenant_mgr::list_tenants() }) .await .map_err(ApiError::from_err)??; diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index be849ce35f..d78974aebb 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -637,7 +637,7 @@ impl postgres_backend::Handler for PageServerHandler { .write_message_noflush(&BeMessage::DataRow(&[Some(&branches_buf)]))? .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?; } else if query_string.starts_with("tenant_list") { - let tenants = crate::branches::get_tenants(self.conf)?; + let tenants = crate::tenant_mgr::list_tenants()?; let tenants_buf = serde_json::to_vec(&tenants)?; pgb.write_message_noflush(&SINGLE_COL_ROWDESC)? diff --git a/pageserver/src/tenant_mgr.rs b/pageserver/src/tenant_mgr.rs index be3a36fda4..7b9924efae 100644 --- a/pageserver/src/tenant_mgr.rs +++ b/pageserver/src/tenant_mgr.rs @@ -9,6 +9,7 @@ use crate::PageServerConf; use anyhow::{anyhow, bail, Context, Result}; use lazy_static::lazy_static; use log::{debug, info}; +use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fmt; use std::fs; @@ -27,8 +28,8 @@ struct Tenant { repo: Option>, } -#[derive(Debug)] -enum TenantState { +#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +pub enum TenantState { // This tenant only exists in cloud storage. It cannot be accessed. CloudOnly, // This tenant exists in cloud storage, and we are currently downloading it to local disk. @@ -247,3 +248,23 @@ fn list_tenantids() -> Result> { }) .collect() } + +#[derive(Serialize, Deserialize, Clone)] +pub struct TenantInfo { + #[serde(with = "hex")] + pub id: ZTenantId, + pub state: TenantState, +} + +pub fn list_tenants() -> Result> { + let m = access_tenants(); + m.iter() + .map(|v| { + let (id, tenant) = v; + Ok(TenantInfo { + id: *id, + state: tenant.state, + }) + }) + .collect() +} diff --git a/test_runner/batch_others/test_pageserver_api.py b/test_runner/batch_others/test_pageserver_api.py index 95b0172e4c..407b78fff6 100644 --- a/test_runner/batch_others/test_pageserver_api.py +++ b/test_runner/batch_others/test_pageserver_api.py @@ -56,7 +56,7 @@ def test_branch_list_psql(pageserver: ZenithPageserver, zenith_cli): def test_tenant_list_psql(pageserver: ZenithPageserver, zenith_cli): res = zenith_cli.run(["tenant", "list"]) res.check_returncode() - tenants = res.stdout.splitlines() + tenants = sorted(map(lambda t: t.split()[0], res.stdout.splitlines())) assert tenants == [pageserver.initial_tenant] conn = pageserver.connect() @@ -74,7 +74,7 @@ def test_tenant_list_psql(pageserver: ZenithPageserver, zenith_cli): cur.execute('tenant_list') # compare tenants list - new_tenants = sorted(json.loads(cur.fetchone()[0])) + new_tenants = sorted(map(lambda t: t['id'], json.loads(cur.fetchone()[0]))) assert sorted([pageserver.initial_tenant, tenant1]) == new_tenants @@ -82,12 +82,12 @@ def check_client(client: ZenithPageserverHttpClient, initial_tenant: str): client.check_status() # check initial tenant is there - assert initial_tenant in set(client.tenant_list()) + assert initial_tenant in {t['id'] for t in client.tenant_list()} # create new tenant and check it is also there tenant_id = uuid4() client.tenant_create(tenant_id) - assert tenant_id.hex in set(client.tenant_list()) + assert tenant_id.hex in {t['id'] for t in client.tenant_list()} # create branch branch_name = uuid4().hex diff --git a/test_runner/batch_others/test_zenith_cli.py b/test_runner/batch_others/test_zenith_cli.py index 7379cf2981..a328f1a607 100644 --- a/test_runner/batch_others/test_zenith_cli.py +++ b/test_runner/batch_others/test_zenith_cli.py @@ -60,11 +60,11 @@ def test_cli_branch_list(pageserver: ZenithPageserver, zenith_cli): def helper_compare_tenant_list(page_server_cur, zenith_cli: ZenithCli): page_server_cur.execute(f'tenant_list') - tenants_api = sorted(json.loads(page_server_cur.fetchone()[0])) + tenants_api = sorted(map(lambda t: t['id'], json.loads(page_server_cur.fetchone()[0]))) res = zenith_cli.run(["tenant", "list"]) assert res.stderr == '' - tenants_cli = sorted(res.stdout.splitlines()) + tenants_cli = sorted(map(lambda t: t.split()[0], res.stdout.splitlines())) assert tenants_api == tenants_cli @@ -94,7 +94,7 @@ def test_cli_tenant_list(pageserver: ZenithPageserver, zenith_cli: ZenithCli): res = zenith_cli.run(["tenant", "list"]) res.check_returncode() - tenants = sorted(res.stdout.splitlines()) + tenants = sorted(map(lambda t: t.split()[0], res.stdout.splitlines())) assert pageserver.initial_tenant in tenants assert tenant1 in tenants diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index b343828006..2ea321a0af 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -269,7 +269,7 @@ class ZenithPageserverHttpClient(requests.Session): res.raise_for_status() return res.json() - def tenant_list(self) -> List[str]: + def tenant_list(self) -> List[Dict]: res = self.get(f"http://localhost:{self.port}/v1/tenant") res.raise_for_status() return res.json() @@ -401,7 +401,7 @@ class ZenithPageserver(PgProtocol): self.zenith_cli.run(['start']) self.running = True # get newly created tenant id - self.initial_tenant = self.zenith_cli.run(['tenant', 'list']).stdout.strip() + self.initial_tenant = self.zenith_cli.run(['tenant', 'list']).stdout.split()[0] return self def stop(self, immediate=False) -> 'ZenithPageserver': diff --git a/zenith/src/main.rs b/zenith/src/main.rs index e79d42377e..214e71c1c7 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -384,8 +384,8 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &local_env::LocalEnv) -> Result let pageserver = PageServerNode::from_env(env); match tenant_match.subcommand() { ("list", Some(_)) => { - for tenant in pageserver.tenant_list()? { - println!("{}", tenant); + for t in pageserver.tenant_list()? { + println!("{} {}", t.id, t.state); } } ("create", Some(create_match)) => {