From 26a68612d92608fd1768bdf65dc9cb4725d87edc Mon Sep 17 00:00:00 2001 From: Andrey Taranik Date: Sun, 27 Feb 2022 01:36:40 +0300 Subject: [PATCH 01/42] safekeeper to cosnole call fix (#1333) --- .circleci/ansible/deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/ansible/deploy.yaml b/.circleci/ansible/deploy.yaml index 06385aa0d3..c95524a8a5 100644 --- a/.circleci/ansible/deploy.yaml +++ b/.circleci/ansible/deploy.yaml @@ -169,6 +169,6 @@ shell: cmd: | INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) - curl -sfS -d '{"version": {{ current_version }} }' -X POST {{ hostvars.localhost.zenith.console_mgmt_base_url }}/api/v1/safekeepers/$INSTANCE_ID + curl -sfS -d '{"version": {{ current_version }} }' -X POST {{ console_mgmt_base_url }}/api/v1/safekeepers/$INSTANCE_ID tags: - safekeeper From 949f8b463330e318d72c3944fe78958b38305cb3 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Tue, 1 Mar 2022 22:42:57 +0200 Subject: [PATCH 02/42] Fix 1.59 rustc clippy warnings --- compute_tools/src/pg_helpers.rs | 2 +- compute_tools/src/spec.rs | 2 +- pageserver/src/layered_repository.rs | 9 ++++----- pageserver/src/layered_repository/inmemory_layer.rs | 9 ++++----- pageserver/src/walrecord.rs | 12 +++++------- proxy/src/main.rs | 2 +- 6 files changed, 16 insertions(+), 20 deletions(-) diff --git a/compute_tools/src/pg_helpers.rs b/compute_tools/src/pg_helpers.rs index 8b6dc04069..6a22b865fa 100644 --- a/compute_tools/src/pg_helpers.rs +++ b/compute_tools/src/pg_helpers.rs @@ -171,7 +171,7 @@ impl PgQuote for PgIdent { /// always quotes provided string with `""` and escapes every `"`. Not idempotent, /// i.e. if string is already escaped it will be escaped again. fn quote(&self) -> String { - let result = format!("\"{}\"", self.replace("\"", "\"\"")); + let result = format!("\"{}\"", self.replace('"', "\"\"")); result } } diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index 41e4174bf0..1dd7c0044e 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -215,7 +215,7 @@ pub fn handle_databases(spec: &ClusterSpec, client: &mut Client) -> Result<()> { if let Some(r) = pg_db { // XXX: db owner name is returned as quoted string from Postgres, // when quoting is needed. - let new_owner = if r.owner.starts_with('\"') { + let new_owner = if r.owner.starts_with('"') { db.owner.quote() } else { db.owner.clone() diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index 5dae1902c1..975b2f5d2b 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -893,12 +893,11 @@ impl Timeline for LayeredTimeline { let seg = SegmentTag { rel, segno: 0 }; - let result; - if let Some((layer, lsn)) = self.get_layer_for_read(seg, lsn)? { - result = layer.get_seg_exists(lsn)?; + let result = if let Some((layer, lsn)) = self.get_layer_for_read(seg, lsn)? { + layer.get_seg_exists(lsn)? } else { - result = false; - } + false + }; trace!("get_rel_exists: {} at {} -> {}", rel, lsn, result); Ok(result) diff --git a/pageserver/src/layered_repository/inmemory_layer.rs b/pageserver/src/layered_repository/inmemory_layer.rs index 17b061b20e..6e24bf6022 100644 --- a/pageserver/src/layered_repository/inmemory_layer.rs +++ b/pageserver/src/layered_repository/inmemory_layer.rs @@ -170,12 +170,11 @@ impl Layer for InMemoryLayer { fn filename(&self) -> PathBuf { let inner = self.inner.read().unwrap(); - let end_lsn; - if let Some(drop_lsn) = inner.end_lsn { - end_lsn = drop_lsn; + let end_lsn = if let Some(drop_lsn) = inner.end_lsn { + drop_lsn } else { - end_lsn = Lsn(u64::MAX); - } + Lsn(u64::MAX) + }; let delta_filename = DeltaFileName { seg: self.seg, diff --git a/pageserver/src/walrecord.rs b/pageserver/src/walrecord.rs index 378a015d4a..ca9107cdbf 100644 --- a/pageserver/src/walrecord.rs +++ b/pageserver/src/walrecord.rs @@ -268,12 +268,11 @@ impl XlXactParsedRecord { let info = xl_info & pg_constants::XLOG_XACT_OPMASK; // The record starts with time of commit/abort let xact_time = buf.get_i64_le(); - let xinfo; - if xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 { - xinfo = buf.get_u32_le(); + let xinfo = if xl_info & pg_constants::XLOG_XACT_HAS_INFO != 0 { + buf.get_u32_le() } else { - xinfo = 0; - } + 0 + }; let db_id; let ts_id; if xinfo & pg_constants::XACT_XINFO_HAS_DBINFO != 0 { @@ -502,7 +501,6 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord { 0..=pg_constants::XLR_MAX_BLOCK_ID => { /* XLogRecordBlockHeader */ let mut blk = DecodedBkpBlock::new(); - let fork_flags: u8; if block_id <= max_block_id { // TODO @@ -515,7 +513,7 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord { } max_block_id = block_id; - fork_flags = buf.get_u8(); + let fork_flags: u8 = buf.get_u8(); blk.forknum = fork_flags & pg_constants::BKPBLOCK_FORK_MASK; blk.flags = fork_flags; blk.has_image = (fork_flags & pg_constants::BKPBLOCK_HAS_IMAGE) != 0; diff --git a/proxy/src/main.rs b/proxy/src/main.rs index fb3bf725b8..de618ccde9 100644 --- a/proxy/src/main.rs +++ b/proxy/src/main.rs @@ -122,7 +122,7 @@ async fn main() -> anyhow::Result<()> { None => RouterConfig::Dynamic(auth_method), Some(addr) => { if let ClientAuthMethod::Password = auth_method { - let (host, port) = addr.split_once(":").unwrap(); + let (host, port) = addr.split_once(':').unwrap(); RouterConfig::Static { host: host.to_string(), port: port.parse().unwrap(), From 1d90b1b205023c3bd404de8b361dda69cef6a502 Mon Sep 17 00:00:00 2001 From: Dmitry Rodionov Date: Fri, 4 Mar 2022 01:10:42 +0300 Subject: [PATCH 03/42] add node id to pageserver (#1310) * Add --id argument to safekeeper setting its unique u64 id. In preparation for storage node messaging. IDs are supposed to be monotonically assigned by the console. In tests it is issued by ZenithEnv; at the zenith cli level and fixtures, string name is completely replaced by integer id. Example TOML configs are adjusted accordingly. Sequential ids are chosen over Zid mainly because they are compact and easy to type/remember. * add node id to pageserver This adds node id parameter to pageserver configuration. Also I use a simple builder to construct pageserver config struct to avoid setting node id to some temporary invalid value. Some of the changes in test fixtures are needed to split init and start operations for envrionment. Co-authored-by: Arseny Sher --- control_plane/safekeepers.conf | 6 +- control_plane/simple.conf | 2 +- control_plane/src/local_env.rs | 17 +- control_plane/src/safekeeper.rs | 16 +- control_plane/src/storage.rs | 3 + docker-entrypoint.sh | 2 +- pageserver/src/bin/pageserver.rs | 9 +- pageserver/src/config.rs | 281 +++++++++++++++--- pageserver/src/http/models.rs | 6 + pageserver/src/http/openapi_spec.yml | 5 + pageserver/src/http/routes.rs | 14 +- test_runner/README.md | 2 +- test_runner/batch_others/test_auth.py | 4 +- test_runner/batch_others/test_backpressure.py | 2 +- .../batch_others/test_branch_behind.py | 2 +- test_runner/batch_others/test_next_xid.py | 2 +- .../batch_others/test_pageserver_api.py | 17 +- .../batch_others/test_pageserver_catchup.py | 2 +- .../batch_others/test_pageserver_restart.py | 2 +- .../batch_others/test_remote_storage.py | 2 +- .../batch_others/test_restart_compute.py | 2 +- .../batch_others/test_tenant_relocation.py | 2 +- test_runner/batch_others/test_tenants.py | 2 +- .../batch_others/test_timeline_size.py | 2 +- test_runner/batch_others/test_wal_acceptor.py | 38 ++- .../batch_others/test_wal_acceptor_async.py | 2 +- test_runner/batch_others/test_zenith_cli.py | 4 +- test_runner/fixtures/zenith_fixtures.py | 70 +++-- .../performance/test_bulk_tenant_create.py | 2 +- walkeeper/src/bin/safekeeper.rs | 81 ++++- walkeeper/src/http/routes.rs | 12 +- walkeeper/src/lib.rs | 4 +- zenith/src/main.rs | 41 +-- zenith_utils/src/zid.rs | 12 + 34 files changed, 501 insertions(+), 169 deletions(-) diff --git a/control_plane/safekeepers.conf b/control_plane/safekeepers.conf index 828d5a5a1e..df7dd2adca 100644 --- a/control_plane/safekeepers.conf +++ b/control_plane/safekeepers.conf @@ -5,16 +5,16 @@ listen_http_addr = '127.0.0.1:9898' auth_type = 'Trust' [[safekeepers]] -name = 'sk1' +id = 1 pg_port = 5454 http_port = 7676 [[safekeepers]] -name = 'sk2' +id = 2 pg_port = 5455 http_port = 7677 [[safekeepers]] -name = 'sk3' +id = 3 pg_port = 5456 http_port = 7678 diff --git a/control_plane/simple.conf b/control_plane/simple.conf index 796c6adbd9..2243a0a5f8 100644 --- a/control_plane/simple.conf +++ b/control_plane/simple.conf @@ -6,6 +6,6 @@ listen_http_addr = '127.0.0.1:9898' auth_type = 'Trust' [[safekeepers]] -name = 'single' +id = 1 pg_port = 5454 http_port = 7676 diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index b80e137cb9..55d0b00496 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -12,7 +12,9 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use zenith_utils::auth::{encode_from_key_file, Claims, Scope}; use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{opt_display_serde, ZTenantId}; +use zenith_utils::zid::{opt_display_serde, ZNodeId, ZTenantId}; + +use crate::safekeeper::SafekeeperNode; // // This data structures represents zenith CLI config @@ -62,6 +64,8 @@ pub struct LocalEnv { #[derive(Serialize, Deserialize, Clone, Debug)] #[serde(default)] pub struct PageServerConf { + // node id + pub id: ZNodeId, // Pageserver connection settings pub listen_pg_addr: String, pub listen_http_addr: String, @@ -76,6 +80,7 @@ pub struct PageServerConf { impl Default for PageServerConf { fn default() -> Self { Self { + id: ZNodeId(0), listen_pg_addr: String::new(), listen_http_addr: String::new(), auth_type: AuthType::Trust, @@ -87,7 +92,7 @@ impl Default for PageServerConf { #[derive(Serialize, Deserialize, Clone, Debug)] #[serde(default)] pub struct SafekeeperConf { - pub name: String, + pub id: ZNodeId, pub pg_port: u16, pub http_port: u16, pub sync: bool, @@ -96,7 +101,7 @@ pub struct SafekeeperConf { impl Default for SafekeeperConf { fn default() -> Self { Self { - name: String::new(), + id: ZNodeId(0), pg_port: 0, http_port: 0, sync: true, @@ -136,8 +141,8 @@ impl LocalEnv { self.base_data_dir.clone() } - pub fn safekeeper_data_dir(&self, node_name: &str) -> PathBuf { - self.base_data_dir.join("safekeepers").join(node_name) + pub fn safekeeper_data_dir(&self, data_dir_name: &str) -> PathBuf { + self.base_data_dir.join("safekeepers").join(data_dir_name) } /// Create a LocalEnv from a config file. @@ -285,7 +290,7 @@ impl LocalEnv { fs::create_dir_all(self.pg_data_dirs_path())?; for safekeeper in &self.safekeepers { - fs::create_dir_all(self.safekeeper_data_dir(&safekeeper.name))?; + fs::create_dir_all(SafekeeperNode::datadir_path_by_id(self, safekeeper.id))?; } let mut conf_content = String::new(); diff --git a/control_plane/src/safekeeper.rs b/control_plane/src/safekeeper.rs index f5478b5922..351d1efbbc 100644 --- a/control_plane/src/safekeeper.rs +++ b/control_plane/src/safekeeper.rs @@ -15,6 +15,7 @@ use reqwest::blocking::{Client, RequestBuilder, Response}; use reqwest::{IntoUrl, Method}; use thiserror::Error; use zenith_utils::http::error::HttpErrorBody; +use zenith_utils::zid::ZNodeId; use crate::local_env::{LocalEnv, SafekeeperConf}; use crate::storage::PageServerNode; @@ -61,7 +62,7 @@ impl ResponseErrorMessageExt for Response { // #[derive(Debug)] pub struct SafekeeperNode { - pub name: String, + pub id: ZNodeId, pub conf: SafekeeperConf, @@ -77,10 +78,10 @@ impl SafekeeperNode { pub fn from_env(env: &LocalEnv, conf: &SafekeeperConf) -> SafekeeperNode { let pageserver = Arc::new(PageServerNode::from_env(env)); - println!("initializing for {} for {}", conf.name, conf.http_port); + println!("initializing for sk {} for {}", conf.id, conf.http_port); SafekeeperNode { - name: conf.name.clone(), + id: conf.id, conf: conf.clone(), pg_connection_config: Self::safekeeper_connection_config(conf.pg_port), env: env.clone(), @@ -98,8 +99,12 @@ impl SafekeeperNode { .unwrap() } + pub fn datadir_path_by_id(env: &LocalEnv, sk_id: ZNodeId) -> PathBuf { + env.safekeeper_data_dir(format!("sk{}", sk_id).as_ref()) + } + pub fn datadir_path(&self) -> PathBuf { - self.env.safekeeper_data_dir(&self.name) + SafekeeperNode::datadir_path_by_id(&self.env, self.id) } pub fn pid_file(&self) -> PathBuf { @@ -120,6 +125,7 @@ impl SafekeeperNode { let mut cmd = Command::new(self.env.safekeeper_bin()?); fill_rust_env_vars( cmd.args(&["-D", self.datadir_path().to_str().unwrap()]) + .args(&["--id", self.id.to_string().as_ref()]) .args(&["--listen-pg", &listen_pg]) .args(&["--listen-http", &listen_http]) .args(&["--recall", "1 second"]) @@ -183,7 +189,7 @@ impl SafekeeperNode { pub fn stop(&self, immediate: bool) -> anyhow::Result<()> { let pid_file = self.pid_file(); if !pid_file.exists() { - println!("Safekeeper {} is already stopped", self.name); + println!("Safekeeper {} is already stopped", self.id); return Ok(()); } let pid = read_pidfile(&pid_file)?; diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index be594889ab..cd429e3f7a 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -103,6 +103,8 @@ impl PageServerNode { ) -> anyhow::Result<()> { let mut cmd = Command::new(self.env.pageserver_bin()?); + let id = format!("id={}", self.env.pageserver.id); + // FIXME: the paths should be shell-escaped to handle paths with spaces, quotas etc. let base_data_dir_param = self.env.base_data_dir.display().to_string(); let pg_distrib_dir_param = @@ -122,6 +124,7 @@ impl PageServerNode { args.extend(["-c", &authg_type_param]); args.extend(["-c", &listen_http_addr_param]); args.extend(["-c", &listen_pg_addr_param]); + args.extend(["-c", &id]); for config_override in config_overrides { args.extend(["-c", config_override]); diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 45c41b4c19..93bb5f9cd7 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -4,7 +4,7 @@ set -eux if [ "$1" = 'pageserver' ]; then if [ ! -d "/data/tenants" ]; then echo "Initializing pageserver data directory" - pageserver --init -D /data -c "pg_distrib_dir='/usr/local'" + pageserver --init -D /data -c "pg_distrib_dir='/usr/local'" -c "id=10" fi echo "Staring pageserver at 0.0.0.0:6400" pageserver -c "listen_pg_addr='0.0.0.0:6400'" -c "listen_http_addr='0.0.0.0:9898'" -D /data diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index fb8baa28f6..d8d4033340 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -61,7 +61,7 @@ fn main() -> Result<()> { .number_of_values(1) .multiple_occurrences(true) .help("Additional configuration overrides of the ones from the toml config file (or new ones to add there). - Any option has to be a valid toml document, example: `-c \"foo='hey'\"` `-c \"foo={value=1}\"`"), + Any option has to be a valid toml document, example: `-c=\"foo='hey'\"` `-c=\"foo={value=1}\"`"), ) .get_matches(); @@ -115,7 +115,14 @@ fn main() -> Result<()> { option_line ) })?; + for (key, item) in doc.iter() { + if key == "id" { + anyhow::ensure!( + init, + "node id can only be set during pageserver init and cannot be overridden" + ); + } toml.insert(key, item.clone()); } } diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 8b65e7e2e6..3deabb7521 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -8,7 +8,7 @@ use anyhow::{bail, ensure, Context, Result}; use toml_edit; use toml_edit::{Document, Item}; use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{ZTenantId, ZTimelineId}; +use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId}; use std::convert::TryInto; use std::env; @@ -78,6 +78,10 @@ pub mod defaults { #[derive(Debug, Clone, PartialEq, Eq)] pub struct PageServerConf { + // Identifier of that particular pageserver so e g safekeepers + // can safely distinguish different pageservers + pub id: ZNodeId, + /// Example (default): 127.0.0.1:64000 pub listen_pg_addr: String, /// Example (default): 127.0.0.1:9898 @@ -118,6 +122,206 @@ pub struct PageServerConf { pub remote_storage_config: Option, } +// use dedicated enum for builder to better indicate the intention +// and avoid possible confusion with nested options +pub enum BuilderValue { + Set(T), + NotSet, +} + +impl BuilderValue { + pub fn ok_or(self, err: E) -> Result { + match self { + Self::Set(v) => Ok(v), + Self::NotSet => Err(err), + } + } +} + +// needed to simplify config construction +struct PageServerConfigBuilder { + listen_pg_addr: BuilderValue, + + listen_http_addr: BuilderValue, + + checkpoint_distance: BuilderValue, + checkpoint_period: BuilderValue, + + gc_horizon: BuilderValue, + gc_period: BuilderValue, + + wait_lsn_timeout: BuilderValue, + wal_redo_timeout: BuilderValue, + + superuser: BuilderValue, + + page_cache_size: BuilderValue, + max_file_descriptors: BuilderValue, + + workdir: BuilderValue, + + pg_distrib_dir: BuilderValue, + + auth_type: BuilderValue, + + // + auth_validation_public_key_path: BuilderValue>, + remote_storage_config: BuilderValue>, + + id: BuilderValue, +} + +impl Default for PageServerConfigBuilder { + fn default() -> Self { + use self::BuilderValue::*; + use defaults::*; + Self { + listen_pg_addr: Set(DEFAULT_PG_LISTEN_ADDR.to_string()), + listen_http_addr: Set(DEFAULT_HTTP_LISTEN_ADDR.to_string()), + checkpoint_distance: Set(DEFAULT_CHECKPOINT_DISTANCE), + checkpoint_period: Set(humantime::parse_duration(DEFAULT_CHECKPOINT_PERIOD) + .expect("cannot parse default checkpoint period")), + gc_horizon: Set(DEFAULT_GC_HORIZON), + gc_period: Set(humantime::parse_duration(DEFAULT_GC_PERIOD) + .expect("cannot parse default gc period")), + wait_lsn_timeout: Set(humantime::parse_duration(DEFAULT_WAIT_LSN_TIMEOUT) + .expect("cannot parse default wait lsn timeout")), + wal_redo_timeout: Set(humantime::parse_duration(DEFAULT_WAL_REDO_TIMEOUT) + .expect("cannot parse default wal redo timeout")), + superuser: Set(DEFAULT_SUPERUSER.to_string()), + page_cache_size: Set(DEFAULT_PAGE_CACHE_SIZE), + max_file_descriptors: Set(DEFAULT_MAX_FILE_DESCRIPTORS), + workdir: Set(PathBuf::new()), + pg_distrib_dir: Set(env::current_dir() + .expect("cannot access current directory") + .join("tmp_install")), + auth_type: Set(AuthType::Trust), + auth_validation_public_key_path: Set(None), + remote_storage_config: Set(None), + id: NotSet, + } + } +} + +impl PageServerConfigBuilder { + pub fn listen_pg_addr(&mut self, listen_pg_addr: String) { + self.listen_pg_addr = BuilderValue::Set(listen_pg_addr) + } + + pub fn listen_http_addr(&mut self, listen_http_addr: String) { + self.listen_http_addr = BuilderValue::Set(listen_http_addr) + } + + pub fn checkpoint_distance(&mut self, checkpoint_distance: u64) { + self.checkpoint_distance = BuilderValue::Set(checkpoint_distance) + } + + pub fn checkpoint_period(&mut self, checkpoint_period: Duration) { + self.checkpoint_period = BuilderValue::Set(checkpoint_period) + } + + pub fn gc_horizon(&mut self, gc_horizon: u64) { + self.gc_horizon = BuilderValue::Set(gc_horizon) + } + + pub fn gc_period(&mut self, gc_period: Duration) { + self.gc_period = BuilderValue::Set(gc_period) + } + + pub fn wait_lsn_timeout(&mut self, wait_lsn_timeout: Duration) { + self.wait_lsn_timeout = BuilderValue::Set(wait_lsn_timeout) + } + + pub fn wal_redo_timeout(&mut self, wal_redo_timeout: Duration) { + self.wal_redo_timeout = BuilderValue::Set(wal_redo_timeout) + } + + pub fn superuser(&mut self, superuser: String) { + self.superuser = BuilderValue::Set(superuser) + } + + pub fn page_cache_size(&mut self, page_cache_size: usize) { + self.page_cache_size = BuilderValue::Set(page_cache_size) + } + + pub fn max_file_descriptors(&mut self, max_file_descriptors: usize) { + self.max_file_descriptors = BuilderValue::Set(max_file_descriptors) + } + + pub fn workdir(&mut self, workdir: PathBuf) { + self.workdir = BuilderValue::Set(workdir) + } + + pub fn pg_distrib_dir(&mut self, pg_distrib_dir: PathBuf) { + self.pg_distrib_dir = BuilderValue::Set(pg_distrib_dir) + } + + pub fn auth_type(&mut self, auth_type: AuthType) { + self.auth_type = BuilderValue::Set(auth_type) + } + + pub fn auth_validation_public_key_path( + &mut self, + auth_validation_public_key_path: Option, + ) { + self.auth_validation_public_key_path = BuilderValue::Set(auth_validation_public_key_path) + } + + pub fn remote_storage_config(&mut self, remote_storage_config: Option) { + self.remote_storage_config = BuilderValue::Set(remote_storage_config) + } + + pub fn id(&mut self, node_id: ZNodeId) { + self.id = BuilderValue::Set(node_id) + } + + pub fn build(self) -> Result { + Ok(PageServerConf { + listen_pg_addr: self + .listen_pg_addr + .ok_or(anyhow::anyhow!("missing listen_pg_addr"))?, + listen_http_addr: self + .listen_http_addr + .ok_or(anyhow::anyhow!("missing listen_http_addr"))?, + checkpoint_distance: self + .checkpoint_distance + .ok_or(anyhow::anyhow!("missing checkpoint_distance"))?, + checkpoint_period: self + .checkpoint_period + .ok_or(anyhow::anyhow!("missing checkpoint_period"))?, + gc_horizon: self + .gc_horizon + .ok_or(anyhow::anyhow!("missing gc_horizon"))?, + gc_period: self.gc_period.ok_or(anyhow::anyhow!("missing gc_period"))?, + wait_lsn_timeout: self + .wait_lsn_timeout + .ok_or(anyhow::anyhow!("missing wait_lsn_timeout"))?, + wal_redo_timeout: self + .wal_redo_timeout + .ok_or(anyhow::anyhow!("missing wal_redo_timeout"))?, + superuser: self.superuser.ok_or(anyhow::anyhow!("missing superuser"))?, + page_cache_size: self + .page_cache_size + .ok_or(anyhow::anyhow!("missing page_cache_size"))?, + max_file_descriptors: self + .max_file_descriptors + .ok_or(anyhow::anyhow!("missing max_file_descriptors"))?, + workdir: self.workdir.ok_or(anyhow::anyhow!("missing workdir"))?, + pg_distrib_dir: self + .pg_distrib_dir + .ok_or(anyhow::anyhow!("missing pg_distrib_dir"))?, + auth_type: self.auth_type.ok_or(anyhow::anyhow!("missing auth_type"))?, + auth_validation_public_key_path: self + .auth_validation_public_key_path + .ok_or(anyhow::anyhow!("missing auth_validation_public_key_path"))?, + remote_storage_config: self + .remote_storage_config + .ok_or(anyhow::anyhow!("missing remote_storage_config"))?, + id: self.id.ok_or(anyhow::anyhow!("missing id"))?, + }) + } +} + /// External backup storage configuration, enough for creating a client for that storage. #[derive(Debug, Clone, PartialEq, Eq)] pub struct RemoteStorageConfig { @@ -233,61 +437,41 @@ impl PageServerConf { /// /// This leaves any options not present in the file in the built-in defaults. pub fn parse_and_validate(toml: &Document, workdir: &Path) -> Result { - use defaults::*; - - let mut conf = PageServerConf { - workdir: workdir.to_path_buf(), - - listen_pg_addr: DEFAULT_PG_LISTEN_ADDR.to_string(), - listen_http_addr: DEFAULT_HTTP_LISTEN_ADDR.to_string(), - checkpoint_distance: DEFAULT_CHECKPOINT_DISTANCE, - checkpoint_period: humantime::parse_duration(DEFAULT_CHECKPOINT_PERIOD)?, - gc_horizon: DEFAULT_GC_HORIZON, - gc_period: humantime::parse_duration(DEFAULT_GC_PERIOD)?, - wait_lsn_timeout: humantime::parse_duration(DEFAULT_WAIT_LSN_TIMEOUT)?, - wal_redo_timeout: humantime::parse_duration(DEFAULT_WAL_REDO_TIMEOUT)?, - page_cache_size: DEFAULT_PAGE_CACHE_SIZE, - max_file_descriptors: DEFAULT_MAX_FILE_DESCRIPTORS, - - pg_distrib_dir: PathBuf::new(), - auth_validation_public_key_path: None, - auth_type: AuthType::Trust, - - remote_storage_config: None, - - superuser: DEFAULT_SUPERUSER.to_string(), - }; + let mut builder = PageServerConfigBuilder::default(); + builder.workdir(workdir.to_owned()); for (key, item) in toml.iter() { match key { - "listen_pg_addr" => conf.listen_pg_addr = parse_toml_string(key, item)?, - "listen_http_addr" => conf.listen_http_addr = parse_toml_string(key, item)?, - "checkpoint_distance" => conf.checkpoint_distance = parse_toml_u64(key, item)?, - "checkpoint_period" => conf.checkpoint_period = parse_toml_duration(key, item)?, - "gc_horizon" => conf.gc_horizon = parse_toml_u64(key, item)?, - "gc_period" => conf.gc_period = parse_toml_duration(key, item)?, - "wait_lsn_timeout" => conf.wait_lsn_timeout = parse_toml_duration(key, item)?, - "wal_redo_timeout" => conf.wal_redo_timeout = parse_toml_duration(key, item)?, - "initial_superuser_name" => conf.superuser = parse_toml_string(key, item)?, - "page_cache_size" => conf.page_cache_size = parse_toml_u64(key, item)? as usize, + "listen_pg_addr" => builder.listen_pg_addr(parse_toml_string(key, item)?), + "listen_http_addr" => builder.listen_http_addr(parse_toml_string(key, item)?), + "checkpoint_distance" => builder.checkpoint_distance(parse_toml_u64(key, item)?), + "checkpoint_period" => builder.checkpoint_period(parse_toml_duration(key, item)?), + "gc_horizon" => builder.gc_horizon(parse_toml_u64(key, item)?), + "gc_period" => builder.gc_period(parse_toml_duration(key, item)?), + "wait_lsn_timeout" => builder.wait_lsn_timeout(parse_toml_duration(key, item)?), + "wal_redo_timeout" => builder.wal_redo_timeout(parse_toml_duration(key, item)?), + "initial_superuser_name" => builder.superuser(parse_toml_string(key, item)?), + "page_cache_size" => builder.page_cache_size(parse_toml_u64(key, item)? as usize), "max_file_descriptors" => { - conf.max_file_descriptors = parse_toml_u64(key, item)? as usize + builder.max_file_descriptors(parse_toml_u64(key, item)? as usize) } "pg_distrib_dir" => { - conf.pg_distrib_dir = PathBuf::from(parse_toml_string(key, item)?) + builder.pg_distrib_dir(PathBuf::from(parse_toml_string(key, item)?)) } - "auth_validation_public_key_path" => { - conf.auth_validation_public_key_path = - Some(PathBuf::from(parse_toml_string(key, item)?)) - } - "auth_type" => conf.auth_type = parse_toml_auth_type(key, item)?, + "auth_validation_public_key_path" => builder.auth_validation_public_key_path(Some( + PathBuf::from(parse_toml_string(key, item)?), + )), + "auth_type" => builder.auth_type(parse_toml_auth_type(key, item)?), "remote_storage" => { - conf.remote_storage_config = Some(Self::parse_remote_storage_config(item)?) + builder.remote_storage_config(Some(Self::parse_remote_storage_config(item)?)) } + "id" => builder.id(ZNodeId(parse_toml_u64(key, item)?)), _ => bail!("unrecognized pageserver option '{}'", key), } } + let mut conf = builder.build().context("invalid config")?; + if conf.auth_type == AuthType::ZenithJWT { let auth_validation_public_key_path = conf .auth_validation_public_key_path @@ -301,9 +485,6 @@ impl PageServerConf { ); } - if conf.pg_distrib_dir == PathBuf::new() { - conf.pg_distrib_dir = env::current_dir()?.join("tmp_install") - }; if !conf.pg_distrib_dir.join("bin/postgres").exists() { bail!( "Can't find postgres binary at {}", @@ -398,6 +579,7 @@ impl PageServerConf { #[cfg(test)] pub fn dummy_conf(repo_dir: PathBuf) -> Self { PageServerConf { + id: ZNodeId(0), checkpoint_distance: defaults::DEFAULT_CHECKPOINT_DISTANCE, checkpoint_period: Duration::from_secs(10), gc_horizon: defaults::DEFAULT_GC_HORIZON, @@ -482,15 +664,16 @@ max_file_descriptors = 333 # initial superuser role name to use when creating a new tenant initial_superuser_name = 'zzzz' +id = 10 - "#; +"#; #[test] fn parse_defaults() -> anyhow::Result<()> { let tempdir = tempdir()?; let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?; // we have to create dummy pathes to overcome the validation errors - let config_string = format!("pg_distrib_dir='{}'", pg_distrib_dir.display()); + let config_string = format!("pg_distrib_dir='{}'\nid=10", pg_distrib_dir.display()); let toml = config_string.parse()?; let parsed_config = @@ -501,6 +684,7 @@ initial_superuser_name = 'zzzz' assert_eq!( parsed_config, PageServerConf { + id: ZNodeId(10), listen_pg_addr: defaults::DEFAULT_PG_LISTEN_ADDR.to_string(), listen_http_addr: defaults::DEFAULT_HTTP_LISTEN_ADDR.to_string(), checkpoint_distance: defaults::DEFAULT_CHECKPOINT_DISTANCE, @@ -544,6 +728,7 @@ initial_superuser_name = 'zzzz' assert_eq!( parsed_config, PageServerConf { + id: ZNodeId(10), listen_pg_addr: "127.0.0.1:64000".to_string(), listen_http_addr: "127.0.0.1:9898".to_string(), checkpoint_distance: 111, diff --git a/pageserver/src/http/models.rs b/pageserver/src/http/models.rs index 6ce377c535..5d7398ef03 100644 --- a/pageserver/src/http/models.rs +++ b/pageserver/src/http/models.rs @@ -1,6 +1,7 @@ use serde::{Deserialize, Serialize}; use crate::ZTenantId; +use zenith_utils::zid::ZNodeId; #[derive(Serialize, Deserialize)] pub struct BranchCreateRequest { @@ -15,3 +16,8 @@ pub struct TenantCreateRequest { #[serde(with = "hex")] pub tenant_id: ZTenantId, } + +#[derive(Serialize)] +pub struct StatusResponse { + pub id: ZNodeId, +} diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index dcb81849e0..baf81fcf21 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -17,6 +17,11 @@ paths: application/json: schema: type: object + required: + - id + properties: + id: + type: integer /v1/timeline/{tenant_id}: parameters: - name: tenant_id diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index b13a45750e..4fc41d6e82 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -1,7 +1,6 @@ use std::sync::Arc; use anyhow::{Context, Result}; -use hyper::header; use hyper::StatusCode; use hyper::{Body, Request, Response, Uri}; use serde::Serialize; @@ -23,6 +22,7 @@ use zenith_utils::lsn::Lsn; use zenith_utils::zid::{opt_display_serde, ZTimelineId}; use super::models::BranchCreateRequest; +use super::models::StatusResponse; use super::models::TenantCreateRequest; use crate::branches::BranchInfo; use crate::repository::RepositoryTimeline; @@ -64,12 +64,12 @@ fn get_config(request: &Request) -> &'static PageServerConf { } // healthcheck handler -async fn status_handler(_: Request) -> Result, ApiError> { - Ok(Response::builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, "application/json") - .body(Body::from("{}")) - .map_err(ApiError::from_err)?) +async fn status_handler(request: Request) -> Result, ApiError> { + let config = get_config(&request); + Ok(json_response( + StatusCode::OK, + StatusResponse { id: config.id }, + )?) } async fn branch_create_handler(mut request: Request) -> Result, ApiError> { diff --git a/test_runner/README.md b/test_runner/README.md index 514c5f1e3a..a56c2df2c0 100644 --- a/test_runner/README.md +++ b/test_runner/README.md @@ -89,7 +89,7 @@ def test_foobar(zenith_env_builder: ZenithEnvBuilder): # Now create the environment. This initializes the repository, and starts # up the page server and the safekeepers - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() # Run the test ... diff --git a/test_runner/batch_others/test_auth.py b/test_runner/batch_others/test_auth.py index 7f86986e2e..ee1a09c917 100644 --- a/test_runner/batch_others/test_auth.py +++ b/test_runner/batch_others/test_auth.py @@ -8,7 +8,7 @@ import pytest def test_pageserver_auth(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.pageserver_auth_enabled = True - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() ps = env.pageserver @@ -51,7 +51,7 @@ def test_compute_auth_to_pageserver(zenith_env_builder: ZenithEnvBuilder, with_w zenith_env_builder.pageserver_auth_enabled = True if with_wal_acceptors: zenith_env_builder.num_safekeepers = 3 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() branch = f"test_compute_auth_to_pageserver{with_wal_acceptors}" env.zenith_cli.create_branch(branch, "main") diff --git a/test_runner/batch_others/test_backpressure.py b/test_runner/batch_others/test_backpressure.py index 23af5b90ed..2b064c9fa8 100644 --- a/test_runner/batch_others/test_backpressure.py +++ b/test_runner/batch_others/test_backpressure.py @@ -93,7 +93,7 @@ def check_backpressure(pg: Postgres, stop_event: threading.Event, polling_interv def test_backpressure_received_lsn_lag(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 1 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() # Create a branch for us env.zenith_cli.create_branch("test_backpressure", "main") diff --git a/test_runner/batch_others/test_branch_behind.py b/test_runner/batch_others/test_branch_behind.py index 860db51c8a..509c46975e 100644 --- a/test_runner/batch_others/test_branch_behind.py +++ b/test_runner/batch_others/test_branch_behind.py @@ -19,7 +19,7 @@ def test_branch_behind(zenith_env_builder: ZenithEnvBuilder): # # See https://github.com/zenithdb/zenith/issues/1068 zenith_env_builder.num_safekeepers = 1 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() # Branch at the point where only 100 rows were inserted env.zenith_cli.create_branch("test_branch_behind", "main") diff --git a/test_runner/batch_others/test_next_xid.py b/test_runner/batch_others/test_next_xid.py index 625abc39d3..fd0f761409 100644 --- a/test_runner/batch_others/test_next_xid.py +++ b/test_runner/batch_others/test_next_xid.py @@ -11,7 +11,7 @@ from fixtures.log_helper import log def test_next_xid(zenith_env_builder: ZenithEnvBuilder): # One safekeeper is enough for this test. zenith_env_builder.num_safekeepers = 1 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() pg = env.postgres.create_start('main') diff --git a/test_runner/batch_others/test_pageserver_api.py b/test_runner/batch_others/test_pageserver_api.py index eccffc4d69..ba1f106c4b 100644 --- a/test_runner/batch_others/test_pageserver_api.py +++ b/test_runner/batch_others/test_pageserver_api.py @@ -1,8 +1,15 @@ -import json from uuid import uuid4, UUID -from fixtures.zenith_fixtures import ZenithEnv, ZenithEnvBuilder, ZenithPageserverHttpClient -from typing import cast -import pytest, psycopg2 +import pytest +from fixtures.zenith_fixtures import ZenithEnv, ZenithEnvBuilder, ZenithPageserverHttpClient, zenith_binpath + + +# test that we cannot override node id +def test_pageserver_init_node_id(zenith_env_builder: ZenithEnvBuilder): + env = zenith_env_builder.init() + with pytest.raises( + Exception, + match="node id can only be set during pageserver init and cannot be overridden"): + env.pageserver.start(overrides=['--pageserver-config-override=id=10']) def check_client(client: ZenithPageserverHttpClient, initial_tenant: UUID): @@ -41,7 +48,7 @@ def test_pageserver_http_api_client(zenith_simple_env: ZenithEnv): def test_pageserver_http_api_client_auth_enabled(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.pageserver_auth_enabled = True - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() management_token = env.auth_keys.generate_management_token() diff --git a/test_runner/batch_others/test_pageserver_catchup.py b/test_runner/batch_others/test_pageserver_catchup.py index 97dc0f3260..985d1a3af0 100644 --- a/test_runner/batch_others/test_pageserver_catchup.py +++ b/test_runner/batch_others/test_pageserver_catchup.py @@ -14,7 +14,7 @@ from fixtures.log_helper import log # and new compute node contains all data. def test_pageserver_catchup_while_compute_down(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 3 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() env.zenith_cli.create_branch("test_pageserver_catchup_while_compute_down", "main") pg = env.postgres.create_start('test_pageserver_catchup_while_compute_down') diff --git a/test_runner/batch_others/test_pageserver_restart.py b/test_runner/batch_others/test_pageserver_restart.py index 0cfc50f0ff..ec93c2cf5b 100644 --- a/test_runner/batch_others/test_pageserver_restart.py +++ b/test_runner/batch_others/test_pageserver_restart.py @@ -13,7 +13,7 @@ from fixtures.log_helper import log def test_pageserver_restart(zenith_env_builder: ZenithEnvBuilder): # One safekeeper is enough for this test. zenith_env_builder.num_safekeepers = 1 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() env.zenith_cli.create_branch("test_pageserver_restart", "main") pg = env.postgres.create_start('test_pageserver_restart') diff --git a/test_runner/batch_others/test_remote_storage.py b/test_runner/batch_others/test_remote_storage.py index fa6feaf412..61feb1a5bd 100644 --- a/test_runner/batch_others/test_remote_storage.py +++ b/test_runner/batch_others/test_remote_storage.py @@ -42,7 +42,7 @@ def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder, data_secret = 'very secret secret' ##### First start, insert secret data and upload it to the remote storage - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() pg = env.postgres.create_start() tenant_id = pg.safe_psql("show zenith.zenith_tenant")[0][0] diff --git a/test_runner/batch_others/test_restart_compute.py b/test_runner/batch_others/test_restart_compute.py index f7810be555..d4dd3fb9e2 100644 --- a/test_runner/batch_others/test_restart_compute.py +++ b/test_runner/batch_others/test_restart_compute.py @@ -13,7 +13,7 @@ def test_restart_compute(zenith_env_builder: ZenithEnvBuilder, with_wal_acceptor zenith_env_builder.pageserver_auth_enabled = True if with_wal_acceptors: zenith_env_builder.num_safekeepers = 3 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() env.zenith_cli.create_branch("test_restart_compute", "main") diff --git a/test_runner/batch_others/test_tenant_relocation.py b/test_runner/batch_others/test_tenant_relocation.py index 5c6d78e730..acff3ef62c 100644 --- a/test_runner/batch_others/test_tenant_relocation.py +++ b/test_runner/batch_others/test_tenant_relocation.py @@ -122,7 +122,7 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder, zenith_env_builder.num_safekeepers = 1 zenith_env_builder.enable_local_fs_remote_storage() - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() # create folder for remote storage mock remote_storage_mock_path = env.repo_dir / 'local_fs_remote_storage' diff --git a/test_runner/batch_others/test_tenants.py b/test_runner/batch_others/test_tenants.py index 232c724870..b665ae9022 100644 --- a/test_runner/batch_others/test_tenants.py +++ b/test_runner/batch_others/test_tenants.py @@ -10,7 +10,7 @@ def test_tenants_normal_work(zenith_env_builder: ZenithEnvBuilder, with_wal_acce if with_wal_acceptors: zenith_env_builder.num_safekeepers = 3 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() """Tests tenants with and without wal acceptors""" tenant_1 = env.create_tenant() tenant_2 = env.create_tenant() diff --git a/test_runner/batch_others/test_timeline_size.py b/test_runner/batch_others/test_timeline_size.py index b48f830528..2c31267922 100644 --- a/test_runner/batch_others/test_timeline_size.py +++ b/test_runner/batch_others/test_timeline_size.py @@ -67,7 +67,7 @@ def wait_for_pageserver_catchup(pgmain: Postgres, polling_interval=1, timeout=60 def test_timeline_size_quota(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 1 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() env.zenith_cli.create_branch("test_timeline_size_quota", "main") client = env.pageserver.http_client() diff --git a/test_runner/batch_others/test_wal_acceptor.py b/test_runner/batch_others/test_wal_acceptor.py index 4d9e18bb58..c375c9626a 100644 --- a/test_runner/batch_others/test_wal_acceptor.py +++ b/test_runner/batch_others/test_wal_acceptor.py @@ -22,7 +22,7 @@ from typing import List, Optional, Any # succeed and data is written def test_normal_work(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 3 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() env.zenith_cli.create_branch("test_wal_acceptors_normal_work", "main") @@ -51,7 +51,7 @@ class BranchMetrics: # against different timelines. def test_many_timelines(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 3 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() n_timelines = 3 @@ -181,7 +181,7 @@ def test_restarts(zenith_env_builder: ZenithEnvBuilder): n_acceptors = 3 zenith_env_builder.num_safekeepers = n_acceptors - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() env.zenith_cli.create_branch("test_wal_acceptors_restarts", "main") pg = env.postgres.create_start('test_wal_acceptors_restarts') @@ -218,7 +218,7 @@ def delayed_wal_acceptor_start(wa): # When majority of acceptors is offline, commits are expected to be frozen def test_unavailability(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 2 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() env.zenith_cli.create_branch("test_wal_acceptors_unavailability", "main") pg = env.postgres.create_start('test_wal_acceptors_unavailability') @@ -289,7 +289,7 @@ def stop_value(): def test_race_conditions(zenith_env_builder: ZenithEnvBuilder, stop_value): zenith_env_builder.num_safekeepers = 3 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() env.zenith_cli.create_branch("test_wal_acceptors_race_conditions", "main") pg = env.postgres.create_start('test_wal_acceptors_race_conditions') @@ -404,7 +404,7 @@ def test_sync_safekeepers(zenith_env_builder: ZenithEnvBuilder, # We don't really need the full environment for this test, just the # safekeepers would be enough. zenith_env_builder.num_safekeepers = 3 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() timeline_id = uuid.uuid4() tenant_id = uuid.uuid4() @@ -454,7 +454,7 @@ def test_sync_safekeepers(zenith_env_builder: ZenithEnvBuilder, def test_timeline_status(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 1 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() env.zenith_cli.create_branch("test_timeline_status", "main") pg = env.postgres.create_start('test_timeline_status') @@ -521,12 +521,7 @@ class SafekeeperEnv: http=self.port_distributor.get_port(), ) - if self.num_safekeepers == 1: - name = "single" - else: - name = f"sk{i}" - - safekeeper_dir = os.path.join(self.repo_dir, name) + safekeeper_dir = os.path.join(self.repo_dir, f"sk{i}") mkdir_if_needed(safekeeper_dir) args = [ @@ -537,6 +532,8 @@ class SafekeeperEnv: f"127.0.0.1:{port.http}", "-D", safekeeper_dir, + "--id", + str(i), "--daemonize" ] @@ -604,9 +601,8 @@ def test_safekeeper_without_pageserver(test_output_dir: str, def test_replace_safekeeper(zenith_env_builder: ZenithEnvBuilder): - def safekeepers_guc(env: ZenithEnv, sk_names: List[str]) -> str: - return ','.join( - [f'localhost:{sk.port.pg}' for sk in env.safekeepers if sk.name in sk_names]) + def safekeepers_guc(env: ZenithEnv, sk_names: List[int]) -> str: + return ','.join([f'localhost:{sk.port.pg}' for sk in env.safekeepers if sk.id in sk_names]) def execute_payload(pg: Postgres): with closing(pg.connect()) as conn: @@ -628,17 +624,17 @@ def test_replace_safekeeper(zenith_env_builder: ZenithEnvBuilder): http_cli = sk.http_client() try: status = http_cli.timeline_status(tenant_id, timeline_id) - log.info(f"Safekeeper {sk.name} status: {status}") + log.info(f"Safekeeper {sk.id} status: {status}") except Exception as e: - log.info(f"Safekeeper {sk.name} status error: {e}") + log.info(f"Safekeeper {sk.id} status error: {e}") zenith_env_builder.num_safekeepers = 4 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() env.zenith_cli.create_branch("test_replace_safekeeper", "main") log.info("Use only first 3 safekeepers") env.safekeepers[3].stop() - active_safekeepers = ['sk1', 'sk2', 'sk3'] + active_safekeepers = [1, 2, 3] pg = env.postgres.create('test_replace_safekeeper') pg.adjust_for_wal_acceptors(safekeepers_guc(env, active_safekeepers)) pg.start() @@ -678,7 +674,7 @@ def test_replace_safekeeper(zenith_env_builder: ZenithEnvBuilder): log.info("Recreate postgres to replace failed sk1 with new sk4") pg.stop_and_destroy().create('test_replace_safekeeper') - active_safekeepers = ['sk2', 'sk3', 'sk4'] + active_safekeepers = [2, 3, 4] env.safekeepers[3].start() pg.adjust_for_wal_acceptors(safekeepers_guc(env, active_safekeepers)) pg.start() diff --git a/test_runner/batch_others/test_wal_acceptor_async.py b/test_runner/batch_others/test_wal_acceptor_async.py index 1d2a186eb7..4b6a27f73d 100644 --- a/test_runner/batch_others/test_wal_acceptor_async.py +++ b/test_runner/batch_others/test_wal_acceptor_async.py @@ -200,7 +200,7 @@ async def run_restarts_under_load(pg: Postgres, acceptors: List[Safekeeper], n_w # restart acceptors one by one, while executing and validating bank transactions def test_restarts_under_load(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 3 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() env.zenith_cli.create_branch("test_wal_acceptors_restarts_under_load", "main") pg = env.postgres.create_start('test_wal_acceptors_restarts_under_load') diff --git a/test_runner/batch_others/test_zenith_cli.py b/test_runner/batch_others/test_zenith_cli.py index ce051dfd6e..f1897e4b6f 100644 --- a/test_runner/batch_others/test_zenith_cli.py +++ b/test_runner/batch_others/test_zenith_cli.py @@ -97,7 +97,7 @@ def test_cli_tenant_list(zenith_simple_env: ZenithEnv): def test_cli_ipv4_listeners(zenith_env_builder: ZenithEnvBuilder): # Start with single sk zenith_env_builder.num_safekeepers = 1 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() # Connect to sk port on v4 loopback res = requests.get(f'http://127.0.0.1:{env.safekeepers[0].port.http}/v1/status') @@ -114,7 +114,7 @@ def test_cli_ipv4_listeners(zenith_env_builder: ZenithEnvBuilder): def test_cli_start_stop(zenith_env_builder: ZenithEnvBuilder): # Start with single sk zenith_env_builder.num_safekeepers = 1 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() # Stop default ps/sk env.zenith_cli.pageserver_stop() diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index b4b3de1db3..252ca9b3c1 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -27,7 +27,7 @@ from dataclasses import dataclass # Type-related stuff from psycopg2.extensions import connection as PgConnection -from typing import Any, Callable, Dict, Iterator, List, Optional, TypeVar, cast, Union, Tuple +from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, TypeVar, cast, Union, Tuple from typing_extensions import Literal import pytest @@ -434,6 +434,14 @@ class ZenithEnvBuilder: self.env = ZenithEnv(self) return self.env + def start(self): + self.env.start() + + def init_start(self) -> ZenithEnv: + env = self.init() + self.start() + return env + """ Sets up the pageserver to use the local fs at the `test_dir/local_fs_remote_storage` path. Errors, if the pageserver has some remote storage configuration already, unless `force_enable` is not set to `True`. @@ -549,6 +557,7 @@ class ZenithEnv: toml += textwrap.dedent(f""" [pageserver] + id=1 listen_pg_addr = 'localhost:{pageserver_port.pg}' listen_http_addr = 'localhost:{pageserver_port.http}' auth_type = '{pageserver_auth_type}' @@ -566,25 +575,22 @@ class ZenithEnv: pg=self.port_distributor.get_port(), http=self.port_distributor.get_port(), ) - - if config.num_safekeepers == 1: - name = "single" - else: - name = f"sk{i}" - toml += f""" -[[safekeepers]] -name = '{name}' -pg_port = {port.pg} -http_port = {port.http} -sync = false # Disable fsyncs to make the tests go faster - """ - safekeeper = Safekeeper(env=self, name=name, port=port) + id = i # assign ids sequentially + toml += textwrap.dedent(f""" + [[safekeepers]] + id = {id} + pg_port = {port.pg} + http_port = {port.http} + sync = false # Disable fsyncs to make the tests go faster + """) + safekeeper = Safekeeper(env=self, id=id, port=port) self.safekeepers.append(safekeeper) log.info(f"Config: {toml}") self.zenith_cli.init(toml) + def start(self): # Start up the page server and all the safekeepers self.pageserver.start() @@ -625,7 +631,7 @@ def _shared_simple_env(request: Any, port_distributor) -> Iterator[ZenithEnv]: with ZenithEnvBuilder(Path(repo_dir), port_distributor) as builder: - env = builder.init() + env = builder.init_start() # For convenience in tests, create a branch from the freshly-initialized cluster. env.zenith_cli.create_branch("empty", "main") @@ -659,7 +665,7 @@ def zenith_env_builder(test_output_dir, port_distributor) -> Iterator[ZenithEnvB To use, define 'zenith_env_builder' fixture in your test to get access to the builder object. Set properties on it to describe the environment. Finally, initialize and start up the environment by calling - zenith_env_builder.init(). + zenith_env_builder.init_start(). After the initialization, you can launch compute nodes by calling the functions in the 'env.postgres' factory object, stop/start the @@ -847,8 +853,8 @@ class ZenithCli: return self.raw_cli(cmd) - def pageserver_start(self) -> 'subprocess.CompletedProcess[str]': - start_args = ['pageserver', 'start'] + def pageserver_start(self, overrides=()) -> 'subprocess.CompletedProcess[str]': + start_args = ['pageserver', 'start', *overrides] append_pageserver_param_overrides(start_args, self.env.pageserver.remote_storage, self.env.pageserver.config_override) @@ -862,17 +868,17 @@ class ZenithCli: log.info(f"Stopping pageserver with {cmd}") return self.raw_cli(cmd) - def safekeeper_start(self, name: str) -> 'subprocess.CompletedProcess[str]': - return self.raw_cli(['safekeeper', 'start', name]) + def safekeeper_start(self, id: int) -> 'subprocess.CompletedProcess[str]': + return self.raw_cli(['safekeeper', 'start', str(id)]) def safekeeper_stop(self, - name: Optional[str] = None, + id: Optional[int] = None, immediate=False) -> 'subprocess.CompletedProcess[str]': args = ['safekeeper', 'stop'] + if id is not None: + args.extend(str(id)) if immediate: args.extend(['-m', 'immediate']) - if name is not None: - args.append(name) return self.raw_cli(args) def pg_create( @@ -1005,14 +1011,15 @@ class ZenithPageserver(PgProtocol): self.remote_storage = remote_storage self.config_override = config_override - def start(self) -> 'ZenithPageserver': + def start(self, overrides=()) -> 'ZenithPageserver': """ Start the page server. + `overrides` allows to add some config to this pageserver start. Returns self. """ assert self.running == False - self.env.zenith_cli.pageserver_start() + self.env.zenith_cli.pageserver_start(overrides=overrides) self.running = True return self @@ -1466,12 +1473,14 @@ class Safekeeper: """ An object representing a running safekeeper daemon. """ env: ZenithEnv port: SafekeeperPort - name: str # identifier for logging + id: int auth_token: Optional[str] = None + running: bool = False def start(self) -> 'Safekeeper': - self.env.zenith_cli.safekeeper_start(self.name) - + assert self.running == False + self.env.zenith_cli.safekeeper_start(self.id) + self.running = True # wait for wal acceptor start by checking its status started_at = time.time() while True: @@ -1489,8 +1498,9 @@ class Safekeeper: return self def stop(self, immediate=False) -> 'Safekeeper': - log.info('Stopping safekeeper {}'.format(self.name)) - self.env.zenith_cli.safekeeper_stop(self.name, immediate) + log.info('Stopping safekeeper {}'.format(self.id)) + self.env.zenith_cli.safekeeper_stop(self.id, immediate) + self.running = False return self def append_logical_message(self, diff --git a/test_runner/performance/test_bulk_tenant_create.py b/test_runner/performance/test_bulk_tenant_create.py index 6fd77f3020..0247385211 100644 --- a/test_runner/performance/test_bulk_tenant_create.py +++ b/test_runner/performance/test_bulk_tenant_create.py @@ -23,7 +23,7 @@ def test_bulk_tenant_create( """Measure tenant creation time (with and without wal acceptors)""" if use_wal_acceptors == 'with_wa': zenith_env_builder.num_safekeepers = 3 - env = zenith_env_builder.init() + env = zenith_env_builder.init_start() time_slices = [] diff --git a/walkeeper/src/bin/safekeeper.rs b/walkeeper/src/bin/safekeeper.rs index ea5d0cba14..48de1481d4 100644 --- a/walkeeper/src/bin/safekeeper.rs +++ b/walkeeper/src/bin/safekeeper.rs @@ -1,17 +1,19 @@ // // Main entry point for the safekeeper executable // -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use clap::{App, Arg}; use const_format::formatcp; use daemonize::Daemonize; use fs2::FileExt; -use std::fs::File; +use std::fs::{self, File}; +use std::io::{ErrorKind, Write}; use std::path::{Path, PathBuf}; use std::thread; use tracing::*; use walkeeper::control_file::{self, CreateControlFile}; use zenith_utils::http::endpoint; +use zenith_utils::zid::ZNodeId; use zenith_utils::{logging, tcp_listener, GIT_VERSION}; use tokio::sync::mpsc; @@ -25,6 +27,7 @@ use zenith_utils::shutdown::exit_now; use zenith_utils::signals; const LOCK_FILE_NAME: &str = "safekeeper.lock"; +const ID_FILE_NAME: &str = "safekeeper.id"; fn main() -> Result<()> { zenith_metrics::set_common_metrics_prefix("safekeeper"); @@ -38,6 +41,12 @@ fn main() -> Result<()> { .takes_value(true) .help("Path to the safekeeper data directory"), ) + .arg( + Arg::new("init") + .long("init") + .takes_value(false) + .help("Initialize safekeeper with ID"), + ) .arg( Arg::new("listen-pg") .short('l') @@ -93,6 +102,9 @@ fn main() -> Result<()> { .takes_value(true) .help("Dump control file at path specifed by this argument and exit"), ) + .arg( + Arg::new("id").long("id").takes_value(true).help("safekeeper node id: integer") + ) .get_matches(); if let Some(addr) = arg_matches.value_of("dump-control-file") { @@ -136,10 +148,19 @@ fn main() -> Result<()> { conf.recall_period = humantime::parse_duration(recall)?; } - start_safekeeper(conf) + let mut given_id = None; + if let Some(given_id_str) = arg_matches.value_of("id") { + given_id = Some(ZNodeId( + given_id_str + .parse() + .context("failed to parse safekeeper id")?, + )); + } + + start_safekeeper(conf, given_id, arg_matches.is_present("init")) } -fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { +fn start_safekeeper(mut conf: SafeKeeperConf, given_id: Option, init: bool) -> Result<()> { let log_file = logging::init("safekeeper.log", conf.daemonize)?; info!("version: {}", GIT_VERSION); @@ -154,6 +175,12 @@ fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { ) })?; + // Set or read our ID. + set_id(&mut conf, given_id)?; + if init { + return Ok(()); + } + let http_listener = tcp_listener::bind(conf.listen_http_addr.clone()).map_err(|e| { error!("failed to bind to address {}: {}", conf.listen_http_addr, e); e @@ -260,3 +287,49 @@ fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { std::process::exit(111); }) } + +/// Determine safekeeper id and set it in config. +fn set_id(conf: &mut SafeKeeperConf, given_id: Option) -> Result<()> { + let id_file_path = conf.workdir.join(ID_FILE_NAME); + + let my_id: ZNodeId; + // If ID exists, read it in; otherwise set one passed + match fs::read(&id_file_path) { + Ok(id_serialized) => { + my_id = ZNodeId( + std::str::from_utf8(&id_serialized) + .context("failed to parse safekeeper id")? + .parse() + .context("failed to parse safekeeper id")?, + ); + if let Some(given_id) = given_id { + if given_id != my_id { + bail!( + "safekeeper already initialized with id {}, can't set {}", + my_id, + given_id + ); + } + } + info!("safekeeper ID {}", my_id); + } + Err(error) => match error.kind() { + ErrorKind::NotFound => { + my_id = if let Some(given_id) = given_id { + given_id + } else { + bail!("safekeeper id is not specified"); + }; + let mut f = File::create(&id_file_path)?; + f.write_all(my_id.to_string().as_bytes())?; + f.sync_all()?; + info!("initialized safekeeper ID {}", my_id); + } + _ => { + return Err(error.into()); + } + }, + } + conf.my_id = my_id; + Ok(()) +} diff --git a/walkeeper/src/http/routes.rs b/walkeeper/src/http/routes.rs index 11a29ac6d3..bc992c6a6f 100644 --- a/walkeeper/src/http/routes.rs +++ b/walkeeper/src/http/routes.rs @@ -5,6 +5,7 @@ use std::fmt::Display; use std::sync::Arc; use zenith_utils::http::{RequestExt, RouterBuilder}; use zenith_utils::lsn::Lsn; +use zenith_utils::zid::ZNodeId; use zenith_utils::zid::ZTenantTimelineId; use crate::control_file::CreateControlFile; @@ -18,9 +19,16 @@ use zenith_utils::http::json::json_response; use zenith_utils::http::request::parse_request_param; use zenith_utils::zid::{ZTenantId, ZTimelineId}; +#[derive(Debug, Serialize)] +struct SafekeeperStatus { + id: ZNodeId, +} + /// Healthcheck handler. -async fn status_handler(_: Request) -> Result, ApiError> { - Ok(json_response(StatusCode::OK, "")?) +async fn status_handler(request: Request) -> Result, ApiError> { + let conf = get_conf(&request); + let status = SafekeeperStatus { id: conf.my_id }; + Ok(json_response(StatusCode::OK, status)?) } fn get_conf(request: &Request) -> &SafeKeeperConf { diff --git a/walkeeper/src/lib.rs b/walkeeper/src/lib.rs index 6c3e0b264e..dfd71e4de2 100644 --- a/walkeeper/src/lib.rs +++ b/walkeeper/src/lib.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use std::time::Duration; -use zenith_utils::zid::ZTenantTimelineId; +use zenith_utils::zid::{ZNodeId, ZTenantTimelineId}; pub mod callmemaybe; pub mod control_file; @@ -46,6 +46,7 @@ pub struct SafeKeeperConf { pub listen_http_addr: String, pub ttl: Option, pub recall_period: Duration, + pub my_id: ZNodeId, } impl SafeKeeperConf { @@ -69,6 +70,7 @@ impl Default for SafeKeeperConf { listen_http_addr: defaults::DEFAULT_HTTP_LISTEN_ADDR.to_string(), ttl: None, recall_period: defaults::DEFAULT_RECALL_PERIOD, + my_id: ZNodeId(0), } } } diff --git a/zenith/src/main.rs b/zenith/src/main.rs index a2a762f5be..5500d924ea 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -18,32 +18,35 @@ use walkeeper::defaults::{ }; use zenith_utils::auth::{Claims, Scope}; use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{ZTenantId, ZTimelineId}; +use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId}; use zenith_utils::GIT_VERSION; use pageserver::branches::BranchInfo; -// Default name of a safekeeper node, if not specified on the command line. -const DEFAULT_SAFEKEEPER_NAME: &str = "single"; +// Default id of a safekeeper node, if not specified on the command line. +const DEFAULT_SAFEKEEPER_ID: ZNodeId = ZNodeId(1); +const DEFAULT_PAGESERVER_ID: ZNodeId = ZNodeId(1); fn default_conf() -> String { format!( r#" # Default built-in configuration, defined in main.rs [pageserver] +id = {pageserver_id} listen_pg_addr = '{pageserver_pg_addr}' listen_http_addr = '{pageserver_http_addr}' auth_type = '{pageserver_auth_type}' [[safekeepers]] -name = '{safekeeper_name}' +id = {safekeeper_id} pg_port = {safekeeper_pg_port} http_port = {safekeeper_http_port} "#, + pageserver_id = DEFAULT_PAGESERVER_ID, pageserver_pg_addr = DEFAULT_PAGESERVER_PG_ADDR, pageserver_http_addr = DEFAULT_PAGESERVER_HTTP_ADDR, pageserver_auth_type = AuthType::Trust, - safekeeper_name = DEFAULT_SAFEKEEPER_NAME, + safekeeper_id = DEFAULT_SAFEKEEPER_ID, safekeeper_pg_port = DEFAULT_SAFEKEEPER_PG_PORT, safekeeper_http_port = DEFAULT_SAFEKEEPER_HTTP_PORT, ) @@ -74,9 +77,9 @@ fn main() -> Result<()> { .required(true); #[rustfmt::skip] - let safekeeper_node_arg = Arg::new("node") + let safekeeper_id_arg = Arg::new("id") .index(1) - .help("Node name") + .help("safekeeper id") .required(false); let timeline_arg = Arg::new("timeline") @@ -154,16 +157,16 @@ fn main() -> Result<()> { .about("Manage safekeepers") .subcommand(App::new("start") .about("Start local safekeeper") - .arg(safekeeper_node_arg.clone()) + .arg(safekeeper_id_arg.clone()) ) .subcommand(App::new("stop") .about("Stop local safekeeper") - .arg(safekeeper_node_arg.clone()) + .arg(safekeeper_id_arg.clone()) .arg(stop_mode_arg.clone()) ) .subcommand(App::new("restart") .about("Restart local safekeeper") - .arg(safekeeper_node_arg.clone()) + .arg(safekeeper_id_arg.clone()) .arg(stop_mode_arg.clone()) ) ) @@ -628,11 +631,11 @@ fn handle_pageserver(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul Ok(()) } -fn get_safekeeper(env: &local_env::LocalEnv, name: &str) -> Result { - if let Some(node) = env.safekeepers.iter().find(|node| node.name == name) { +fn get_safekeeper(env: &local_env::LocalEnv, id: ZNodeId) -> Result { + if let Some(node) = env.safekeepers.iter().find(|node| node.id == id) { Ok(SafekeeperNode::from_env(env, node)) } else { - bail!("could not find safekeeper '{}'", name) + bail!("could not find safekeeper '{}'", id) } } @@ -643,8 +646,12 @@ fn handle_safekeeper(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul }; // All the commands take an optional safekeeper name argument - let node_name = sub_args.value_of("node").unwrap_or(DEFAULT_SAFEKEEPER_NAME); - let safekeeper = get_safekeeper(env, node_name)?; + let sk_id = if let Some(id_str) = sub_args.value_of("id") { + ZNodeId(id_str.parse().context("while parsing safekeeper id")?) + } else { + DEFAULT_SAFEKEEPER_ID + }; + let safekeeper = get_safekeeper(env, sk_id)?; match sub_name { "start" => { @@ -697,7 +704,7 @@ fn handle_start_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result for node in env.safekeepers.iter() { let safekeeper = SafekeeperNode::from_env(env, node); if let Err(e) = safekeeper.start() { - eprintln!("safekeeper '{}' start failed: {}", safekeeper.name, e); + eprintln!("safekeeper '{}' start failed: {}", safekeeper.id, e); exit(1); } } @@ -724,7 +731,7 @@ fn handle_stop_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result< for node in env.safekeepers.iter() { let safekeeper = SafekeeperNode::from_env(env, node); if let Err(e) = safekeeper.stop(immediate) { - eprintln!("safekeeper '{}' stop failed: {}", safekeeper.name, e); + eprintln!("safekeeper '{}' stop failed: {}", safekeeper.id, e); } } Ok(()) diff --git a/zenith_utils/src/zid.rs b/zenith_utils/src/zid.rs index 2e93ab596c..7dfffd96d7 100644 --- a/zenith_utils/src/zid.rs +++ b/zenith_utils/src/zid.rs @@ -221,6 +221,18 @@ impl fmt::Display for ZTenantTimelineId { } } +// Unique ID of a storage node (safekeeper or pageserver). Supposed to be issued +// by the console. +#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Debug, Serialize, Deserialize)] +#[serde(transparent)] +pub struct ZNodeId(pub u64); + +impl fmt::Display for ZNodeId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + #[cfg(test)] mod tests { use std::fmt::Display; From 9424bfae22d6a808371959c87aa1106701a34ad5 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Tue, 1 Mar 2022 22:34:42 +0200 Subject: [PATCH 04/42] Use a separate newtype for ZId that (de)serialize as hex strings --- control_plane/src/local_env.rs | 7 +- pageserver/src/branches.rs | 3 +- pageserver/src/http/routes.rs | 10 +- zenith/src/main.rs | 4 +- zenith_utils/src/auth.rs | 44 ++------ zenith_utils/src/zid.rs | 198 ++++++++++++++++++++++++++------- 6 files changed, 179 insertions(+), 87 deletions(-) diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 55d0b00496..238c78821e 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -12,7 +12,7 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use zenith_utils::auth::{encode_from_key_file, Claims, Scope}; use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{opt_display_serde, ZNodeId, ZTenantId}; +use zenith_utils::zid::{HexZTenantId, ZNodeId, ZTenantId}; use crate::safekeeper::SafekeeperNode; @@ -47,9 +47,8 @@ pub struct LocalEnv { // Default tenant ID to use with the 'zenith' command line utility, when // --tenantid is not explicitly specified. - #[serde(with = "opt_display_serde")] #[serde(default)] - pub default_tenantid: Option, + pub default_tenantid: Option, // used to issue tokens during e.g pg start #[serde(default)] @@ -185,7 +184,7 @@ impl LocalEnv { // If no initial tenant ID was given, generate it. if env.default_tenantid.is_none() { - env.default_tenantid = Some(ZTenantId::generate()); + env.default_tenantid = Some(HexZTenantId::from(ZTenantId::generate())); } env.base_data_dir = base_path(); diff --git a/pageserver/src/branches.rs b/pageserver/src/branches.rs index 8a411060de..43f27af5ea 100644 --- a/pageserver/src/branches.rs +++ b/pageserver/src/branches.rs @@ -16,10 +16,9 @@ use std::{ }; use tracing::*; -use zenith_utils::crashsafe_dir; -use zenith_utils::logging; use zenith_utils::lsn::Lsn; use zenith_utils::zid::{ZTenantId, ZTimelineId}; +use zenith_utils::{crashsafe_dir, logging}; use crate::walredo::WalRedoManager; use crate::CheckpointConfig; diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 4fc41d6e82..26d473efaf 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -19,7 +19,8 @@ use zenith_utils::http::{ }; use zenith_utils::http::{RequestExt, RouterBuilder}; use zenith_utils::lsn::Lsn; -use zenith_utils::zid::{opt_display_serde, ZTimelineId}; +use zenith_utils::zid::HexZTimelineId; +use zenith_utils::zid::ZTimelineId; use super::models::BranchCreateRequest; use super::models::StatusResponse; @@ -198,8 +199,7 @@ enum TimelineInfo { timeline_id: ZTimelineId, #[serde(with = "hex")] tenant_id: ZTenantId, - #[serde(with = "opt_display_serde")] - ancestor_timeline_id: Option, + ancestor_timeline_id: Option, last_record_lsn: Lsn, prev_record_lsn: Lsn, disk_consistent_lsn: Lsn, @@ -232,7 +232,9 @@ async fn timeline_detail_handler(request: Request) -> Result TimelineInfo::Local { timeline_id, tenant_id, - ancestor_timeline_id: timeline.get_ancestor_timeline_id(), + ancestor_timeline_id: timeline + .get_ancestor_timeline_id() + .map(HexZTimelineId::from), disk_consistent_lsn: timeline.get_disk_consistent_lsn(), last_record_lsn: timeline.get_last_record_lsn(), prev_record_lsn: timeline.get_prev_record_lsn(), diff --git a/zenith/src/main.rs b/zenith/src/main.rs index 5500d924ea..bc42af5943 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -392,7 +392,7 @@ fn get_tenantid(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result Result<()> { let pageserver = PageServerNode::from_env(&env); if let Err(e) = pageserver.init( // default_tenantid was generated by the `env.init()` call above - Some(&env.default_tenantid.unwrap().to_string()), + Some(&ZTenantId::from(env.default_tenantid.unwrap()).to_string()), &pageserver_config_overrides(init_match), ) { eprintln!("pageserver init failed: {}", e); diff --git a/zenith_utils/src/auth.rs b/zenith_utils/src/auth.rs index 274dd13bee..cbc4fcee61 100644 --- a/zenith_utils/src/auth.rs +++ b/zenith_utils/src/auth.rs @@ -5,9 +5,7 @@ // The second one is that we wanted to use ed25519 keys, but they are also not supported until next version. So we go with RSA keys for now. // Relevant issue: https://github.com/Keats/jsonwebtoken/issues/162 -use hex::{self, FromHex}; -use serde::de::Error; -use serde::{self, Deserializer, Serializer}; +use serde; use std::fs; use std::path::Path; @@ -17,7 +15,7 @@ use jsonwebtoken::{ }; use serde::{Deserialize, Serialize}; -use crate::zid::ZTenantId; +use crate::zid::{HexZTenantId, ZTenantId}; const JWT_ALGORITHM: Algorithm = Algorithm::RS256; @@ -28,44 +26,18 @@ pub enum Scope { PageServerApi, } -pub fn to_hex_option(value: &Option, serializer: S) -> Result -where - S: Serializer, -{ - match value { - Some(tid) => hex::serialize(tid, serializer), - None => Option::serialize(value, serializer), - } -} - -fn from_hex_option<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let opt: Option = Option::deserialize(deserializer)?; - match opt { - Some(tid) => Ok(Some(ZTenantId::from_hex(tid).map_err(Error::custom)?)), - None => Ok(None), - } -} - #[derive(Debug, Serialize, Deserialize, Clone)] pub struct Claims { - // this custom serialize/deserialize_with is needed because Option is not transparent to serde - // so clearest option is serde(with = "hex") but it is not working, for details see https://github.com/serde-rs/serde/issues/1301 - #[serde( - default, - skip_serializing_if = "Option::is_none", - serialize_with = "to_hex_option", - deserialize_with = "from_hex_option" - )] - pub tenant_id: Option, + pub tenant_id: Option, pub scope: Scope, } impl Claims { pub fn new(tenant_id: Option, scope: Scope) -> Self { - Self { tenant_id, scope } + Self { + tenant_id: tenant_id.map(HexZTenantId::from), + scope, + } } } @@ -75,7 +47,7 @@ pub fn check_permission(claims: &Claims, tenantid: Option) -> Result< bail!("Attempt to access management api with tenant scope. Permission denied") } (Scope::Tenant, Some(tenantid)) => { - if claims.tenant_id.unwrap() != tenantid { + if ZTenantId::from(claims.tenant_id.unwrap()) != tenantid { bail!("Tenant id mismatch. Permission denied") } Ok(()) diff --git a/zenith_utils/src/zid.rs b/zenith_utils/src/zid.rs index 7dfffd96d7..813eb3f8f4 100644 --- a/zenith_utils/src/zid.rs +++ b/zenith_utils/src/zid.rs @@ -2,13 +2,100 @@ use std::{fmt, str::FromStr}; use hex::FromHex; use rand::Rng; -use serde::{Deserialize, Serialize}; +use serde::{ + de::{self, Visitor}, + Deserialize, Serialize, +}; -// Zenith ID is a 128-bit random ID. -// Used to represent various identifiers. Provides handy utility methods and impls. +macro_rules! mutual_from { + ($id1:ident, $id2:ident) => { + impl From<$id1> for $id2 { + fn from(id1: $id1) -> Self { + Self(id1.0.into()) + } + } + + impl From<$id2> for $id1 { + fn from(id2: $id2) -> Self { + Self(id2.0.into()) + } + } + }; +} + +/// Zenith ID is a 128-bit random ID. +/// Used to represent various identifiers. Provides handy utility methods and impls. +/// +/// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look +/// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`. +/// Use [`HexZId`] to serialize it as hex string instead: `ad50847381e248feaac9876cc71ae418`. #[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)] struct ZId([u8; 16]); +/// [`ZId`] version that serializes and deserializes as a hex string. +/// Useful for various json serializations, where hex byte array from original id is not convenient. +/// +/// Plain `ZId` could be (de)serialized into hex string with `#[serde(with = "hex")]` attribute. +/// This however won't work on nested types like `Option` or `Vec`, see https://github.com/serde-rs/serde/issues/723 for the details. +/// Every separate type currently needs a new (de)serializing method for every type separately. +/// +/// To provide a generic way to serialize the ZId as a hex string where `#[serde(with = "hex")]` is not enough, this wrapper is created. +/// The default wrapper serialization is left unchanged due to +/// * byte array (de)serialization being faster and simpler +/// * byte deserialization being used in Safekeeper already, with those bytes coming from compute (see `ProposerGreeting` in safekeeper) +/// * current `HexZId`'s deserialization impl breaks on compute byte array deserialization, having it by default is dangerous +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +struct HexZId([u8; 16]); + +impl Serialize for HexZId { + fn serialize(&self, ser: S) -> Result + where + S: serde::Serializer, + { + hex::encode(self.0).serialize(ser) + } +} + +impl<'de> Deserialize<'de> for HexZId { + fn deserialize(de: D) -> Result + where + D: serde::Deserializer<'de>, + { + de.deserialize_bytes(HexVisitor) + } +} + +struct HexVisitor; + +impl<'de> Visitor<'de> for HexVisitor { + type Value = HexZId; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "A hexadecimal representation of a 128-bit random Zenith ID" + ) + } + + fn visit_bytes(self, hex_bytes: &[u8]) -> Result + where + E: de::Error, + { + ZId::from_hex(hex_bytes) + .map(HexZId::from) + .map_err(de::Error::custom) + } + + fn visit_str(self, hex_bytes_str: &str) -> Result + where + E: de::Error, + { + Self::visit_bytes(self, hex_bytes_str.as_bytes()) + } +} + +mutual_from!(ZId, HexZId); + impl ZId { pub fn get_from_buf(buf: &mut dyn bytes::Buf) -> ZId { let mut arr = [0u8; 16]; @@ -155,46 +242,80 @@ macro_rules! zid_newtype { /// is separate from PostgreSQL timelines, and doesn't have those /// limitations. A zenith timeline is identified by a 128-bit ID, which /// is usually printed out as a hex string. +/// +/// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look +/// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`. +/// Use [`HexZTimelineId`] to serialize it as hex string instead: `ad50847381e248feaac9876cc71ae418`. #[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize)] pub struct ZTimelineId(ZId); -zid_newtype!(ZTimelineId); +/// A [`ZTimelineId`] version that gets (de)serialized as a hex string. +/// Use in complex types, where `#[serde(with = "hex")]` does not work. +/// See [`HexZId`] for more details. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize)] +pub struct HexZTimelineId(HexZId); -// Zenith Tenant Id represents identifiar of a particular tenant. -// Is used for distinguishing requests and data belonging to different users. +impl std::fmt::Debug for HexZTimelineId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + ZTimelineId::from(*self).fmt(f) + } +} + +impl std::fmt::Display for HexZTimelineId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + ZTimelineId::from(*self).fmt(f) + } +} + +impl FromStr for HexZTimelineId { + type Err = ::Err; + + fn from_str(s: &str) -> Result { + Ok(HexZTimelineId::from(ZTimelineId::from_str(s)?)) + } +} + +zid_newtype!(ZTimelineId); +mutual_from!(ZTimelineId, HexZTimelineId); + +/// Zenith Tenant Id represents identifiar of a particular tenant. +/// Is used for distinguishing requests and data belonging to different users. +/// +/// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look +/// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`. +/// Use [`HexZTenantId`] to serialize it as hex string instead: `ad50847381e248feaac9876cc71ae418`. #[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)] pub struct ZTenantId(ZId); -zid_newtype!(ZTenantId); +/// A [`ZTenantId`] version that gets (de)serialized as a hex string. +/// Use in complex types, where `#[serde(with = "hex")]` does not work. +/// See [`HexZId`] for more details. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize)] +pub struct HexZTenantId(HexZId); -/// Serde routines for Option (de)serialization, using `T:Display` representations for inner values. -/// Useful for Option and Option to get their hex representations into serialized string and deserialize them back. -pub mod opt_display_serde { - use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; - use std::{fmt::Display, str::FromStr}; - - pub fn serialize(id: &Option, ser: S) -> Result - where - S: Serializer, - Id: Display, - { - id.as_ref().map(ToString::to_string).serialize(ser) - } - - pub fn deserialize<'de, D, Id>(des: D) -> Result, D::Error> - where - D: Deserializer<'de>, - Id: FromStr, - ::Err: Display, - { - Ok(if let Some(s) = Option::::deserialize(des)? { - Some(Id::from_str(&s).map_err(de::Error::custom)?) - } else { - None - }) +impl std::fmt::Debug for HexZTenantId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + ZTenantId::from(*self).fmt(f) } } +impl std::fmt::Display for HexZTenantId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + ZTenantId::from(*self).fmt(f) + } +} + +impl FromStr for HexZTenantId { + type Err = ::Err; + + fn from_str(s: &str) -> Result { + Ok(HexZTenantId::from(ZTenantId::from_str(s)?)) + } +} + +zid_newtype!(ZTenantId); +mutual_from!(ZTenantId, HexZTenantId); + // A pair uniquely identifying Zenith instance. #[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash)] pub struct ZTenantTimelineId { @@ -243,16 +364,15 @@ mod tests { #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct TestStruct + Display> { - #[serde(with = "opt_display_serde")] field: Option, } #[test] fn test_hex_serializations_tenant_id() { let original_struct = TestStruct { - field: Some(ZTenantId::from_array(hex!( + field: Some(HexZTenantId::from(ZTenantId::from_array(hex!( "11223344556677881122334455667788" - ))), + )))), }; let serialized_string = serde_json::to_string(&original_struct).unwrap(); @@ -261,7 +381,7 @@ mod tests { r#"{"field":"11223344556677881122334455667788"}"# ); - let deserialized_struct: TestStruct = + let deserialized_struct: TestStruct = serde_json::from_str(&serialized_string).unwrap(); assert_eq!(original_struct, deserialized_struct); } @@ -269,9 +389,9 @@ mod tests { #[test] fn test_hex_serializations_timeline_id() { let original_struct = TestStruct { - field: Some(ZTimelineId::from_array(hex!( + field: Some(HexZTimelineId::from(ZTimelineId::from_array(hex!( "AA223344556677881122334455667788" - ))), + )))), }; let serialized_string = serde_json::to_string(&original_struct).unwrap(); @@ -280,7 +400,7 @@ mod tests { r#"{"field":"aa223344556677881122334455667788"}"# ); - let deserialized_struct: TestStruct = + let deserialized_struct: TestStruct = serde_json::from_str(&serialized_string).unwrap(); assert_eq!(original_struct, deserialized_struct); } From 66eb2a1dd32403405414e0986c457588d8d45609 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Wed, 2 Mar 2022 13:27:39 +0200 Subject: [PATCH 05/42] Replace zenith/build build image with zimg/* ones --- Dockerfile | 6 +++--- Dockerfile.build | 16 ---------------- docs/docker.md | 24 +++--------------------- 3 files changed, 6 insertions(+), 40 deletions(-) delete mode 100644 Dockerfile.build diff --git a/Dockerfile b/Dockerfile index dd0dba60ca..18abae5327 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ # Build Postgres separately --- this layer will be rebuilt only if one of # mentioned paths will get any changes. # -FROM zenithdb/build:buster AS pg-build +FROM zimg/rust:1.56 AS pg-build WORKDIR /zenith COPY ./vendor/postgres vendor/postgres COPY ./Makefile Makefile @@ -20,7 +20,7 @@ RUN rm -rf postgres_install/build # TODO: build cargo deps as separate layer. We used cargo-chef before but that was # net time waste in a lot of cases. Copying Cargo.lock with empty lib.rs should do the work. # -FROM zenithdb/build:buster AS build +FROM zimg/rust:1.56 AS build ARG GIT_VERSION RUN if [ -z "$GIT_VERSION" ]; then echo "GIT_VERSION is reqired, use build_arg to pass it"; exit 1; fi @@ -34,7 +34,7 @@ RUN GIT_VERSION=$GIT_VERSION cargo build --release # # Copy binaries to resulting image. # -FROM debian:buster-slim +FROM debian:bullseye-slim WORKDIR /data RUN apt-get update && apt-get -yq install libreadline-dev libseccomp-dev openssl ca-certificates && \ diff --git a/Dockerfile.build b/Dockerfile.build deleted file mode 100644 index a9fd2cb0af..0000000000 --- a/Dockerfile.build +++ /dev/null @@ -1,16 +0,0 @@ -# -# Image with all the required dependencies to build https://github.com/zenithdb/zenith -# and Postgres from https://github.com/zenithdb/postgres -# Also includes some rust development and build tools. -# NB: keep in sync with rust image version in .circle/config.yml -# -FROM rust:1.56.1-slim-buster -WORKDIR /zenith - -# Install postgres and zenith build dependencies -# clang is for rocksdb -RUN apt-get update && apt-get -yq install automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \ - libseccomp-dev pkg-config libssl-dev clang - -# Install rust tools -RUN rustup component add clippy && cargo install cargo-audit diff --git a/docs/docker.md b/docs/docker.md index 14ba2146cb..cc54d012dd 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -7,32 +7,14 @@ Currently we build two main images: - [zenithdb/zenith](https://hub.docker.com/repository/docker/zenithdb/zenith) — image with pre-built `pageserver`, `safekeeper` and `proxy` binaries and all the required runtime dependencies. Built from [/Dockerfile](/Dockerfile). - [zenithdb/compute-node](https://hub.docker.com/repository/docker/zenithdb/compute-node) — compute node image with pre-built Postgres binaries from [zenithdb/postgres](https://github.com/zenithdb/postgres). -And two intermediate images used either to reduce build time or to deliver some additional binary tools from other repos: +And additional intermediate images: -- [zenithdb/build](https://hub.docker.com/repository/docker/zenithdb/build) — image with all the dependencies required to build Zenith and compute node images. This image is based on `rust:slim-buster`, so it also has a proper `rust` environment. Built from [/Dockerfile.build](/Dockerfile.build). - [zenithdb/compute-tools](https://hub.docker.com/repository/docker/zenithdb/compute-tools) — compute node configuration management tools. ## Building pipeline 1. Image `zenithdb/compute-tools` is re-built automatically. -2. Image `zenithdb/build` is built manually. If you want to introduce any new compile time dependencies to Zenith or compute node you have to update this image as well, build it and push to Docker Hub. +2. Image `zenithdb/compute-node` is built independently in the [zenithdb/postgres](https://github.com/zenithdb/postgres) repo. -Build: -```sh -docker build -t zenithdb/build:buster -f Dockerfile.build . -``` - -Login: -```sh -docker login -``` - -Push to Docker Hub: -```sh -docker push zenithdb/build:buster -``` - -3. Image `zenithdb/compute-node` is built independently in the [zenithdb/postgres](https://github.com/zenithdb/postgres) repo. - -4. Image `zenithdb/zenith` is built in this repo after a successful `release` tests run and pushed to Docker Hub automatically. +3. Image `zenithdb/zenith` is built in this repo after a successful `release` tests run and pushed to Docker Hub automatically. From f86cf93435133ee11f8c4bc53b1470e2dada3ce0 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Tue, 15 Feb 2022 20:10:10 +0300 Subject: [PATCH 06/42] Refactor timeline creation on safekeepers, allowing storing peer ids. Have separate routine and http endpoint to create timeline on safekeepers. It is not used yet, i.e. timeline is still created implicitly, but we'll change that once infrastructure for learning which tlis are assigned to which safekeepers will be ready, preventing accidental creation by compute. Changes format of safekeeper control file, allowing to store set of peers. Knowing peers provides a part of foundation for peer recovery (calculating min horizons like truncate_lsn for WAL truncation and commit_lsn for sync-safekeepers replacement) and proper membership change; similarly, we don't yet use it for now. Employing cf file version bump, extracts tenant_id and timeline_id to top level where it is more suitable. Also adds a bunch of LSNs there and rename truncate_lsn to more specific peer_horizon_lsn. --- Cargo.lock | 1 + control_plane/Cargo.toml | 1 + control_plane/src/safekeeper.rs | 24 ++++- walkeeper/src/bin/safekeeper.rs | 7 +- walkeeper/src/control_file.rs | 104 ++++++++---------- walkeeper/src/control_file_upgrade.rs | 82 +++++++++++++-- walkeeper/src/handler.rs | 38 +++---- walkeeper/src/http/mod.rs | 1 + walkeeper/src/http/models.rs | 9 ++ walkeeper/src/http/routes.rs | 32 +++++- walkeeper/src/safekeeper.rs | 145 ++++++++++++++++++-------- walkeeper/src/timeline.rs | 128 ++++++++++++++++------- walkeeper/src/wal_storage.rs | 23 ++-- zenith_utils/src/zid.rs | 4 + 14 files changed, 404 insertions(+), 195 deletions(-) create mode 100644 walkeeper/src/http/models.rs diff --git a/Cargo.lock b/Cargo.lock index ba3c6729d6..ad38a41d91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -424,6 +424,7 @@ dependencies = [ "thiserror", "toml", "url", + "walkeeper", "workspace_hack", "zenith_utils", ] diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index 5e972200c2..eff6b3ef2d 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -17,5 +17,6 @@ url = "2.2.2" reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "rustls-tls"] } pageserver = { path = "../pageserver" } +walkeeper = { path = "../walkeeper" } zenith_utils = { path = "../zenith_utils" } workspace_hack = { path = "../workspace_hack" } diff --git a/control_plane/src/safekeeper.rs b/control_plane/src/safekeeper.rs index 351d1efbbc..969e2cd531 100644 --- a/control_plane/src/safekeeper.rs +++ b/control_plane/src/safekeeper.rs @@ -14,8 +14,9 @@ use postgres::Config; use reqwest::blocking::{Client, RequestBuilder, Response}; use reqwest::{IntoUrl, Method}; use thiserror::Error; +use walkeeper::http::models::TimelineCreateRequest; use zenith_utils::http::error::HttpErrorBody; -use zenith_utils::zid::ZNodeId; +use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId}; use crate::local_env::{LocalEnv, SafekeeperConf}; use crate::storage::PageServerNode; @@ -261,4 +262,25 @@ impl SafekeeperNode { .error_from_body()?; Ok(()) } + + pub fn timeline_create( + &self, + tenant_id: ZTenantId, + timeline_id: ZTimelineId, + peer_ids: Vec, + ) -> Result<()> { + Ok(self + .http_request( + Method::POST, + format!("{}/{}", self.http_base_url, "timeline"), + ) + .json(&TimelineCreateRequest { + tenant_id, + timeline_id, + peer_ids, + }) + .send()? + .error_from_body()? + .json()?) + } } diff --git a/walkeeper/src/bin/safekeeper.rs b/walkeeper/src/bin/safekeeper.rs index 48de1481d4..6c45115e5f 100644 --- a/walkeeper/src/bin/safekeeper.rs +++ b/walkeeper/src/bin/safekeeper.rs @@ -11,7 +11,7 @@ use std::io::{ErrorKind, Write}; use std::path::{Path, PathBuf}; use std::thread; use tracing::*; -use walkeeper::control_file::{self, CreateControlFile}; +use walkeeper::control_file::{self}; use zenith_utils::http::endpoint; use zenith_utils::zid::ZNodeId; use zenith_utils::{logging, tcp_listener, GIT_VERSION}; @@ -108,10 +108,7 @@ fn main() -> Result<()> { .get_matches(); if let Some(addr) = arg_matches.value_of("dump-control-file") { - let state = control_file::FileStorage::load_control_file( - Path::new(addr), - CreateControlFile::False, - )?; + let state = control_file::FileStorage::load_control_file(Path::new(addr))?; let json = serde_json::to_string(&state)?; print!("{}", json); return Ok(()); diff --git a/walkeeper/src/control_file.rs b/walkeeper/src/control_file.rs index 6016e00d1d..8b4e618661 100644 --- a/walkeeper/src/control_file.rs +++ b/walkeeper/src/control_file.rs @@ -27,13 +27,6 @@ const CONTROL_FILE_NAME: &str = "safekeeper.control"; const CONTROL_FILE_NAME_PARTIAL: &str = "safekeeper.control.partial"; pub const CHECKSUM_SIZE: usize = std::mem::size_of::(); -// A named boolean. -#[derive(Debug)] -pub enum CreateControlFile { - True, - False, -} - lazy_static! { static ref PERSIST_CONTROL_FILE_SECONDS: HistogramVec = register_histogram_vec!( "safekeeper_persist_control_file_seconds", @@ -94,28 +87,22 @@ impl FileStorage { pub fn load_control_file_conf( conf: &SafeKeeperConf, zttid: &ZTenantTimelineId, - create: CreateControlFile, ) -> Result { let path = conf.timeline_dir(zttid).join(CONTROL_FILE_NAME); - Self::load_control_file(path, create) + Self::load_control_file(path) } /// Read in the control file. /// If create=false and file doesn't exist, bails out. - pub fn load_control_file>( - control_file_path: P, - create: CreateControlFile, - ) -> Result { + pub fn load_control_file>(control_file_path: P) -> Result { info!( - "loading control file {}, create={:?}", + "loading control file {}", control_file_path.as_ref().display(), - create, ); let mut control_file = OpenOptions::new() .read(true) .write(true) - .create(matches!(create, CreateControlFile::True)) .open(&control_file_path) .with_context(|| { format!( @@ -124,41 +111,32 @@ impl FileStorage { ) })?; - // Empty file is legit on 'create', don't try to deser from it. - let state = if control_file.metadata().unwrap().len() == 0 { - if let CreateControlFile::False = create { - bail!("control file is empty"); - } - SafeKeeperState::new() - } else { - let mut buf = Vec::new(); - control_file - .read_to_end(&mut buf) - .context("failed to read control file")?; + let mut buf = Vec::new(); + control_file + .read_to_end(&mut buf) + .context("failed to read control file")?; - let calculated_checksum = crc32c::crc32c(&buf[..buf.len() - CHECKSUM_SIZE]); + let calculated_checksum = crc32c::crc32c(&buf[..buf.len() - CHECKSUM_SIZE]); - let expected_checksum_bytes: &[u8; CHECKSUM_SIZE] = - buf[buf.len() - CHECKSUM_SIZE..].try_into()?; - let expected_checksum = u32::from_le_bytes(*expected_checksum_bytes); + let expected_checksum_bytes: &[u8; CHECKSUM_SIZE] = + buf[buf.len() - CHECKSUM_SIZE..].try_into()?; + let expected_checksum = u32::from_le_bytes(*expected_checksum_bytes); - ensure!( - calculated_checksum == expected_checksum, + ensure!( + calculated_checksum == expected_checksum, + format!( + "safekeeper control file checksum mismatch: expected {} got {}", + expected_checksum, calculated_checksum + ) + ); + + let state = FileStorage::deser_sk_state(&mut &buf[..buf.len() - CHECKSUM_SIZE]) + .with_context(|| { format!( - "safekeeper control file checksum mismatch: expected {} got {}", - expected_checksum, calculated_checksum + "while reading control file {}", + control_file_path.as_ref().display(), ) - ); - - FileStorage::deser_sk_state(&mut &buf[..buf.len() - CHECKSUM_SIZE]).with_context( - || { - format!( - "while reading control file {}", - control_file_path.as_ref().display(), - ) - }, - )? - }; + })?; Ok(state) } } @@ -247,31 +225,38 @@ mod test { fn load_from_control_file( conf: &SafeKeeperConf, zttid: &ZTenantTimelineId, - create: CreateControlFile, ) -> Result<(FileStorage, SafeKeeperState)> { fs::create_dir_all(&conf.timeline_dir(zttid)).expect("failed to create timeline dir"); Ok(( FileStorage::new(zttid, conf), - FileStorage::load_control_file_conf(conf, zttid, create)?, + FileStorage::load_control_file_conf(conf, zttid)?, )) } + fn create( + conf: &SafeKeeperConf, + zttid: &ZTenantTimelineId, + ) -> Result<(FileStorage, SafeKeeperState)> { + fs::create_dir_all(&conf.timeline_dir(zttid)).expect("failed to create timeline dir"); + let state = SafeKeeperState::empty(); + let mut storage = FileStorage::new(zttid, conf); + storage.persist(&state)?; + Ok((storage, state)) + } + #[test] fn test_read_write_safekeeper_state() { let conf = stub_conf(); let zttid = ZTenantTimelineId::generate(); { - let (mut storage, mut state) = - load_from_control_file(&conf, &zttid, CreateControlFile::True) - .expect("failed to read state"); + let (mut storage, mut state) = create(&conf, &zttid).expect("failed to create state"); // change something - state.wal_start_lsn = Lsn(42); + state.commit_lsn = Lsn(42); storage.persist(&state).expect("failed to persist state"); } - let (_, state) = load_from_control_file(&conf, &zttid, CreateControlFile::False) - .expect("failed to read state"); - assert_eq!(state.wal_start_lsn, Lsn(42)); + let (_, state) = load_from_control_file(&conf, &zttid).expect("failed to read state"); + assert_eq!(state.commit_lsn, Lsn(42)); } #[test] @@ -279,11 +264,10 @@ mod test { let conf = stub_conf(); let zttid = ZTenantTimelineId::generate(); { - let (mut storage, mut state) = - load_from_control_file(&conf, &zttid, CreateControlFile::True) - .expect("failed to read state"); + let (mut storage, mut state) = create(&conf, &zttid).expect("failed to read state"); + // change something - state.wal_start_lsn = Lsn(42); + state.commit_lsn = Lsn(42); storage.persist(&state).expect("failed to persist state"); } let control_path = conf.timeline_dir(&zttid).join(CONTROL_FILE_NAME); @@ -291,7 +275,7 @@ mod test { data[0] += 1; // change the first byte of the file to fail checksum validation fs::write(&control_path, &data).expect("failed to write control file"); - match load_from_control_file(&conf, &zttid, CreateControlFile::False) { + match load_from_control_file(&conf, &zttid) { Err(err) => assert!(err .to_string() .contains("safekeeper control file checksum mismatch")), diff --git a/walkeeper/src/control_file_upgrade.rs b/walkeeper/src/control_file_upgrade.rs index 913bd02c1e..9effe42f8d 100644 --- a/walkeeper/src/control_file_upgrade.rs +++ b/walkeeper/src/control_file_upgrade.rs @@ -1,6 +1,6 @@ //! Code to deal with safekeeper control file upgrades use crate::safekeeper::{ - AcceptorState, PgUuid, SafeKeeperState, ServerInfo, Term, TermHistory, TermSwitchEntry, + AcceptorState, Peers, PgUuid, SafeKeeperState, ServerInfo, Term, TermHistory, TermSwitchEntry, }; use anyhow::{bail, Result}; use serde::{Deserialize, Serialize}; @@ -26,7 +26,7 @@ struct SafeKeeperStateV1 { /// persistent acceptor state acceptor_state: AcceptorStateV1, /// information about server - server: ServerInfo, + server: ServerInfoV2, /// Unique id of the last *elected* proposer we dealed with. Not needed /// for correctness, exists for monitoring purposes. proposer_uuid: PgUuid, @@ -70,6 +70,39 @@ pub struct SafeKeeperStateV2 { pub wal_start_lsn: Lsn, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ServerInfoV3 { + /// Postgres server version + pub pg_version: u32, + pub system_id: SystemId, + #[serde(with = "hex")] + pub tenant_id: ZTenantId, + /// Zenith timelineid + #[serde(with = "hex")] + pub timeline_id: ZTimelineId, + pub wal_seg_size: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SafeKeeperStateV3 { + /// persistent acceptor state + pub acceptor_state: AcceptorState, + /// information about server + pub server: ServerInfoV3, + /// Unique id of the last *elected* proposer we dealed with. Not needed + /// for correctness, exists for monitoring purposes. + #[serde(with = "hex")] + pub proposer_uuid: PgUuid, + /// part of WAL acknowledged by quorum and available locally + pub commit_lsn: Lsn, + /// minimal LSN which may be needed for recovery of some safekeeper (end_lsn + /// of last record streamed to everyone) + pub truncate_lsn: Lsn, + // Safekeeper starts receiving WAL from this LSN, zeros before it ought to + // be skipped during decoding. + pub wal_start_lsn: Lsn, +} + pub fn upgrade_control_file(buf: &[u8], version: u32) -> Result { // migrate to storing full term history if version == 1 { @@ -83,12 +116,20 @@ pub fn upgrade_control_file(buf: &[u8], version: u32) -> Result }]), }; return Ok(SafeKeeperState { + tenant_id: oldstate.server.tenant_id, + timeline_id: oldstate.server.ztli, acceptor_state: ac, - server: oldstate.server.clone(), + server: ServerInfo { + pg_version: oldstate.server.pg_version, + system_id: oldstate.server.system_id, + wal_seg_size: oldstate.server.wal_seg_size, + }, proposer_uuid: oldstate.proposer_uuid, commit_lsn: oldstate.commit_lsn, - truncate_lsn: oldstate.truncate_lsn, - wal_start_lsn: oldstate.wal_start_lsn, + s3_wal_lsn: Lsn(0), + peer_horizon_lsn: oldstate.truncate_lsn, + remote_consistent_lsn: Lsn(0), + peers: Peers(vec![]), }); // migrate to hexing some zids } else if version == 2 { @@ -97,17 +138,40 @@ pub fn upgrade_control_file(buf: &[u8], version: u32) -> Result let server = ServerInfo { pg_version: oldstate.server.pg_version, system_id: oldstate.server.system_id, - tenant_id: oldstate.server.tenant_id, - timeline_id: oldstate.server.ztli, wal_seg_size: oldstate.server.wal_seg_size, }; return Ok(SafeKeeperState { + tenant_id: oldstate.server.tenant_id, + timeline_id: oldstate.server.ztli, acceptor_state: oldstate.acceptor_state, server, proposer_uuid: oldstate.proposer_uuid, commit_lsn: oldstate.commit_lsn, - truncate_lsn: oldstate.truncate_lsn, - wal_start_lsn: oldstate.wal_start_lsn, + s3_wal_lsn: Lsn(0), + peer_horizon_lsn: oldstate.truncate_lsn, + remote_consistent_lsn: Lsn(0), + peers: Peers(vec![]), + }); + // migrate to moving ztenantid/ztli to the top and adding some lsns + } else if version == 3 { + info!("reading safekeeper control file version {}", version); + let oldstate = SafeKeeperStateV3::des(&buf[..buf.len()])?; + let server = ServerInfo { + pg_version: oldstate.server.pg_version, + system_id: oldstate.server.system_id, + wal_seg_size: oldstate.server.wal_seg_size, + }; + return Ok(SafeKeeperState { + tenant_id: oldstate.server.tenant_id, + timeline_id: oldstate.server.timeline_id, + acceptor_state: oldstate.acceptor_state, + server, + proposer_uuid: oldstate.proposer_uuid, + commit_lsn: oldstate.commit_lsn, + s3_wal_lsn: Lsn(0), + peer_horizon_lsn: oldstate.truncate_lsn, + remote_consistent_lsn: Lsn(0), + peers: Peers(vec![]), }); } bail!("unsupported safekeeper control file version {}", version) diff --git a/walkeeper/src/handler.rs b/walkeeper/src/handler.rs index d1ead5cb37..ead6fab9fb 100644 --- a/walkeeper/src/handler.rs +++ b/walkeeper/src/handler.rs @@ -13,6 +13,7 @@ use postgres_ffi::xlog_utils::PG_TLI; use regex::Regex; use std::str::FromStr; use std::sync::Arc; +use tracing::info; use zenith_utils::lsn::Lsn; use zenith_utils::postgres_backend; use zenith_utils::postgres_backend::PostgresBackend; @@ -20,7 +21,6 @@ use zenith_utils::pq_proto::{BeMessage, FeStartupPacket, RowDescriptor, INT4_OID use zenith_utils::zid::{ZTenantId, ZTenantTimelineId, ZTimelineId}; use crate::callmemaybe::CallmeEvent; -use crate::control_file::CreateControlFile; use tokio::sync::mpsc::UnboundedSender; /// Safekeeper handler of postgres commands @@ -101,29 +101,19 @@ impl postgres_backend::Handler for SafekeeperPostgresHandler { fn process_query(&mut self, pgb: &mut PostgresBackend, query_string: &str) -> Result<()> { let cmd = parse_cmd(query_string)?; - // Is this command is ztimeline scoped? - match cmd { - SafekeeperPostgresCommand::StartWalPush { .. } - | SafekeeperPostgresCommand::StartReplication { .. } - | SafekeeperPostgresCommand::IdentifySystem - | SafekeeperPostgresCommand::JSONCtrl { .. } => { - let tenantid = self.ztenantid.context("tenantid is required")?; - let timelineid = self.ztimelineid.context("timelineid is required")?; - if self.timeline.is_none() { - // START_WAL_PUSH is the only command that initializes the timeline in production. - // There is also JSON_CTRL command, which should initialize the timeline for testing. - let create_control_file = match cmd { - SafekeeperPostgresCommand::StartWalPush { .. } - | SafekeeperPostgresCommand::JSONCtrl { .. } => CreateControlFile::True, - _ => CreateControlFile::False, - }; - self.timeline.set( - &self.conf, - ZTenantTimelineId::new(tenantid, timelineid), - create_control_file, - )?; - } - } + info!("got query {:?}", query_string); + + let create = !(matches!(cmd, SafekeeperPostgresCommand::StartReplication { .. }) + || matches!(cmd, SafekeeperPostgresCommand::IdentifySystem)); + + let tenantid = self.ztenantid.context("tenantid is required")?; + let timelineid = self.ztimelineid.context("timelineid is required")?; + if self.timeline.is_none() { + self.timeline.set( + &self.conf, + ZTenantTimelineId::new(tenantid, timelineid), + create, + )?; } match cmd { diff --git a/walkeeper/src/http/mod.rs b/walkeeper/src/http/mod.rs index c82d1c0362..4c0be17ecd 100644 --- a/walkeeper/src/http/mod.rs +++ b/walkeeper/src/http/mod.rs @@ -1,2 +1,3 @@ +pub mod models; pub mod routes; pub use routes::make_router; diff --git a/walkeeper/src/http/models.rs b/walkeeper/src/http/models.rs new file mode 100644 index 0000000000..8a6ed7a812 --- /dev/null +++ b/walkeeper/src/http/models.rs @@ -0,0 +1,9 @@ +use serde::{Deserialize, Serialize}; +use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId}; + +#[derive(Serialize, Deserialize)] +pub struct TimelineCreateRequest { + pub tenant_id: ZTenantId, + pub timeline_id: ZTimelineId, + pub peer_ids: Vec, +} diff --git a/walkeeper/src/http/routes.rs b/walkeeper/src/http/routes.rs index bc992c6a6f..74f7f4a735 100644 --- a/walkeeper/src/http/routes.rs +++ b/walkeeper/src/http/routes.rs @@ -1,14 +1,15 @@ use hyper::{Body, Request, Response, StatusCode}; + use serde::Serialize; use serde::Serializer; use std::fmt::Display; use std::sync::Arc; +use zenith_utils::http::json::json_request; use zenith_utils::http::{RequestExt, RouterBuilder}; use zenith_utils::lsn::Lsn; use zenith_utils::zid::ZNodeId; use zenith_utils::zid::ZTenantTimelineId; -use crate::control_file::CreateControlFile; use crate::safekeeper::Term; use crate::safekeeper::TermHistory; use crate::timeline::GlobalTimelines; @@ -19,6 +20,8 @@ use zenith_utils::http::json::json_response; use zenith_utils::http::request::parse_request_param; use zenith_utils::zid::{ZTenantId, ZTimelineId}; +use super::models::TimelineCreateRequest; + #[derive(Debug, Serialize)] struct SafekeeperStatus { id: ZNodeId, @@ -66,7 +69,11 @@ struct TimelineStatus { #[serde(serialize_with = "display_serialize")] commit_lsn: Lsn, #[serde(serialize_with = "display_serialize")] - truncate_lsn: Lsn, + s3_wal_lsn: Lsn, + #[serde(serialize_with = "display_serialize")] + peer_horizon_lsn: Lsn, + #[serde(serialize_with = "display_serialize")] + remote_consistent_lsn: Lsn, #[serde(serialize_with = "display_serialize")] flush_lsn: Lsn, } @@ -78,8 +85,7 @@ async fn timeline_status_handler(request: Request) -> Result) -> Result) -> Result, ApiError> { + let request_data: TimelineCreateRequest = json_request(&mut request).await?; + + let zttid = ZTenantTimelineId { + tenant_id: request_data.tenant_id, + timeline_id: request_data.timeline_id, + }; + GlobalTimelines::create(get_conf(&request), zttid, request_data.peer_ids) + .map_err(ApiError::from_err)?; + + Ok(json_response(StatusCode::CREATED, ())?) +} + /// Safekeeper http router. pub fn make_router(conf: SafeKeeperConf) -> RouterBuilder { let router = endpoint::make_router(); @@ -110,4 +131,5 @@ pub fn make_router(conf: SafeKeeperConf) -> RouterBuilder "/v1/timeline/:tenant_id/:timeline_id", timeline_status_handler, ) + .post("/v1/timeline", timeline_create_handler) } diff --git a/walkeeper/src/safekeeper.rs b/walkeeper/src/safekeeper.rs index fa624bb18f..f8b12530d8 100644 --- a/walkeeper/src/safekeeper.rs +++ b/walkeeper/src/safekeeper.rs @@ -10,6 +10,8 @@ use std::cmp::min; use std::fmt; use std::io::Read; use tracing::*; +use zenith_utils::zid::ZNodeId; +use zenith_utils::zid::ZTenantTimelineId; use lazy_static::lazy_static; @@ -25,12 +27,13 @@ use zenith_utils::pq_proto::ZenithFeedback; use zenith_utils::zid::{ZTenantId, ZTimelineId}; pub const SK_MAGIC: u32 = 0xcafeceefu32; -pub const SK_FORMAT_VERSION: u32 = 3; +pub const SK_FORMAT_VERSION: u32 = 4; const SK_PROTOCOL_VERSION: u32 = 1; const UNKNOWN_SERVER_VERSION: u32 = 0; /// Consensus logical timestamp. pub type Term = u64; +const INVALID_TERM: Term = 0; #[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct TermSwitchEntry { @@ -128,18 +131,47 @@ pub struct ServerInfo { /// Postgres server version pub pg_version: u32, pub system_id: SystemId, - #[serde(with = "hex")] - pub tenant_id: ZTenantId, - /// Zenith timelineid - #[serde(with = "hex")] - pub timeline_id: ZTimelineId, pub wal_seg_size: u32, } +/// Data published by safekeeper to the peers +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PeerInfo { + /// LSN up to which safekeeper offloaded WAL to s3. + s3_wal_lsn: Lsn, + /// Term of the last entry. + term: Term, + /// LSN of the last record. + flush_lsn: Lsn, + /// Up to which LSN safekeeper regards its WAL as committed. + commit_lsn: Lsn, +} + +impl PeerInfo { + fn new() -> Self { + Self { + s3_wal_lsn: Lsn(0), + term: INVALID_TERM, + flush_lsn: Lsn(0), + commit_lsn: Lsn(0), + } + } +} + +// vector-based node id -> peer state map with very limited functionality we +// need/ +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Peers(pub Vec<(ZNodeId, PeerInfo)>); + /// Persistent information stored on safekeeper node /// On disk data is prefixed by magic and format version and followed by checksum. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SafeKeeperState { + #[serde(with = "hex")] + pub tenant_id: ZTenantId, + /// Zenith timelineid + #[serde(with = "hex")] + pub timeline_id: ZTimelineId, /// persistent acceptor state pub acceptor_state: AcceptorState, /// information about server @@ -148,19 +180,33 @@ pub struct SafeKeeperState { /// for correctness, exists for monitoring purposes. #[serde(with = "hex")] pub proposer_uuid: PgUuid, - /// part of WAL acknowledged by quorum and available locally + /// Part of WAL acknowledged by quorum and available locally. Always points + /// to record boundary. pub commit_lsn: Lsn, - /// minimal LSN which may be needed for recovery of some safekeeper (end_lsn - /// of last record streamed to everyone) - pub truncate_lsn: Lsn, - // Safekeeper starts receiving WAL from this LSN, zeros before it ought to - // be skipped during decoding. - pub wal_start_lsn: Lsn, + /// First LSN not yet offloaded to s3. Useful to persist to avoid finding + /// out offloading progress on boot. + pub s3_wal_lsn: Lsn, + /// Minimal LSN which may be needed for recovery of some safekeeper (end_lsn + /// of last record streamed to everyone). Persisting it helps skipping + /// recovery in walproposer, generally we compute it from peers. In + /// walproposer proto called 'truncate_lsn'. + pub peer_horizon_lsn: Lsn, + /// LSN of the oldest known checkpoint made by pageserver and successfully + /// pushed to s3. We don't remove WAL beyond it. Persisted only for + /// informational purposes, we receive it from pageserver. + pub remote_consistent_lsn: Lsn, + // Peers and their state as we remember it. Knowing peers themselves is + // fundamental; but state is saved here only for informational purposes and + // obviously can be stale. (Currently not saved at all, but let's provision + // place to have less file version upgrades). + pub peers: Peers, } impl SafeKeeperState { - pub fn new() -> SafeKeeperState { + pub fn new(zttid: &ZTenantTimelineId, peers: Vec) -> SafeKeeperState { SafeKeeperState { + tenant_id: zttid.tenant_id, + timeline_id: zttid.timeline_id, acceptor_state: AcceptorState { term: 0, term_history: TermHistory::empty(), @@ -168,21 +214,20 @@ impl SafeKeeperState { server: ServerInfo { pg_version: UNKNOWN_SERVER_VERSION, /* Postgres server version */ system_id: 0, /* Postgres system identifier */ - tenant_id: ZTenantId::from([0u8; 16]), - timeline_id: ZTimelineId::from([0u8; 16]), wal_seg_size: 0, }, proposer_uuid: [0; 16], - commit_lsn: Lsn(0), /* part of WAL acknowledged by quorum */ - truncate_lsn: Lsn(0), /* minimal LSN which may be needed for recovery of some safekeeper */ - wal_start_lsn: Lsn(0), + commit_lsn: Lsn(0), + s3_wal_lsn: Lsn(0), + peer_horizon_lsn: Lsn(0), + remote_consistent_lsn: Lsn(0), + peers: Peers(peers.iter().map(|p| (*p, PeerInfo::new())).collect()), } } -} -impl Default for SafeKeeperState { - fn default() -> Self { - Self::new() + #[cfg(test)] + pub fn empty() -> Self { + SafeKeeperState::new(&ZTenantTimelineId::empty(), vec![]) } } @@ -421,6 +466,7 @@ lazy_static! { struct SafeKeeperMetrics { commit_lsn: Gauge, + // WAL-related metrics are in WalStorageMetrics } impl SafeKeeperMetrics { @@ -443,7 +489,7 @@ pub struct SafeKeeper { /// not-yet-flushed pairs of same named fields in s.* pub commit_lsn: Lsn, - pub truncate_lsn: Lsn, + pub peer_horizon_lsn: Lsn, pub s: SafeKeeperState, // persistent part pub control_store: CTRL, @@ -462,16 +508,14 @@ where wal_store: WAL, state: SafeKeeperState, ) -> SafeKeeper { - if state.server.timeline_id != ZTimelineId::from([0u8; 16]) - && ztli != state.server.timeline_id - { - panic!("Calling SafeKeeper::new with inconsistent ztli ({}) and SafeKeeperState.server.timeline_id ({})", ztli, state.server.timeline_id); + if state.timeline_id != ZTimelineId::from([0u8; 16]) && ztli != state.timeline_id { + panic!("Calling SafeKeeper::new with inconsistent ztli ({}) and SafeKeeperState.server.timeline_id ({})", ztli, state.timeline_id); } SafeKeeper { - metrics: SafeKeeperMetrics::new(state.server.tenant_id, ztli, state.commit_lsn), + metrics: SafeKeeperMetrics::new(state.tenant_id, ztli, state.commit_lsn), commit_lsn: state.commit_lsn, - truncate_lsn: state.truncate_lsn, + peer_horizon_lsn: state.peer_horizon_lsn, s: state, control_store, wal_store, @@ -532,12 +576,24 @@ where msg.pg_version, self.s.server.pg_version ); } + if msg.tenant_id != self.s.tenant_id { + bail!( + "invalid tenant ID, got {}, expected {}", + msg.tenant_id, + self.s.tenant_id + ); + } + if msg.ztli != self.s.timeline_id { + bail!( + "invalid timeline ID, got {}, expected {}", + msg.ztli, + self.s.timeline_id + ); + } // set basic info about server, if not yet // TODO: verify that is doesn't change after self.s.server.system_id = msg.system_id; - self.s.server.tenant_id = msg.tenant_id; - self.s.server.timeline_id = msg.ztli; self.s.server.wal_seg_size = msg.wal_seg_size; self.control_store .persist(&self.s) @@ -568,7 +624,7 @@ where term: self.s.acceptor_state.term, vote_given: false as u64, flush_lsn: self.wal_store.flush_lsn(), - truncate_lsn: self.s.truncate_lsn, + truncate_lsn: self.s.peer_horizon_lsn, term_history: self.get_term_history(), }; if self.s.acceptor_state.term < msg.term { @@ -655,10 +711,11 @@ where if !msg.wal_data.is_empty() { self.wal_store.write_wal(msg.h.begin_lsn, &msg.wal_data)?; - // If this was the first record we ever receieved, remember LSN to help - // find_end_of_wal skip the hole in the beginning. - if self.s.wal_start_lsn == Lsn(0) { - self.s.wal_start_lsn = msg.h.begin_lsn; + // If this was the first record we ever receieved, initialize + // commit_lsn to help find_end_of_wal skip the hole in the + // beginning. + if self.s.commit_lsn == Lsn(0) { + self.s.commit_lsn = msg.h.begin_lsn; sync_control_file = true; require_flush = true; } @@ -685,15 +742,15 @@ where .set(u64::from(self.commit_lsn) as f64); } - self.truncate_lsn = msg.h.truncate_lsn; + self.peer_horizon_lsn = msg.h.truncate_lsn; // Update truncate and commit LSN in control file. // To avoid negative impact on performance of extra fsync, do it only // when truncate_lsn delta exceeds WAL segment size. sync_control_file |= - self.s.truncate_lsn + (self.s.server.wal_seg_size as u64) < self.truncate_lsn; + self.s.peer_horizon_lsn + (self.s.server.wal_seg_size as u64) < self.peer_horizon_lsn; if sync_control_file { self.s.commit_lsn = self.commit_lsn; - self.s.truncate_lsn = self.truncate_lsn; + self.s.peer_horizon_lsn = self.peer_horizon_lsn; } if sync_control_file { @@ -774,11 +831,11 @@ mod tests { #[test] fn test_voting() { let storage = InMemoryState { - persisted_state: SafeKeeperState::new(), + persisted_state: SafeKeeperState::empty(), }; let wal_store = DummyWalStore { lsn: Lsn(0) }; let ztli = ZTimelineId::from([0u8; 16]); - let mut sk = SafeKeeper::new(ztli, storage, wal_store, SafeKeeperState::new()); + let mut sk = SafeKeeper::new(ztli, storage, wal_store, SafeKeeperState::empty()); // check voting for 1 is ok let vote_request = ProposerAcceptorMessage::VoteRequest(VoteRequest { term: 1 }); @@ -806,11 +863,11 @@ mod tests { #[test] fn test_epoch_switch() { let storage = InMemoryState { - persisted_state: SafeKeeperState::new(), + persisted_state: SafeKeeperState::empty(), }; let wal_store = DummyWalStore { lsn: Lsn(0) }; let ztli = ZTimelineId::from([0u8; 16]); - let mut sk = SafeKeeper::new(ztli, storage, wal_store, SafeKeeperState::new()); + let mut sk = SafeKeeper::new(ztli, storage, wal_store, SafeKeeperState::empty()); let mut ar_hdr = AppendRequestHeader { term: 1, diff --git a/walkeeper/src/timeline.rs b/walkeeper/src/timeline.rs index c639e81b79..ea8308b95e 100644 --- a/walkeeper/src/timeline.rs +++ b/walkeeper/src/timeline.rs @@ -1,7 +1,7 @@ //! This module contains timeline id -> safekeeper state map with file-backed //! persistence and support for interaction between sending and receiving wal. -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use lazy_static::lazy_static; @@ -9,22 +9,24 @@ use std::cmp::{max, min}; use std::collections::HashMap; use std::fs::{self}; -use std::sync::{Arc, Condvar, Mutex}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; use tracing::*; use zenith_utils::lsn::Lsn; -use zenith_utils::zid::ZTenantTimelineId; +use zenith_utils::zid::{ZNodeId, ZTenantTimelineId}; use crate::callmemaybe::{CallmeEvent, SubscriptionStateKey}; -use crate::control_file::{self, CreateControlFile}; +use crate::control_file; +use crate::control_file::Storage as cf_storage; use crate::safekeeper::{ AcceptorProposerMessage, ProposerAcceptorMessage, SafeKeeper, SafeKeeperState, }; use crate::send_wal::HotStandbyFeedback; -use crate::wal_storage::{self, Storage}; +use crate::wal_storage; +use crate::wal_storage::Storage as wal_storage_iface; use crate::SafeKeeperConf; use zenith_utils::pq_proto::ZenithFeedback; @@ -87,21 +89,39 @@ struct SharedState { } impl SharedState { - /// Restore SharedState from control file. - /// If create=false and file doesn't exist, bails out. - fn create_restore( + /// Initialize timeline state, creating control file + fn create( conf: &SafeKeeperConf, zttid: &ZTenantTimelineId, - create: CreateControlFile, + peer_ids: Vec, ) -> Result { - let state = control_file::FileStorage::load_control_file_conf(conf, zttid, create) + let state = SafeKeeperState::new(zttid, peer_ids); + let control_store = control_file::FileStorage::new(zttid, conf); + let wal_store = wal_storage::PhysicalStorage::new(zttid, conf); + let mut sk = SafeKeeper::new(zttid.timeline_id, control_store, wal_store, state); + sk.control_store.persist(&sk.s)?; + + Ok(Self { + notified_commit_lsn: Lsn(0), + sk, + replicas: Vec::new(), + active: false, + num_computes: 0, + pageserver_connstr: None, + }) + } + + /// Restore SharedState from control file. + /// If file doesn't exist, bails out. + fn restore(conf: &SafeKeeperConf, zttid: &ZTenantTimelineId) -> Result { + let state = control_file::FileStorage::load_control_file_conf(conf, zttid) .context("failed to load from control file")?; let control_store = control_file::FileStorage::new(zttid, conf); let wal_store = wal_storage::PhysicalStorage::new(zttid, conf); - info!("timeline {} created or restored", zttid.timeline_id); + info!("timeline {} restored", zttid.timeline_id); Ok(Self { notified_commit_lsn: Lsn(0), @@ -418,26 +438,13 @@ impl Timeline { // Utilities needed by various Connection-like objects pub trait TimelineTools { - fn set( - &mut self, - conf: &SafeKeeperConf, - zttid: ZTenantTimelineId, - create: CreateControlFile, - ) -> Result<()>; + fn set(&mut self, conf: &SafeKeeperConf, zttid: ZTenantTimelineId, create: bool) -> Result<()>; fn get(&self) -> &Arc; } impl TimelineTools for Option> { - fn set( - &mut self, - conf: &SafeKeeperConf, - zttid: ZTenantTimelineId, - create: CreateControlFile, - ) -> Result<()> { - // We will only set the timeline once. If it were to ever change, - // anyone who cloned the Arc would be out of date. - assert!(self.is_none()); + fn set(&mut self, conf: &SafeKeeperConf, zttid: ZTenantTimelineId, create: bool) -> Result<()> { *self = Some(GlobalTimelines::get(conf, zttid, create)?); Ok(()) } @@ -456,30 +463,73 @@ lazy_static! { pub struct GlobalTimelines; impl GlobalTimelines { + fn create_internal( + mut timelines: MutexGuard>>, + conf: &SafeKeeperConf, + zttid: ZTenantTimelineId, + peer_ids: Vec, + ) -> Result> { + match timelines.get(&zttid) { + Some(_) => bail!("timeline {} already exists", zttid), + None => { + // TODO: check directory existence + let dir = conf.timeline_dir(&zttid); + fs::create_dir_all(dir)?; + let shared_state = SharedState::create(conf, &zttid, peer_ids) + .context("failed to create shared state")?; + + let new_tli = Arc::new(Timeline::new(zttid, shared_state)); + timelines.insert(zttid, Arc::clone(&new_tli)); + Ok(new_tli) + } + } + } + + pub fn create( + conf: &SafeKeeperConf, + zttid: ZTenantTimelineId, + peer_ids: Vec, + ) -> Result> { + let timelines = TIMELINES.lock().unwrap(); + GlobalTimelines::create_internal(timelines, conf, zttid, peer_ids) + } + /// Get a timeline with control file loaded from the global TIMELINES map. - /// If control file doesn't exist and create=false, bails out. + /// If control file doesn't exist, bails out. pub fn get( conf: &SafeKeeperConf, zttid: ZTenantTimelineId, - create: CreateControlFile, + create: bool, ) -> Result> { let mut timelines = TIMELINES.lock().unwrap(); match timelines.get(&zttid) { Some(result) => Ok(Arc::clone(result)), None => { - if let CreateControlFile::True = create { - let dir = conf.timeline_dir(&zttid); - info!( - "creating timeline dir {}, create is {:?}", - dir.display(), - create - ); - fs::create_dir_all(dir)?; - } + let shared_state = + SharedState::restore(conf, &zttid).context("failed to restore shared state"); - let shared_state = SharedState::create_restore(conf, &zttid, create) - .context("failed to restore shared state")?; + let shared_state = match shared_state { + Ok(shared_state) => shared_state, + Err(error) => { + // TODO: always create timeline explicitly + if error + .root_cause() + .to_string() + .contains("No such file or directory") + && create + { + return GlobalTimelines::create_internal( + timelines, + conf, + zttid, + vec![], + ); + } else { + return Err(error); + } + } + }; let new_tli = Arc::new(Timeline::new(zttid, shared_state)); timelines.insert(zttid, Arc::clone(&new_tli)); diff --git a/walkeeper/src/wal_storage.rs b/walkeeper/src/wal_storage.rs index 73eccd0ae8..7cef525bee 100644 --- a/walkeeper/src/wal_storage.rs +++ b/walkeeper/src/wal_storage.rs @@ -301,7 +301,8 @@ impl Storage for PhysicalStorage { /// allows to postpone its initialization. fn init_storage(&mut self, state: &SafeKeeperState) -> Result<()> { if state.server.wal_seg_size == 0 { - // wal_seg_size is still unknown + // wal_seg_size is still unknown. This is dead path normally, should + // be used only in tests. return Ok(()); } @@ -315,9 +316,13 @@ impl Storage for PhysicalStorage { let wal_seg_size = state.server.wal_seg_size as usize; self.wal_seg_size = Some(wal_seg_size); - // we need to read WAL from disk to know which LSNs are stored on disk - self.write_lsn = - Lsn(find_end_of_wal(&self.timeline_dir, wal_seg_size, true, state.wal_start_lsn)?.0); + // Find out where stored WAL ends, starting at commit_lsn which is a + // known recent record boundary (unless we don't have WAL at all). + self.write_lsn = if state.commit_lsn == Lsn(0) { + Lsn(0) + } else { + Lsn(find_end_of_wal(&self.timeline_dir, wal_seg_size, true, state.commit_lsn)?.0) + }; self.write_record_lsn = self.write_lsn; @@ -326,11 +331,13 @@ impl Storage for PhysicalStorage { self.update_flush_lsn(); info!( - "initialized storage for timeline {}, flush_lsn={}, commit_lsn={}, truncate_lsn={}", - self.zttid.timeline_id, self.flush_record_lsn, state.commit_lsn, state.truncate_lsn, + "initialized storage for timeline {}, flush_lsn={}, commit_lsn={}, peer_horizon_lsn={}", + self.zttid.timeline_id, self.flush_record_lsn, state.commit_lsn, state.peer_horizon_lsn, ); - if self.flush_record_lsn < state.commit_lsn || self.flush_record_lsn < state.truncate_lsn { - warn!("timeline {} potential data loss: flush_lsn by find_end_of_wal is less than either commit_lsn or truncate_lsn from control file", self.zttid.timeline_id); + if self.flush_record_lsn < state.commit_lsn + || self.flush_record_lsn < state.peer_horizon_lsn + { + warn!("timeline {} potential data loss: flush_lsn by find_end_of_wal is less than either commit_lsn or peer_horizon_lsn from control file", self.zttid.timeline_id); } Ok(()) diff --git a/zenith_utils/src/zid.rs b/zenith_utils/src/zid.rs index 813eb3f8f4..a740d4fb48 100644 --- a/zenith_utils/src/zid.rs +++ b/zenith_utils/src/zid.rs @@ -334,6 +334,10 @@ impl ZTenantTimelineId { pub fn generate() -> Self { Self::new(ZTenantId::generate(), ZTimelineId::generate()) } + + pub fn empty() -> Self { + Self::new(ZTenantId::from([0u8; 16]), ZTimelineId::from([0u8; 16])) + } } impl fmt::Display for ZTenantTimelineId { From 8e37d345a8fd4e7e1b25fe1e88af95de5e163ee3 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Tue, 8 Mar 2022 08:07:00 +0300 Subject: [PATCH 07/42] Adjust safekeeper detailed logging to batch fsyncing. --- walkeeper/src/safekeeper.rs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/walkeeper/src/safekeeper.rs b/walkeeper/src/safekeeper.rs index f8b12530d8..53fd6f5588 100644 --- a/walkeeper/src/safekeeper.rs +++ b/walkeeper/src/safekeeper.rs @@ -649,14 +649,16 @@ where /// Form AppendResponse from current state. fn append_response(&self) -> AppendResponse { - AppendResponse { + let ar = AppendResponse { term: self.s.acceptor_state.term, flush_lsn: self.wal_store.flush_lsn(), commit_lsn: self.s.commit_lsn, // will be filled by the upper code to avoid bothering safekeeper hs_feedback: HotStandbyFeedback::empty(), zenith_feedback: ZenithFeedback::empty(), - } + }; + trace!("formed AppendResponse {:?}", ar); + ar } fn handle_elected(&mut self, msg: &ProposerElected) -> Result> { @@ -757,20 +759,21 @@ where self.control_store.persist(&self.s)?; } + trace!( + "processed AppendRequest of len {}, end_lsn={:?}, commit_lsn={:?}, truncate_lsn={:?}, flushed={:?}", + msg.wal_data.len(), + msg.h.end_lsn, + msg.h.commit_lsn, + msg.h.truncate_lsn, + require_flush, + ); + // If flush_lsn hasn't updated, AppendResponse is not very useful. if !require_flush { return Ok(None); } let resp = self.append_response(); - trace!( - "processed AppendRequest of len {}, end_lsn={:?}, commit_lsn={:?}, truncate_lsn={:?}, resp {:?}", - msg.wal_data.len(), - msg.h.end_lsn, - msg.h.commit_lsn, - msg.h.truncate_lsn, - &resp, - ); Ok(Some(AcceptorProposerMessage::AppendResponse(resp))) } From cffac59a41e59f2f92e68b183bd64bdaf7f17fa7 Mon Sep 17 00:00:00 2001 From: Andrey Taranik Date: Tue, 8 Mar 2022 23:19:49 +0300 Subject: [PATCH 08/42] Docker improvement (#1345) * dockerfile fix, rust cache in docker build flow * check rust cachepot * another check rust cachepot * cleanup --- .circleci/config.yml | 30 +++++++++++----- Dockerfile | 81 +++++++++++++++++++++----------------------- 2 files changed, 61 insertions(+), 50 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index db9fc31334..d342e7c9f4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -440,8 +440,14 @@ jobs: command: | echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin DOCKER_TAG=$(git log --oneline|wc -l) - docker build --build-arg GIT_VERSION=$CIRCLE_SHA1 -t zenithdb/zenith:latest . && docker push zenithdb/zenith:latest - docker tag zenithdb/zenith:latest zenithdb/zenith:${DOCKER_TAG} && docker push zenithdb/zenith:${DOCKER_TAG} + docker build \ + --pull \ + --build-arg GIT_VERSION=${CIRCLE_SHA1} \ + --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \ + --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \ + --tag zenithdb/zenith:${DOCKER_TAG} --tag zenithdb/zenith:latest . + docker push zenithdb/zenith:${DOCKER_TAG} + docker push zenithdb/zenith:latest # Build zenithdb/compute-node:latest image and push it to Docker hub docker-image-compute: @@ -468,8 +474,9 @@ jobs: command: | echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin DOCKER_TAG=$(git log --oneline|wc -l) - docker build -t zenithdb/compute-node:latest vendor/postgres && docker push zenithdb/compute-node:latest - docker tag zenithdb/compute-node:latest zenithdb/compute-node:${DOCKER_TAG} && docker push zenithdb/compute-node:${DOCKER_TAG} + docker build --tag zenithdb/compute-node:${DOCKER_TAG} --tag zenithdb/compute-node:latest vendor/postgres + docker push zenithdb/compute-node:${DOCKER_TAG} + docker push zenithdb/compute-node:latest # Build production zenithdb/zenith:release image and push it to Docker hub docker-image-release: @@ -487,8 +494,14 @@ jobs: command: | echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin DOCKER_TAG="release-$(git log --oneline|wc -l)" - docker build --build-arg GIT_VERSION=$CIRCLE_SHA1 -t zenithdb/zenith:release . && docker push zenithdb/zenith:release - docker tag zenithdb/zenith:release zenithdb/zenith:${DOCKER_TAG} && docker push zenithdb/zenith:${DOCKER_TAG} + docker build \ + --pull \ + --build-arg GIT_VERSION=${CIRCLE_SHA1} \ + --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \ + --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \ + --tag zenithdb/zenith:${DOCKER_TAG} --tag zenithdb/zenith:release . + docker push zenithdb/zenith:${DOCKER_TAG} + docker push zenithdb/zenith:release # Build production zenithdb/compute-node:release image and push it to Docker hub docker-image-compute-release: @@ -515,8 +528,9 @@ jobs: command: | echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin DOCKER_TAG="release-$(git log --oneline|wc -l)" - docker build -t zenithdb/compute-node:release vendor/postgres && docker push zenithdb/compute-node:release - docker tag zenithdb/compute-node:release zenithdb/compute-node:${DOCKER_TAG} && docker push zenithdb/compute-node:${DOCKER_TAG} + docker build --tag zenithdb/compute-node:${DOCKER_TAG} --tag zenithdb/compute-node:release vendor/postgres + docker push zenithdb/compute-node:${DOCKER_TAG} + docker push zenithdb/compute-node:release deploy-staging: docker: diff --git a/Dockerfile b/Dockerfile index 18abae5327..c568cb27b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,62 +1,59 @@ -# -# Docker image for console integration testing. -# - -# -# Build Postgres separately --- this layer will be rebuilt only if one of -# mentioned paths will get any changes. +# Build Postgres # FROM zimg/rust:1.56 AS pg-build -WORKDIR /zenith -COPY ./vendor/postgres vendor/postgres -COPY ./Makefile Makefile -ENV BUILD_TYPE release -RUN make -j $(getconf _NPROCESSORS_ONLN) -s postgres -RUN rm -rf postgres_install/build +WORKDIR /pg + +USER root + +COPY vendor/postgres vendor/postgres +COPY Makefile Makefile + +ENV BUILD_TYPE release +RUN set -e \ + && make -j $(nproc) -s postgres \ + && rm -rf tmp_install/build \ + && tar -C tmp_install -czf /postgres_install.tar.gz . -# # Build zenith binaries # -# TODO: build cargo deps as separate layer. We used cargo-chef before but that was -# net time waste in a lot of cases. Copying Cargo.lock with empty lib.rs should do the work. -# FROM zimg/rust:1.56 AS build +ARG GIT_VERSION=local -ARG GIT_VERSION -RUN if [ -z "$GIT_VERSION" ]; then echo "GIT_VERSION is reqired, use build_arg to pass it"; exit 1; fi - -WORKDIR /zenith -COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server +ARG CACHEPOT_BUCKET=zenith-rust-cachepot +ARG AWS_ACCESS_KEY_ID +ARG AWS_SECRET_ACCESS_KEY +ENV RUSTC_WRAPPER cachepot +COPY --from=pg-build /pg/tmp_install/include/postgresql/server tmp_install/include/postgresql/server COPY . . -RUN GIT_VERSION=$GIT_VERSION cargo build --release -# -# Copy binaries to resulting image. +RUN cargo build --release + +# Build final image # FROM debian:bullseye-slim WORKDIR /data -RUN apt-get update && apt-get -yq install libreadline-dev libseccomp-dev openssl ca-certificates && \ - mkdir zenith_install +RUN set -e \ + && apt-get update \ + && apt-get install -y \ + libreadline-dev \ + libseccomp-dev \ + openssl \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && useradd -d /data zenith \ + && chown -R zenith:zenith /data + +COPY --from=build --chown=zenith:zenith /home/circleci/project/target/release/pageserver /usr/local/bin +COPY --from=build --chown=zenith:zenith /home/circleci/project/target/release/safekeeper /usr/local/bin +COPY --from=build --chown=zenith:zenith /home/circleci/project/target/release/proxy /usr/local/bin + +COPY --from=pg-build /pg/tmp_install/ /usr/local/ +COPY --from=pg-build /postgres_install.tar.gz /data/ -COPY --from=build /zenith/target/release/pageserver /usr/local/bin -COPY --from=build /zenith/target/release/safekeeper /usr/local/bin -COPY --from=build /zenith/target/release/proxy /usr/local/bin -COPY --from=pg-build /zenith/tmp_install postgres_install COPY docker-entrypoint.sh /docker-entrypoint.sh -# Remove build artifacts (~ 500 MB) -RUN rm -rf postgres_install/build && \ - # 'Install' Postgres binaries locally - cp -r postgres_install/* /usr/local/ && \ - # Prepare an archive of Postgres binaries (should be around 11 MB) - # and keep it inside container for an ease of deploy pipeline. - cd postgres_install && tar -czf /data/postgres_install.tar.gz . && cd .. && \ - rm -rf postgres_install - -RUN useradd -d /data zenith && chown -R zenith:zenith /data - VOLUME ["/data"] USER zenith EXPOSE 6400 From 934bbcba0fd8eb5c654109acf8934c2b62ee12e1 Mon Sep 17 00:00:00 2001 From: Andrey Taranik Date: Wed, 9 Mar 2022 10:13:46 +0300 Subject: [PATCH 09/42] revert docker build to debian:buster based rust (#1347) * dockerfile fix, rust cache in docker build flow * check rust cachepot * another check rust cachepot * cleanup * revert docker build to debian:buster based rust to avoid libc6 version mismatch --- Dockerfile | 9 ++++++--- Dockerfile.build | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 Dockerfile.build diff --git a/Dockerfile b/Dockerfile index c568cb27b0..9ee6abaa8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ # Build Postgres # -FROM zimg/rust:1.56 AS pg-build +#FROM zimg/rust:1.56 AS pg-build +FROM zenithdb/build:buster-20220309 AS pg-build WORKDIR /pg USER root @@ -16,13 +17,15 @@ RUN set -e \ # Build zenith binaries # -FROM zimg/rust:1.56 AS build +#FROM zimg/rust:1.56 AS build +FROM zenithdb/build:buster-20220309 AS build ARG GIT_VERSION=local ARG CACHEPOT_BUCKET=zenith-rust-cachepot ARG AWS_ACCESS_KEY_ID ARG AWS_SECRET_ACCESS_KEY -ENV RUSTC_WRAPPER cachepot +#ENV RUSTC_WRAPPER cachepot +ENV RUSTC_WRAPPER /usr/local/cargo/bin/cachepot COPY --from=pg-build /pg/tmp_install/include/postgresql/server tmp_install/include/postgresql/server COPY . . diff --git a/Dockerfile.build b/Dockerfile.build new file mode 100644 index 0000000000..44a2aaafb9 --- /dev/null +++ b/Dockerfile.build @@ -0,0 +1,23 @@ +FROM rust:1.56.1-slim-buster +WORKDIR /home/circleci/project + +RUN set -e \ + && apt-get update \ + && apt-get -yq install \ + automake \ + libtool \ + build-essential \ + bison \ + flex \ + libreadline-dev \ + zlib1g-dev \ + libxml2-dev \ + libseccomp-dev \ + pkg-config \ + libssl-dev \ + clang + +RUN set -e \ + && rustup component add clippy \ + && cargo install cargo-audit \ + && cargo install --git https://github.com/paritytech/cachepot From 15b19a0a5713eec7cc740f5725df971f407d73e4 Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Wed, 9 Mar 2022 14:47:06 -0500 Subject: [PATCH 10/42] [proxy] Test connstr options (#1344) * Add proxy test * Fix typo --- test_runner/batch_others/test_proxy.py | 13 +++++++++++ test_runner/fixtures/zenith_fixtures.py | 29 ++++++++++++++++++------- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/test_runner/batch_others/test_proxy.py b/test_runner/batch_others/test_proxy.py index 9510e880b2..d2039f9758 100644 --- a/test_runner/batch_others/test_proxy.py +++ b/test_runner/batch_others/test_proxy.py @@ -1,2 +1,15 @@ +import pytest + + def test_proxy_select_1(static_proxy): static_proxy.safe_psql("select 1;") + + +@pytest.mark.xfail # Proxy eats the extra connection options +def test_proxy_options(static_proxy): + schema_name = "tmp_schema_1" + with static_proxy.connect(schema=schema_name) as conn: + with conn.cursor() as cur: + cur.execute("SHOW search_path;") + search_path = cur.fetchall()[0][0] + assert schema_name == search_path diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index 252ca9b3c1..4d6e84048c 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -242,15 +242,20 @@ class PgProtocol: host: str, port: int, username: Optional[str] = None, - password: Optional[str] = None): + password: Optional[str] = None, + dbname: Optional[str] = None, + schema: Optional[str] = None): self.host = host self.port = port self.username = username self.password = password + self.dbname = dbname + self.schema = schema def connstr(self, *, - dbname: str = 'postgres', + dbname: Optional[str] = None, + schema: Optional[str] = None, username: Optional[str] = None, password: Optional[str] = None) -> str: """ @@ -259,6 +264,8 @@ class PgProtocol: username = username or self.username password = password or self.password + dbname = dbname or self.dbname or "postgres" + schema = schema or self.schema res = f'host={self.host} port={self.port} dbname={dbname}' if username: @@ -267,13 +274,17 @@ class PgProtocol: if password: res = f'{res} password={password}' + if schema: + res = f"{res} options='-c search_path={schema}'" + return res # autocommit=True here by default because that's what we need most of the time def connect(self, *, autocommit=True, - dbname: str = 'postgres', + dbname: Optional[str] = None, + schema: Optional[str] = None, username: Optional[str] = None, password: Optional[str] = None) -> PgConnection: """ @@ -282,11 +293,13 @@ class PgProtocol: This method passes all extra params to connstr. """ - conn = psycopg2.connect(self.connstr( - dbname=dbname, - username=username, - password=password, - )) + conn = psycopg2.connect( + self.connstr( + dbname=dbname, + schema=schema, + username=username, + password=password, + )) # WARNING: this setting affects *all* tests! conn.autocommit = autocommit return conn From 5b34afe89326e337a256eef109990621e73dfd83 Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 23 Feb 2022 21:37:10 +0300 Subject: [PATCH 11/42] Bump vendor/postgres to use local relation cache for smgr_exists --- vendor/postgres | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/postgres b/vendor/postgres index 31dc24ab29..bf6797aab5 160000 --- a/vendor/postgres +++ b/vendor/postgres @@ -1 +1 @@ -Subproject commit 31dc24ab29e6bdd5cfb85920a9c728f759c01b29 +Subproject commit bf6797aab54f1a7b865491262328598ae1869c1f From 87f306c516e8fa1a8b43778971d570ac3201ad19 Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 23 Feb 2022 21:58:32 +0300 Subject: [PATCH 12/42] Tune backpressure in python tests to make them more stable --- test_runner/fixtures/zenith_fixtures.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index 4d6e84048c..06f75aa604 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -1285,6 +1285,10 @@ class Postgres(PgProtocol): if config_lines is None: config_lines = [] + + # set small 'max_replication_write_lag' to enable backpressure + # and make tests more stable. + config_lines = ['max_replication_write_lag=15MB'] + config_lines self.config(config_lines) return self From 2883a25761431eeb4ec53945c3ea4f6fbf2aa6b2 Mon Sep 17 00:00:00 2001 From: Anastasia Lubennikova Date: Thu, 10 Mar 2022 17:31:43 +0400 Subject: [PATCH 13/42] Bump vendor/postgres to use local relation cache for smgr_exists --- vendor/postgres | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/postgres b/vendor/postgres index bf6797aab5..093aa160e5 160000 --- a/vendor/postgres +++ b/vendor/postgres @@ -1 +1 @@ -Subproject commit bf6797aab54f1a7b865491262328598ae1869c1f +Subproject commit 093aa160e5df19814ff19b995d36dd5ee03c7f8b From 10f811e886292e258adec931945f7f6bdce4b412 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Fri, 4 Feb 2022 10:37:39 -0500 Subject: [PATCH 14/42] Use `timeline` instead of `branch` in pageserver's API --- control_plane/src/compute.rs | 38 ++-- control_plane/src/storage.rs | 45 ++-- pageserver/src/bin/pageserver.rs | 5 +- pageserver/src/config.rs | 8 - pageserver/src/http/models.rs | 6 +- pageserver/src/http/openapi_spec.yml | 141 +++---------- pageserver/src/http/routes.rs | 123 +++-------- pageserver/src/layered_repository.rs | 45 ++-- pageserver/src/lib.rs | 2 +- pageserver/src/remote_storage/README.md | 8 - pageserver/src/remote_storage/storage_sync.rs | 62 +----- .../remote_storage/storage_sync/download.rs | 100 +-------- .../src/remote_storage/storage_sync/index.rs | 37 +--- .../src/remote_storage/storage_sync/upload.rs | 94 +-------- pageserver/src/repository.rs | 21 +- pageserver/src/tenant_mgr.rs | 4 +- pageserver/src/{branches.rs => timelines.rs} | 194 ++++++------------ zenith/src/main.rs | 178 ++++++++-------- 18 files changed, 311 insertions(+), 800 deletions(-) rename pageserver/src/{branches.rs => timelines.rs} (70%) diff --git a/control_plane/src/compute.rs b/control_plane/src/compute.rs index a61191e7a4..3569cc1dbb 100644 --- a/control_plane/src/compute.rs +++ b/control_plane/src/compute.rs @@ -73,39 +73,43 @@ impl ComputeControlPlane { .unwrap_or(self.base_port) } - // FIXME: see also parse_point_in_time in branches.rs. + // FIXME: see also parse_point_in_time in timelines.rs. fn parse_point_in_time( &self, - tenantid: ZTenantId, + tenant_id: ZTenantId, s: &str, ) -> Result<(ZTimelineId, Option)> { - let mut strings = s.split('@'); - let name = strings.next().unwrap(); + let _strings = s.split('@'); + // let name = strings.next().unwrap(); - let lsn = strings - .next() - .map(Lsn::from_str) - .transpose() - .context("invalid LSN in point-in-time specification")?; + // let lsn = strings + // .next() + // .map(Lsn::from_str) + // .transpose() + // .context("invalid LSN in point-in-time specification")?; - // Resolve the timeline ID, given the human-readable branch name - let timeline_id = self - .pageserver - .branch_get_by_name(&tenantid, name)? - .timeline_id; + // // Resolve the timeline ID, given the human-readable branch name + // let timeline_id = self + // .pageserver + // .branch_get_by_name(&tenant_id, name)? + // .timeline_id; - Ok((timeline_id, lsn)) + // Ok((timeline_id, lsn)) + todo!("TODO kb check more about the '@name' format") } pub fn new_node( &mut self, tenantid: ZTenantId, name: &str, - timeline_spec: &str, + timeline_spec: Option<&str>, port: Option, ) -> Result> { // Resolve the human-readable timeline spec into timeline ID and LSN - let (timelineid, lsn) = self.parse_point_in_time(tenantid, timeline_spec)?; + let (timelineid, lsn) = match timeline_spec { + Some(timeline_spec) => self.parse_point_in_time(tenantid, timeline_spec)?, + None => (ZTimelineId::generate(), None), + }; let port = port.unwrap_or_else(|| self.get_port()); let node = Arc::new(PostgresNode { diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index cd429e3f7a..aed9a757d4 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -9,18 +9,18 @@ use anyhow::bail; use nix::errno::Errno; use nix::sys::signal::{kill, Signal}; use nix::unistd::Pid; -use pageserver::http::models::{BranchCreateRequest, TenantCreateRequest}; +use pageserver::http::models::{TenantCreateRequest, TimelineCreateRequest}; +use pageserver::timelines::TimelineInfo; use postgres::{Config, NoTls}; use reqwest::blocking::{Client, RequestBuilder, Response}; use reqwest::{IntoUrl, Method}; use thiserror::Error; use zenith_utils::http::error::HttpErrorBody; use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::ZTenantId; +use zenith_utils::zid::{ZTenantId, ZTimelineId}; use crate::local_env::LocalEnv; use crate::{fill_rust_env_vars, read_pidfile}; -use pageserver::branches::BranchInfo; use pageserver::tenant_mgr::TenantInfo; use zenith_utils::connstring::connection_address; @@ -335,47 +335,32 @@ impl PageServerNode { .json()?) } - pub fn branch_list(&self, tenantid: &ZTenantId) -> Result> { + pub fn timeline_list(&self, tenantid: &ZTenantId) -> Result> { Ok(self .http_request( Method::GET, - format!("{}/branch/{}", self.http_base_url, tenantid), + format!("{}/timeline/{}", self.http_base_url, tenantid), ) .send()? .error_from_body()? .json()?) } - pub fn branch_create( + pub fn timeline_create( &self, - branch_name: &str, - startpoint: &str, - tenantid: &ZTenantId, - ) -> Result { + timeline_id: ZTimelineId, + start_point: String, + tenant_id: ZTenantId, + ) -> Result { Ok(self - .http_request(Method::POST, format!("{}/branch", self.http_base_url)) - .json(&BranchCreateRequest { - tenant_id: tenantid.to_owned(), - name: branch_name.to_owned(), - start_point: startpoint.to_owned(), + .http_request(Method::POST, format!("{}/timeline", self.http_base_url)) + .json(&TimelineCreateRequest { + tenant_id, + timeline_id, + start_point, }) .send()? .error_from_body()? .json()?) } - - pub fn branch_get_by_name( - &self, - tenantid: &ZTenantId, - branch_name: &str, - ) -> Result { - Ok(self - .http_request( - Method::GET, - format!("{}/branch/{}/{}", self.http_base_url, tenantid, branch_name), - ) - .send()? - .error_for_status()? - .json()?) - } } diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index d8d4033340..2fa772af58 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -10,11 +10,10 @@ use clap::{App, Arg}; use daemonize::Daemonize; use pageserver::{ - branches, config::{defaults::*, PageServerConf}, http, page_cache, page_service, remote_storage, tenant_mgr, thread_mgr, thread_mgr::ThreadKind, - virtual_file, LOG_FILE_NAME, + timelines, virtual_file, LOG_FILE_NAME, }; use zenith_utils::http::endpoint; use zenith_utils::postgres_backend; @@ -143,7 +142,7 @@ fn main() -> Result<()> { // Create repo and exit if init was requested if init { - branches::init_pageserver(conf, create_tenant).context("Failed to init pageserver")?; + timelines::init_pageserver(conf, create_tenant).context("Failed to init pageserver")?; // write the config file std::fs::write(&cfg_file_path, toml.to_string()).with_context(|| { format!( diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 3deabb7521..5a9c7557cc 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -400,14 +400,6 @@ impl PageServerConf { self.tags_path(tenantid).join(tag_name) } - pub fn branches_path(&self, tenantid: &ZTenantId) -> PathBuf { - self.tenant_path(tenantid).join("refs").join("branches") - } - - pub fn branch_path(&self, branch_name: &str, tenantid: &ZTenantId) -> PathBuf { - self.branches_path(tenantid).join(branch_name) - } - pub fn timelines_path(&self, tenantid: &ZTenantId) -> PathBuf { self.tenant_path(tenantid).join(TIMELINES_SEGMENT_NAME) } diff --git a/pageserver/src/http/models.rs b/pageserver/src/http/models.rs index 5d7398ef03..a6dce33c03 100644 --- a/pageserver/src/http/models.rs +++ b/pageserver/src/http/models.rs @@ -1,13 +1,15 @@ use serde::{Deserialize, Serialize}; +use zenith_utils::zid::ZTimelineId; use crate::ZTenantId; use zenith_utils::zid::ZNodeId; #[derive(Serialize, Deserialize)] -pub struct BranchCreateRequest { +pub struct TimelineCreateRequest { #[serde(with = "hex")] pub tenant_id: ZTenantId, - pub name: String, + #[serde(with = "hex")] + pub timeline_id: ZTimelineId, pub start_point: String, } diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index baf81fcf21..7f3bf97bfe 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -30,19 +30,22 @@ paths: schema: type: string format: hex + - name: include-non-incremental-logical-size + in: query + schema: + type: string + description: Controls calculation of current_logical_size_non_incremental get: - description: List tenant timelines + description: Get timelines for tenant responses: "200": - description: array of brief timeline descriptions + description: TimelineInfo content: application/json: schema: type: array items: - # currently, just a timeline id string, but when remote index gets to be accessed - # remote/local timeline field would be added at least - type: string + $ref: "#/components/schemas/TimelineInfo" "400": description: Error when no tenant id found in path content: @@ -81,8 +84,13 @@ paths: schema: type: string format: hex + - name: include-non-incremental-logical-size + in: query + schema: + type: string + description: Controls calculation of current_logical_size_non_incremental get: - description: Get timeline info for tenant's remote timeline + description: Get timelines for tenant responses: "200": description: TimelineInfo @@ -91,7 +99,7 @@ paths: schema: $ref: "#/components/schemas/TimelineInfo" "400": - description: Error when no tenant id found in path or no branch name + description: Error when no tenant id found in path or no timeline id content: application/json: schema: @@ -114,108 +122,9 @@ paths: application/json: schema: $ref: "#/components/schemas/Error" - /v1/branch/{tenant_id}: - parameters: - - name: tenant_id - in: path - required: true - schema: - type: string - format: hex - - name: include-non-incremental-logical-size - in: query - schema: - type: string - description: Controls calculation of current_logical_size_non_incremental - get: - description: Get branches for tenant - responses: - "200": - description: BranchInfo - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/BranchInfo" - "400": - description: Error when no tenant id found in path - content: - application/json: - schema: - $ref: "#/components/schemas/Error" - "401": - description: Unauthorized Error - content: - application/json: - schema: - $ref: "#/components/schemas/UnauthorizedError" - "403": - description: Forbidden Error - content: - application/json: - schema: - $ref: "#/components/schemas/ForbiddenError" - "500": - description: Generic operation error - content: - application/json: - schema: - $ref: "#/components/schemas/Error" - /v1/branch/{tenant_id}/{branch_name}: - parameters: - - name: tenant_id - in: path - required: true - schema: - type: string - format: hex - - name: branch_name - in: path - required: true - schema: - type: string - - name: include-non-incremental-logical-size - in: query - schema: - type: string - description: Controls calculation of current_logical_size_non_incremental - get: - description: Get branches for tenant - responses: - "200": - description: BranchInfo - content: - application/json: - schema: - $ref: "#/components/schemas/BranchInfo" - "400": - description: Error when no tenant id found in path or no branch name - content: - application/json: - schema: - $ref: "#/components/schemas/Error" - "401": - description: Unauthorized Error - content: - application/json: - schema: - $ref: "#/components/schemas/UnauthorizedError" - "403": - description: Forbidden Error - content: - application/json: - schema: - $ref: "#/components/schemas/ForbiddenError" - "500": - description: Generic operation error - content: - application/json: - schema: - $ref: "#/components/schemas/Error" - /v1/branch/: + /v1/timeline/: post: - description: Create branch + description: Create timeline requestBody: content: application/json: @@ -223,25 +132,26 @@ paths: type: object required: - "tenant_id" - - "name" + - "timeline_id" - "start_point" properties: tenant_id: type: string format: hex - name: + timeline_id: type: string + format: hex start_point: type: string responses: "201": - description: BranchInfo + description: TimelineInfo content: application/json: schema: - $ref: "#/components/schemas/BranchInfo" + $ref: "#/components/schemas/TImelineInfo" "400": - description: Malformed branch create request + description: Malformed timeline create request content: application/json: schema: @@ -358,16 +268,13 @@ components: type: string state: type: string - BranchInfo: + TimelineInfo: type: object required: - - name - timeline_id - latest_valid_lsn - current_logical_size properties: - name: - type: string timeline_id: type: string format: hex diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 26d473efaf..5ab1576aa6 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use anyhow::{Context, Result}; +use anyhow::Result; use hyper::StatusCode; use hyper::{Body, Request, Response, Uri}; use serde::Serialize; @@ -14,7 +14,6 @@ use zenith_utils::http::{ endpoint, error::HttpErrorBody, json::{json_request, json_response}, - request::get_request_param, request::parse_request_param, }; use zenith_utils::http::{RequestExt, RouterBuilder}; @@ -22,13 +21,12 @@ use zenith_utils::lsn::Lsn; use zenith_utils::zid::HexZTimelineId; use zenith_utils::zid::ZTimelineId; -use super::models::BranchCreateRequest; use super::models::StatusResponse; use super::models::TenantCreateRequest; -use crate::branches::BranchInfo; +use super::models::TimelineCreateRequest; use crate::repository::RepositoryTimeline; use crate::repository::TimelineSyncState; -use crate::{branches, config::PageServerConf, tenant_mgr, ZTenantId}; +use crate::{config::PageServerConf, tenant_mgr, timelines, ZTenantId}; #[derive(Debug)] struct State { @@ -73,18 +71,18 @@ async fn status_handler(request: Request) -> Result, ApiErr )?) } -async fn branch_create_handler(mut request: Request) -> Result, ApiError> { - let request_data: BranchCreateRequest = json_request(&mut request).await?; +async fn timeline_create_handler(mut request: Request) -> Result, ApiError> { + let request_data: TimelineCreateRequest = json_request(&mut request).await?; check_permission(&request, Some(request_data.tenant_id))?; let response_data = tokio::task::spawn_blocking(move || { - let _enter = info_span!("/branch_create", name = %request_data.name, tenant = %request_data.tenant_id, startpoint=%request_data.start_point).entered(); - branches::create_branch( + let _enter = info_span!("/timeline_create", timeline = %request_data.timeline_id, tenant = %request_data.tenant_id, startpoint=%request_data.start_point).entered(); + timelines::create_timeline( get_config(&request), - &request_data.name, &request_data.start_point, - &request_data.tenant_id, + request_data.tenant_id, + request_data.timeline_id, ) }) .await @@ -92,6 +90,19 @@ async fn branch_create_handler(mut request: Request) -> Result) -> Result, ApiError> { + let tenant_id: ZTenantId = parse_request_param(&request, "tenant_id")?; + check_permission(&request, Some(tenant_id))?; + let include_non_incremental_logical_size = get_include_non_incremental_logical_size(&request); + let response_data = tokio::task::spawn_blocking(move || { + let _enter = info_span!("timeline_list", tenant = %tenant_id).entered(); + crate::timelines::get_timelines(tenant_id, include_non_incremental_logical_size) + }) + .await + .map_err(ApiError::from_err)??; + Ok(json_response(StatusCode::OK, response_data)?) +} + // Gate non incremental logical size calculation behind a flag // after pgbench -i -s100 calculation took 28ms so if multiplied by the number of timelines // and tenants it can take noticeable amount of time. Also the value currently used only in tests @@ -107,90 +118,6 @@ fn get_include_non_incremental_logical_size(request: &Request) -> bool { .unwrap_or(false) } -async fn branch_list_handler(request: Request) -> Result, ApiError> { - let tenantid: ZTenantId = parse_request_param(&request, "tenant_id")?; - - let include_non_incremental_logical_size = get_include_non_incremental_logical_size(&request); - - check_permission(&request, Some(tenantid))?; - - let response_data = tokio::task::spawn_blocking(move || { - let _enter = info_span!("branch_list", tenant = %tenantid).entered(); - crate::branches::get_branches( - get_config(&request), - &tenantid, - include_non_incremental_logical_size, - ) - }) - .await - .map_err(ApiError::from_err)??; - Ok(json_response(StatusCode::OK, response_data)?) -} - -async fn branch_detail_handler(request: Request) -> Result, ApiError> { - let tenantid: ZTenantId = parse_request_param(&request, "tenant_id")?; - let branch_name: String = get_request_param(&request, "branch_name")?.to_string(); - let conf = get_state(&request).conf; - let path = conf.branch_path(&branch_name, &tenantid); - - let include_non_incremental_logical_size = get_include_non_incremental_logical_size(&request); - - let response_data = tokio::task::spawn_blocking(move || { - let _enter = info_span!("branch_detail", tenant = %tenantid, branch=%branch_name).entered(); - let repo = tenant_mgr::get_repository_for_tenant(tenantid)?; - BranchInfo::from_path(path, &repo, include_non_incremental_logical_size) - }) - .await - .map_err(ApiError::from_err)??; - - Ok(json_response(StatusCode::OK, response_data)?) -} - -async fn timeline_list_handler(request: Request) -> Result, ApiError> { - let tenant_id: ZTenantId = parse_request_param(&request, "tenant_id")?; - check_permission(&request, Some(tenant_id))?; - - let conf = get_state(&request).conf; - let timelines_dir = conf.timelines_path(&tenant_id); - - let mut timelines_dir_contents = - tokio::fs::read_dir(&timelines_dir).await.with_context(|| { - format!( - "Failed to list timelines dir '{}' contents", - timelines_dir.display() - ) - })?; - - let mut local_timelines = Vec::new(); - while let Some(entry) = timelines_dir_contents.next_entry().await.with_context(|| { - format!( - "Failed to list timelines dir '{}' contents", - timelines_dir.display() - ) - })? { - let entry_path = entry.path(); - let entry_type = entry.file_type().await.with_context(|| { - format!( - "Failed to get file type of timeline dirs' entry '{}'", - entry_path.display() - ) - })?; - - if entry_type.is_dir() { - match entry.file_name().to_string_lossy().parse::() { - Ok(timeline_id) => local_timelines.push(timeline_id.to_string()), - Err(e) => error!( - "Failed to get parse timeline id from timeline dirs' entry '{}': {}", - entry_path.display(), - e - ), - } - } - } - - Ok(json_response(StatusCode::OK, local_timelines)?) -} - #[derive(Debug, Serialize)] #[serde(tag = "type")] enum TimelineInfo { @@ -260,7 +187,7 @@ async fn timeline_attach_handler(request: Request) -> Result { + RepositoryTimeline::Local { .. } => { anyhow::bail!("Timeline with id {} is already local", timeline_id) } RepositoryTimeline::Remote { @@ -369,9 +296,7 @@ pub fn make_router( "/v1/timeline/:tenant_id/:timeline_id/detach", timeline_detach_handler, ) - .get("/v1/branch/:tenant_id", branch_list_handler) - .get("/v1/branch/:tenant_id/:branch_name", branch_detail_handler) - .post("/v1/branch", branch_create_handler) + .post("/v1/timeline", timeline_create_handler) .get("/v1/tenant", tenant_list_handler) .post("/v1/tenant", tenant_create_handler) .any(handler_404) diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index 975b2f5d2b..c3d42d1829 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -137,19 +137,20 @@ pub struct LayeredRepository { /// Public interface impl Repository for LayeredRepository { fn get_timeline(&self, timelineid: ZTimelineId) -> Result { - let mut timelines = self.timelines.lock().unwrap(); - Ok( - match self.get_or_init_timeline(timelineid, &mut timelines)? { - LayeredTimelineEntry::Local(local) => RepositoryTimeline::Local(local), - LayeredTimelineEntry::Remote { - id, - disk_consistent_lsn, - } => RepositoryTimeline::Remote { - id, - disk_consistent_lsn, - }, - }, - ) + Ok(RepositoryTimeline::from(self.get_or_init_timeline( + timelineid, + &mut self.timelines.lock().unwrap(), + )?)) + } + + fn list_timelines(&self) -> Result> { + Ok(self + .timelines + .lock() + .unwrap() + .values() + .map(|timeline_entry| RepositoryTimeline::from(timeline_entry.clone())) + .collect()) } fn create_empty_timeline( @@ -428,6 +429,24 @@ impl LayeredTimelineEntry { } } +impl From for RepositoryTimeline { + fn from(layered_timeline: LayeredTimelineEntry) -> Self { + match layered_timeline { + LayeredTimelineEntry::Local(timeline) => RepositoryTimeline::Local { + id: timeline.timelineid, + timeline, + }, + LayeredTimelineEntry::Remote { + id, + disk_consistent_lsn, + } => RepositoryTimeline::Remote { + id, + disk_consistent_lsn, + }, + } + } +} + /// Private functions impl LayeredRepository { // Implementation of the public `get_timeline` function. This differs from the public diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index 3a68f56187..3d66192c80 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -1,5 +1,4 @@ pub mod basebackup; -pub mod branches; pub mod config; pub mod http; pub mod import_datadir; @@ -12,6 +11,7 @@ pub mod repository; pub mod tenant_mgr; pub mod tenant_threads; pub mod thread_mgr; +pub mod timelines; pub mod virtual_file; pub mod walingest; pub mod walreceiver; diff --git a/pageserver/src/remote_storage/README.md b/pageserver/src/remote_storage/README.md index 1c718acf06..3c77275da8 100644 --- a/pageserver/src/remote_storage/README.md +++ b/pageserver/src/remote_storage/README.md @@ -62,11 +62,3 @@ Based on previous evaluation, even `rusoto-s3` could be a better choice over thi So far, we don't adjust the remote storage based on GC thread loop results, only checkpointer loop affects the remote storage. Index module could be used as a base to implement a deferred GC mechanism, a "defragmentation" that repacks archives into new ones after GC is done removing the files from the archives. - -* bracnhes implementaion could be improved - -Currently, there's a code to sync the branches along with the timeline files: on upload, every local branch files that are missing remotely are uploaded, -on the timeline download, missing remote branch files are downlaoded. - -A branch is a per-tenant entity, yet a current implementaion requires synchronizing a timeline first to get the branch files locally. -Currently, there's no other way to know about the remote branch files, neither the file contents is verified and updated. diff --git a/pageserver/src/remote_storage/storage_sync.rs b/pageserver/src/remote_storage/storage_sync.rs index 6b588c8e5f..d14f849e15 100644 --- a/pageserver/src/remote_storage/storage_sync.rs +++ b/pageserver/src/remote_storage/storage_sync.rs @@ -14,13 +14,6 @@ //! Only GC removes local timeline files, the GC support is not added to sync currently, //! yet downloading extra files is not critically bad at this stage, GC can remove those again. //! -//! Along the timeline files, branch files are uploaded and downloaded every time a corresponding sync task is processed. -//! For simplicity, branch files are also treated as immutable: only missing files are uploaded or downloaded, no removals, amendments or file contents checks are done. -//! Also, the branches are copied as separate files, with no extra compressions done. -//! Despite branches information currently belonging to tenants, a tenants' timeline sync is required to upload or download the branch files, also, there's no way to know -//! the branch sync state outside of the sync loop. -//! This implementation is currently considered as temporary and is a subjec to change later. -//! //! During the loop startup, an initial [`RemoteTimelineIndex`] state is constructed via listing the remote storage contents. //! It's enough to poll the remote state once on startup only, due to agreement that the pageserver has //! an exclusive write access to the remote storage: new files appear in the storage only after the same @@ -66,7 +59,6 @@ //! NOTE: No real contents or checksum check happens right now and is a subject to improve later. //! //! After the whole timeline is downloaded, [`crate::tenant_mgr::set_timeline_states`] function is used to update pageserver memory stage for the timeline processed. -//! No extra branch registration is done. //! //! When pageserver signals shutdown, current sync task gets finished and the loop exists. @@ -77,7 +69,7 @@ pub mod index; mod upload; use std::{ - collections::{BTreeSet, HashMap, HashSet, VecDeque}, + collections::{BTreeSet, HashMap, VecDeque}, num::{NonZeroU32, NonZeroUsize}, path::{Path, PathBuf}, sync::Arc, @@ -87,7 +79,6 @@ use anyhow::{bail, Context}; use futures::stream::{FuturesUnordered, StreamExt}; use lazy_static::lazy_static; use tokio::{ - fs, runtime::Runtime, sync::{ mpsc::{self, UnboundedReceiver}, @@ -101,8 +92,7 @@ use self::{ compression::ArchiveHeader, download::{download_timeline, DownloadedTimeline}, index::{ - ArchiveDescription, ArchiveId, RelativePath, RemoteTimeline, RemoteTimelineIndex, - TimelineIndexEntry, + ArchiveDescription, ArchiveId, RemoteTimeline, RemoteTimelineIndex, TimelineIndexEntry, }, upload::upload_timeline_checkpoint, }; @@ -843,28 +833,6 @@ async fn download_archive_header< Ok(header) } -async fn tenant_branch_files( - conf: &'static PageServerConf, - tenant_id: ZTenantId, -) -> anyhow::Result> { - let branches_dir = conf.branches_path(&tenant_id); - if !branches_dir.exists() { - return Ok(HashSet::new()); - } - - let mut branch_entries = fs::read_dir(&branches_dir) - .await - .context("Failed to list tenant branches dir contents")?; - - let mut branch_files = HashSet::new(); - while let Some(branch_entry) = branch_entries.next_entry().await? { - if branch_entry.file_type().await?.is_file() { - branch_files.insert(RelativePath::new(&branches_dir, branch_entry.path())?); - } - } - Ok(branch_files) -} - #[cfg(test)] mod test_utils { use std::{ @@ -971,30 +939,9 @@ mod test_utils { "Index contains unexpected sync ids" ); - let mut actual_branches = BTreeMap::new(); - let mut expected_branches = BTreeMap::new(); let mut actual_timeline_entries = BTreeMap::new(); let mut expected_timeline_entries = BTreeMap::new(); for sync_id in actual_sync_ids { - actual_branches.insert( - sync_id.tenant_id, - index_read - .branch_files(sync_id.tenant_id) - .into_iter() - .flat_map(|branch_paths| branch_paths.iter()) - .cloned() - .collect::>(), - ); - expected_branches.insert( - sync_id.tenant_id, - expected_index_with_descriptions - .branch_files(sync_id.tenant_id) - .into_iter() - .flat_map(|branch_paths| branch_paths.iter()) - .cloned() - .collect::>(), - ); - actual_timeline_entries.insert( sync_id, index_read.timeline_entry(&sync_id).unwrap().clone(), @@ -1009,11 +956,6 @@ mod test_utils { } drop(index_read); - assert_eq!( - actual_branches, expected_branches, - "Index contains unexpected branches" - ); - for (sync_id, actual_timeline_entry) in actual_timeline_entries { let expected_timeline_description = expected_timeline_entries .remove(&sync_id) diff --git a/pageserver/src/remote_storage/storage_sync/download.rs b/pageserver/src/remote_storage/storage_sync/download.rs index f268fc442a..00115ba8d5 100644 --- a/pageserver/src/remote_storage/storage_sync/download.rs +++ b/pageserver/src/remote_storage/storage_sync/download.rs @@ -1,10 +1,8 @@ //! Timeline synchrnonization logic to put files from archives on remote storage into pageserver's local directory. -//! Currently, tenant branch files are also downloaded, but this does not appear final. use std::{borrow::Cow, collections::BTreeSet, path::PathBuf, sync::Arc}; use anyhow::{ensure, Context}; -use futures::{stream::FuturesUnordered, StreamExt}; use tokio::{fs, sync::RwLock}; use tracing::{debug, error, trace, warn}; use zenith_utils::{lsn::Lsn, zid::ZTenantId}; @@ -14,8 +12,8 @@ use crate::{ layered_repository::metadata::{metadata_path, TimelineMetadata}, remote_storage::{ storage_sync::{ - compression, index::TimelineIndexEntry, sync_queue, tenant_branch_files, - update_index_description, SyncKind, SyncTask, + compression, index::TimelineIndexEntry, sync_queue, update_index_description, SyncKind, + SyncTask, }, RemoteStorage, ZTenantTimelineId, }, @@ -42,8 +40,6 @@ pub(super) enum DownloadedTimeline { /// Timeline files that already exist locally are skipped during the download, but the local metadata file is /// updated in the end of every checkpoint archive extraction. /// -/// Before any archives are considered, the branch files are checked locally and remotely, all remote-only files are downloaded. -/// /// On an error, bumps the retries count and reschedules the download, with updated archive skip list /// (for any new successful archive downloads and extractions). pub(super) async fn download_timeline< @@ -113,22 +109,6 @@ pub(super) async fn download_timeline< } }; - if let Err(e) = download_missing_branches(conf, remote_assets.as_ref(), sync_id.tenant_id).await - { - error!( - "Failed to download missing branches for sync id {}: {:?}", - sync_id, e - ); - sync_queue::push(SyncTask::new( - sync_id, - retries, - SyncKind::Download(download), - )); - return DownloadedTimeline::FailedAndRescheduled { - disk_consistent_lsn, - }; - } - debug!("Downloading timeline archives"); let archives_to_download = remote_timeline .checkpoints() @@ -250,82 +230,6 @@ async fn read_local_metadata( .context("Failed to read local metadata files bytes")?) } -async fn download_missing_branches< - P: std::fmt::Debug + Send + Sync + 'static, - S: RemoteStorage + Send + Sync + 'static, ->( - conf: &'static PageServerConf, - (storage, index): &(S, RwLock), - tenant_id: ZTenantId, -) -> anyhow::Result<()> { - let local_branches = tenant_branch_files(conf, tenant_id) - .await - .context("Failed to list local branch files for the tenant")?; - let local_branches_dir = conf.branches_path(&tenant_id); - if !local_branches_dir.exists() { - fs::create_dir_all(&local_branches_dir) - .await - .with_context(|| { - format!( - "Failed to create local branches directory at path '{}'", - local_branches_dir.display() - ) - })?; - } - - if let Some(remote_branches) = index.read().await.branch_files(tenant_id) { - let mut remote_only_branches_downloads = remote_branches - .difference(&local_branches) - .map(|remote_only_branch| async move { - let branches_dir = conf.branches_path(&tenant_id); - let remote_branch_path = remote_only_branch.as_path(&branches_dir); - let storage_path = - storage.storage_path(&remote_branch_path).with_context(|| { - format!( - "Failed to derive a storage path for branch with local path '{}'", - remote_branch_path.display() - ) - })?; - let mut target_file = fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&remote_branch_path) - .await - .with_context(|| { - format!( - "Failed to create local branch file at '{}'", - remote_branch_path.display() - ) - })?; - storage - .download(&storage_path, &mut target_file) - .await - .with_context(|| { - format!( - "Failed to download branch file from the remote path {:?}", - storage_path - ) - })?; - Ok::<_, anyhow::Error>(()) - }) - .collect::>(); - - let mut branch_downloads_failed = false; - while let Some(download_result) = remote_only_branches_downloads.next().await { - if let Err(e) = download_result { - branch_downloads_failed = true; - error!("Failed to download a branch file: {:?}", e); - } - } - ensure!( - !branch_downloads_failed, - "Failed to download all branch files" - ); - } - - Ok(()) -} - #[cfg(test)] mod tests { use std::collections::BTreeSet; diff --git a/pageserver/src/remote_storage/storage_sync/index.rs b/pageserver/src/remote_storage/storage_sync/index.rs index 3d2680948d..8ff92ed55e 100644 --- a/pageserver/src/remote_storage/storage_sync/index.rs +++ b/pageserver/src/remote_storage/storage_sync/index.rs @@ -5,7 +5,7 @@ //! This way in the future, the index could be restored fast from its serialized stored form. use std::{ - collections::{BTreeMap, BTreeSet, HashMap, HashSet}, + collections::{BTreeMap, BTreeSet, HashMap}, path::{Path, PathBuf}, }; @@ -49,10 +49,9 @@ impl RelativePath { } /// An index to track tenant files that exist on the remote storage. -/// Currently, timeline archives and branch files are tracked. +/// Currently, timeline archives files are tracked only. #[derive(Debug, Clone)] pub struct RemoteTimelineIndex { - branch_files: HashMap>, timeline_files: HashMap, } @@ -65,7 +64,6 @@ impl RemoteTimelineIndex { paths: impl Iterator, ) -> Self { let mut index = Self { - branch_files: HashMap::new(), timeline_files: HashMap::new(), }; for path in paths { @@ -98,17 +96,6 @@ impl RemoteTimelineIndex { pub fn all_sync_ids(&self) -> impl Iterator + '_ { self.timeline_files.keys().copied() } - - pub fn add_branch_file(&mut self, tenant_id: ZTenantId, path: RelativePath) { - self.branch_files - .entry(tenant_id) - .or_insert_with(HashSet::new) - .insert(path); - } - - pub fn branch_files(&self, tenant_id: ZTenantId) -> Option<&HashSet> { - self.branch_files.get(&tenant_id) - } } #[derive(Debug, Clone, PartialEq, Eq)] @@ -306,20 +293,9 @@ fn try_parse_index_entry( .parse::() .with_context(|| format!("Failed to parse tenant id from path '{}'", path.display()))?; - let branches_path = conf.branches_path(&tenant_id); let timelines_path = conf.timelines_path(&tenant_id); - match ( - RelativePath::new(&branches_path, &path), - path.strip_prefix(&timelines_path), - ) { - (Ok(_), Ok(_)) => bail!( - "Path '{}' cannot start with both branches '{}' and the timelines '{}' prefixes", - path.display(), - branches_path.display(), - timelines_path.display() - ), - (Ok(branches_entry), Err(_)) => index.add_branch_file(tenant_id, branches_entry), - (Err(_), Ok(timelines_subpath)) => { + match path.strip_prefix(&timelines_path) { + Ok(timelines_subpath) => { let mut segments = timelines_subpath.iter(); let timeline_id = segments .next() @@ -375,11 +351,10 @@ fn try_parse_index_entry( } } } - (Err(branches_error), Err(timelines_strip_error)) => { + Err(timelines_strip_error) => { bail!( - "Path '{}' is not an index entry: it's neither parsable as a branch entry '{:#}' nor as an archive entry '{}'", + "Path '{}' is not an archive entry '{}'", path.display(), - branches_error, timelines_strip_error, ) } diff --git a/pageserver/src/remote_storage/storage_sync/upload.rs b/pageserver/src/remote_storage/storage_sync/upload.rs index 0f57d714dd..d064039ecc 100644 --- a/pageserver/src/remote_storage/storage_sync/upload.rs +++ b/pageserver/src/remote_storage/storage_sync/upload.rs @@ -1,13 +1,10 @@ //! Timeline synchronization logic to compress and upload to the remote storage all new timeline files from the checkpoints. -//! Currently, tenant branch files are also uploaded, but this does not appear final. use std::{borrow::Cow, collections::BTreeSet, path::PathBuf, sync::Arc}; -use anyhow::{ensure, Context}; -use futures::{stream::FuturesUnordered, StreamExt}; -use tokio::{fs, sync::RwLock}; +use anyhow::ensure; +use tokio::sync::RwLock; use tracing::{debug, error, warn}; -use zenith_utils::zid::ZTenantId; use crate::{ config::PageServerConf, @@ -15,7 +12,7 @@ use crate::{ storage_sync::{ compression, index::{RemoteTimeline, TimelineIndexEntry}, - sync_queue, tenant_branch_files, update_index_description, SyncKind, SyncTask, + sync_queue, update_index_description, SyncKind, SyncTask, }, RemoteStorage, ZTenantTimelineId, }, @@ -26,8 +23,6 @@ use super::{compression::ArchiveHeader, index::RemoteTimelineIndex, NewCheckpoin /// Attempts to compress and upload given checkpoint files. /// No extra checks for overlapping files is made: download takes care of that, ensuring no non-metadata local timeline files are overwritten. /// -/// Before the checkpoint files are uploaded, branch files are uploaded, if any local ones are missing remotely. -/// /// On an error, bumps the retries count and reschedules the entire task. /// On success, populates index data with new downloads. pub(super) async fn upload_timeline_checkpoint< @@ -41,19 +36,6 @@ pub(super) async fn upload_timeline_checkpoint< retries: u32, ) -> Option { debug!("Uploading checkpoint for sync id {}", sync_id); - if let Err(e) = upload_missing_branches(config, remote_assets.as_ref(), sync_id.tenant_id).await - { - error!( - "Failed to upload missing branches for sync id {}: {:?}", - sync_id, e - ); - sync_queue::push(SyncTask::new( - sync_id, - retries, - SyncKind::Upload(new_checkpoint), - )); - return Some(false); - } let new_upload_lsn = new_checkpoint.metadata.disk_consistent_lsn(); let index = &remote_assets.1; @@ -201,76 +183,6 @@ async fn try_upload_checkpoint< .map(|(header, header_size, _)| (header, header_size)) } -async fn upload_missing_branches< - P: std::fmt::Debug + Send + Sync + 'static, - S: RemoteStorage + Send + Sync + 'static, ->( - config: &'static PageServerConf, - (storage, index): &(S, RwLock), - tenant_id: ZTenantId, -) -> anyhow::Result<()> { - let local_branches = tenant_branch_files(config, tenant_id) - .await - .context("Failed to list local branch files for the tenant")?; - let index_read = index.read().await; - let remote_branches = index_read - .branch_files(tenant_id) - .cloned() - .unwrap_or_default(); - drop(index_read); - - let mut branch_uploads = local_branches - .difference(&remote_branches) - .map(|local_only_branch| async move { - let local_branch_path = local_only_branch.as_path(&config.branches_path(&tenant_id)); - let storage_path = storage.storage_path(&local_branch_path).with_context(|| { - format!( - "Failed to derive a storage path for branch with local path '{}'", - local_branch_path.display() - ) - })?; - let local_branch_file = fs::OpenOptions::new() - .read(true) - .open(&local_branch_path) - .await - .with_context(|| { - format!( - "Failed to open local branch file {} for reading", - local_branch_path.display() - ) - })?; - storage - .upload(local_branch_file, &storage_path) - .await - .with_context(|| { - format!( - "Failed to upload branch file to the remote path {:?}", - storage_path - ) - })?; - Ok::<_, anyhow::Error>(local_only_branch) - }) - .collect::>(); - - let mut branch_uploads_failed = false; - while let Some(upload_result) = branch_uploads.next().await { - match upload_result { - Ok(local_only_branch) => index - .write() - .await - .add_branch_file(tenant_id, local_only_branch.clone()), - Err(e) => { - error!("Failed to upload branch file: {:?}", e); - branch_uploads_failed = true; - } - } - } - - ensure!(!branch_uploads_failed, "Failed to upload all branch files"); - - Ok(()) -} - #[cfg(test)] mod tests { use tempfile::tempdir; diff --git a/pageserver/src/repository.rs b/pageserver/src/repository.rs index 6142953a58..674d447624 100644 --- a/pageserver/src/repository.rs +++ b/pageserver/src/repository.rs @@ -36,6 +36,10 @@ pub trait Repository: Send + Sync { /// Get Timeline handle for given zenith timeline ID. fn get_timeline(&self, timelineid: ZTimelineId) -> Result; + /// Lists timelines the repository contains. + /// Up to repository's implementation to omit certain timelines that ar not considered ready for use. + fn list_timelines(&self) -> Result>; + /// Create a new, empty timeline. The caller is responsible for loading data into it /// Initdb lsn is provided for timeline impl to be able to perform checks for some operations against it. fn create_empty_timeline( @@ -72,7 +76,10 @@ pub trait Repository: Send + Sync { pub enum RepositoryTimeline { /// Timeline, with its files present locally in pageserver's working directory. /// Loaded into pageserver's memory and ready to be used. - Local(Arc), + Local { + id: ZTimelineId, + timeline: Arc, + }, /// Timeline, found on the pageserver's remote storage, but not yet downloaded locally. Remote { id: ZTimelineId, @@ -83,12 +90,19 @@ pub enum RepositoryTimeline { impl RepositoryTimeline { pub fn local_timeline(&self) -> Option> { - if let Self::Local(local_timeline) = self { - Some(Arc::clone(local_timeline)) + if let Self::Local { timeline, .. } = self { + Some(Arc::clone(timeline)) } else { None } } + + pub fn id(&self) -> ZTimelineId { + match self { + Self::Local { id, .. } => *id, + Self::Remote { id, .. } => *id, + } + } } /// A state of the timeline synchronization with the remote storage. @@ -390,7 +404,6 @@ pub mod repo_harness { let tenant_id = ZTenantId::generate(); fs::create_dir_all(conf.tenant_path(&tenant_id))?; - fs::create_dir_all(conf.branches_path(&tenant_id))?; Ok(Self { conf, tenant_id }) } diff --git a/pageserver/src/tenant_mgr.rs b/pageserver/src/tenant_mgr.rs index d60b5fefd3..98777e5e4b 100644 --- a/pageserver/src/tenant_mgr.rs +++ b/pageserver/src/tenant_mgr.rs @@ -1,12 +1,12 @@ //! This module acts as a switchboard to access different repositories managed by this //! page server. -use crate::branches; use crate::config::PageServerConf; use crate::layered_repository::LayeredRepository; use crate::repository::{Repository, Timeline, TimelineSyncState}; use crate::thread_mgr; use crate::thread_mgr::ThreadKind; +use crate::timelines; use crate::walredo::PostgresRedoManager; use crate::CheckpointConfig; use anyhow::{bail, Context, Result}; @@ -182,7 +182,7 @@ pub fn create_repository_for_tenant( tenantid: ZTenantId, ) -> Result<()> { let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenantid)); - let repo = branches::create_repo(conf, tenantid, wal_redo_manager)?; + let repo = timelines::create_repo(conf, tenantid, wal_redo_manager)?; match access_tenants().entry(tenantid) { hash_map::Entry::Occupied(_) => bail!("tenant {} already exists", tenantid), diff --git a/pageserver/src/branches.rs b/pageserver/src/timelines.rs similarity index 70% rename from pageserver/src/branches.rs rename to pageserver/src/timelines.rs index 43f27af5ea..4a84b434a9 100644 --- a/pageserver/src/branches.rs +++ b/pageserver/src/timelines.rs @@ -1,5 +1,5 @@ //! -//! Branch management code +//! Timeline management code //! // TODO: move all paths construction to conf impl // @@ -27,8 +27,7 @@ use crate::{import_datadir, LOG_FILE_NAME}; use crate::{repository::RepositoryTimeline, tenant_mgr}; #[derive(Serialize, Deserialize, Clone)] -pub struct BranchInfo { - pub name: String, +pub struct TimelineInfo { #[serde(with = "hex")] pub timeline_id: ZTimelineId, pub latest_valid_lsn: Lsn, @@ -38,59 +37,6 @@ pub struct BranchInfo { pub current_logical_size_non_incremental: Option, } -impl BranchInfo { - pub fn from_path>( - path: T, - repo: &Arc, - include_non_incremental_logical_size: bool, - ) -> Result { - let path = path.as_ref(); - let name = path.file_name().unwrap().to_string_lossy().to_string(); - let timeline_id = std::fs::read_to_string(path) - .with_context(|| { - format!( - "Failed to read branch file contents at path '{}'", - path.display() - ) - })? - .parse::()?; - - let timeline = match repo.get_timeline(timeline_id)? { - RepositoryTimeline::Local(local_entry) => local_entry, - RepositoryTimeline::Remote { .. } => { - bail!("Timeline {} is remote, no branches to display", timeline_id) - } - }; - - // we use ancestor lsn zero if we don't have an ancestor, so turn this into an option based on timeline id - let (ancestor_id, ancestor_lsn) = match timeline.get_ancestor_timeline_id() { - Some(ancestor_id) => ( - Some(ancestor_id.to_string()), - Some(timeline.get_ancestor_lsn().to_string()), - ), - None => (None, None), - }; - - // non incremental size calculation can be heavy, so let it be optional - // needed for tests to check size calculation - let current_logical_size_non_incremental = include_non_incremental_logical_size - .then(|| { - timeline.get_current_logical_size_non_incremental(timeline.get_last_record_lsn()) - }) - .transpose()?; - - Ok(BranchInfo { - name, - timeline_id, - latest_valid_lsn: timeline.get_last_record_lsn(), - ancestor_id, - ancestor_lsn, - current_logical_size: timeline.get_current_logical_size(), - current_logical_size_non_incremental, - }) - } -} - #[derive(Debug, Clone, Copy)] pub struct PointInTime { pub timelineid: ZTimelineId, @@ -140,7 +86,6 @@ pub fn create_repo( .with_context(|| format!("could not create directory {}", repo_dir.display()))?; crashsafe_dir::create_dir(conf.timelines_path(&tenantid))?; - crashsafe_dir::create_dir_all(conf.branches_path(&tenantid))?; crashsafe_dir::create_dir_all(conf.tags_path(&tenantid))?; info!("created directory structure in {}", repo_dir.display()); @@ -198,7 +143,7 @@ fn run_initdb(conf: &'static PageServerConf, initdbpath: &Path) -> Result<()> { .output() .context("failed to execute initdb")?; if !initdb_output.status.success() { - anyhow::bail!( + bail!( "initdb failed: '{}'", String::from_utf8_lossy(&initdb_output.stderr) ); @@ -245,65 +190,80 @@ fn bootstrap_timeline( timeline.get_last_record_lsn() ); - let data = tli.to_string(); - fs::write(conf.branch_path("main", &tenantid), data)?; - println!("created main branch"); - // Remove temp dir. We don't need it anymore fs::remove_dir_all(pgdata_path)?; Ok(()) } -pub(crate) fn get_branches( - conf: &PageServerConf, - tenantid: &ZTenantId, +pub(crate) fn get_timelines( + tenant_id: ZTenantId, include_non_incremental_logical_size: bool, -) -> Result> { - let repo = tenant_mgr::get_repository_for_tenant(*tenantid)?; +) -> Result> { + let repo = tenant_mgr::get_repository_for_tenant(tenant_id) + .with_context(|| format!("Failed to get repo for tenant {}", tenant_id))?; - // Each branch has a corresponding record (text file) in the refs/branches - // with timeline_id. - let branches_dir = conf.branches_path(tenantid); - - std::fs::read_dir(&branches_dir) - .with_context(|| { - format!( - "Found no branches directory '{}' for tenant {}", - branches_dir.display(), - tenantid - ) - })? - .map(|dir_entry_res| { - let dir_entry = dir_entry_res.with_context(|| { - format!( - "Failed to list branches directory '{}' content for tenant {}", - branches_dir.display(), - tenantid - ) - })?; - BranchInfo::from_path( - dir_entry.path(), - &repo, - include_non_incremental_logical_size, - ) + Ok(repo + .list_timelines() + .with_context(|| format!("Failed to list timelines for tenant {}", tenant_id))? + .into_iter() + .filter_map(|timeline| match timeline { + RepositoryTimeline::Local { timeline, id } => Some((id, timeline)), + RepositoryTimeline::Remote { .. } => None, }) - .collect() + .map(|(timeline_id, timeline)| { + let (ancestor_id, ancestor_lsn) = match timeline.get_ancestor_timeline_id() { + Some(ancestor_id) => ( + Some(ancestor_id.to_string()), + Some(timeline.get_ancestor_lsn().to_string()), + ), + None => (None, None), + }; + + let current_logical_size_non_incremental = if include_non_incremental_logical_size { + match timeline + .get_current_logical_size_non_incremental(timeline.get_last_record_lsn()) + { + Ok(size) => Some(size), + Err(e) => { + error!( + "Failed to get current logical size for timeline {}: {:?}", + timeline_id, e + ); + None + } + } + } else { + None + }; + + TimelineInfo { + timeline_id, + latest_valid_lsn: timeline.get_last_record_lsn(), + ancestor_id, + ancestor_lsn, + current_logical_size: timeline.get_current_logical_size(), + // non incremental size calculation can be heavy, so let it be optional + // needed for tests to check size calculation + current_logical_size_non_incremental, + } + }) + .collect()) } -pub(crate) fn create_branch( +pub(crate) fn create_timeline( conf: &PageServerConf, - branchname: &str, startpoint_str: &str, - tenantid: &ZTenantId, -) -> Result { - let repo = tenant_mgr::get_repository_for_tenant(*tenantid)?; + tenant_id: ZTenantId, + timeline_id: ZTimelineId, +) -> Result { + let repo = tenant_mgr::get_repository_for_tenant(tenant_id)?; - if conf.branch_path(branchname, tenantid).exists() { - anyhow::bail!("branch {} already exists", branchname); + if conf.timeline_path(&timeline_id, &tenant_id).exists() { + bail!("timeline {} already exists", timeline_id); } - let mut startpoint = parse_point_in_time(conf, startpoint_str, tenantid)?; + let mut startpoint = parse_point_in_time(conf, startpoint_str, &tenant_id)?; let timeline = repo .get_timeline(startpoint.timelineid)? .local_timeline() @@ -325,10 +285,10 @@ pub(crate) fn create_branch( startpoint.lsn = startpoint.lsn.align(); if timeline.get_ancestor_lsn() > startpoint.lsn { // can we safely just branch from the ancestor instead? - anyhow::bail!( - "invalid startpoint {} for the branch {}: less than timeline ancestor lsn {:?}", + bail!( + "invalid startpoint {} for the timeline {}: less than timeline ancestor lsn {:?}", startpoint.lsn, - branchname, + timeline_id, timeline.get_ancestor_lsn() ); } @@ -342,11 +302,11 @@ pub(crate) fn create_branch( // Remember the human-readable branch name for the new timeline. // FIXME: there's a race condition, if you create a branch with the same // name concurrently. + // TODO kb timeline creation needs more let data = new_timeline_id.to_string(); - fs::write(conf.branch_path(branchname, tenantid), data)?; + fs::write(conf.timeline_path(&timeline_id, &tenant_id), data)?; - Ok(BranchInfo { - name: branchname.to_string(), + Ok(TimelineInfo { timeline_id: new_timeline_id, latest_valid_lsn: startpoint.lsn, ancestor_id: Some(startpoint.timelineid.to_string()), @@ -367,14 +327,6 @@ pub(crate) fn create_branch( // A specific LSN on a timeline: // bc62e7d612d0e6fe8f99a6dd2f281f9d@2/15D3DD8 // -// Same, with a human-friendly branch name: -// main -// main@2/15D3DD8 -// -// Human-friendly tag name: -// mytag -// -// fn parse_point_in_time( conf: &PageServerConf, s: &str, @@ -399,18 +351,6 @@ fn parse_point_in_time( } } - // Check if it's a branch - // Check if it's branch @ LSN - let branchpath = conf.branch_path(name, tenantid); - if branchpath.exists() { - let pointstr = fs::read_to_string(branchpath)?; - - let mut result = parse_point_in_time(conf, &pointstr, tenantid)?; - - result.lsn = lsn.unwrap_or(Lsn(0)); - return Ok(result); - } - // Check if it's a timelineid // Check if it's timelineid @ LSN if let Ok(timelineid) = ZTimelineId::from_str(name) { diff --git a/zenith/src/main.rs b/zenith/src/main.rs index bc42af5943..9f8996a540 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -21,7 +21,7 @@ use zenith_utils::postgres_backend::AuthType; use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId}; use zenith_utils::GIT_VERSION; -use pageserver::branches::BranchInfo; +use pageserver::timelines::TimelineInfo; // Default id of a safekeeper node, if not specified on the command line. const DEFAULT_SAFEKEEPER_ID: ZNodeId = ZNodeId(1); @@ -53,12 +53,12 @@ http_port = {safekeeper_http_port} } /// -/// Branches tree element used as a value in the HashMap. +/// Timelines tree element used as a value in the HashMap. /// -struct BranchTreeEl { - /// `BranchInfo` received from the `pageserver` via the `branch_list` libpq API call. - pub info: BranchInfo, - /// Holds all direct children of this branch referenced using `timeline_id`. +struct TimelineTreeEl { + /// `TimelineInfo` received from the `pageserver` via the `timeline_list` libpq API call. + pub info: TimelineInfo, + /// Holds all direct children of this timeline referenced using `timeline_id`. pub children: Vec, } @@ -84,7 +84,7 @@ fn main() -> Result<()> { let timeline_arg = Arg::new("timeline") .index(2) - .help("Branch name or a point-in time specification") + .help("Timeline id or a point-in time specification") .required(false); let tenantid_arg = Arg::new("tenantid") @@ -129,9 +129,9 @@ fn main() -> Result<()> { ) ) .subcommand( - App::new("branch") - .about("Create a new branch") - .arg(Arg::new("branchname").required(false).index(1)) + App::new("timeline") + .about("Create a new timeline") + .arg(Arg::new("timeline-name").required(false).index(1)) .arg(Arg::new("start-point").required(false).index(2)) .arg(tenantid_arg.clone()), ).subcommand( @@ -239,7 +239,7 @@ fn main() -> Result<()> { match sub_name { "tenant" => handle_tenant(sub_args, &env), - "branch" => handle_branch(sub_args, &env), + "timeline" => handle_timeline(sub_args, &env), "start" => handle_start_all(sub_args, &env), "stop" => handle_stop_all(sub_args, &env), "pageserver" => handle_pageserver(sub_args, &env), @@ -257,43 +257,42 @@ fn main() -> Result<()> { } /// -/// Prints branches list as a tree-like structure. +/// Prints timelines list as a tree-like structure. /// -fn print_branches_tree(branches: Vec) -> Result<()> { - let mut branches_hash: HashMap = HashMap::new(); +fn print_timelines_tree(timelines: Vec) -> Result<()> { + let mut timelines_hash: HashMap = timelines + .iter() + .map(|t| { + ( + t.timeline_id.to_string(), + TimelineTreeEl { + info: t.clone(), + children: Vec::new(), + }, + ) + }) + .collect(); - // Form a hash table of branch timeline_id -> BranchTreeEl. - for branch in &branches { - branches_hash.insert( - branch.timeline_id.to_string(), - BranchTreeEl { - info: branch.clone(), - children: Vec::new(), - }, - ); - } - - // Memorize all direct children of each branch. - for branch in &branches { - if let Some(tid) = &branch.ancestor_id { - branches_hash + // Memorize all direct children of each timeline. + for timeline in &timelines { + if let Some(tid) = &timeline.ancestor_id { + timelines_hash .get_mut(tid) - .context("missing branch info in the HashMap")? + .context("missing timeline info in the HashMap")? .children - .push(branch.timeline_id.to_string()); + .push(timeline.timeline_id.to_string()); } } // Sort children by tid to bring some minimal order. - for branch in &mut branches_hash.values_mut() { - branch.children.sort(); + for timeline in &mut timelines_hash.values_mut() { + timeline.children.sort(); } - for branch in branches_hash.values() { - // Start with root branches (no ancestors) first. - // Now there is 'main' branch only, but things may change. - if branch.info.ancestor_id.is_none() { - print_branch(0, &Vec::from([true]), branch, &branches_hash)?; + for timeline in timelines_hash.values() { + // Start with root timelines (no ancestors) first. + if timeline.info.ancestor_id.is_none() { + print_timeline(0, &Vec::from([true]), timeline, &timelines_hash)?; } } @@ -301,27 +300,27 @@ fn print_branches_tree(branches: Vec) -> Result<()> { } /// -/// Recursively prints branch info with all its children. +/// Recursively prints timeline info with all its children. /// -fn print_branch( +fn print_timeline( nesting_level: usize, is_last: &[bool], - branch: &BranchTreeEl, - branches: &HashMap, + timeline: &TimelineTreeEl, + timelines: &HashMap, ) -> Result<()> { // Draw main padding print!(" "); if nesting_level > 0 { - let lsn = branch + let lsn = timeline .info .ancestor_lsn .as_ref() - .context("missing branch info in the HashMap")?; + .context("missing timeline info in the HashMap")?; let mut br_sym = "┣━"; // Draw each nesting padding with proper style - // depending on whether its branch ended or not. + // depending on whether its timeline ended or not. if nesting_level > 1 { for l in &is_last[1..is_last.len() - 1] { if *l { @@ -332,7 +331,7 @@ fn print_branch( } } - // We are the last in this sub-branch + // We are the last in this sub-timeline if *is_last.last().unwrap() { br_sym = "┗━"; } @@ -340,51 +339,51 @@ fn print_branch( print!("{} @{}: ", br_sym, lsn); } - // Finally print a branch name with new line - println!("{}", branch.info.name); + // Finally print a timeline name with new line + println!("{}", timeline.info.timeline_id); - let len = branch.children.len(); + let len = timeline.children.len(); let mut i: usize = 0; let mut is_last_new = Vec::from(is_last); is_last_new.push(false); - for child in &branch.children { + for child in &timeline.children { i += 1; - // Mark that the last padding is the end of the branch + // Mark that the last padding is the end of the timeline if i == len { if let Some(last) = is_last_new.last_mut() { *last = true; } } - print_branch( + print_timeline( nesting_level + 1, &is_last_new, - branches + timelines .get(child) - .context("missing branch info in the HashMap")?, - branches, + .context("missing timeline info in the HashMap")?, + timelines, )?; } Ok(()) } -/// Returns a map of timeline IDs to branch_name@lsn strings. +/// Returns a map of timeline IDs to timeline_id@lsn strings. /// Connects to the pageserver to query this information. -fn get_branch_infos( +fn get_timeline_infos( env: &local_env::LocalEnv, tenantid: &ZTenantId, -) -> Result> { +) -> Result> { let page_server = PageServerNode::from_env(env); - let branch_infos: Vec = page_server.branch_list(tenantid)?; - let branch_infos: HashMap = branch_infos + let timeline_infos: Vec = page_server.timeline_list(tenantid)?; + let timeline_infos: HashMap = timeline_infos .into_iter() - .map(|branch_info| (branch_info.timeline_id, branch_info)) + .map(|timeline_info| (timeline_info.timeline_id, timeline_info)) .collect(); - Ok(branch_infos) + Ok(timeline_infos) } // Helper function to parse --tenantid option, or get the default from config file @@ -459,24 +458,28 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &local_env::LocalEnv) -> Result Ok(()) } -fn handle_branch(branch_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { +fn handle_timeline(timeline_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { let pageserver = PageServerNode::from_env(env); - let tenantid = get_tenantid(branch_match, env)?; + let tenant_id = get_tenantid(timeline_match, env)?; - if let Some(branchname) = branch_match.value_of("branchname") { - let startpoint_str = branch_match + if let Some(timeline_id) = timeline_match.value_of("timeline-id") { + let startpoint_str = timeline_match .value_of("start-point") .context("Missing start-point")?; - let branch = pageserver.branch_create(branchname, startpoint_str, &tenantid)?; + let timeline_id = timeline_id + .parse::() + .context("Failed to parse timeline id from the request")?; + let timeline = + pageserver.timeline_create(timeline_id, startpoint_str.to_owned(), tenant_id)?; println!( - "Created branch '{}' at {:?} for tenant: {}", - branch.name, branch.latest_valid_lsn, tenantid, + "Created timeline '{}' at {:?} for tenant: {}", + timeline.timeline_id, timeline.latest_valid_lsn, tenant_id, ); } else { - // No arguments, list branches for tenant - let branches = pageserver.branch_list(&tenantid)?; - print_branches_tree(branches)?; + // No arguments, list timelines for tenant + let timelines = pageserver.timeline_list(&tenant_id)?; + print_timelines_tree(timelines)?; } Ok(()) @@ -495,12 +498,12 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { match sub_name { "list" => { - let branch_infos = get_branch_infos(env, &tenantid).unwrap_or_else(|e| { - eprintln!("Failed to load branch info: {}", e); + let timeline_infos = get_timeline_infos(env, &tenantid).unwrap_or_else(|e| { + eprintln!("Failed to load timeline info: {}", e); HashMap::new() }); - println!("NODE\tADDRESS\t\tBRANCH\tLSN\t\tSTATUS"); + println!("NODE\tADDRESS\t\tTIMELINE\tLSN\t\tSTATUS"); for ((_, node_name), node) in cplane .nodes .iter() @@ -509,7 +512,7 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { // FIXME: This shows the LSN at the end of the timeline. It's not the // right thing to do for read-only nodes that might be anchored at an // older point in time, or following but lagging behind the primary. - let lsn_str = branch_infos + let lsn_str = timeline_infos .get(&node.timelineid) .map(|bi| bi.latest_valid_lsn.to_string()) .unwrap_or_else(|| "?".to_string()); @@ -518,7 +521,7 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { "{}\t{}\t{}\t{}\t{}", node_name, node.address, - node.timelineid, // FIXME: resolve human-friendly branch name + node.timelineid, lsn_str, node.status(), ); @@ -526,17 +529,17 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { } "create" => { let node_name = sub_args.value_of("node").unwrap_or("main"); - let timeline_name = sub_args.value_of("timeline").unwrap_or(node_name); + let timeline_spec = sub_args.value_of("timeline"); let port: Option = match sub_args.value_of("port") { Some(p) => Some(p.parse()?), None => None, }; - cplane.new_node(tenantid, node_name, timeline_name, port)?; + cplane.new_node(tenantid, node_name, timeline_spec, port)?; } "start" => { let node_name = sub_args.value_of("node").unwrap_or("main"); - let timeline_name = sub_args.value_of("timeline"); + let timeline_spec = sub_args.value_of("timeline"); let port: Option = match sub_args.value_of("port") { Some(p) => Some(p.parse()?), @@ -554,8 +557,8 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { }; if let Some(node) = node { - if timeline_name.is_some() { - println!("timeline name ignored because node exists already"); + if timeline_spec.is_some() { + println!("timeline spec ignored because its node exists already"); } println!("Starting existing postgres {}...", node_name); node.start(&auth_token)?; @@ -565,12 +568,11 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { // start --port X // stop // start <-- will also use port X even without explicit port argument - let timeline_name = timeline_name.unwrap_or(node_name); println!( - "Starting new postgres {} on {}...", - node_name, timeline_name + "Starting new postgres {} on timeline {:?} ...", + node_name, timeline_spec ); - let node = cplane.new_node(tenantid, node_name, timeline_name, port)?; + let node = cplane.new_node(tenantid, node_name, timeline_spec, port)?; node.start(&auth_token)?; } } @@ -585,9 +587,7 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { node.stop(destroy)?; } - _ => { - bail!("Unexpected pg subcommand '{}'", sub_name) - } + _ => bail!("Unexpected pg subcommand '{}'", sub_name), } Ok(()) From 0c91091c637d167e65b21e7456c344cafaeb6016 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Sun, 13 Feb 2022 23:49:32 +0200 Subject: [PATCH 15/42] Avoid point in time concept on pageserver level --- control_plane/src/compute.rs | 67 ++++++++++++++++++++------------- control_plane/src/storage.rs | 7 ++-- pageserver/src/config.rs | 12 ------ pageserver/src/http/models.rs | 4 +- pageserver/src/http/routes.rs | 4 +- pageserver/src/timelines.rs | 71 ++++++----------------------------- zenith/src/main.rs | 12 +++--- 7 files changed, 67 insertions(+), 110 deletions(-) diff --git a/control_plane/src/compute.rs b/control_plane/src/compute.rs index 3569cc1dbb..3381ca4a04 100644 --- a/control_plane/src/compute.rs +++ b/control_plane/src/compute.rs @@ -10,7 +10,7 @@ use std::str::FromStr; use std::sync::Arc; use std::time::Duration; -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use zenith_utils::connstring::connection_host_port; use zenith_utils::lsn::Lsn; use zenith_utils::postgres_backend::AuthType; @@ -73,31 +73,6 @@ impl ComputeControlPlane { .unwrap_or(self.base_port) } - // FIXME: see also parse_point_in_time in timelines.rs. - fn parse_point_in_time( - &self, - tenant_id: ZTenantId, - s: &str, - ) -> Result<(ZTimelineId, Option)> { - let _strings = s.split('@'); - // let name = strings.next().unwrap(); - - // let lsn = strings - // .next() - // .map(Lsn::from_str) - // .transpose() - // .context("invalid LSN in point-in-time specification")?; - - // // Resolve the timeline ID, given the human-readable branch name - // let timeline_id = self - // .pageserver - // .branch_get_by_name(&tenant_id, name)? - // .timeline_id; - - // Ok((timeline_id, lsn)) - todo!("TODO kb check more about the '@name' format") - } - pub fn new_node( &mut self, tenantid: ZTenantId, @@ -107,7 +82,7 @@ impl ComputeControlPlane { ) -> Result> { // Resolve the human-readable timeline spec into timeline ID and LSN let (timelineid, lsn) = match timeline_spec { - Some(timeline_spec) => self.parse_point_in_time(tenantid, timeline_spec)?, + Some(timeline_spec) => parse_point_in_time(timeline_spec)?, None => (ZTimelineId::generate(), None), }; @@ -134,6 +109,44 @@ impl ComputeControlPlane { } } +// Parse user-given string that represents a point-in-time. +// +// Variants suported: +// +// Raw timeline id in hex, meaning the end of that timeline: +// bc62e7d612d0e6fe8f99a6dd2f281f9d +// +// A specific LSN on a timeline: +// bc62e7d612d0e6fe8f99a6dd2f281f9d@2/15D3DD8 +// +fn parse_point_in_time(timeline_spec: &str) -> anyhow::Result<(ZTimelineId, Option)> { + let mut strings = timeline_spec.split('@'); + + let name = match strings.next() { + Some(n) => n, + None => bail!("invalid timeline specification: {}", timeline_spec), + }; + let timeline_id = ZTimelineId::from_str(name).with_context(|| { + format!( + "failed to parse the timeline id from specification: {}", + timeline_spec + ) + })?; + + let lsn = strings + .next() + .map(Lsn::from_str) + .transpose() + .with_context(|| { + format!( + "failed to parse the Lsn from timeline specification: {}", + timeline_spec + ) + })?; + + Ok((timeline_id, lsn)) +} + /////////////////////////////////////////////////////////////////////////////// #[derive(Debug)] diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index aed9a757d4..d550bfc064 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -16,6 +16,7 @@ use reqwest::blocking::{Client, RequestBuilder, Response}; use reqwest::{IntoUrl, Method}; use thiserror::Error; use zenith_utils::http::error::HttpErrorBody; +use zenith_utils::lsn::Lsn; use zenith_utils::postgres_backend::AuthType; use zenith_utils::zid::{ZTenantId, ZTimelineId}; @@ -348,16 +349,16 @@ impl PageServerNode { pub fn timeline_create( &self, - timeline_id: ZTimelineId, - start_point: String, tenant_id: ZTenantId, + timeline_id: ZTimelineId, + start_lsn: Option, ) -> Result { Ok(self .http_request(Method::POST, format!("{}/timeline", self.http_base_url)) .json(&TimelineCreateRequest { tenant_id, timeline_id, - start_point, + start_lsn, }) .send()? .error_from_body()? diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 5a9c7557cc..dc85c83c17 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -392,14 +392,6 @@ impl PageServerConf { self.tenants_path().join(tenantid.to_string()) } - pub fn tags_path(&self, tenantid: &ZTenantId) -> PathBuf { - self.tenant_path(tenantid).join("refs").join("tags") - } - - pub fn tag_path(&self, tag_name: &str, tenantid: &ZTenantId) -> PathBuf { - self.tags_path(tenantid).join(tag_name) - } - pub fn timelines_path(&self, tenantid: &ZTenantId) -> PathBuf { self.tenant_path(tenantid).join(TIMELINES_SEGMENT_NAME) } @@ -408,10 +400,6 @@ impl PageServerConf { self.timelines_path(tenantid).join(timelineid.to_string()) } - pub fn ancestor_path(&self, timelineid: &ZTimelineId, tenantid: &ZTenantId) -> PathBuf { - self.timeline_path(timelineid, tenantid).join("ancestor") - } - // // Postgres distribution paths // diff --git a/pageserver/src/http/models.rs b/pageserver/src/http/models.rs index a6dce33c03..bc0d46a96c 100644 --- a/pageserver/src/http/models.rs +++ b/pageserver/src/http/models.rs @@ -1,5 +1,5 @@ use serde::{Deserialize, Serialize}; -use zenith_utils::zid::ZTimelineId; +use zenith_utils::{lsn::Lsn, zid::ZTimelineId}; use crate::ZTenantId; use zenith_utils::zid::ZNodeId; @@ -10,7 +10,7 @@ pub struct TimelineCreateRequest { pub tenant_id: ZTenantId, #[serde(with = "hex")] pub timeline_id: ZTimelineId, - pub start_point: String, + pub start_lsn: Option, } #[derive(Serialize, Deserialize)] diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 5ab1576aa6..34a61cab9c 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -77,12 +77,12 @@ async fn timeline_create_handler(mut request: Request) -> Result, ) -> Result { - let repo = tenant_mgr::get_repository_for_tenant(tenant_id)?; - if conf.timeline_path(&timeline_id, &tenant_id).exists() { bail!("timeline {} already exists", timeline_id); } - let mut startpoint = parse_point_in_time(conf, startpoint_str, &tenant_id)?; + let repo = tenant_mgr::get_repository_for_tenant(tenant_id)?; + + let mut startpoint = PointInTime { + timeline_id, + lsn: start_lsn.unwrap_or(Lsn(0)), + }; + let timeline = repo - .get_timeline(startpoint.timelineid)? + .get_timeline(startpoint.timeline_id)? .local_timeline() .context("Cannot branch off the timeline that's not present locally")?; if startpoint.lsn == Lsn(0) { @@ -297,7 +300,7 @@ pub(crate) fn create_timeline( // Forward entire timeline creation routine to repository // backend, so it can do all needed initialization - repo.branch_timeline(startpoint.timelineid, new_timeline_id, startpoint.lsn)?; + repo.branch_timeline(startpoint.timeline_id, new_timeline_id, startpoint.lsn)?; // Remember the human-readable branch name for the new timeline. // FIXME: there's a race condition, if you create a branch with the same @@ -309,59 +312,9 @@ pub(crate) fn create_timeline( Ok(TimelineInfo { timeline_id: new_timeline_id, latest_valid_lsn: startpoint.lsn, - ancestor_id: Some(startpoint.timelineid.to_string()), + ancestor_id: Some(startpoint.timeline_id.to_string()), ancestor_lsn: Some(startpoint.lsn.to_string()), current_logical_size: 0, current_logical_size_non_incremental: Some(0), }) } - -// -// Parse user-given string that represents a point-in-time. -// -// We support multiple variants: -// -// Raw timeline id in hex, meaning the end of that timeline: -// bc62e7d612d0e6fe8f99a6dd2f281f9d -// -// A specific LSN on a timeline: -// bc62e7d612d0e6fe8f99a6dd2f281f9d@2/15D3DD8 -// -fn parse_point_in_time( - conf: &PageServerConf, - s: &str, - tenantid: &ZTenantId, -) -> Result { - let mut strings = s.split('@'); - let name = strings.next().unwrap(); - - let lsn = strings - .next() - .map(Lsn::from_str) - .transpose() - .context("invalid LSN in point-in-time specification")?; - - // Check if it's a tag - if lsn.is_none() { - let tagpath = conf.tag_path(name, tenantid); - if tagpath.exists() { - let pointstr = fs::read_to_string(tagpath)?; - - return parse_point_in_time(conf, &pointstr, tenantid); - } - } - - // Check if it's a timelineid - // Check if it's timelineid @ LSN - if let Ok(timelineid) = ZTimelineId::from_str(name) { - let tlipath = conf.timeline_path(&timelineid, tenantid); - if tlipath.exists() { - return Ok(PointInTime { - timelineid, - lsn: lsn.unwrap_or(Lsn(0)), - }); - } - } - - bail!("could not parse point-in-time {}", s); -} diff --git a/zenith/src/main.rs b/zenith/src/main.rs index 9f8996a540..7170653754 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -17,6 +17,7 @@ use walkeeper::defaults::{ DEFAULT_PG_LISTEN_PORT as DEFAULT_SAFEKEEPER_PG_PORT, }; use zenith_utils::auth::{Claims, Scope}; +use zenith_utils::lsn::Lsn; use zenith_utils::postgres_backend::AuthType; use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId}; use zenith_utils::GIT_VERSION; @@ -464,14 +465,15 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &local_env::LocalEnv) -> Re let tenant_id = get_tenantid(timeline_match, env)?; if let Some(timeline_id) = timeline_match.value_of("timeline-id") { - let startpoint_str = timeline_match - .value_of("start-point") - .context("Missing start-point")?; + let start_lsn = timeline_match + .value_of("start-lsn") + .map(|lsn| lsn.parse::()) + .transpose() + .context("Failed to parse start Lsn from the request")?; let timeline_id = timeline_id .parse::() .context("Failed to parse timeline id from the request")?; - let timeline = - pageserver.timeline_create(timeline_id, startpoint_str.to_owned(), tenant_id)?; + let timeline = pageserver.timeline_create(tenant_id, timeline_id, start_lsn)?; println!( "Created timeline '{}' at {:?} for tenant: {}", timeline.timeline_id, timeline.latest_valid_lsn, tenant_id, From f49990ed433616270a7db33c3d554d9ed4cf4135 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Mon, 14 Feb 2022 00:53:00 +0200 Subject: [PATCH 16/42] Allow creating timelines by branching off ancestors --- control_plane/src/compute.rs | 81 +---- control_plane/src/local_env.rs | 16 +- control_plane/src/storage.rs | 4 +- pageserver/src/http/models.rs | 9 +- pageserver/src/http/routes.rs | 58 +--- .../src/remote_storage/storage_sync/index.rs | 2 +- pageserver/src/repository.rs | 2 +- pageserver/src/tenant_mgr.rs | 6 +- pageserver/src/timelines.rs | 297 ++++++++++------ test_runner/batch_others/test_auth.py | 19 +- .../batch_others/test_branch_behind.py | 33 +- .../batch_others/test_clog_truncate.py | 15 +- test_runner/batch_others/test_config.py | 8 +- test_runner/batch_others/test_createdropdb.py | 27 +- test_runner/batch_others/test_createuser.py | 11 +- .../batch_others/test_gc_aggressive.py | 5 +- test_runner/batch_others/test_multixact.py | 10 +- .../batch_others/test_old_request_lsn.py | 4 +- .../batch_others/test_pageserver_api.py | 14 +- .../batch_others/test_pageserver_catchup.py | 8 +- .../batch_others/test_pageserver_restart.py | 4 +- .../batch_others/test_parallel_copy.py | 6 +- test_runner/batch_others/test_pgbench.py | 4 +- .../batch_others/test_readonly_node.py | 18 +- .../batch_others/test_restart_compute.py | 11 +- test_runner/batch_others/test_snapfiles_gc.py | 4 +- test_runner/batch_others/test_subxacts.py | 4 +- .../batch_others/test_tenant_relocation.py | 12 +- test_runner/batch_others/test_tenants.py | 22 +- .../batch_others/test_timeline_size.py | 17 +- test_runner/batch_others/test_twophase.py | 11 +- test_runner/batch_others/test_vm_bits.py | 10 +- test_runner/batch_others/test_wal_acceptor.py | 86 ++--- .../batch_others/test_wal_acceptor_async.py | 5 +- test_runner/batch_others/test_zenith_cli.py | 60 ++-- .../batch_pg_regress/test_isolation.py | 6 +- .../batch_pg_regress/test_pg_regress.py | 4 +- .../batch_pg_regress/test_zenith_regress.py | 4 +- test_runner/fixtures/compare_fixtures.py | 5 +- test_runner/fixtures/zenith_fixtures.py | 243 +++++++------ .../performance/test_bulk_tenant_create.py | 11 +- .../performance/test_parallel_copy_to.py | 1 - test_runner/test_broken.py | 4 +- zenith/src/main.rs | 327 ++++++++++++------ 44 files changed, 855 insertions(+), 653 deletions(-) diff --git a/control_plane/src/compute.rs b/control_plane/src/compute.rs index 3381ca4a04..5d225a67fa 100644 --- a/control_plane/src/compute.rs +++ b/control_plane/src/compute.rs @@ -10,7 +10,7 @@ use std::str::FromStr; use std::sync::Arc; use std::time::Duration; -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result}; use zenith_utils::connstring::connection_host_port; use zenith_utils::lsn::Lsn; use zenith_utils::postgres_backend::AuthType; @@ -37,7 +37,7 @@ impl ComputeControlPlane { // pgdatadirs // |- tenants // | |- - // | | |- + // | | |- pub fn load(env: LocalEnv) -> Result { let pageserver = Arc::new(PageServerNode::from_env(&env)); @@ -52,7 +52,7 @@ impl ComputeControlPlane { .with_context(|| format!("failed to list {}", tenant_dir.path().display()))? { let node = PostgresNode::from_dir_entry(timeline_dir?, &env, &pageserver)?; - nodes.insert((node.tenantid, node.name.clone()), Arc::new(node)); + nodes.insert((node.tenant_id, node.name.clone()), Arc::new(node)); } } @@ -75,17 +75,12 @@ impl ComputeControlPlane { pub fn new_node( &mut self, - tenantid: ZTenantId, + tenant_id: ZTenantId, name: &str, - timeline_spec: Option<&str>, + timeline_id: ZTimelineId, + lsn: Option, port: Option, ) -> Result> { - // Resolve the human-readable timeline spec into timeline ID and LSN - let (timelineid, lsn) = match timeline_spec { - Some(timeline_spec) => parse_point_in_time(timeline_spec)?, - None => (ZTimelineId::generate(), None), - }; - let port = port.unwrap_or_else(|| self.get_port()); let node = Arc::new(PostgresNode { name: name.to_owned(), @@ -93,9 +88,9 @@ impl ComputeControlPlane { env: self.env.clone(), pageserver: Arc::clone(&self.pageserver), is_test: false, - timelineid, + timeline_id, lsn, - tenantid, + tenant_id, uses_wal_proposer: false, }); @@ -103,50 +98,12 @@ impl ComputeControlPlane { node.setup_pg_conf(self.env.pageserver.auth_type)?; self.nodes - .insert((tenantid, node.name.clone()), Arc::clone(&node)); + .insert((tenant_id, node.name.clone()), Arc::clone(&node)); Ok(node) } } -// Parse user-given string that represents a point-in-time. -// -// Variants suported: -// -// Raw timeline id in hex, meaning the end of that timeline: -// bc62e7d612d0e6fe8f99a6dd2f281f9d -// -// A specific LSN on a timeline: -// bc62e7d612d0e6fe8f99a6dd2f281f9d@2/15D3DD8 -// -fn parse_point_in_time(timeline_spec: &str) -> anyhow::Result<(ZTimelineId, Option)> { - let mut strings = timeline_spec.split('@'); - - let name = match strings.next() { - Some(n) => n, - None => bail!("invalid timeline specification: {}", timeline_spec), - }; - let timeline_id = ZTimelineId::from_str(name).with_context(|| { - format!( - "failed to parse the timeline id from specification: {}", - timeline_spec - ) - })?; - - let lsn = strings - .next() - .map(Lsn::from_str) - .transpose() - .with_context(|| { - format!( - "failed to parse the Lsn from timeline specification: {}", - timeline_spec - ) - })?; - - Ok((timeline_id, lsn)) -} - /////////////////////////////////////////////////////////////////////////////// #[derive(Debug)] @@ -156,9 +113,9 @@ pub struct PostgresNode { pub env: LocalEnv, pageserver: Arc, is_test: bool, - pub timelineid: ZTimelineId, + pub timeline_id: ZTimelineId, pub lsn: Option, // if it's a read-only node. None for primary - pub tenantid: ZTenantId, + pub tenant_id: ZTenantId, uses_wal_proposer: bool, } @@ -191,7 +148,7 @@ impl PostgresNode { let context = format!("in config file {}", cfg_path_str); let port: u16 = conf.parse_field("port", &context)?; let timelineid: ZTimelineId = conf.parse_field("zenith.zenith_timeline", &context)?; - let tenantid: ZTenantId = conf.parse_field("zenith.zenith_tenant", &context)?; + let tenant_id: ZTenantId = conf.parse_field("zenith.zenith_tenant", &context)?; let uses_wal_proposer = conf.get("wal_acceptors").is_some(); // parse recovery_target_lsn, if any @@ -205,9 +162,9 @@ impl PostgresNode { env: env.clone(), pageserver: Arc::clone(pageserver), is_test: false, - timelineid, + timeline_id: timelineid, lsn: recovery_target_lsn, - tenantid, + tenant_id, uses_wal_proposer, }) } @@ -258,9 +215,9 @@ impl PostgresNode { ); let sql = if let Some(lsn) = lsn { - format!("basebackup {} {} {}", self.tenantid, self.timelineid, lsn) + format!("basebackup {} {} {}", self.tenant_id, self.timeline_id, lsn) } else { - format!("basebackup {} {}", self.tenantid, self.timelineid) + format!("basebackup {} {}", self.tenant_id, self.timeline_id) }; let mut client = self @@ -346,8 +303,8 @@ impl PostgresNode { conf.append("shared_preload_libraries", "zenith"); conf.append_line(""); conf.append("zenith.page_server_connstring", &pageserver_connstr); - conf.append("zenith.zenith_tenant", &self.tenantid.to_string()); - conf.append("zenith.zenith_timeline", &self.timelineid.to_string()); + conf.append("zenith.zenith_tenant", &self.tenant_id.to_string()); + conf.append("zenith.zenith_timeline", &self.timeline_id.to_string()); if let Some(lsn) = self.lsn { conf.append("recovery_target_lsn", &lsn.to_string()); } @@ -425,7 +382,7 @@ impl PostgresNode { } pub fn pgdata(&self) -> PathBuf { - self.env.pg_data_dir(&self.tenantid, &self.name) + self.env.pg_data_dir(&self.tenant_id, &self.name) } pub fn status(&self) -> &str { diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 238c78821e..98b6379106 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -5,6 +5,7 @@ use anyhow::{bail, Context}; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use std::env; use std::fmt::Write; use std::fs; @@ -12,7 +13,7 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use zenith_utils::auth::{encode_from_key_file, Claims, Scope}; use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{HexZTenantId, ZNodeId, ZTenantId}; +use zenith_utils::zid::{HexZTenantId, ZNodeId, ZTenantId, ZTimelineId}; use crate::safekeeper::SafekeeperNode; @@ -48,7 +49,7 @@ pub struct LocalEnv { // Default tenant ID to use with the 'zenith' command line utility, when // --tenantid is not explicitly specified. #[serde(default)] - pub default_tenantid: Option, + pub default_tenant_id: Option, // used to issue tokens during e.g pg start #[serde(default)] @@ -58,6 +59,13 @@ pub struct LocalEnv { #[serde(default)] pub safekeepers: Vec, + + /// Every tenant has a first timeline created for it, currently the only one ancestor-less for this tenant. + /// It is used as a default timeline for branching, if no ancestor timeline is specified. + #[serde(default)] + // TODO kb this does not survive calls between invocations, so will have to persist it. + // Then it comes back to names again? + pub initial_timelines: HashMap, } #[derive(Serialize, Deserialize, Clone, Debug)] @@ -183,8 +191,8 @@ impl LocalEnv { } // If no initial tenant ID was given, generate it. - if env.default_tenantid.is_none() { - env.default_tenantid = Some(HexZTenantId::from(ZTenantId::generate())); + if env.default_tenant_id.is_none() { + env.default_tenant_id = Some(HexZTenantId::from(ZTenantId::generate())); } env.base_data_dir = base_path(); diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index d550bfc064..9d5a88784d 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -325,7 +325,7 @@ impl PageServerNode { .json()?) } - pub fn tenant_create(&self, tenantid: ZTenantId) -> Result<()> { + pub fn tenant_create(&self, tenantid: ZTenantId) -> Result { Ok(self .http_request(Method::POST, format!("{}/{}", self.http_base_url, "tenant")) .json(&TenantCreateRequest { @@ -352,6 +352,7 @@ impl PageServerNode { tenant_id: ZTenantId, timeline_id: ZTimelineId, start_lsn: Option, + ancestor_timeline_id: Option, ) -> Result { Ok(self .http_request(Method::POST, format!("{}/timeline", self.http_base_url)) @@ -359,6 +360,7 @@ impl PageServerNode { tenant_id, timeline_id, start_lsn, + ancestor_timeline_id, }) .send()? .error_from_body()? diff --git a/pageserver/src/http/models.rs b/pageserver/src/http/models.rs index bc0d46a96c..7f95c64527 100644 --- a/pageserver/src/http/models.rs +++ b/pageserver/src/http/models.rs @@ -1,8 +1,9 @@ use serde::{Deserialize, Serialize}; -use zenith_utils::{lsn::Lsn, zid::ZTimelineId}; - -use crate::ZTenantId; use zenith_utils::zid::ZNodeId; +use zenith_utils::{ + lsn::Lsn, + zid::{opt_display_serde, ZTenantId, ZTimelineId}, +}; #[derive(Serialize, Deserialize)] pub struct TimelineCreateRequest { @@ -10,6 +11,8 @@ pub struct TimelineCreateRequest { pub tenant_id: ZTenantId, #[serde(with = "hex")] pub timeline_id: ZTimelineId, + #[serde(with = "opt_display_serde")] + pub ancestor_timeline_id: Option, pub start_lsn: Option, } diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 34a61cab9c..f332e59135 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -3,7 +3,6 @@ use std::sync::Arc; use anyhow::Result; use hyper::StatusCode; use hyper::{Body, Request, Response, Uri}; -use serde::Serialize; use tracing::*; use zenith_utils::auth::JwtAuth; use zenith_utils::http::endpoint::attach_openapi_ui; @@ -17,15 +16,13 @@ use zenith_utils::http::{ request::parse_request_param, }; use zenith_utils::http::{RequestExt, RouterBuilder}; -use zenith_utils::lsn::Lsn; -use zenith_utils::zid::HexZTimelineId; -use zenith_utils::zid::ZTimelineId; +use zenith_utils::zid::{HexZTimelineId, ZTimelineId}; use super::models::StatusResponse; use super::models::TenantCreateRequest; use super::models::TimelineCreateRequest; use crate::repository::RepositoryTimeline; -use crate::repository::TimelineSyncState; +use crate::timelines::TimelineInfo; use crate::{config::PageServerConf, tenant_mgr, timelines, ZTenantId}; #[derive(Debug)] @@ -82,6 +79,7 @@ async fn timeline_create_handler(mut request: Request) -> Result) -> bool { .unwrap_or(false) } -#[derive(Debug, Serialize)] -#[serde(tag = "type")] -enum TimelineInfo { - Local { - #[serde(with = "hex")] - timeline_id: ZTimelineId, - #[serde(with = "hex")] - tenant_id: ZTenantId, - ancestor_timeline_id: Option, - last_record_lsn: Lsn, - prev_record_lsn: Lsn, - disk_consistent_lsn: Lsn, - timeline_state: Option, - }, - Remote { - #[serde(with = "hex")] - timeline_id: ZTimelineId, - #[serde(with = "hex")] - tenant_id: ZTenantId, - }, -} - async fn timeline_detail_handler(request: Request) -> Result, ApiError> { let tenant_id: ZTenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; @@ -151,23 +127,13 @@ async fn timeline_detail_handler(request: Request) -> Result(match repo.get_timeline(timeline_id)?.local_timeline() { - None => TimelineInfo::Remote { - timeline_id, - tenant_id, - }, - Some(timeline) => TimelineInfo::Local { - timeline_id, - tenant_id, - ancestor_timeline_id: timeline - .get_ancestor_timeline_id() - .map(HexZTimelineId::from), - disk_consistent_lsn: timeline.get_disk_consistent_lsn(), - last_record_lsn: timeline.get_last_record_lsn(), - prev_record_lsn: timeline.get_prev_record_lsn(), - timeline_state: repo.get_timeline_state(timeline_id), - }, - }) + let include_non_incremental_logical_size = + get_include_non_incremental_logical_size(&request); + Ok::<_, anyhow::Error>(TimelineInfo::from_repo_timeline( + tenant_id, + repo.get_timeline(timeline_id)?, + include_non_incremental_logical_size, + )) }) .await .map_err(ApiError::from_err)??; @@ -247,13 +213,13 @@ async fn tenant_create_handler(mut request: Request) -> Result) -> Result, ApiError> { diff --git a/pageserver/src/remote_storage/storage_sync/index.rs b/pageserver/src/remote_storage/storage_sync/index.rs index 8ff92ed55e..81c99754c9 100644 --- a/pageserver/src/remote_storage/storage_sync/index.rs +++ b/pageserver/src/remote_storage/storage_sync/index.rs @@ -49,7 +49,7 @@ impl RelativePath { } /// An index to track tenant files that exist on the remote storage. -/// Currently, timeline archives files are tracked only. +/// Currently, timeline archive files are tracked only. #[derive(Debug, Clone)] pub struct RemoteTimelineIndex { timeline_files: HashMap, diff --git a/pageserver/src/repository.rs b/pageserver/src/repository.rs index 674d447624..be937b8d26 100644 --- a/pageserver/src/repository.rs +++ b/pageserver/src/repository.rs @@ -107,7 +107,7 @@ impl RepositoryTimeline { /// A state of the timeline synchronization with the remote storage. /// Contains `disk_consistent_lsn` of the corresponding remote timeline (latest checkpoint's disk_consistent_lsn). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum TimelineSyncState { /// No further downloads from the remote storage are needed. /// The timeline state is up-to-date or ahead of the remote storage one, diff --git a/pageserver/src/tenant_mgr.rs b/pageserver/src/tenant_mgr.rs index 98777e5e4b..f7f694d833 100644 --- a/pageserver/src/tenant_mgr.rs +++ b/pageserver/src/tenant_mgr.rs @@ -180,9 +180,9 @@ pub fn shutdown_all_tenants() { pub fn create_repository_for_tenant( conf: &'static PageServerConf, tenantid: ZTenantId, -) -> Result<()> { +) -> Result { let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenantid)); - let repo = timelines::create_repo(conf, tenantid, wal_redo_manager)?; + let (initial_timeline_id, repo) = timelines::create_repo(conf, tenantid, wal_redo_manager)?; match access_tenants().entry(tenantid) { hash_map::Entry::Occupied(_) => bail!("tenant {} already exists", tenantid), @@ -194,7 +194,7 @@ pub fn create_repository_for_tenant( } } - Ok(()) + Ok(initial_timeline_id) } pub fn get_tenant_state(tenantid: ZTenantId) -> Option { diff --git a/pageserver/src/timelines.rs b/pageserver/src/timelines.rs index 1e54fe3897..fc29767ddd 100644 --- a/pageserver/src/timelines.rs +++ b/pageserver/src/timelines.rs @@ -17,24 +17,133 @@ use std::{ use tracing::*; use zenith_utils::lsn::Lsn; -use zenith_utils::zid::{ZTenantId, ZTimelineId}; +use zenith_utils::zid::{opt_display_serde, ZTenantId, ZTimelineId}; use zenith_utils::{crashsafe_dir, logging}; use crate::walredo::WalRedoManager; -use crate::CheckpointConfig; use crate::{config::PageServerConf, repository::Repository}; use crate::{import_datadir, LOG_FILE_NAME}; use crate::{repository::RepositoryTimeline, tenant_mgr}; +use crate::{repository::Timeline, CheckpointConfig}; #[derive(Serialize, Deserialize, Clone)] -pub struct TimelineInfo { - #[serde(with = "hex")] - pub timeline_id: ZTimelineId, - pub latest_valid_lsn: Lsn, - pub ancestor_id: Option, - pub ancestor_lsn: Option, - pub current_logical_size: usize, - pub current_logical_size_non_incremental: Option, +#[serde(tag = "type")] +pub enum TimelineInfo { + Local { + #[serde(with = "hex")] + timeline_id: ZTimelineId, + #[serde(with = "hex")] + tenant_id: ZTenantId, + last_record_lsn: Lsn, + prev_record_lsn: Lsn, + #[serde(with = "opt_display_serde")] + ancestor_timeline_id: Option, + ancestor_lsn: Option, + disk_consistent_lsn: Lsn, + current_logical_size: usize, + current_logical_size_non_incremental: Option, + }, + Remote { + #[serde(with = "hex")] + timeline_id: ZTimelineId, + #[serde(with = "hex")] + tenant_id: ZTenantId, + disk_consistent_lsn: Lsn, + }, +} + +impl TimelineInfo { + pub fn from_repo_timeline( + tenant_id: ZTenantId, + repo_timeline: RepositoryTimeline, + include_non_incremental_logical_size: bool, + ) -> Self { + match repo_timeline { + RepositoryTimeline::Local { id, timeline } => { + let ancestor_timeline_id = timeline.get_ancestor_timeline_id(); + let ancestor_lsn = if ancestor_timeline_id.is_some() { + Some(timeline.get_ancestor_lsn()) + } else { + None + }; + + Self::Local { + timeline_id: id, + tenant_id, + last_record_lsn: timeline.get_last_record_lsn(), + prev_record_lsn: timeline.get_prev_record_lsn(), + ancestor_timeline_id, + ancestor_lsn, + disk_consistent_lsn: timeline.get_disk_consistent_lsn(), + current_logical_size: timeline.get_current_logical_size(), + current_logical_size_non_incremental: get_current_logical_size_non_incremental( + include_non_incremental_logical_size, + timeline.as_ref(), + ), + } + } + RepositoryTimeline::Remote { + id, + disk_consistent_lsn, + } => Self::Remote { + timeline_id: id, + tenant_id, + disk_consistent_lsn, + }, + } + } + + pub fn from_dyn_timeline( + tenant_id: ZTenantId, + timeline_id: ZTimelineId, + timeline: &dyn Timeline, + include_non_incremental_logical_size: bool, + ) -> Self { + let ancestor_timeline_id = timeline.get_ancestor_timeline_id(); + let ancestor_lsn = if ancestor_timeline_id.is_some() { + Some(timeline.get_ancestor_lsn()) + } else { + None + }; + + Self::Local { + timeline_id, + tenant_id, + last_record_lsn: timeline.get_last_record_lsn(), + prev_record_lsn: timeline.get_prev_record_lsn(), + ancestor_timeline_id, + ancestor_lsn, + disk_consistent_lsn: timeline.get_disk_consistent_lsn(), + current_logical_size: timeline.get_current_logical_size(), + current_logical_size_non_incremental: get_current_logical_size_non_incremental( + include_non_incremental_logical_size, + timeline, + ), + } + } + + pub fn timeline_id(&self) -> ZTimelineId { + match *self { + TimelineInfo::Local { timeline_id, .. } => timeline_id, + TimelineInfo::Remote { timeline_id, .. } => timeline_id, + } + } +} + +fn get_current_logical_size_non_incremental( + include_non_incremental_logical_size: bool, + timeline: &dyn Timeline, +) -> Option { + if !include_non_incremental_logical_size { + return None; + } + match timeline.get_current_logical_size_non_incremental(timeline.get_last_record_lsn()) { + Ok(size) => Some(size), + Err(e) => { + error!("Failed to get non-incremental logical size: {:?}", e); + None + } + } } #[derive(Debug, Clone, Copy)] @@ -75,7 +184,7 @@ pub fn create_repo( conf: &'static PageServerConf, tenantid: ZTenantId, wal_redo_manager: Arc, -) -> Result> { +) -> Result<(ZTimelineId, Arc)> { let repo_dir = conf.tenant_path(&tenantid); if repo_dir.exists() { bail!("repo for {} already exists", tenantid) @@ -107,7 +216,7 @@ pub fn create_repo( // move data loading out of create_repo() bootstrap_timeline(conf, tenantid, timeline_id, repo.as_ref())?; - Ok(repo) + Ok((timeline_id, repo)) } // Returns checkpoint LSN from controlfile @@ -160,7 +269,7 @@ fn bootstrap_timeline( tenantid: ZTenantId, tli: ZTimelineId, repo: &dyn Repository, -) -> Result<()> { +) -> Result> { let _enter = info_span!("bootstrapping", timeline = %tli, tenant = %tenantid).entered(); let initdb_path = conf.tenant_path(&tenantid).join("tmp"); @@ -192,7 +301,7 @@ fn bootstrap_timeline( // Remove temp dir. We don't need it anymore fs::remove_dir_all(pgdata_path)?; - Ok(()) + Ok(timeline) } pub(crate) fn get_timelines( @@ -211,110 +320,86 @@ pub(crate) fn get_timelines( RepositoryTimeline::Remote { .. } => None, }) .map(|(timeline_id, timeline)| { - let (ancestor_id, ancestor_lsn) = match timeline.get_ancestor_timeline_id() { - Some(ancestor_id) => ( - Some(ancestor_id.to_string()), - Some(timeline.get_ancestor_lsn().to_string()), - ), - None => (None, None), - }; - - let current_logical_size_non_incremental = if include_non_incremental_logical_size { - match timeline - .get_current_logical_size_non_incremental(timeline.get_last_record_lsn()) - { - Ok(size) => Some(size), - Err(e) => { - error!( - "Failed to get current logical size for timeline {}: {:?}", - timeline_id, e - ); - None - } - } - } else { - None - }; - - TimelineInfo { + TimelineInfo::from_dyn_timeline( + tenant_id, timeline_id, - latest_valid_lsn: timeline.get_last_record_lsn(), - ancestor_id, - ancestor_lsn, - current_logical_size: timeline.get_current_logical_size(), - // non incremental size calculation can be heavy, so let it be optional - // needed for tests to check size calculation - current_logical_size_non_incremental, - } + timeline.as_ref(), + include_non_incremental_logical_size, + ) }) .collect()) } pub(crate) fn create_timeline( - conf: &PageServerConf, + conf: &'static PageServerConf, tenant_id: ZTenantId, - timeline_id: ZTimelineId, - start_lsn: Option, + new_timeline_id: ZTimelineId, + ancestor_timeline_id: Option, + ancestor_start_lsn: Option, ) -> Result { - if conf.timeline_path(&timeline_id, &tenant_id).exists() { - bail!("timeline {} already exists", timeline_id); + if conf.timeline_path(&new_timeline_id, &tenant_id).exists() { + bail!("timeline {} already exists", new_timeline_id); } let repo = tenant_mgr::get_repository_for_tenant(tenant_id)?; + let mut start_lsn = ancestor_start_lsn.unwrap_or(Lsn(0)); - let mut startpoint = PointInTime { - timeline_id, - lsn: start_lsn.unwrap_or(Lsn(0)), - }; + match ancestor_timeline_id { + Some(ancestor_timeline_id) => { + let ancestor_timeline = repo + .get_timeline(ancestor_timeline_id) + .with_context(|| format!("Cannot get ancestor timeline {}", ancestor_timeline_id))? + .local_timeline() + .with_context(|| { + format!( + "Cannot branch off the timeline {} that's not present locally", + ancestor_timeline_id + ) + })?; - let timeline = repo - .get_timeline(startpoint.timeline_id)? - .local_timeline() - .context("Cannot branch off the timeline that's not present locally")?; - if startpoint.lsn == Lsn(0) { - // Find end of WAL on the old timeline - let end_of_wal = timeline.get_last_record_lsn(); - info!("branching at end of WAL: {}", end_of_wal); - startpoint.lsn = end_of_wal; - } else { - // Wait for the WAL to arrive and be processed on the parent branch up - // to the requested branch point. The repository code itself doesn't - // require it, but if we start to receive WAL on the new timeline, - // decoding the new WAL might need to look up previous pages, relation - // sizes etc. and that would get confused if the previous page versions - // are not in the repository yet. - timeline.wait_lsn(startpoint.lsn)?; + if start_lsn == Lsn(0) { + // Find end of WAL on the old timeline + let end_of_wal = ancestor_timeline.get_last_record_lsn(); + info!("branching at end of WAL: {}", end_of_wal); + start_lsn = end_of_wal; + } else { + // Wait for the WAL to arrive and be processed on the parent branch up + // to the requested branch point. The repository code itself doesn't + // require it, but if we start to receive WAL on the new timeline, + // decoding the new WAL might need to look up previous pages, relation + // sizes etc. and that would get confused if the previous page versions + // are not in the repository yet. + ancestor_timeline.wait_lsn(start_lsn)?; + } + start_lsn = start_lsn.align(); + + let ancestor_ancestor_lsn = ancestor_timeline.get_ancestor_lsn(); + if ancestor_ancestor_lsn > start_lsn { + // can we safely just branch from the ancestor instead? + anyhow::bail!( + "invalid start lsn {} for ancestor timeline {}: less than timeline ancestor lsn {}", + start_lsn, + ancestor_timeline_id, + ancestor_ancestor_lsn, + ); + } + repo.branch_timeline(ancestor_timeline_id, new_timeline_id, start_lsn)?; + // load the timeline into memory + let loaded_timeline = repo.get_timeline(new_timeline_id)?; + Ok(TimelineInfo::from_repo_timeline( + tenant_id, + loaded_timeline, + false, + )) + } + None => { + let new_timeline = bootstrap_timeline(conf, tenant_id, new_timeline_id, repo.as_ref())?; + Ok(TimelineInfo::from_dyn_timeline( + tenant_id, + new_timeline_id, + new_timeline.as_ref(), + false, + )) + } } - startpoint.lsn = startpoint.lsn.align(); - if timeline.get_ancestor_lsn() > startpoint.lsn { - // can we safely just branch from the ancestor instead? - bail!( - "invalid startpoint {} for the timeline {}: less than timeline ancestor lsn {:?}", - startpoint.lsn, - timeline_id, - timeline.get_ancestor_lsn() - ); - } - - let new_timeline_id = ZTimelineId::generate(); - - // Forward entire timeline creation routine to repository - // backend, so it can do all needed initialization - repo.branch_timeline(startpoint.timeline_id, new_timeline_id, startpoint.lsn)?; - - // Remember the human-readable branch name for the new timeline. - // FIXME: there's a race condition, if you create a branch with the same - // name concurrently. - // TODO kb timeline creation needs more - let data = new_timeline_id.to_string(); - fs::write(conf.timeline_path(&timeline_id, &tenant_id), data)?; - - Ok(TimelineInfo { - timeline_id: new_timeline_id, - latest_valid_lsn: startpoint.lsn, - ancestor_id: Some(startpoint.timeline_id.to_string()), - ancestor_lsn: Some(startpoint.lsn.to_string()), - current_logical_size: 0, - current_logical_size_non_incremental: Some(0), - }) } diff --git a/test_runner/batch_others/test_auth.py b/test_runner/batch_others/test_auth.py index ee1a09c917..4d1d0847ed 100644 --- a/test_runner/batch_others/test_auth.py +++ b/test_runner/batch_others/test_auth.py @@ -1,8 +1,8 @@ from contextlib import closing from typing import Iterator from uuid import UUID, uuid4 -import psycopg2 from fixtures.zenith_fixtures import ZenithEnvBuilder, ZenithPageserverApiException +from requests.exceptions import HTTPError import pytest @@ -26,14 +26,20 @@ def test_pageserver_auth(zenith_env_builder: ZenithEnvBuilder): ps.safe_psql("set FOO", password=management_token) # tenant can create branches - tenant_http_client.branch_create(env.initial_tenant, 'new1', 'main') + tenant_http_client.timeline_create(timeline_id=uuid4(), + tenant_id=env.initial_tenant, + ancestor_timeline_id=env.initial_timeline) # console can create branches for tenant - management_http_client.branch_create(env.initial_tenant, 'new2', 'main') + management_http_client.timeline_create(timeline_id=uuid4(), + tenant_id=env.initial_tenant, + ancestor_timeline_id=env.initial_timeline) # fail to create branch using token with different tenant_id with pytest.raises(ZenithPageserverApiException, match='Forbidden: Tenant id mismatch. Permission denied'): - invalid_tenant_http_client.branch_create(env.initial_tenant, "new3", "main") + invalid_tenant_http_client.timeline_create(timeline_id=uuid4(), + tenant_id=env.initial_tenant, + ancestor_timeline_id=env.initial_timeline) # create tenant using management token management_http_client.tenant_create(uuid4()) @@ -54,9 +60,8 @@ def test_compute_auth_to_pageserver(zenith_env_builder: ZenithEnvBuilder, with_w env = zenith_env_builder.init_start() branch = f"test_compute_auth_to_pageserver{with_wal_acceptors}" - env.zenith_cli.create_branch(branch, "main") - - pg = env.postgres.create_start(branch) + new_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start(branch, timeline_id=new_timeline_id) with closing(pg.connect()) as conn: with conn.cursor() as cur: diff --git a/test_runner/batch_others/test_branch_behind.py b/test_runner/batch_others/test_branch_behind.py index 509c46975e..f8ff1741b4 100644 --- a/test_runner/batch_others/test_branch_behind.py +++ b/test_runner/batch_others/test_branch_behind.py @@ -22,9 +22,9 @@ def test_branch_behind(zenith_env_builder: ZenithEnvBuilder): env = zenith_env_builder.init_start() # Branch at the point where only 100 rows were inserted - env.zenith_cli.create_branch("test_branch_behind", "main") - - pgmain = env.postgres.create_start('test_branch_behind') + test_branch_behind_timeline_id = env.zenith_cli.branch_timeline() + pgmain = env.postgres.create_start('test_branch_behind', + timeline_id=test_branch_behind_timeline_id) log.info("postgres is running on 'test_branch_behind' branch") main_pg_conn = pgmain.connect() @@ -60,7 +60,8 @@ def test_branch_behind(zenith_env_builder: ZenithEnvBuilder): log.info(f'LSN after 200100 rows: {lsn_b}') # Branch at the point where only 100 rows were inserted - env.zenith_cli.create_branch("test_branch_behind_hundred", "test_branch_behind@" + lsn_a) + test_branch_behind_hundred_timeline_id = env.zenith_cli.branch_timeline( + ancestor_timeline_id=test_branch_behind_timeline_id, ancestor_start_lsn=lsn_a) # Insert many more rows. This generates enough WAL to fill a few segments. main_cur.execute(''' @@ -75,10 +76,13 @@ def test_branch_behind(zenith_env_builder: ZenithEnvBuilder): log.info(f'LSN after 400100 rows: {lsn_c}') # Branch at the point where only 200100 rows were inserted - env.zenith_cli.create_branch("test_branch_behind_more", "test_branch_behind@" + lsn_b) + test_branch_behind_more_timeline_id = env.zenith_cli.branch_timeline( + ancestor_timeline_id=test_branch_behind_timeline_id, ancestor_start_lsn=lsn_b) - pg_hundred = env.postgres.create_start("test_branch_behind_hundred") - pg_more = env.postgres.create_start("test_branch_behind_more") + pg_hundred = env.postgres.create_start("test_branch_behind_hundred", + timeline_id=test_branch_behind_hundred_timeline_id) + pg_more = env.postgres.create_start("test_branch_behind_more", + timeline_id=test_branch_behind_more_timeline_id) # On the 'hundred' branch, we should see only 100 rows hundred_pg_conn = pg_hundred.connect() @@ -99,19 +103,23 @@ def test_branch_behind(zenith_env_builder: ZenithEnvBuilder): # Check bad lsn's for branching # branch at segment boundary - env.zenith_cli.create_branch("test_branch_segment_boundary", "test_branch_behind@0/3000000") - pg = env.postgres.create_start("test_branch_segment_boundary") + test_branch_segment_boundary_timeline_id = env.zenith_cli.branch_timeline( + ancestor_timeline_id=test_branch_behind_timeline_id, ancestor_start_lsn="0/3000000") + pg = env.postgres.create_start("test_branch_segment_boundary", + timeline_id=test_branch_segment_boundary_timeline_id) cur = pg.connect().cursor() cur.execute('SELECT 1') assert cur.fetchone() == (1, ) # branch at pre-initdb lsn with pytest.raises(Exception, match="invalid branch start lsn"): - env.zenith_cli.create_branch("test_branch_preinitdb", "main@0/42") + env.zenith_cli.branch_timeline(ancestor_timeline_id=env.initial_timeline, + ancestor_start_lsn="0/42") # branch at pre-ancestor lsn with pytest.raises(Exception, match="less than timeline ancestor lsn"): - env.zenith_cli.create_branch("test_branch_preinitdb", "test_branch_behind@0/42") + env.zenith_cli.branch_timeline(ancestor_timeline_id=test_branch_behind_timeline_id, + ancestor_start_lsn="0/42") # check that we cannot create branch based on garbage collected data with closing(env.pageserver.connect()) as psconn: @@ -123,7 +131,8 @@ def test_branch_behind(zenith_env_builder: ZenithEnvBuilder): with pytest.raises(Exception, match="invalid branch start lsn"): # this gced_lsn is pretty random, so if gc is disabled this woudln't fail - env.zenith_cli.create_branch("test_branch_create_fail", f"test_branch_behind@{gced_lsn}") + env.zenith_cli.branch_timeline(ancestor_timeline_id=test_branch_behind_timeline_id, + ancestor_start_lsn=gced_lsn) # check that after gc everything is still there hundred_cur.execute('SELECT count(*) FROM foo') diff --git a/test_runner/batch_others/test_clog_truncate.py b/test_runner/batch_others/test_clog_truncate.py index 504f455936..9d3927aa84 100644 --- a/test_runner/batch_others/test_clog_truncate.py +++ b/test_runner/batch_others/test_clog_truncate.py @@ -12,7 +12,7 @@ from fixtures.log_helper import log # def test_clog_truncate(zenith_simple_env: ZenithEnv): env = zenith_simple_env - env.zenith_cli.create_branch("test_clog_truncate", "empty") + test_clog_truncate_timeline_id = env.zenith_cli.branch_timeline() # set agressive autovacuum to make sure that truncation will happen config = [ @@ -25,7 +25,9 @@ def test_clog_truncate(zenith_simple_env: ZenithEnv): 'autovacuum_freeze_max_age=100000' ] - pg = env.postgres.create_start('test_clog_truncate', config_lines=config) + pg = env.postgres.create_start('test_clog_truncate', + config_lines=config, + timeline_id=test_clog_truncate_timeline_id) log.info('postgres is running on test_clog_truncate branch') # Install extension containing function needed for test @@ -62,10 +64,11 @@ def test_clog_truncate(zenith_simple_env: ZenithEnv): # create new branch after clog truncation and start a compute node on it log.info(f'create branch at lsn_after_truncation {lsn_after_truncation}') - env.zenith_cli.create_branch("test_clog_truncate_new", - "test_clog_truncate@" + lsn_after_truncation) - - pg2 = env.postgres.create_start('test_clog_truncate_new') + test_clog_truncate_new_timeline_id = env.zenith_cli.branch_timeline( + ancestor_timeline_id=test_clog_truncate_timeline_id, + ancestor_start_lsn=lsn_after_truncation) + pg2 = env.postgres.create_start('test_clog_truncate_new', + timeline_id=test_clog_truncate_new_timeline_id) log.info('postgres is running on test_clog_truncate_new branch') # check that new node doesn't contain truncated segment diff --git a/test_runner/batch_others/test_config.py b/test_runner/batch_others/test_config.py index fd2b3b4e99..bd1f8b487f 100644 --- a/test_runner/batch_others/test_config.py +++ b/test_runner/batch_others/test_config.py @@ -9,10 +9,10 @@ from fixtures.log_helper import log # def test_config(zenith_simple_env: ZenithEnv): env = zenith_simple_env - env.zenith_cli.create_branch("test_config", "empty") - - # change config - pg = env.postgres.create_start('test_config', config_lines=['log_min_messages=debug1']) + new_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_config', + config_lines=['log_min_messages=debug1'], + timeline_id=new_timeline_id) log.info('postgres is running on test_config branch') with closing(pg.connect()) as conn: diff --git a/test_runner/batch_others/test_createdropdb.py b/test_runner/batch_others/test_createdropdb.py index 38243b298b..e77e1928b8 100644 --- a/test_runner/batch_others/test_createdropdb.py +++ b/test_runner/batch_others/test_createdropdb.py @@ -11,9 +11,9 @@ from fixtures.log_helper import log # def test_createdb(zenith_simple_env: ZenithEnv): env = zenith_simple_env - env.zenith_cli.create_branch("test_createdb", "empty") + test_createdb_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_createdb') + pg = env.postgres.create_start('test_createdb', timeline_id=test_createdb_timeline_id) log.info("postgres is running on 'test_createdb' branch") with closing(pg.connect()) as conn: @@ -27,9 +27,9 @@ def test_createdb(zenith_simple_env: ZenithEnv): lsn = cur.fetchone()[0] # Create a branch - env.zenith_cli.create_branch("test_createdb2", "test_createdb@" + lsn) - - pg2 = env.postgres.create_start('test_createdb2') + test_createdb2_timeline_id = env.zenith_cli.branch_timeline( + ancestor_timeline_id=test_createdb_timeline_id, ancestor_start_lsn=lsn) + pg2 = env.postgres.create_start('test_createdb2', timeline_id=test_createdb2_timeline_id) # Test that you can connect to the new database on both branches for db in (pg, pg2): @@ -41,9 +41,8 @@ def test_createdb(zenith_simple_env: ZenithEnv): # def test_dropdb(zenith_simple_env: ZenithEnv, test_output_dir): env = zenith_simple_env - env.zenith_cli.create_branch("test_dropdb", "empty") - - pg = env.postgres.create_start('test_dropdb') + test_dropdb_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_dropdb', timeline_id=test_dropdb_timeline_id) log.info("postgres is running on 'test_dropdb' branch") with closing(pg.connect()) as conn: @@ -66,11 +65,15 @@ def test_dropdb(zenith_simple_env: ZenithEnv, test_output_dir): lsn_after_drop = cur.fetchone()[0] # Create two branches before and after database drop. - env.zenith_cli.create_branch("test_before_dropdb", "test_dropdb@" + lsn_before_drop) - pg_before = env.postgres.create_start('test_before_dropdb') + test_before_dropdb_timeline_db = env.zenith_cli.branch_timeline( + ancestor_timeline_id=test_dropdb_timeline_id, ancestor_start_lsn=lsn_before_drop) + pg_before = env.postgres.create_start('test_before_dropdb', + timeline_id=test_before_dropdb_timeline_db) - env.zenith_cli.create_branch("test_after_dropdb", "test_dropdb@" + lsn_after_drop) - pg_after = env.postgres.create_start('test_after_dropdb') + test_after_dropdb_timeline_id = env.zenith_cli.branch_timeline( + ancestor_timeline_id=test_dropdb_timeline_id, ancestor_start_lsn=lsn_after_drop) + pg_after = env.postgres.create_start('test_after_dropdb', + timeline_id=test_after_dropdb_timeline_id) # Test that database exists on the branch before drop pg_before.connect(dbname='foodb').close() diff --git a/test_runner/batch_others/test_createuser.py b/test_runner/batch_others/test_createuser.py index 1959b47dcc..8f825a0a1a 100644 --- a/test_runner/batch_others/test_createuser.py +++ b/test_runner/batch_others/test_createuser.py @@ -9,9 +9,8 @@ from fixtures.log_helper import log # def test_createuser(zenith_simple_env: ZenithEnv): env = zenith_simple_env - env.zenith_cli.create_branch("test_createuser", "empty") - - pg = env.postgres.create_start('test_createuser') + test_createuser_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_createuser', timeline_id=test_createuser_timeline_id) log.info("postgres is running on 'test_createuser' branch") with closing(pg.connect()) as conn: @@ -25,9 +24,9 @@ def test_createuser(zenith_simple_env: ZenithEnv): lsn = cur.fetchone()[0] # Create a branch - env.zenith_cli.create_branch("test_createuser2", "test_createuser@" + lsn) - - pg2 = env.postgres.create_start('test_createuser2') + test_createuser2_timeline_id = env.zenith_cli.branch_timeline( + ancestor_timeline_id=test_createuser_timeline_id, ancestor_start_lsn=lsn) + pg2 = env.postgres.create_start('test_createuser2', timeline_id=test_createuser2_timeline_id) # Test that you can connect to new branch as a new user assert pg2.safe_psql('select current_user', username='testuser') == [('testuser', )] diff --git a/test_runner/batch_others/test_gc_aggressive.py b/test_runner/batch_others/test_gc_aggressive.py index 9de6ba9f59..7dd38a5799 100644 --- a/test_runner/batch_others/test_gc_aggressive.py +++ b/test_runner/batch_others/test_gc_aggressive.py @@ -1,7 +1,6 @@ from contextlib import closing import asyncio -import asyncpg import random from fixtures.zenith_fixtures import ZenithEnv, Postgres, Safekeeper @@ -55,8 +54,8 @@ async def update_and_gc(env: ZenithEnv, pg: Postgres, timeline: str): # def test_gc_aggressive(zenith_simple_env: ZenithEnv): env = zenith_simple_env - env.zenith_cli.create_branch("test_gc_aggressive", "empty") - pg = env.postgres.create_start('test_gc_aggressive') + new_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_gc_aggressive', timeline_id=new_timeline_id) log.info('postgres is running on test_gc_aggressive branch') conn = pg.connect() diff --git a/test_runner/batch_others/test_multixact.py b/test_runner/batch_others/test_multixact.py index 6a2afd2ede..11f8000226 100644 --- a/test_runner/batch_others/test_multixact.py +++ b/test_runner/batch_others/test_multixact.py @@ -10,8 +10,8 @@ from fixtures.log_helper import log # def test_multixact(zenith_simple_env: ZenithEnv, test_output_dir): env = zenith_simple_env - env.zenith_cli.create_branch("test_multixact", "empty") - pg = env.postgres.create_start('test_multixact') + test_multixact_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_multixact', timeline_id=test_multixact_timeline_id) log.info("postgres is running on 'test_multixact' branch") pg_conn = pg.connect() @@ -60,8 +60,10 @@ def test_multixact(zenith_simple_env: ZenithEnv, test_output_dir): assert int(next_multixact_id) > int(next_multixact_id_old) # Branch at this point - env.zenith_cli.create_branch("test_multixact_new", "test_multixact@" + lsn) - pg_new = env.postgres.create_start('test_multixact_new') + test_multixact_new_timeline_id = env.zenith_cli.branch_timeline( + ancestor_timeline_id=test_multixact_timeline_id, ancestor_start_lsn=lsn) + pg_new = env.postgres.create_start('test_multixact_new', + timeline_id=test_multixact_new_timeline_id) log.info("postgres is running on 'test_multixact_new' branch") pg_new_conn = pg_new.connect() diff --git a/test_runner/batch_others/test_old_request_lsn.py b/test_runner/batch_others/test_old_request_lsn.py index d09fb24913..f0701dfe4f 100644 --- a/test_runner/batch_others/test_old_request_lsn.py +++ b/test_runner/batch_others/test_old_request_lsn.py @@ -16,8 +16,8 @@ from fixtures.log_helper import log # def test_old_request_lsn(zenith_simple_env: ZenithEnv): env = zenith_simple_env - env.zenith_cli.create_branch("test_old_request_lsn", "empty") - pg = env.postgres.create_start('test_old_request_lsn') + new_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_old_request_lsn', timeline_id=new_timeline_id) log.info('postgres is running on test_old_request_lsn branch') pg_conn = pg.connect() diff --git a/test_runner/batch_others/test_pageserver_api.py b/test_runner/batch_others/test_pageserver_api.py index ba1f106c4b..4c3b98e838 100644 --- a/test_runner/batch_others/test_pageserver_api.py +++ b/test_runner/batch_others/test_pageserver_api.py @@ -26,18 +26,20 @@ def check_client(client: ZenithPageserverHttpClient, initial_tenant: UUID): # check its timelines timelines = client.timeline_list(tenant_id) assert len(timelines) > 0 - for timeline_id_str in timelines: - timeline_details = client.timeline_detail(tenant_id, UUID(timeline_id_str)) + for timeline in timelines: + timeline_id_str = str(timeline['timeline_id']) + timeline_details = client.timeline_detail(tenant_id=tenant_id, + timeline_id=UUID(timeline_id_str)) assert timeline_details['type'] == 'Local' assert timeline_details['tenant_id'] == tenant_id.hex assert timeline_details['timeline_id'] == timeline_id_str - # create branch - branch_name = uuid4().hex - client.branch_create(tenant_id, branch_name, "main") + # create timeline + timeline_id = uuid4() + client.timeline_create(tenant_id=tenant_id, timeline_id=timeline_id) # check it is there - assert branch_name in {b['name'] for b in client.branch_list(tenant_id)} + assert timeline_id.hex in {b['timeline_id'] for b in client.timeline_list(tenant_id)} def test_pageserver_http_api_client(zenith_simple_env: ZenithEnv): diff --git a/test_runner/batch_others/test_pageserver_catchup.py b/test_runner/batch_others/test_pageserver_catchup.py index 985d1a3af0..ba77a4a321 100644 --- a/test_runner/batch_others/test_pageserver_catchup.py +++ b/test_runner/batch_others/test_pageserver_catchup.py @@ -16,8 +16,9 @@ def test_pageserver_catchup_while_compute_down(zenith_env_builder: ZenithEnvBuil zenith_env_builder.num_safekeepers = 3 env = zenith_env_builder.init_start() - env.zenith_cli.create_branch("test_pageserver_catchup_while_compute_down", "main") - pg = env.postgres.create_start('test_pageserver_catchup_while_compute_down') + new_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_pageserver_catchup_while_compute_down', + timeline_id=new_timeline_id) pg_conn = pg.connect() cur = pg_conn.cursor() @@ -59,7 +60,8 @@ def test_pageserver_catchup_while_compute_down(zenith_env_builder: ZenithEnvBuil env.safekeepers[2].start() # restart compute node - pg.stop_and_destroy().create_start('test_pageserver_catchup_while_compute_down') + pg.stop_and_destroy().create_start('test_pageserver_catchup_while_compute_down', + timeline_id=new_timeline_id) # Ensure that basebackup went correct and pageserver returned all data pg_conn = pg.connect() diff --git a/test_runner/batch_others/test_pageserver_restart.py b/test_runner/batch_others/test_pageserver_restart.py index ec93c2cf5b..f1d154408c 100644 --- a/test_runner/batch_others/test_pageserver_restart.py +++ b/test_runner/batch_others/test_pageserver_restart.py @@ -15,8 +15,8 @@ def test_pageserver_restart(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 1 env = zenith_env_builder.init_start() - env.zenith_cli.create_branch("test_pageserver_restart", "main") - pg = env.postgres.create_start('test_pageserver_restart') + new_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_pageserver_restart', timeline_id=new_timeline_id) pg_conn = pg.connect() cur = pg_conn.cursor() diff --git a/test_runner/batch_others/test_parallel_copy.py b/test_runner/batch_others/test_parallel_copy.py index 6f87bc4a36..8e954a8e51 100644 --- a/test_runner/batch_others/test_parallel_copy.py +++ b/test_runner/batch_others/test_parallel_copy.py @@ -1,7 +1,5 @@ from io import BytesIO import asyncio -import asyncpg -import subprocess from fixtures.zenith_fixtures import ZenithEnv, Postgres from fixtures.log_helper import log @@ -37,8 +35,8 @@ async def parallel_load_same_table(pg: Postgres, n_parallel: int): # Load data into one table with COPY TO from 5 parallel connections def test_parallel_copy(zenith_simple_env: ZenithEnv, n_parallel=5): env = zenith_simple_env - env.zenith_cli.create_branch("test_parallel_copy", "empty") - pg = env.postgres.create_start('test_parallel_copy') + new_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_parallel_copy', timeline_id=new_timeline_id) log.info("postgres is running on 'test_parallel_copy' branch") # Create test table diff --git a/test_runner/batch_others/test_pgbench.py b/test_runner/batch_others/test_pgbench.py index 09713023bc..207f1e1e2c 100644 --- a/test_runner/batch_others/test_pgbench.py +++ b/test_runner/batch_others/test_pgbench.py @@ -4,8 +4,8 @@ from fixtures.log_helper import log def test_pgbench(zenith_simple_env: ZenithEnv, pg_bin): env = zenith_simple_env - env.zenith_cli.create_branch("test_pgbench", "empty") - pg = env.postgres.create_start('test_pgbench') + new_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_pgbench', timeline_id=new_timeline_id) log.info("postgres is running on 'test_pgbench' branch") connstr = pg.connstr() diff --git a/test_runner/batch_others/test_readonly_node.py b/test_runner/batch_others/test_readonly_node.py index ba256e71f7..2998ea7528 100644 --- a/test_runner/batch_others/test_readonly_node.py +++ b/test_runner/batch_others/test_readonly_node.py @@ -11,9 +11,9 @@ from fixtures.zenith_fixtures import ZenithEnv # def test_readonly_node(zenith_simple_env: ZenithEnv): env = zenith_simple_env - env.zenith_cli.create_branch("test_readonly_node", "empty") - - pgmain = env.postgres.create_start('test_readonly_node') + test_readonly_node_timeline_id = env.zenith_cli.branch_timeline() + pgmain = env.postgres.create_start('test_readonly_node', + timeline_id=test_readonly_node_timeline_id) log.info("postgres is running on 'test_readonly_node' branch") main_pg_conn = pgmain.connect() @@ -54,11 +54,13 @@ def test_readonly_node(zenith_simple_env: ZenithEnv): # Create first read-only node at the point where only 100 rows were inserted pg_hundred = env.postgres.create_start("test_readonly_node_hundred", - branch=f'test_readonly_node@{lsn_a}') + timeline_id=test_readonly_node_timeline_id, + lsn=lsn_a) # And another at the point where 200100 rows were inserted pg_more = env.postgres.create_start("test_readonly_node_more", - branch=f'test_readonly_node@{lsn_b}') + timeline_id=test_readonly_node_timeline_id, + lsn=lsn_b) # On the 'hundred' node, we should see only 100 rows hundred_pg_conn = pg_hundred.connect() @@ -78,7 +80,8 @@ def test_readonly_node(zenith_simple_env: ZenithEnv): # Check creating a node at segment boundary pg = env.postgres.create_start("test_branch_segment_boundary", - branch="test_readonly_node@0/3000000") + timeline_id=test_readonly_node_timeline_id, + lsn='0/3000000') cur = pg.connect().cursor() cur.execute('SELECT 1') assert cur.fetchone() == (1, ) @@ -87,4 +90,5 @@ def test_readonly_node(zenith_simple_env: ZenithEnv): with pytest.raises(Exception, match="invalid basebackup lsn"): # compute node startup with invalid LSN should fail env.zenith_cli.pg_start("test_readonly_node_preinitdb", - timeline_spec="test_readonly_node@0/42") + timeline_id=test_readonly_node_timeline_id, + lsn="0/42") diff --git a/test_runner/batch_others/test_restart_compute.py b/test_runner/batch_others/test_restart_compute.py index d4dd3fb9e2..baa1f787df 100644 --- a/test_runner/batch_others/test_restart_compute.py +++ b/test_runner/batch_others/test_restart_compute.py @@ -15,9 +15,8 @@ def test_restart_compute(zenith_env_builder: ZenithEnvBuilder, with_wal_acceptor zenith_env_builder.num_safekeepers = 3 env = zenith_env_builder.init_start() - env.zenith_cli.create_branch("test_restart_compute", "main") - - pg = env.postgres.create_start('test_restart_compute') + new_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_restart_compute', timeline_id=new_timeline_id) log.info("postgres is running on 'test_restart_compute' branch") with closing(pg.connect()) as conn: @@ -30,7 +29,7 @@ def test_restart_compute(zenith_env_builder: ZenithEnvBuilder, with_wal_acceptor log.info(f"res = {r}") # Remove data directory and restart - pg.stop_and_destroy().create_start('test_restart_compute') + pg.stop_and_destroy().create_start('test_restart_compute', timeline_id=new_timeline_id) with closing(pg.connect()) as conn: with conn.cursor() as cur: @@ -49,7 +48,7 @@ def test_restart_compute(zenith_env_builder: ZenithEnvBuilder, with_wal_acceptor log.info(f"res = {r}") # Again remove data directory and restart - pg.stop_and_destroy().create_start('test_restart_compute') + pg.stop_and_destroy().create_start('test_restart_compute', timeline_id=new_timeline_id) # That select causes lots of FPI's and increases probability of wakeepers # lagging behind after query completion @@ -63,7 +62,7 @@ def test_restart_compute(zenith_env_builder: ZenithEnvBuilder, with_wal_acceptor log.info(f"res = {r}") # And again remove data directory and restart - pg.stop_and_destroy().create_start('test_restart_compute') + pg.stop_and_destroy().create_start('test_restart_compute', timeline_id=new_timeline_id) with closing(pg.connect()) as conn: with conn.cursor() as cur: diff --git a/test_runner/batch_others/test_snapfiles_gc.py b/test_runner/batch_others/test_snapfiles_gc.py index c6d4512bc9..fb02e54be2 100644 --- a/test_runner/batch_others/test_snapfiles_gc.py +++ b/test_runner/batch_others/test_snapfiles_gc.py @@ -14,8 +14,8 @@ from fixtures.log_helper import log # def test_layerfiles_gc(zenith_simple_env: ZenithEnv): env = zenith_simple_env - env.zenith_cli.create_branch("test_layerfiles_gc", "empty") - pg = env.postgres.create_start('test_layerfiles_gc') + new_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_layerfiles_gc', timeline_id=new_timeline_id) with closing(pg.connect()) as conn: with conn.cursor() as cur: diff --git a/test_runner/batch_others/test_subxacts.py b/test_runner/batch_others/test_subxacts.py index bed1c4be63..6153bd1fe2 100644 --- a/test_runner/batch_others/test_subxacts.py +++ b/test_runner/batch_others/test_subxacts.py @@ -10,8 +10,8 @@ from fixtures.log_helper import log # CLOG. def test_subxacts(zenith_simple_env: ZenithEnv, test_output_dir): env = zenith_simple_env - env.zenith_cli.create_branch("test_subxacts", "empty") - pg = env.postgres.create_start('test_subxacts') + new_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_subxacts', timeline_id=new_timeline_id) log.info("postgres is running on 'test_subxacts' branch") pg_conn = pg.connect() diff --git a/test_runner/batch_others/test_tenant_relocation.py b/test_runner/batch_others/test_tenant_relocation.py index acff3ef62c..429aee8488 100644 --- a/test_runner/batch_others/test_tenant_relocation.py +++ b/test_runner/batch_others/test_tenant_relocation.py @@ -127,16 +127,14 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder, # create folder for remote storage mock remote_storage_mock_path = env.repo_dir / 'local_fs_remote_storage' - tenant = env.create_tenant(UUID("74ee8b079a0e437eb0afea7d26a07209")) + (tenant, _) = env.zenith_cli.create_tenant(UUID("74ee8b079a0e437eb0afea7d26a07209")) log.info("tenant to relocate %s", tenant) - env.zenith_cli.create_branch("test_tenant_relocation", "main", tenant_id=tenant) + new_timeline_id = env.zenith_cli.branch_timeline(tenant_id=tenant) - tenant_pg = env.postgres.create_start( - "test_tenant_relocation", - "main", # branch name, None means same as node name - tenant_id=tenant, - ) + tenant_pg = env.postgres.create_start("test_tenant_relocation", + tenant_id=tenant, + timeline_id=new_timeline_id) # insert some data with closing(tenant_pg.connect()) as conn: diff --git a/test_runner/batch_others/test_tenants.py b/test_runner/batch_others/test_tenants.py index b665ae9022..20a910e9ce 100644 --- a/test_runner/batch_others/test_tenants.py +++ b/test_runner/batch_others/test_tenants.py @@ -12,25 +12,23 @@ def test_tenants_normal_work(zenith_env_builder: ZenithEnvBuilder, with_wal_acce env = zenith_env_builder.init_start() """Tests tenants with and without wal acceptors""" - tenant_1 = env.create_tenant() - tenant_2 = env.create_tenant() + (tenant_1, initial_timeline_1) = env.zenith_cli.create_tenant() + (tenant_2, initial_timeline_2) = env.zenith_cli.create_tenant() - env.zenith_cli.create_branch(f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}", - "main", - tenant_id=tenant_1) - env.zenith_cli.create_branch(f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}", - "main", - tenant_id=tenant_2) + new_timeline_tenant_1 = env.zenith_cli.branch_timeline(tenant_id=tenant_1, + ancestor_timeline_id=initial_timeline_1) + new_timeline_tenant_2 = env.zenith_cli.branch_timeline(tenant_id=tenant_2, + ancestor_timeline_id=initial_timeline_2) pg_tenant1 = env.postgres.create_start( f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}", - None, # branch name, None means same as node name - tenant_1, + tenant_id=tenant_1, + timeline_id=new_timeline_tenant_1, ) pg_tenant2 = env.postgres.create_start( f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}", - None, # branch name, None means same as node name - tenant_2, + tenant_id=tenant_2, + timeline_id=new_timeline_tenant_2, ) for pg in [pg_tenant1, pg_tenant2]: diff --git a/test_runner/batch_others/test_timeline_size.py b/test_runner/batch_others/test_timeline_size.py index 2c31267922..49143d0000 100644 --- a/test_runner/batch_others/test_timeline_size.py +++ b/test_runner/batch_others/test_timeline_size.py @@ -10,13 +10,14 @@ import time def test_timeline_size(zenith_simple_env: ZenithEnv): env = zenith_simple_env # Branch at the point where only 100 rows were inserted - env.zenith_cli.create_branch("test_timeline_size", "empty") + new_timeline_id = env.zenith_cli.branch_timeline() client = env.pageserver.http_client() - res = client.branch_detail(env.initial_tenant, "test_timeline_size") + res = client.timeline_detail(tenant_id=env.initial_tenant, timeline_id=new_timeline_id) + print(f'@@@@@@@@@@\n{res}\n@@@@@@@@@@@') assert res["current_logical_size"] == res["current_logical_size_non_incremental"] - pgmain = env.postgres.create_start("test_timeline_size") + pgmain = env.postgres.create_start("test_timeline_size", timeline_id=new_timeline_id) log.info("postgres is running on 'test_timeline_size' branch") with closing(pgmain.connect()) as conn: @@ -31,11 +32,11 @@ def test_timeline_size(zenith_simple_env: ZenithEnv): FROM generate_series(1, 10) g """) - res = client.branch_detail(env.initial_tenant, "test_timeline_size") + res = client.timeline_detail(tenant_id=env.initial_tenant, timeline_id=new_timeline_id) assert res["current_logical_size"] == res["current_logical_size_non_incremental"] cur.execute("TRUNCATE foo") - res = client.branch_detail(env.initial_tenant, "test_timeline_size") + res = client.timeline_detail(tenant_id=env.initial_tenant, timeline_id=new_timeline_id) assert res["current_logical_size"] == res["current_logical_size_non_incremental"] @@ -68,17 +69,17 @@ def wait_for_pageserver_catchup(pgmain: Postgres, polling_interval=1, timeout=60 def test_timeline_size_quota(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 1 env = zenith_env_builder.init_start() - env.zenith_cli.create_branch("test_timeline_size_quota", "main") + new_timeline_id = env.zenith_cli.branch_timeline() client = env.pageserver.http_client() - res = client.branch_detail(env.initial_tenant, "test_timeline_size_quota") + res = client.timeline_detail(tenant_id=env.initial_tenant, timeline_id=new_timeline_id) assert res["current_logical_size"] == res["current_logical_size_non_incremental"] pgmain = env.postgres.create_start( "test_timeline_size_quota", # Set small limit for the test config_lines=['zenith.max_cluster_size=30MB'], - ) + timeline_id=new_timeline_id) log.info("postgres is running on 'test_timeline_size_quota' branch") with closing(pgmain.connect()) as conn: diff --git a/test_runner/batch_others/test_twophase.py b/test_runner/batch_others/test_twophase.py index d6a1cd01e8..b479e9de22 100644 --- a/test_runner/batch_others/test_twophase.py +++ b/test_runner/batch_others/test_twophase.py @@ -9,9 +9,10 @@ from fixtures.log_helper import log # def test_twophase(zenith_simple_env: ZenithEnv): env = zenith_simple_env - env.zenith_cli.create_branch("test_twophase", "empty") - - pg = env.postgres.create_start('test_twophase', config_lines=['max_prepared_transactions=5']) + test_twophase_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_twophase', + config_lines=['max_prepared_transactions=5'], + timeline_id=test_twophase_timeline_id) log.info("postgres is running on 'test_twophase' branch") conn = pg.connect() @@ -56,12 +57,14 @@ def test_twophase(zenith_simple_env: ZenithEnv): assert len(twophase_files) == 2 # Create a branch with the transaction in prepared state - env.zenith_cli.create_branch("test_twophase_prepared", "test_twophase") + test_twophase_prepared_timeline_id = env.zenith_cli.branch_timeline( + ancestor_timeline_id=test_twophase_timeline_id) # Start compute on the new branch pg2 = env.postgres.create_start( 'test_twophase_prepared', config_lines=['max_prepared_transactions=5'], + timeline_id=test_twophase_prepared_timeline_id, ) # Check that we restored only needed twophase files diff --git a/test_runner/batch_others/test_vm_bits.py b/test_runner/batch_others/test_vm_bits.py index 49e48dd450..a657b3e3fd 100644 --- a/test_runner/batch_others/test_vm_bits.py +++ b/test_runner/batch_others/test_vm_bits.py @@ -9,8 +9,8 @@ from fixtures.log_helper import log def test_vm_bit_clear(zenith_simple_env: ZenithEnv): env = zenith_simple_env - env.zenith_cli.create_branch("test_vm_bit_clear", "empty") - pg = env.postgres.create_start('test_vm_bit_clear') + test_vm_bit_clear_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_vm_bit_clear', timeline_id=test_vm_bit_clear_timeline_id) log.info("postgres is running on 'test_vm_bit_clear' branch") pg_conn = pg.connect() @@ -33,7 +33,8 @@ def test_vm_bit_clear(zenith_simple_env: ZenithEnv): cur.execute('UPDATE vmtest_update SET id = 5000 WHERE id = 1') # Branch at this point, to test that later - env.zenith_cli.create_branch("test_vm_bit_clear_new", "test_vm_bit_clear") + test_vm_bit_clear_new_timeline_id = env.zenith_cli.branch_timeline( + ancestor_timeline_id=test_vm_bit_clear_timeline_id) # Clear the buffer cache, to force the VM page to be re-fetched from # the page server @@ -61,7 +62,8 @@ def test_vm_bit_clear(zenith_simple_env: ZenithEnv): # a dirty VM page is evicted. If the VM bit was not correctly cleared by the # earlier WAL record, the full-page image hides the problem. Starting a new # server at the right point-in-time avoids that full-page image. - pg_new = env.postgres.create_start('test_vm_bit_clear_new') + pg_new = env.postgres.create_start('test_vm_bit_clear_new', + timeline_id=test_vm_bit_clear_new_timeline_id) log.info("postgres is running on 'test_vm_bit_clear_new' branch") pg_new_conn = pg_new.connect() diff --git a/test_runner/batch_others/test_wal_acceptor.py b/test_runner/batch_others/test_wal_acceptor.py index c375c9626a..3e39228494 100644 --- a/test_runner/batch_others/test_wal_acceptor.py +++ b/test_runner/batch_others/test_wal_acceptor.py @@ -24,9 +24,8 @@ def test_normal_work(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 3 env = zenith_env_builder.init_start() - env.zenith_cli.create_branch("test_wal_acceptors_normal_work", "main") - - pg = env.postgres.create_start('test_wal_acceptors_normal_work') + new_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_wal_acceptors_normal_work', timeline_id=new_timeline_id) with closing(pg.connect()) as conn: with conn.cursor() as cur: @@ -39,9 +38,9 @@ def test_normal_work(zenith_env_builder: ZenithEnvBuilder): @dataclass -class BranchMetrics: - name: str - latest_valid_lsn: int +class TimelineMetrics: + timeline_id: str + last_record_lsn: int # One entry per each Safekeeper, order is the same flush_lsns: List[int] = field(default_factory=list) commit_lsns: List[int] = field(default_factory=list) @@ -55,21 +54,26 @@ def test_many_timelines(zenith_env_builder: ZenithEnvBuilder): n_timelines = 3 - branches = ["test_wal_acceptors_many_timelines_{}".format(tlin) for tlin in range(n_timelines)] + branch_names = [ + "test_wal_acceptors_many_timelines_{}".format(tlin) for tlin in range(n_timelines) + ] + branch_names_to_timeline_ids = {} # start postgres on each timeline pgs = [] - for branch in branches: - env.zenith_cli.create_branch(branch, "main") - pgs.append(env.postgres.create_start(branch)) + for branch_name in branch_names: + new_timeline_id = env.zenith_cli.branch_timeline() + pgs.append(env.postgres.create_start(branch_name, timeline_id=new_timeline_id)) + branch_names_to_timeline_ids[branch_name] = new_timeline_id tenant_id = env.initial_tenant - def collect_metrics(message: str) -> List[BranchMetrics]: + def collect_metrics(message: str) -> List[TimelineMetrics]: with env.pageserver.http_client() as pageserver_http: - branch_details = [ - pageserver_http.branch_detail(tenant_id=tenant_id, name=branch) - for branch in branches + timeline_details = [ + pageserver_http.timeline_detail( + tenant_id=tenant_id, timeline_id=branch_names_to_timeline_ids[branch_name]) + for branch_name in branch_names ] # All changes visible to pageserver (latest_valid_lsn) should be # confirmed by safekeepers first. As we cannot atomically get @@ -80,14 +84,15 @@ def test_many_timelines(zenith_env_builder: ZenithEnvBuilder): # safekeepers' state, it will look contradictory. sk_metrics = [sk.http_client().get_metrics() for sk in env.safekeepers] - branch_metrics = [] + timeline_metrics = [] with env.pageserver.http_client() as pageserver_http: - for branch_detail in branch_details: - timeline_id: str = branch_detail["timeline_id"] + for timeline_detail in timeline_details: + print(f"@@@@@@@@@@@\n{timeline_detail}\n@@@@@@@@@@@") + timeline_id: str = timeline_detail["timeline_id"] - m = BranchMetrics( - name=branch_detail["name"], - latest_valid_lsn=branch_detail["latest_valid_lsn"], + m = TimelineMetrics( + timeline_id=timeline_id, + last_record_lsn=timeline_detail["last_record_lsn"], ) for sk_m in sk_metrics: m.flush_lsns.append(sk_m.flush_lsn_inexact[(tenant_id.hex, timeline_id)]) @@ -99,13 +104,13 @@ def test_many_timelines(zenith_env_builder: ZenithEnvBuilder): # We only call collect_metrics() after a transaction is confirmed by # the compute node, which only happens after a consensus of safekeepers # has confirmed the transaction. We assume majority consensus here. - assert (2 * sum(m.latest_valid_lsn <= lsn + assert (2 * sum(m.last_record_lsn <= lsn for lsn in m.flush_lsns) > zenith_env_builder.num_safekeepers) - assert (2 * sum(m.latest_valid_lsn <= lsn + assert (2 * sum(m.last_record_lsn <= lsn for lsn in m.commit_lsns) > zenith_env_builder.num_safekeepers) - branch_metrics.append(m) - log.info(f"{message}: {branch_metrics}") - return branch_metrics + timeline_metrics.append(m) + log.info(f"{message}: {timeline_metrics}") + return timeline_metrics # TODO: https://github.com/zenithdb/zenith/issues/809 # collect_metrics("before CREATE TABLE") @@ -117,7 +122,7 @@ def test_many_timelines(zenith_env_builder: ZenithEnvBuilder): pg.safe_psql("CREATE TABLE t(key int primary key, value text)") init_m = collect_metrics("after CREATE TABLE") - # Populate data for 2/3 branches + # Populate data for 2/3 timelines class MetricsChecker(threading.Thread): def __init__(self) -> None: super().__init__(daemon=True) @@ -155,15 +160,15 @@ def test_many_timelines(zenith_env_builder: ZenithEnvBuilder): collect_metrics("after INSERT INTO") - # Check data for 2/3 branches + # Check data for 2/3 timelines for pg in pgs[:-1]: res = pg.safe_psql("SELECT sum(key) FROM t") assert res[0] == (5000050000, ) final_m = collect_metrics("after SELECT") - # Assume that LSNs (a) behave similarly in all branches; and (b) INSERT INTO alters LSN significantly. + # Assume that LSNs (a) behave similarly in all timelines; and (b) INSERT INTO alters LSN significantly. # Also assume that safekeepers will not be significantly out of sync in this test. - middle_lsn = (init_m[0].latest_valid_lsn + final_m[0].latest_valid_lsn) // 2 + middle_lsn = (init_m[0].last_record_lsn + final_m[0].last_record_lsn) // 2 assert max(init_m[0].flush_lsns) < middle_lsn < min(final_m[0].flush_lsns) assert max(init_m[0].commit_lsns) < middle_lsn < min(final_m[0].commit_lsns) assert max(init_m[1].flush_lsns) < middle_lsn < min(final_m[1].flush_lsns) @@ -183,8 +188,8 @@ def test_restarts(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = n_acceptors env = zenith_env_builder.init_start() - env.zenith_cli.create_branch("test_wal_acceptors_restarts", "main") - pg = env.postgres.create_start('test_wal_acceptors_restarts') + new_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_wal_acceptors_restarts', timeline_id=new_timeline_id) # we rely upon autocommit after each statement # as waiting for acceptors happens there @@ -220,8 +225,8 @@ def test_unavailability(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 2 env = zenith_env_builder.init_start() - env.zenith_cli.create_branch("test_wal_acceptors_unavailability", "main") - pg = env.postgres.create_start('test_wal_acceptors_unavailability') + new_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_wal_acceptors_unavailability', timeline_id=new_timeline_id) # we rely upon autocommit after each statement # as waiting for acceptors happens there @@ -291,8 +296,9 @@ def test_race_conditions(zenith_env_builder: ZenithEnvBuilder, stop_value): zenith_env_builder.num_safekeepers = 3 env = zenith_env_builder.init_start() - env.zenith_cli.create_branch("test_wal_acceptors_race_conditions", "main") - pg = env.postgres.create_start('test_wal_acceptors_race_conditions') + new_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_wal_acceptors_race_conditions', + timeline_id=new_timeline_id) # we rely upon autocommit after each statement # as waiting for acceptors happens there @@ -456,8 +462,8 @@ def test_timeline_status(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 1 env = zenith_env_builder.init_start() - env.zenith_cli.create_branch("test_timeline_status", "main") - pg = env.postgres.create_start('test_timeline_status') + new_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_timeline_status', timeline_id=new_timeline_id) wa = env.safekeepers[0] wa_http_cli = wa.http_client() @@ -630,12 +636,12 @@ def test_replace_safekeeper(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 4 env = zenith_env_builder.init_start() - env.zenith_cli.create_branch("test_replace_safekeeper", "main") + new_timeline_id = env.zenith_cli.branch_timeline() log.info("Use only first 3 safekeepers") env.safekeepers[3].stop() active_safekeepers = [1, 2, 3] - pg = env.postgres.create('test_replace_safekeeper') + pg = env.postgres.create('test_replace_safekeeper', timeline_id=new_timeline_id) pg.adjust_for_wal_acceptors(safekeepers_guc(env, active_safekeepers)) pg.start() @@ -673,7 +679,7 @@ def test_replace_safekeeper(zenith_env_builder: ZenithEnvBuilder): show_statuses(env.safekeepers, tenant_id, timeline_id) log.info("Recreate postgres to replace failed sk1 with new sk4") - pg.stop_and_destroy().create('test_replace_safekeeper') + pg.stop_and_destroy().create('test_replace_safekeeper', timeline_id=uuid.UUID(timeline_id)) active_safekeepers = [2, 3, 4] env.safekeepers[3].start() pg.adjust_for_wal_acceptors(safekeepers_guc(env, active_safekeepers)) diff --git a/test_runner/batch_others/test_wal_acceptor_async.py b/test_runner/batch_others/test_wal_acceptor_async.py index 4b6a27f73d..719e8c163f 100644 --- a/test_runner/batch_others/test_wal_acceptor_async.py +++ b/test_runner/batch_others/test_wal_acceptor_async.py @@ -202,8 +202,9 @@ def test_restarts_under_load(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 3 env = zenith_env_builder.init_start() - env.zenith_cli.create_branch("test_wal_acceptors_restarts_under_load", "main") - pg = env.postgres.create_start('test_wal_acceptors_restarts_under_load') + new_timeline_id = env.zenith_cli.branch_timeline() + pg = env.postgres.create_start('test_wal_acceptors_restarts_under_load', + timeline_id=new_timeline_id) asyncio.run(run_restarts_under_load(pg, env.safekeepers)) diff --git a/test_runner/batch_others/test_zenith_cli.py b/test_runner/batch_others/test_zenith_cli.py index f1897e4b6f..4f089d4354 100644 --- a/test_runner/batch_others/test_zenith_cli.py +++ b/test_runner/batch_others/test_zenith_cli.py @@ -7,52 +7,47 @@ from fixtures.zenith_fixtures import ZenithEnv, ZenithEnvBuilder, ZenithPageserv from typing import cast -def helper_compare_branch_list(pageserver_http_client: ZenithPageserverHttpClient, - env: ZenithEnv, - initial_tenant: uuid.UUID): +def helper_compare_timeline_list(pageserver_http_client: ZenithPageserverHttpClient, + env: ZenithEnv, + initial_tenant: uuid.UUID): """ - Compare branches list returned by CLI and directly via API. - Filters out branches created by other tests. + Compare timelines list returned by CLI and directly via API. + Filters out timelines created by other tests. """ - branches = pageserver_http_client.branch_list(initial_tenant) - branches_api = sorted(map(lambda b: cast(str, b['name']), branches)) - branches_api = [b for b in branches_api if b.startswith('test_cli_') or b in ('empty', 'main')] - res = env.zenith_cli.list_branches() - branches_cli = sorted(map(lambda b: b.split(':')[-1].strip(), res.stdout.strip().split("\n"))) - branches_cli = [b for b in branches_cli if b.startswith('test_cli_') or b in ('empty', 'main')] - - res = env.zenith_cli.list_branches(tenant_id=initial_tenant) - branches_cli_with_tenant_arg = sorted( - map(lambda b: b.split(':')[-1].strip(), res.stdout.strip().split("\n"))) - branches_cli_with_tenant_arg = [ - b for b in branches_cli if b.startswith('test_cli_') or b in ('empty', 'main') + timelines_cli = env.zenith_cli.list_timelines() + timelines_cli = [ + b for b in timelines_cli if b.startswith('test_cli_') or b in ('empty', 'main') ] - assert branches_api == branches_cli == branches_cli_with_tenant_arg + timelines_cli_with_tenant_arg = env.zenith_cli.list_timelines(initial_tenant) + timelines_cli_with_tenant_arg = [ + b for b in timelines_cli if b.startswith('test_cli_') or b in ('empty', 'main') + ] + + assert timelines_cli == timelines_cli_with_tenant_arg -def test_cli_branch_list(zenith_simple_env: ZenithEnv): +def test_cli_timeline_list(zenith_simple_env: ZenithEnv): env = zenith_simple_env pageserver_http_client = env.pageserver.http_client() # Initial sanity check - helper_compare_branch_list(pageserver_http_client, env, env.initial_tenant) - env.zenith_cli.create_branch("test_cli_branch_list_main", "empty") - helper_compare_branch_list(pageserver_http_client, env, env.initial_tenant) + helper_compare_timeline_list(pageserver_http_client, env, env.initial_tenant) + + # Create a branch for us + main_timeline_id = env.zenith_cli.branch_timeline() + helper_compare_timeline_list(pageserver_http_client, env, env.initial_tenant) # Create a nested branch - res = env.zenith_cli.create_branch("test_cli_branch_list_nested", "test_cli_branch_list_main") - assert res.stderr == '' - helper_compare_branch_list(pageserver_http_client, env, env.initial_tenant) + nested_timeline_id = env.zenith_cli.branch_timeline(ancestor_timeline_id=main_timeline_id) + helper_compare_timeline_list(pageserver_http_client, env, env.initial_tenant) # Check that all new branches are visible via CLI - res = env.zenith_cli.list_branches() - assert res.stderr == '' - branches_cli = sorted(map(lambda b: b.split(':')[-1].strip(), res.stdout.strip().split("\n"))) + timelines_cli = env.zenith_cli.list_timelines() - assert 'test_cli_branch_list_main' in branches_cli - assert 'test_cli_branch_list_nested' in branches_cli + assert main_timeline_id.hex in timelines_cli + assert nested_timeline_id.hex in timelines_cli def helper_compare_tenant_list(pageserver_http_client: ZenithPageserverHttpClient, env: ZenithEnv): @@ -60,7 +55,6 @@ def helper_compare_tenant_list(pageserver_http_client: ZenithPageserverHttpClien tenants_api = sorted(map(lambda t: cast(str, t['id']), tenants)) res = env.zenith_cli.list_tenants() - assert res.stderr == '' tenants_cli = sorted(map(lambda t: t.split()[0], res.stdout.splitlines())) assert tenants_api == tenants_cli @@ -74,14 +68,14 @@ def test_cli_tenant_list(zenith_simple_env: ZenithEnv): # Create new tenant tenant1 = uuid.uuid4() - env.zenith_cli.create_tenant(tenant1) + env.zenith_cli.create_tenant(tenant_id=tenant1) # check tenant1 appeared helper_compare_tenant_list(pageserver_http_client, env) # Create new tenant tenant2 = uuid.uuid4() - env.zenith_cli.create_tenant(tenant2) + env.zenith_cli.create_tenant(tenant_id=tenant2) # check tenant2 appeared helper_compare_tenant_list(pageserver_http_client, env) diff --git a/test_runner/batch_pg_regress/test_isolation.py b/test_runner/batch_pg_regress/test_isolation.py index ddafc3815b..8dce020dc0 100644 --- a/test_runner/batch_pg_regress/test_isolation.py +++ b/test_runner/batch_pg_regress/test_isolation.py @@ -7,10 +7,12 @@ from fixtures.zenith_fixtures import ZenithEnv, base_dir, pg_distrib_dir def test_isolation(zenith_simple_env: ZenithEnv, test_output_dir, pg_bin, capsys): env = zenith_simple_env - env.zenith_cli.create_branch("test_isolation", "empty") + new_timeline_id = env.zenith_cli.branch_timeline() # Connect to postgres and create a database called "regression". # isolation tests use prepared transactions, so enable them - pg = env.postgres.create_start('test_isolation', config_lines=['max_prepared_transactions=100']) + pg = env.postgres.create_start('test_isolation', + config_lines=['max_prepared_transactions=100'], + timeline_id=new_timeline_id) pg.safe_psql('CREATE DATABASE isolation_regression') # Create some local directories for pg_isolation_regress to run in. diff --git a/test_runner/batch_pg_regress/test_pg_regress.py b/test_runner/batch_pg_regress/test_pg_regress.py index 5199f65216..efeb63fce3 100644 --- a/test_runner/batch_pg_regress/test_pg_regress.py +++ b/test_runner/batch_pg_regress/test_pg_regress.py @@ -7,9 +7,9 @@ from fixtures.zenith_fixtures import ZenithEnv, check_restored_datadir_content, def test_pg_regress(zenith_simple_env: ZenithEnv, test_output_dir: str, pg_bin, capsys): env = zenith_simple_env - env.zenith_cli.create_branch("test_pg_regress", "empty") + new_timeline_id = env.zenith_cli.branch_timeline() # Connect to postgres and create a database called "regression". - pg = env.postgres.create_start('test_pg_regress') + pg = env.postgres.create_start('test_pg_regress', timeline_id=new_timeline_id) pg.safe_psql('CREATE DATABASE regression') # Create some local directories for pg_regress to run in. diff --git a/test_runner/batch_pg_regress/test_zenith_regress.py b/test_runner/batch_pg_regress/test_zenith_regress.py index 31d5b07093..2ccbafccfd 100644 --- a/test_runner/batch_pg_regress/test_zenith_regress.py +++ b/test_runner/batch_pg_regress/test_zenith_regress.py @@ -11,9 +11,9 @@ from fixtures.log_helper import log def test_zenith_regress(zenith_simple_env: ZenithEnv, test_output_dir, pg_bin, capsys): env = zenith_simple_env - env.zenith_cli.create_branch("test_zenith_regress", "empty") + new_timeline_id = env.zenith_cli.branch_timeline() # Connect to postgres and create a database called "regression". - pg = env.postgres.create_start('test_zenith_regress') + pg = env.postgres.create_start('test_zenith_regress', timeline_id=new_timeline_id) pg.safe_psql('CREATE DATABASE regression') # Create some local directories for pg_regress to run in. diff --git a/test_runner/fixtures/compare_fixtures.py b/test_runner/fixtures/compare_fixtures.py index 570c787184..66b9fe54ea 100644 --- a/test_runner/fixtures/compare_fixtures.py +++ b/test_runner/fixtures/compare_fixtures.py @@ -64,9 +64,8 @@ class ZenithCompare(PgCompare): self._pg_bin = pg_bin # We only use one branch and one timeline - self.branch = branch_name - self.env.zenith_cli.create_branch(self.branch, "empty") - self._pg = self.env.postgres.create_start(self.branch) + timeline_id = self.env.zenith_cli.branch_timeline() + self._pg = self.env.postgres.create_start("branch", timeline_id=timeline_id) self.timeline = self.pg.safe_psql("SHOW zenith.zenith_timeline")[0][0] # Long-lived cursor, useful for flushing diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index 06f75aa604..7c4d178a3f 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -548,8 +548,7 @@ class ZenithEnv: self.s3_mock_server = config.s3_mock_server self.zenith_cli = ZenithCli(env=self) - self.postgres = PostgresFactory(self) - + self.zenith_cli = ZenithCli(env=self) self.safekeepers: List[Safekeeper] = [] # generate initial tenant ID here instead of letting 'zenith init' generate it, @@ -558,7 +557,7 @@ class ZenithEnv: # Create a config file corresponding to the options toml = textwrap.dedent(f""" - default_tenantid = '{self.initial_tenant.hex}' + default_tenant_id = '{self.initial_tenant.hex}' """) # Create config for pageserver @@ -600,8 +599,9 @@ class ZenithEnv: self.safekeepers.append(safekeeper) log.info(f"Config: {toml}") - - self.zenith_cli.init(toml) + # TODO kb is this a wrong concept? will break for multiple tenant tests + self.initial_timeline = self.zenith_cli.init(toml) + self.postgres = PostgresFactory(self) def start(self): # Start up the page server and all the safekeepers @@ -614,12 +614,6 @@ class ZenithEnv: """ Get list of safekeeper endpoints suitable for wal_acceptors GUC """ return ','.join([f'localhost:{wa.port.pg}' for wa in self.safekeepers]) - def create_tenant(self, tenant_id: Optional[uuid.UUID] = None) -> uuid.UUID: - if tenant_id is None: - tenant_id = uuid.uuid4() - self.zenith_cli.create_tenant(tenant_id) - return tenant_id - @cached_property def auth_keys(self) -> AuthKeys: pub = (Path(self.repo_dir) / 'auth_public_key.pem').read_bytes() @@ -643,14 +637,7 @@ def _shared_simple_env(request: Any, port_distributor) -> Iterator[ZenithEnv]: shutil.rmtree(repo_dir, ignore_errors=True) with ZenithEnvBuilder(Path(repo_dir), port_distributor) as builder: - - env = builder.init_start() - - # For convenience in tests, create a branch from the freshly-initialized cluster. - env.zenith_cli.create_branch("empty", "main") - - # Return the builder to the caller - yield env + yield builder.init_start() @pytest.fixture(scope='function') @@ -729,34 +716,27 @@ class ZenithPageserverHttpClient(requests.Session): f"http://localhost:{self.port}/v1/timeline/{tenant_id.hex}/{timeline_id.hex}/detach", ) self.verbose_error(res) - def branch_list(self, tenant_id: uuid.UUID) -> List[Dict[Any, Any]]: - res = self.get(f"http://localhost:{self.port}/v1/branch/{tenant_id.hex}") - self.verbose_error(res) - res_json = res.json() - assert isinstance(res_json, list) - return res_json - - def branch_create(self, tenant_id: uuid.UUID, name: str, start_point: str) -> Dict[Any, Any]: - res = self.post(f"http://localhost:{self.port}/v1/branch", + def timeline_create(self, + tenant_id: uuid.UUID, + timeline_id: uuid.UUID, + start_lsn: Optional[str] = None, + ancestor_timeline_id: Optional[uuid.UUID] = None) -> Dict[Any, Any]: + res = self.post(f"http://localhost:{self.port}/v1/timeline", json={ - 'tenant_id': tenant_id.hex, - 'name': name, - 'start_point': start_point, + 'tenant_id': + tenant_id.hex, + 'timeline_id': + timeline_id.hex, + 'start_lsn': + start_lsn, + 'ancestor_timeline_id': + ancestor_timeline_id.hex if ancestor_timeline_id else None, }) self.verbose_error(res) res_json = res.json() assert isinstance(res_json, dict) return res_json - def branch_detail(self, tenant_id: uuid.UUID, name: str) -> Dict[Any, Any]: - res = self.get( - f"http://localhost:{self.port}/v1/branch/{tenant_id.hex}/{name}?include-non-incremental-logical-size=1", - ) - self.verbose_error(res) - res_json = res.json() - assert isinstance(res_json, dict) - return res_json - def tenant_list(self) -> List[Dict[Any, Any]]: res = self.get(f"http://localhost:{self.port}/v1/tenant") self.verbose_error(res) @@ -774,7 +754,7 @@ class ZenithPageserverHttpClient(requests.Session): self.verbose_error(res) return res.json() - def timeline_list(self, tenant_id: uuid.UUID) -> List[str]: + def timeline_list(self, tenant_id: uuid.UUID) -> List[Dict[Any, Any]]: res = self.get(f"http://localhost:{self.port}/v1/timeline/{tenant_id.hex}") self.verbose_error(res) res_json = res.json() @@ -783,7 +763,8 @@ class ZenithPageserverHttpClient(requests.Session): def timeline_detail(self, tenant_id: uuid.UUID, timeline_id: uuid.UUID) -> Dict[Any, Any]: res = self.get( - f"http://localhost:{self.port}/v1/timeline/{tenant_id.hex}/{timeline_id.hex}") + f"http://localhost:{self.port}/v1/timeline/{tenant_id.hex}/{timeline_id.hex}?include-non-incremental-logical-size=1" + ) self.verbose_error(res) res_json = res.json() assert isinstance(res_json, dict) @@ -827,34 +808,76 @@ class ZenithCli: self.env = env pass - def create_tenant(self, tenant_id: Optional[uuid.UUID] = None) -> uuid.UUID: + def create_tenant(self, tenant_id: Optional[uuid.UUID] = None) -> tuple[uuid.UUID, uuid.UUID]: + """ + Creates a new tenant, returns its id and its initial timeline's id. + """ if tenant_id is None: tenant_id = uuid.uuid4() - self.raw_cli(['tenant', 'create', tenant_id.hex]) - return tenant_id + res = self.raw_cli(['tenant', 'create', '--tenant-id', tenant_id.hex]) + + initial_timeline_id_extractor = re.compile(r"initial timeline: '(?P[^']+)'", + re.MULTILINE) + matches = initial_timeline_id_extractor.search(res.stdout) + + created_timeline_id = None + if matches is not None: + created_timeline_id = matches.group('timeline_id') + + if created_timeline_id is None: + raise Exception('could not find timeline id after `zenith tenant create` invocation') + else: + return (tenant_id, uuid.UUID(created_timeline_id)) def list_tenants(self) -> 'subprocess.CompletedProcess[str]': - return self.raw_cli(['tenant', 'list']) + res = self.raw_cli(['tenant', 'list']) + res.check_returncode() + return res - def create_branch(self, - branch_name: str, - starting_point: str, - tenant_id: Optional[uuid.UUID] = None) -> 'subprocess.CompletedProcess[str]': - args = ['branch'] - if tenant_id is not None: - args.extend(['--tenantid', tenant_id.hex]) - args.extend([branch_name, starting_point]) + def branch_timeline(self, + tenant_id: Optional[uuid.UUID] = None, + new_timeline_id: Optional[uuid.UUID] = None, + ancestor_timeline_id: Optional[uuid.UUID] = None, + ancestor_start_lsn: Optional[str] = None) -> uuid.UUID: + cmd = [ + 'timeline', + 'branch', + '--tenant-id', + (tenant_id or self.env.initial_tenant).hex, + '--ancestor-timeline-id', + (ancestor_timeline_id or self.env.initial_timeline).hex, + ] + if ancestor_start_lsn is not None: + cmd.extend(['--ancestor-start-lsn', ancestor_start_lsn]) + if new_timeline_id is not None: + cmd.extend(['--timeline-id', new_timeline_id.hex]) - return self.raw_cli(args) + completed_process = self.raw_cli(cmd) + completed_process.check_returncode() + create_timeline_id_extractor = re.compile(r"^Created timeline '(?P[^']+)'", + re.MULTILINE) + matches = create_timeline_id_extractor.search(completed_process.stdout) - def list_branches(self, - tenant_id: Optional[uuid.UUID] = None) -> 'subprocess.CompletedProcess[str]': - args = ['branch'] - if tenant_id is not None: - args.extend(['--tenantid', tenant_id.hex]) - return self.raw_cli(args) + created_timeline_id = None + if matches is not None: + created_timeline_id = matches.group('timeline_id') + + if created_timeline_id is None: + raise Exception('could not find timeline id after `zenith timeline create` invocation') + else: + return uuid.UUID(created_timeline_id) + + def list_timelines(self, tenant_id: Optional[uuid.UUID] = None) -> List[str]: + res = self.raw_cli( + ['timeline', 'list', '--tenant-id', (tenant_id or self.env.initial_tenant).hex]) + branches_cli = sorted( + map(lambda b: b.split(') ')[-1].strip().split(':')[-1].strip(), + res.stdout.strip().split("\n"))) + return branches_cli + + def init(self, config_toml: str) -> uuid.UUID: + initial_timeline = None - def init(self, config_toml: str) -> 'subprocess.CompletedProcess[str]': with tempfile.NamedTemporaryFile(mode='w+') as tmp: tmp.write(config_toml) tmp.flush() @@ -864,7 +887,18 @@ class ZenithCli: self.env.pageserver.remote_storage, self.env.pageserver.config_override) - return self.raw_cli(cmd) + completed_process = self.raw_cli(cmd) + completed_process.check_returncode() + init_timeline_id_extractor = re.compile( + r'^created initial timeline (?P[^\s]+)\s', re.MULTILINE) + matches = init_timeline_id_extractor.search(completed_process.stdout) + if matches is not None: + initial_timeline = matches.group('timeline_id') + + if initial_timeline is None: + raise Exception('could not find timeline id after `zenith init` invocation') + else: + return uuid.UUID(initial_timeline) def pageserver_start(self, overrides=()) -> 'subprocess.CompletedProcess[str]': start_args = ['pageserver', 'start', *overrides] @@ -898,36 +932,50 @@ class ZenithCli: self, node_name: str, tenant_id: Optional[uuid.UUID] = None, - timeline_spec: Optional[str] = None, + timeline_id: Optional[uuid.UUID] = None, + lsn: Optional[str] = None, port: Optional[int] = None, ) -> 'subprocess.CompletedProcess[str]': - args = ['pg', 'create'] - if tenant_id is not None: - args.extend(['--tenantid', tenant_id.hex]) + args = [ + 'pg', + 'create', + '--tenant-id', (tenant_id or self.env.initial_tenant).hex, + '--timeline-id', (timeline_id or self.env.initial_timeline).hex + ] + if lsn is not None: + args.append(f'--lsn={lsn}') if port is not None: args.append(f'--port={port}') args.append(node_name) - if timeline_spec is not None: - args.append(timeline_spec) - return self.raw_cli(args) + res = self.raw_cli(args) + res.check_returncode() + return res def pg_start( self, node_name: str, tenant_id: Optional[uuid.UUID] = None, - timeline_spec: Optional[str] = None, + timeline_id: Optional[uuid.UUID] = None, + lsn: Optional[str] = None, port: Optional[int] = None, ) -> 'subprocess.CompletedProcess[str]': - args = ['pg', 'start'] - if tenant_id is not None: - args.extend(['--tenantid', tenant_id.hex]) + args = [ + 'pg', + 'start', + '--tenant-id', + (tenant_id or self.env.initial_tenant).hex, + '--timeline-id', + (timeline_id or self.env.initial_timeline).hex, + ] + if lsn is not None: + args.append(f'--lsn={lsn}') if port is not None: args.append(f'--port={port}') args.append(node_name) - if timeline_spec is not None: - args.append(timeline_spec) - return self.raw_cli(args) + res = self.raw_cli(args) + res.check_returncode() + return res def pg_stop( self, @@ -935,9 +983,7 @@ class ZenithCli: tenant_id: Optional[uuid.UUID] = None, destroy=False, ) -> 'subprocess.CompletedProcess[str]': - args = ['pg', 'stop'] - if tenant_id is not None: - args.extend(['--tenantid', tenant_id.hex]) + args = ['pg', 'stop', f'--tenant-id={(tenant_id or self.env.initial_tenant).hex}'] if destroy: args.append('--destroy') args.append(node_name) @@ -1044,7 +1090,6 @@ class ZenithPageserver(PgProtocol): if self.running: self.env.zenith_cli.pageserver_stop(immediate) self.running = False - return self def __enter__(self): @@ -1261,7 +1306,8 @@ class Postgres(PgProtocol): def create( self, node_name: str, - branch: Optional[str] = None, + timeline_id: uuid.UUID, + lsn: Optional[str] = None, config_lines: Optional[List[str]] = None, ) -> 'Postgres': """ @@ -1272,13 +1318,11 @@ class Postgres(PgProtocol): if not config_lines: config_lines = [] - if branch is None: - branch = node_name - self.env.zenith_cli.pg_create(node_name, + timeline_id=timeline_id, tenant_id=self.tenant_id, - port=self.port, - timeline_spec=branch) + lsn=lsn, + port=self.port) self.node_name = node_name path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id.hex / self.node_name self.pgdata_dir = os.path.join(self.env.repo_dir, path) @@ -1375,7 +1419,7 @@ class Postgres(PgProtocol): if self.running: assert self.node_name is not None - self.env.zenith_cli.pg_stop(self.node_name, tenant_id=self.tenant_id) + self.env.zenith_cli.pg_stop(self.node_name, self.tenant_id) self.running = False return self @@ -1387,7 +1431,7 @@ class Postgres(PgProtocol): """ assert self.node_name is not None - self.env.zenith_cli.pg_stop(self.node_name, self.tenant_id, destroy=True) + self.env.zenith_cli.pg_stop(self.node_name, self.tenant_id, True) self.node_name = None return self @@ -1395,7 +1439,8 @@ class Postgres(PgProtocol): def create_start( self, node_name: str, - branch: Optional[str] = None, + timeline_id: uuid.UUID, + lsn: Optional[str] = None, config_lines: Optional[List[str]] = None, ) -> 'Postgres': """ @@ -1406,8 +1451,9 @@ class Postgres(PgProtocol): self.create( node_name=node_name, - branch=branch, + timeline_id=timeline_id, config_lines=config_lines, + lsn=lsn, ).start() return self @@ -1428,8 +1474,9 @@ class PostgresFactory: def create_start(self, node_name: str = "main", - branch: Optional[str] = None, tenant_id: Optional[uuid.UUID] = None, + timeline_id: Optional[uuid.UUID] = None, + lsn: Optional[str] = None, config_lines: Optional[List[str]] = None) -> Postgres: pg = Postgres( @@ -1442,14 +1489,16 @@ class PostgresFactory: return pg.create_start( node_name=node_name, - branch=branch, + timeline_id=timeline_id or self.env.initial_timeline, config_lines=config_lines, + lsn=lsn, ) def create(self, node_name: str = "main", - branch: Optional[str] = None, tenant_id: Optional[uuid.UUID] = None, + timeline_id: Optional[uuid.UUID] = None, + lsn: Optional[str] = None, config_lines: Optional[List[str]] = None) -> Postgres: pg = Postgres( @@ -1463,7 +1512,8 @@ class PostgresFactory: return pg.create( node_name=node_name, - branch=branch, + timeline_id=timeline_id or self.env.initial_timeline, + lsn=lsn, config_lines=config_lines, ) @@ -1683,8 +1733,7 @@ def list_files_to_compare(pgdata_dir: str): # pg is the existing and running compute node, that we want to compare with a basebackup def check_restored_datadir_content(test_output_dir: str, env: ZenithEnv, pg: Postgres): - - # Get the timeline ID of our branch. We need it for the 'basebackup' command + # Get the timeline ID. We need it for the 'basebackup' command with closing(pg.connect()) as conn: with conn.cursor() as cur: cur.execute("SHOW zenith.zenith_timeline") diff --git a/test_runner/performance/test_bulk_tenant_create.py b/test_runner/performance/test_bulk_tenant_create.py index 0247385211..dda31ba692 100644 --- a/test_runner/performance/test_bulk_tenant_create.py +++ b/test_runner/performance/test_bulk_tenant_create.py @@ -30,11 +30,9 @@ def test_bulk_tenant_create( for i in range(tenants_count): start = timeit.default_timer() - tenant = env.create_tenant() - env.zenith_cli.create_branch( - f"test_bulk_tenant_create_{tenants_count}_{i}_{use_wal_acceptors}", - "main", - tenant_id=tenant) + (tenant, tenant_initial_timeline_id) = env.zenith_cli.create_tenant() + new_timeline_id = env.zenith_cli.branch_timeline( + tenant_id=tenant, ancestor_timeline_id=tenant_initial_timeline_id) # FIXME: We used to start new safekeepers here. Did that make sense? Should we do it now? #if use_wal_acceptors == 'with_wa': @@ -42,9 +40,8 @@ def test_bulk_tenant_create( pg_tenant = env.postgres.create_start( f"test_bulk_tenant_create_{tenants_count}_{i}_{use_wal_acceptors}", - None, # branch name, None means same as node name tenant, - ) + timeline_id=new_timeline_id) end = timeit.default_timer() time_slices.append(end - start) diff --git a/test_runner/performance/test_parallel_copy_to.py b/test_runner/performance/test_parallel_copy_to.py index e4388ce8e2..0ee0a37ebb 100644 --- a/test_runner/performance/test_parallel_copy_to.py +++ b/test_runner/performance/test_parallel_copy_to.py @@ -1,6 +1,5 @@ from io import BytesIO import asyncio -import asyncpg from fixtures.zenith_fixtures import ZenithEnv, Postgres, PgProtocol from fixtures.log_helper import log from fixtures.benchmark_fixture import MetricReport, ZenithBenchmarker diff --git a/test_runner/test_broken.py b/test_runner/test_broken.py index 56c735e87c..994544666b 100644 --- a/test_runner/test_broken.py +++ b/test_runner/test_broken.py @@ -21,8 +21,8 @@ run_broken = pytest.mark.skipif(os.environ.get('RUN_BROKEN') is None, def test_broken(zenith_simple_env: ZenithEnv, pg_bin): env = zenith_simple_env - env.zenith_cli.create_branch("test_broken", "empty") - env.postgres.create_start("test_broken") + new_timeline_id = env.zenith_cli.branch_timeline() + env.postgres.create_start("test_broken", timeline_id=new_timeline_id) log.info('postgres is running') log.info('THIS NEXT COMMAND WILL FAIL:') diff --git a/zenith/src/main.rs b/zenith/src/main.rs index 7170653754..dcfeb63309 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -9,7 +9,7 @@ use pageserver::config::defaults::{ DEFAULT_HTTP_LISTEN_ADDR as DEFAULT_PAGESERVER_HTTP_ADDR, DEFAULT_PG_LISTEN_ADDR as DEFAULT_PAGESERVER_PG_ADDR, }; -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::process::exit; use std::str::FromStr; use walkeeper::defaults::{ @@ -60,7 +60,7 @@ struct TimelineTreeEl { /// `TimelineInfo` received from the `pageserver` via the `timeline_list` libpq API call. pub info: TimelineInfo, /// Holds all direct children of this timeline referenced using `timeline_id`. - pub children: Vec, + pub children: BTreeSet, } // Main entry point for the 'zenith' CLI utility @@ -71,25 +71,18 @@ struct TimelineTreeEl { // * Providing CLI api to the pageserver // * TODO: export/import to/from usual postgres fn main() -> Result<()> { - #[rustfmt::skip] // rustfmt squashes these into a single line otherwise - let pg_node_arg = Arg::new("node") - .index(1) - .help("Node name") - .required(true); + let pg_node_arg = Arg::new("node").help("Node name").required(true); - #[rustfmt::skip] - let safekeeper_id_arg = Arg::new("id") - .index(1) - .help("safekeeper id") + let safekeeper_id_arg = Arg::new("id").help("safekeeper id").required(false); + + let timeline_id_arg = Arg::new("timeline-id") + .long("timeline-id") + .help("Timeline id. Represented as a hexadecimal string 32 symbols length") + .takes_value(true) .required(false); - let timeline_arg = Arg::new("timeline") - .index(2) - .help("Timeline id or a point-in time specification") - .required(false); - - let tenantid_arg = Arg::new("tenantid") - .long("tenantid") + let tenant_id_arg = Arg::new("tenant-id") + .long("tenant-id") .help("Tenant id. Represented as a hexadecimal string 32 symbols length") .takes_value(true) .required(false); @@ -115,6 +108,12 @@ fn main() -> Result<()> { .help("Additional pageserver's configuration options or overrides, refer to pageserver's 'config-override' CLI parameter docs for more") .required(false); + let lsn_arg = Arg::new("lsn") + .long("lsn") + .help("Specify Lsn on the timeline to start from. By default, end of the timeline would be used.") + .takes_value(true) + .required(false); + let matches = App::new("Zenith CLI") .setting(AppSettings::ArgRequiredElseHelp) .version(GIT_VERSION) @@ -131,16 +130,28 @@ fn main() -> Result<()> { ) .subcommand( App::new("timeline") - .about("Create a new timeline") - .arg(Arg::new("timeline-name").required(false).index(1)) - .arg(Arg::new("start-point").required(false).index(2)) - .arg(tenantid_arg.clone()), + .about("Manage timelines") + .subcommand(App::new("list") + .about("List all timelines, available to this pageserver") + .arg(tenant_id_arg.clone())) + .subcommand(App::new("branch") + .about("Create a new timeline, using another timeline as a base, copying its data") + .arg(tenant_id_arg.clone()) + .arg(timeline_id_arg.clone().help("Id of the new timeline, optional. If not specified, it will be generated randomly")) + .arg(Arg::new("ancestor-timeline-id").long("ancestor-timeline-id").takes_value(true) + .help("Use last Lsn of another timeline (and its data) as base when creating the new timeline").required(false)) + .arg(Arg::new("ancestor-start-lsn").long("ancestor-start-lsn").takes_value(true) + .help("When using another timeline as base, use a specific Lsn in it instead of the latest one").required(false))) + .subcommand(App::new("create") + .about("Create a new blank timeline") + .arg(tenant_id_arg.clone()) + .arg(timeline_id_arg.clone().help("Id of the new timeline, optional. If not specified, it will be generated randomly"))) ).subcommand( App::new("tenant") .setting(AppSettings::ArgRequiredElseHelp) .about("Manage tenants") .subcommand(App::new("list")) - .subcommand(App::new("create").arg(Arg::new("tenantid").required(false).index(1))) + .subcommand(App::new("create").arg(tenant_id_arg.clone())) ) .subcommand( App::new("pageserver") @@ -175,12 +186,13 @@ fn main() -> Result<()> { App::new("pg") .setting(AppSettings::ArgRequiredElseHelp) .about("Manage postgres instances") - .subcommand(App::new("list").arg(tenantid_arg.clone())) + .subcommand(App::new("list").arg(tenant_id_arg.clone())) .subcommand(App::new("create") .about("Create a postgres compute node") .arg(pg_node_arg.clone()) - .arg(timeline_arg.clone()) - .arg(tenantid_arg.clone()) + .arg(timeline_id_arg.clone()) + .arg(tenant_id_arg.clone()) + .arg(lsn_arg.clone()) .arg(port_arg.clone()) .arg( Arg::new("config-only") @@ -191,14 +203,14 @@ fn main() -> Result<()> { .subcommand(App::new("start") .about("Start a postgres compute node.\n This command actually creates new node from scratch, but preserves existing config files") .arg(pg_node_arg.clone()) - .arg(timeline_arg.clone()) - .arg(tenantid_arg.clone()) + .arg(timeline_id_arg.clone()) + .arg(tenant_id_arg.clone()) + .arg(lsn_arg.clone()) .arg(port_arg.clone())) .subcommand( App::new("stop") .arg(pg_node_arg.clone()) - .arg(timeline_arg.clone()) - .arg(tenantid_arg.clone()) + .arg(tenant_id_arg.clone()) .arg( Arg::new("destroy") .help("Also delete data directory (now optional, should be default in future)") @@ -230,7 +242,7 @@ fn main() -> Result<()> { handle_init(sub_args) } else { // all other commands need an existing config - let env = match LocalEnv::load_config() { + let mut env = match LocalEnv::load_config() { Ok(conf) => conf, Err(e) => { eprintln!("Error loading config: {}", e); @@ -239,7 +251,7 @@ fn main() -> Result<()> { }; match sub_name { - "tenant" => handle_tenant(sub_args, &env), + "tenant" => handle_tenant(sub_args, &mut env), "timeline" => handle_timeline(sub_args, &env), "start" => handle_start_all(sub_args, &env), "stop" => handle_stop_all(sub_args, &env), @@ -261,39 +273,44 @@ fn main() -> Result<()> { /// Prints timelines list as a tree-like structure. /// fn print_timelines_tree(timelines: Vec) -> Result<()> { - let mut timelines_hash: HashMap = timelines + let mut timelines_hash = timelines .iter() .map(|t| { ( - t.timeline_id.to_string(), + t.timeline_id(), TimelineTreeEl { info: t.clone(), - children: Vec::new(), + children: BTreeSet::new(), }, ) }) - .collect(); + .collect::>(); // Memorize all direct children of each timeline. for timeline in &timelines { - if let Some(tid) = &timeline.ancestor_id { + if let TimelineInfo::Local { + ancestor_timeline_id: Some(tid), + .. + } = timeline + { timelines_hash .get_mut(tid) .context("missing timeline info in the HashMap")? .children - .push(timeline.timeline_id.to_string()); + .insert(timeline.timeline_id()); } } - // Sort children by tid to bring some minimal order. - for timeline in &mut timelines_hash.values_mut() { - timeline.children.sort(); - } - for timeline in timelines_hash.values() { - // Start with root timelines (no ancestors) first. - if timeline.info.ancestor_id.is_none() { - print_timeline(0, &Vec::from([true]), timeline, &timelines_hash)?; + // Start with root local timelines (no ancestors) first. + if let TimelineInfo::Local { + ancestor_timeline_id, + .. + } = &timeline.info + { + if ancestor_timeline_id.is_none() { + print_timeline(0, &Vec::from([true]), timeline, &timelines_hash)?; + } } } @@ -307,17 +324,22 @@ fn print_timeline( nesting_level: usize, is_last: &[bool], timeline: &TimelineTreeEl, - timelines: &HashMap, + timelines: &HashMap, ) -> Result<()> { + let local_or_remote = match timeline.info { + TimelineInfo::Local { .. } => "(L)", + TimelineInfo::Remote { .. } => "(R)", + }; // Draw main padding - print!(" "); + print!("{} ", local_or_remote); if nesting_level > 0 { - let lsn = timeline - .info - .ancestor_lsn - .as_ref() - .context("missing timeline info in the HashMap")?; + let lsn_string = match timeline.info { + TimelineInfo::Local { ancestor_lsn, .. } => ancestor_lsn + .map(|lsn| lsn.to_string()) + .unwrap_or_else(|| "Unknown local Lsn".to_string()), + TimelineInfo::Remote { .. } => "unknown Lsn (remote)".to_string(), + }; let mut br_sym = "┣━"; // Draw each nesting padding with proper style @@ -337,11 +359,11 @@ fn print_timeline( br_sym = "┗━"; } - print!("{} @{}: ", br_sym, lsn); + print!("{} @{}: ", br_sym, lsn_string); } // Finally print a timeline name with new line - println!("{}", timeline.info.timeline_id); + println!("{}", timeline.info.timeline_id()); let len = timeline.children.len(); let mut i: usize = 0; @@ -375,26 +397,44 @@ fn print_timeline( /// Connects to the pageserver to query this information. fn get_timeline_infos( env: &local_env::LocalEnv, - tenantid: &ZTenantId, + tenant_id: &ZTenantId, ) -> Result> { let page_server = PageServerNode::from_env(env); - let timeline_infos: Vec = page_server.timeline_list(tenantid)?; + let timeline_infos: Vec = page_server.timeline_list(tenant_id)?; let timeline_infos: HashMap = timeline_infos .into_iter() - .map(|timeline_info| (timeline_info.timeline_id, timeline_info)) + .map(|timeline_info| (timeline_info.timeline_id(), timeline_info)) .collect(); Ok(timeline_infos) } -// Helper function to parse --tenantid option, or get the default from config file -fn get_tenantid(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result { - if let Some(tenantid_cmd) = sub_match.value_of("tenantid") { - Ok(ZTenantId::from_str(tenantid_cmd)?) - } else if let Some(tenantid_conf) = env.default_tenantid { +// Helper function to parse --tenant_id option, or get the default from config file +fn get_tenant_id(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow::Result { + if let Some(tenantid_cmd) = sub_match.value_of("tenant-id") { + Ok( + ZTenantId::from_str(tenantid_cmd) + .context("Failed to parse tenant id from arguments")?, + ) + } else if let Some(tenantid_conf) = env.default_tenant_id { Ok(ZTenantId::from(tenantid_conf)) } else { - bail!("No tenantid. Use --tenantid, or set 'default_tenantid' in the config file"); + bail!("No tenant id. Use --tenant-id, or set 'default_tenant_id' in the config file"); + } +} + +fn get_timeline_id( + sub_match: &ArgMatches, + tenant_id: ZTenantId, + env: &local_env::LocalEnv, +) -> anyhow::Result { + if let Some(timeline_id) = sub_match.value_of("timeline-id") { + Ok(ZTimelineId::from_str(timeline_id) + .context("Failed to parse timeline id from arguments")?) + } else if let Some(&initial_timeline_id) = env.initial_timelines.get(&tenant_id) { + Ok(initial_timeline_id) + } else { + bail!("No timeline id, specify one in the subcommand's arguments"); } } @@ -418,7 +458,7 @@ fn handle_init(init_match: &ArgMatches) -> Result<()> { let pageserver = PageServerNode::from_env(&env); if let Err(e) = pageserver.init( // default_tenantid was generated by the `env.init()` call above - Some(&ZTenantId::from(env.default_tenantid.unwrap()).to_string()), + Some(&ZTenantId::from(env.default_tenant_id.unwrap()).to_string()), &pageserver_config_overrides(init_match), ) { eprintln!("pageserver init failed: {}", e); @@ -436,7 +476,7 @@ fn pageserver_config_overrides(init_match: &ArgMatches) -> Vec<&str> { .collect() } -fn handle_tenant(tenant_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { +fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> Result<()> { let pageserver = PageServerNode::from_env(env); match tenant_match.subcommand() { Some(("list", _)) => { @@ -445,13 +485,17 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &local_env::LocalEnv) -> Result } } Some(("create", create_match)) => { - let tenantid = match create_match.value_of("tenantid") { - Some(tenantid) => ZTenantId::from_str(tenantid)?, + let tenant_id = match create_match.value_of("tenant-id") { + Some(id) => ZTenantId::from_str(id)?, None => ZTenantId::generate(), }; - println!("using tenant id {}", tenantid); - pageserver.tenant_create(tenantid)?; - println!("tenant successfully created on the pageserver"); + println!("using tenant id {}", tenant_id); + let initial_timeline_id = pageserver.tenant_create(tenant_id)?; + env.initial_timelines.insert(tenant_id, initial_timeline_id); + println!( + "tenant {} successfully created on the pageserver, initial timeline: '{}'", + tenant_id, initial_timeline_id + ); } Some((sub_name, _)) => bail!("Unexpected tenant subcommand '{}'", sub_name), None => bail!("no tenant subcommand provided"), @@ -462,26 +506,77 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &local_env::LocalEnv) -> Result fn handle_timeline(timeline_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { let pageserver = PageServerNode::from_env(env); - let tenant_id = get_tenantid(timeline_match, env)?; + match timeline_match.subcommand() { + Some(("list", list_match)) => { + let tenant_id = get_tenant_id(list_match, env)?; + let timelines = pageserver.timeline_list(&tenant_id)?; + print_timelines_tree(timelines)?; + } + Some(("create", create_match)) => { + let tenant_id = get_tenant_id(create_match, env)?; + let timeline_id = get_timeline_id(create_match, tenant_id, env) + .unwrap_or_else(|_| ZTimelineId::generate()); + let timeline = pageserver.timeline_create(tenant_id, timeline_id, None, None)?; - if let Some(timeline_id) = timeline_match.value_of("timeline-id") { - let start_lsn = timeline_match - .value_of("start-lsn") - .map(|lsn| lsn.parse::()) - .transpose() - .context("Failed to parse start Lsn from the request")?; - let timeline_id = timeline_id - .parse::() - .context("Failed to parse timeline id from the request")?; - let timeline = pageserver.timeline_create(tenant_id, timeline_id, start_lsn)?; - println!( - "Created timeline '{}' at {:?} for tenant: {}", - timeline.timeline_id, timeline.latest_valid_lsn, tenant_id, - ); - } else { - // No arguments, list timelines for tenant - let timelines = pageserver.timeline_list(&tenant_id)?; - print_timelines_tree(timelines)?; + let last_record_lsn = match timeline { + TimelineInfo::Local { + last_record_lsn, .. + } => last_record_lsn, + TimelineInfo::Remote { .. } => { + bail!("Timeline {} was created as remote, not local", timeline_id) + } + }; + println!( + "Created timeline '{}' at Lsn {} for tenant: {}", + timeline.timeline_id(), + last_record_lsn, + tenant_id, + ); + } + Some(("branch", branch_match)) => { + let tenant_id = get_tenant_id(branch_match, env)?; + let timeline_id = get_timeline_id(branch_match, tenant_id, env) + .unwrap_or_else(|_| ZTimelineId::generate()); + let ancestor_timeline_id = match branch_match + .value_of("ancestor-timeline-id") + .map(ZTimelineId::from_str) + .transpose() + .context("Failed to parse ancestor timeline id from the request")? + .or_else(|| env.initial_timelines.get(&tenant_id).copied()) + { + Some(id) => id, + None => bail!("No ancestor timeline id provided"), + }; + let start_lsn = branch_match + .value_of("ancestor-start-lsn") + .map(Lsn::from_str) + .transpose() + .context("Failed to parse ancestor start Lsn from the request")?; + let timeline = pageserver.timeline_create( + tenant_id, + timeline_id, + start_lsn, + Some(ancestor_timeline_id), + )?; + + let last_record_lsn = match timeline { + TimelineInfo::Local { + last_record_lsn, .. + } => last_record_lsn, + TimelineInfo::Remote { .. } => { + bail!("Timeline {} was created as remote, not local", timeline_id) + } + }; + println!( + "Created timeline '{}' at Lsn {} for tenant: {}. Ancestor timeline: '{}'", + timeline.timeline_id(), + last_record_lsn, + tenant_id, + ancestor_timeline_id, + ); + } + Some((sub_name, _)) => bail!("Unexpected tenant subcommand '{}'", sub_name), + None => bail!("no tenant subcommand provided"), } Ok(()) @@ -495,12 +590,12 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { let mut cplane = ComputeControlPlane::load(env.clone())?; - // All subcommands take an optional --tenantid option - let tenantid = get_tenantid(sub_args, env)?; + // All subcommands take an optional --tenant-id option + let tenant_id = get_tenant_id(sub_args, env)?; match sub_name { "list" => { - let timeline_infos = get_timeline_infos(env, &tenantid).unwrap_or_else(|e| { + let timeline_infos = get_timeline_infos(env, &tenant_id).unwrap_or_else(|e| { eprintln!("Failed to load timeline info: {}", e); HashMap::new() }); @@ -509,21 +604,26 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { for ((_, node_name), node) in cplane .nodes .iter() - .filter(|((node_tenantid, _), _)| node_tenantid == &tenantid) + .filter(|((node_tenant_id, _), _)| node_tenant_id == &tenant_id) { // FIXME: This shows the LSN at the end of the timeline. It's not the // right thing to do for read-only nodes that might be anchored at an // older point in time, or following but lagging behind the primary. let lsn_str = timeline_infos - .get(&node.timelineid) - .map(|bi| bi.latest_valid_lsn.to_string()) - .unwrap_or_else(|| "?".to_string()); + .get(&node.timeline_id) + .map(|bi| match bi { + TimelineInfo::Local { + last_record_lsn, .. + } => last_record_lsn.to_string(), + TimelineInfo::Remote { .. } => "? (remote)".to_string(), + }) + .unwrap_or_else(|| '?'.to_string()); println!( "{}\t{}\t{}\t{}\t{}", node_name, node.address, - node.timelineid, + node.timeline_id, lsn_str, node.status(), ); @@ -531,27 +631,31 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { } "create" => { let node_name = sub_args.value_of("node").unwrap_or("main"); - let timeline_spec = sub_args.value_of("timeline"); + let lsn = sub_args + .value_of("lsn") + .map(Lsn::from_str) + .transpose() + .context("Failed to parse Lsn from the request")?; + let timeline_id = get_timeline_id(sub_args, tenant_id, env)?; let port: Option = match sub_args.value_of("port") { Some(p) => Some(p.parse()?), None => None, }; - cplane.new_node(tenantid, node_name, timeline_spec, port)?; + cplane.new_node(tenant_id, node_name, timeline_id, lsn, port)?; } "start" => { let node_name = sub_args.value_of("node").unwrap_or("main"); - let timeline_spec = sub_args.value_of("timeline"); let port: Option = match sub_args.value_of("port") { Some(p) => Some(p.parse()?), None => None, }; - let node = cplane.nodes.get(&(tenantid, node_name.to_owned())); + let node = cplane.nodes.get(&(tenant_id, node_name.to_owned())); let auth_token = if matches!(env.pageserver.auth_type, AuthType::ZenithJWT) { - let claims = Claims::new(Some(tenantid), Scope::Tenant); + let claims = Claims::new(Some(tenant_id), Scope::Tenant); Some(env.generate_auth_token(&claims)?) } else { @@ -559,22 +663,25 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { }; if let Some(node) = node { - if timeline_spec.is_some() { - println!("timeline spec ignored because its node exists already"); - } println!("Starting existing postgres {}...", node_name); node.start(&auth_token)?; } else { + let timeline_id = get_timeline_id(sub_args, tenant_id, env)?; + let lsn = sub_args + .value_of("lsn") + .map(Lsn::from_str) + .transpose() + .context("Failed to parse Lsn from the request")?; // when used with custom port this results in non obvious behaviour // port is remembered from first start command, i e // start --port X // stop // start <-- will also use port X even without explicit port argument println!( - "Starting new postgres {} on timeline {:?} ...", - node_name, timeline_spec + "Starting new postgres {} on timeline {} ...", + node_name, timeline_id ); - let node = cplane.new_node(tenantid, node_name, timeline_spec, port)?; + let node = cplane.new_node(tenant_id, node_name, timeline_id, lsn, port)?; node.start(&auth_token)?; } } @@ -584,7 +691,7 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { let node = cplane .nodes - .get(&(tenantid, node_name.to_owned())) + .get(&(tenant_id, node_name.to_owned())) .with_context(|| format!("postgres {} is not found", node_name))?; node.stop(destroy)?; } From 4d0f7fd1e4306f387b1606377ba8f574ed40cf4f Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Thu, 24 Feb 2022 13:40:32 +0300 Subject: [PATCH 17/42] Update Zenith CLI config between runs --- .github/workflows/benchmarking.yml | 2 +- control_plane/src/compute.rs | 4 +- control_plane/src/local_env.rs | 97 ++++----- pageserver/src/timelines.rs | 2 - test_runner/batch_others/test_auth.py | 15 +- test_runner/batch_others/test_backpressure.py | 2 +- .../batch_others/test_branch_behind.py | 42 ++-- .../batch_others/test_clog_truncate.py | 15 +- test_runner/batch_others/test_config.py | 8 +- test_runner/batch_others/test_createdropdb.py | 29 ++- test_runner/batch_others/test_createuser.py | 9 +- .../batch_others/test_gc_aggressive.py | 5 +- test_runner/batch_others/test_multixact.py | 10 +- .../batch_others/test_old_request_lsn.py | 4 +- .../batch_others/test_pageserver_catchup.py | 8 +- .../batch_others/test_pageserver_restart.py | 4 +- .../batch_others/test_parallel_copy.py | 4 +- test_runner/batch_others/test_pgbench.py | 4 +- .../batch_others/test_readonly_node.py | 21 +- .../batch_others/test_restart_compute.py | 10 +- test_runner/batch_others/test_snapfiles_gc.py | 4 +- test_runner/batch_others/test_subxacts.py | 4 +- .../batch_others/test_tenant_relocation.py | 8 +- test_runner/batch_others/test_tenants.py | 18 +- .../batch_others/test_timeline_size.py | 10 +- test_runner/batch_others/test_twophase.py | 10 +- test_runner/batch_others/test_vm_bits.py | 10 +- test_runner/batch_others/test_wal_acceptor.py | 32 ++- .../batch_others/test_wal_acceptor_async.py | 5 +- test_runner/batch_others/test_zenith_cli.py | 11 +- .../batch_pg_regress/test_isolation.py | 6 +- .../batch_pg_regress/test_pg_regress.py | 4 +- .../batch_pg_regress/test_zenith_regress.py | 4 +- test_runner/fixtures/compare_fixtures.py | 4 +- test_runner/fixtures/zenith_fixtures.py | 186 ++++++++---------- .../performance/test_bulk_tenant_create.py | 10 +- .../performance/test_parallel_copy_to.py | 1 + test_runner/test_broken.py | 4 +- zenith/src/main.rs | 158 ++++++++------- zenith_utils/src/zid.rs | 2 +- 40 files changed, 371 insertions(+), 415 deletions(-) diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index dd23440afb..36df35297d 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -48,7 +48,7 @@ jobs: echo Python python3 --version poetry run python3 --version - echo Pipenv + echo Poetry poetry --version echo Pgbench $PG_BIN/pgbench --version diff --git a/control_plane/src/compute.rs b/control_plane/src/compute.rs index 5d225a67fa..e8baffdc74 100644 --- a/control_plane/src/compute.rs +++ b/control_plane/src/compute.rs @@ -147,7 +147,7 @@ impl PostgresNode { // Read a few options from the config file let context = format!("in config file {}", cfg_path_str); let port: u16 = conf.parse_field("port", &context)?; - let timelineid: ZTimelineId = conf.parse_field("zenith.zenith_timeline", &context)?; + let timeline_id: ZTimelineId = conf.parse_field("zenith.zenith_timeline", &context)?; let tenant_id: ZTenantId = conf.parse_field("zenith.zenith_tenant", &context)?; let uses_wal_proposer = conf.get("wal_acceptors").is_some(); @@ -162,7 +162,7 @@ impl PostgresNode { env: env.clone(), pageserver: Arc::clone(pageserver), is_test: false, - timeline_id: timelineid, + timeline_id, lsn: recovery_target_lsn, tenant_id, uses_wal_proposer, diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 98b6379106..9b50a6b9e4 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -3,17 +3,16 @@ //! Now it also provides init method which acts like a stub for proper installation //! script which will use local paths. -use anyhow::{bail, Context}; +use anyhow::{bail, ensure, Context}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::env; -use std::fmt::Write; use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use zenith_utils::auth::{encode_from_key_file, Claims, Scope}; use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{HexZTenantId, ZNodeId, ZTenantId, ZTimelineId}; +use zenith_utils::zid::{HexZTenantId, ZNodeId, ZTenantId, ZTenantTimelineId}; use crate::safekeeper::SafekeeperNode; @@ -24,7 +23,7 @@ use crate::safekeeper::SafekeeperNode; // to 'zenith init --config=' option. See control_plane/simple.conf for // an example. // -#[derive(Serialize, Deserialize, Clone, Debug)] +#[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] pub struct LocalEnv { // Base directory for all the nodes (the pageserver, safekeepers and // compute nodes). @@ -63,12 +62,10 @@ pub struct LocalEnv { /// Every tenant has a first timeline created for it, currently the only one ancestor-less for this tenant. /// It is used as a default timeline for branching, if no ancestor timeline is specified. #[serde(default)] - // TODO kb this does not survive calls between invocations, so will have to persist it. - // Then it comes back to names again? - pub initial_timelines: HashMap, + pub branch_name_mappings: HashMap, } -#[derive(Serialize, Deserialize, Clone, Debug)] +#[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] #[serde(default)] pub struct PageServerConf { // node id @@ -96,7 +93,7 @@ impl Default for PageServerConf { } } -#[derive(Serialize, Deserialize, Clone, Debug)] +#[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] #[serde(default)] pub struct SafekeeperConf { pub id: ZNodeId, @@ -222,6 +219,39 @@ impl LocalEnv { Ok(env) } + pub fn persist_config(&self, base_path: &Path) -> anyhow::Result<()> { + // Currently, the user first passes a config file with 'zenith init --config=' + // We read that in, in `create_config`, and fill any missing defaults. Then it's saved + // to .zenith/config. TODO: We lose any formatting and comments along the way, which is + // a bit sad. + let mut conf_content = r#"# This file describes a locale deployment of the page server +# and safekeeeper node. It is read by the 'zenith' command-line +# utility. +"# + .to_string(); + + // Convert the LocalEnv to a toml file. + // + // This could be as simple as this: + // + // conf_content += &toml::to_string_pretty(env)?; + // + // But it results in a "values must be emitted before tables". I'm not sure + // why, AFAICS the table, i.e. 'safekeepers: Vec' is last. + // Maybe rust reorders the fields to squeeze avoid padding or something? + // In any case, converting to toml::Value first, and serializing that, works. + // See https://github.com/alexcrichton/toml-rs/issues/142 + conf_content += &toml::to_string_pretty(&toml::Value::try_from(self)?)?; + + let target_config_path = base_path.join("config"); + fs::write(&target_config_path, conf_content).with_context(|| { + format!( + "Failed to write config file into path '{}'", + target_config_path.display() + ) + }) + } + // this function is used only for testing purposes in CLI e g generate tokens during init pub fn generate_auth_token(&self, claims: &Claims) -> anyhow::Result { let private_key_path = if self.private_key_path.is_absolute() { @@ -240,15 +270,15 @@ impl LocalEnv { pub fn init(&mut self) -> anyhow::Result<()> { // check if config already exists let base_path = &self.base_data_dir; - if base_path == Path::new("") { - bail!("repository base path is missing"); - } - if base_path.exists() { - bail!( - "directory '{}' already exists. Perhaps already initialized?", - base_path.to_str().unwrap() - ); - } + ensure!( + base_path != Path::new(""), + "repository base path is missing" + ); + ensure!( + !base_path.exists(), + "directory '{}' already exists. Perhaps already initialized?", + base_path.display() + ); fs::create_dir(&base_path)?; @@ -300,36 +330,7 @@ impl LocalEnv { fs::create_dir_all(SafekeeperNode::datadir_path_by_id(self, safekeeper.id))?; } - let mut conf_content = String::new(); - - // Currently, the user first passes a config file with 'zenith init --config=' - // We read that in, in `create_config`, and fill any missing defaults. Then it's saved - // to .zenith/config. TODO: We lose any formatting and comments along the way, which is - // a bit sad. - write!( - &mut conf_content, - r#"# This file describes a locale deployment of the page server -# and safekeeeper node. It is read by the 'zenith' command-line -# utility. -"# - )?; - - // Convert the LocalEnv to a toml file. - // - // This could be as simple as this: - // - // conf_content += &toml::to_string_pretty(env)?; - // - // But it results in a "values must be emitted before tables". I'm not sure - // why, AFAICS the table, i.e. 'safekeepers: Vec' is last. - // Maybe rust reorders the fields to squeeze avoid padding or something? - // In any case, converting to toml::Value first, and serializing that, works. - // See https://github.com/alexcrichton/toml-rs/issues/142 - conf_content += &toml::to_string_pretty(&toml::Value::try_from(&self)?)?; - - fs::write(base_path.join("config"), conf_content)?; - - Ok(()) + self.persist_config(base_path) } } diff --git a/pageserver/src/timelines.rs b/pageserver/src/timelines.rs index fc29767ddd..8b4dc57342 100644 --- a/pageserver/src/timelines.rs +++ b/pageserver/src/timelines.rs @@ -1,7 +1,5 @@ //! //! Timeline management code -//! -// TODO: move all paths construction to conf impl // use anyhow::{bail, Context, Result}; diff --git a/test_runner/batch_others/test_auth.py b/test_runner/batch_others/test_auth.py index 4d1d0847ed..e92eb2e044 100644 --- a/test_runner/batch_others/test_auth.py +++ b/test_runner/batch_others/test_auth.py @@ -25,21 +25,24 @@ def test_pageserver_auth(zenith_env_builder: ZenithEnvBuilder): ps.safe_psql("set FOO", password=tenant_token) ps.safe_psql("set FOO", password=management_token) + new_timeline_id = env.zenith_cli.create_branch('test_pageserver_auth', + tenant_id=env.initial_tenant) + # tenant can create branches tenant_http_client.timeline_create(timeline_id=uuid4(), tenant_id=env.initial_tenant, - ancestor_timeline_id=env.initial_timeline) + ancestor_timeline_id=new_timeline_id) # console can create branches for tenant management_http_client.timeline_create(timeline_id=uuid4(), tenant_id=env.initial_tenant, - ancestor_timeline_id=env.initial_timeline) + ancestor_timeline_id=new_timeline_id) # fail to create branch using token with different tenant_id with pytest.raises(ZenithPageserverApiException, match='Forbidden: Tenant id mismatch. Permission denied'): invalid_tenant_http_client.timeline_create(timeline_id=uuid4(), tenant_id=env.initial_tenant, - ancestor_timeline_id=env.initial_timeline) + ancestor_timeline_id=new_timeline_id) # create tenant using management token management_http_client.tenant_create(uuid4()) @@ -59,9 +62,9 @@ def test_compute_auth_to_pageserver(zenith_env_builder: ZenithEnvBuilder, with_w zenith_env_builder.num_safekeepers = 3 env = zenith_env_builder.init_start() - branch = f"test_compute_auth_to_pageserver{with_wal_acceptors}" - new_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start(branch, timeline_id=new_timeline_id) + branch = f'test_compute_auth_to_pageserver{with_wal_acceptors}' + env.zenith_cli.create_branch(branch) + pg = env.postgres.create_start(branch) with closing(pg.connect()) as conn: with conn.cursor() as cur: diff --git a/test_runner/batch_others/test_backpressure.py b/test_runner/batch_others/test_backpressure.py index 2b064c9fa8..ff34121327 100644 --- a/test_runner/batch_others/test_backpressure.py +++ b/test_runner/batch_others/test_backpressure.py @@ -95,7 +95,7 @@ def test_backpressure_received_lsn_lag(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 1 env = zenith_env_builder.init_start() # Create a branch for us - env.zenith_cli.create_branch("test_backpressure", "main") + env.zenith_cli.create_branch('test_backpressure') pg = env.postgres.create_start('test_backpressure', config_lines=['max_replication_write_lag=30MB']) diff --git a/test_runner/batch_others/test_branch_behind.py b/test_runner/batch_others/test_branch_behind.py index f8ff1741b4..4e2be352f4 100644 --- a/test_runner/batch_others/test_branch_behind.py +++ b/test_runner/batch_others/test_branch_behind.py @@ -22,9 +22,8 @@ def test_branch_behind(zenith_env_builder: ZenithEnvBuilder): env = zenith_env_builder.init_start() # Branch at the point where only 100 rows were inserted - test_branch_behind_timeline_id = env.zenith_cli.branch_timeline() - pgmain = env.postgres.create_start('test_branch_behind', - timeline_id=test_branch_behind_timeline_id) + env.zenith_cli.create_branch('test_branch_behind') + pgmain = env.postgres.create_start('test_branch_behind') log.info("postgres is running on 'test_branch_behind' branch") main_pg_conn = pgmain.connect() @@ -60,8 +59,9 @@ def test_branch_behind(zenith_env_builder: ZenithEnvBuilder): log.info(f'LSN after 200100 rows: {lsn_b}') # Branch at the point where only 100 rows were inserted - test_branch_behind_hundred_timeline_id = env.zenith_cli.branch_timeline( - ancestor_timeline_id=test_branch_behind_timeline_id, ancestor_start_lsn=lsn_a) + env.zenith_cli.create_branch('test_branch_behind_hundred', + 'test_branch_behind', + ancestor_start_lsn=lsn_a) # Insert many more rows. This generates enough WAL to fill a few segments. main_cur.execute(''' @@ -76,13 +76,12 @@ def test_branch_behind(zenith_env_builder: ZenithEnvBuilder): log.info(f'LSN after 400100 rows: {lsn_c}') # Branch at the point where only 200100 rows were inserted - test_branch_behind_more_timeline_id = env.zenith_cli.branch_timeline( - ancestor_timeline_id=test_branch_behind_timeline_id, ancestor_start_lsn=lsn_b) + env.zenith_cli.create_branch('test_branch_behind_more', + 'test_branch_behind', + ancestor_start_lsn=lsn_b) - pg_hundred = env.postgres.create_start("test_branch_behind_hundred", - timeline_id=test_branch_behind_hundred_timeline_id) - pg_more = env.postgres.create_start("test_branch_behind_more", - timeline_id=test_branch_behind_more_timeline_id) + pg_hundred = env.postgres.create_start('test_branch_behind_hundred') + pg_more = env.postgres.create_start('test_branch_behind_more') # On the 'hundred' branch, we should see only 100 rows hundred_pg_conn = pg_hundred.connect() @@ -103,23 +102,23 @@ def test_branch_behind(zenith_env_builder: ZenithEnvBuilder): # Check bad lsn's for branching # branch at segment boundary - test_branch_segment_boundary_timeline_id = env.zenith_cli.branch_timeline( - ancestor_timeline_id=test_branch_behind_timeline_id, ancestor_start_lsn="0/3000000") - pg = env.postgres.create_start("test_branch_segment_boundary", - timeline_id=test_branch_segment_boundary_timeline_id) + env.zenith_cli.create_branch('test_branch_segment_boundary', + 'test_branch_behind', + ancestor_start_lsn="0/3000000") + pg = env.postgres.create_start('test_branch_segment_boundary') cur = pg.connect().cursor() cur.execute('SELECT 1') assert cur.fetchone() == (1, ) # branch at pre-initdb lsn with pytest.raises(Exception, match="invalid branch start lsn"): - env.zenith_cli.branch_timeline(ancestor_timeline_id=env.initial_timeline, - ancestor_start_lsn="0/42") + env.zenith_cli.create_branch('test_branch_preinitdb', ancestor_start_lsn="0/42") # branch at pre-ancestor lsn with pytest.raises(Exception, match="less than timeline ancestor lsn"): - env.zenith_cli.branch_timeline(ancestor_timeline_id=test_branch_behind_timeline_id, - ancestor_start_lsn="0/42") + env.zenith_cli.create_branch('test_branch_preinitdb', + 'test_branch_behind', + ancestor_start_lsn="0/42") # check that we cannot create branch based on garbage collected data with closing(env.pageserver.connect()) as psconn: @@ -131,8 +130,9 @@ def test_branch_behind(zenith_env_builder: ZenithEnvBuilder): with pytest.raises(Exception, match="invalid branch start lsn"): # this gced_lsn is pretty random, so if gc is disabled this woudln't fail - env.zenith_cli.branch_timeline(ancestor_timeline_id=test_branch_behind_timeline_id, - ancestor_start_lsn=gced_lsn) + env.zenith_cli.create_branch('test_branch_create_fail', + 'test_branch_behind', + ancestor_start_lsn=gced_lsn) # check that after gc everything is still there hundred_cur.execute('SELECT count(*) FROM foo') diff --git a/test_runner/batch_others/test_clog_truncate.py b/test_runner/batch_others/test_clog_truncate.py index 9d3927aa84..b7eeedb23e 100644 --- a/test_runner/batch_others/test_clog_truncate.py +++ b/test_runner/batch_others/test_clog_truncate.py @@ -12,7 +12,7 @@ from fixtures.log_helper import log # def test_clog_truncate(zenith_simple_env: ZenithEnv): env = zenith_simple_env - test_clog_truncate_timeline_id = env.zenith_cli.branch_timeline() + env.zenith_cli.create_branch('test_clog_truncate', 'empty') # set agressive autovacuum to make sure that truncation will happen config = [ @@ -25,9 +25,7 @@ def test_clog_truncate(zenith_simple_env: ZenithEnv): 'autovacuum_freeze_max_age=100000' ] - pg = env.postgres.create_start('test_clog_truncate', - config_lines=config, - timeline_id=test_clog_truncate_timeline_id) + pg = env.postgres.create_start('test_clog_truncate', config_lines=config) log.info('postgres is running on test_clog_truncate branch') # Install extension containing function needed for test @@ -64,11 +62,10 @@ def test_clog_truncate(zenith_simple_env: ZenithEnv): # create new branch after clog truncation and start a compute node on it log.info(f'create branch at lsn_after_truncation {lsn_after_truncation}') - test_clog_truncate_new_timeline_id = env.zenith_cli.branch_timeline( - ancestor_timeline_id=test_clog_truncate_timeline_id, - ancestor_start_lsn=lsn_after_truncation) - pg2 = env.postgres.create_start('test_clog_truncate_new', - timeline_id=test_clog_truncate_new_timeline_id) + env.zenith_cli.create_branch('test_clog_truncate_new', + 'test_clog_truncate', + ancestor_start_lsn=lsn_after_truncation) + pg2 = env.postgres.create_start('test_clog_truncate_new') log.info('postgres is running on test_clog_truncate_new branch') # check that new node doesn't contain truncated segment diff --git a/test_runner/batch_others/test_config.py b/test_runner/batch_others/test_config.py index bd1f8b487f..fd2b3b4e99 100644 --- a/test_runner/batch_others/test_config.py +++ b/test_runner/batch_others/test_config.py @@ -9,10 +9,10 @@ from fixtures.log_helper import log # def test_config(zenith_simple_env: ZenithEnv): env = zenith_simple_env - new_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_config', - config_lines=['log_min_messages=debug1'], - timeline_id=new_timeline_id) + env.zenith_cli.create_branch("test_config", "empty") + + # change config + pg = env.postgres.create_start('test_config', config_lines=['log_min_messages=debug1']) log.info('postgres is running on test_config branch') with closing(pg.connect()) as conn: diff --git a/test_runner/batch_others/test_createdropdb.py b/test_runner/batch_others/test_createdropdb.py index e77e1928b8..88937fa0dc 100644 --- a/test_runner/batch_others/test_createdropdb.py +++ b/test_runner/batch_others/test_createdropdb.py @@ -11,9 +11,9 @@ from fixtures.log_helper import log # def test_createdb(zenith_simple_env: ZenithEnv): env = zenith_simple_env - test_createdb_timeline_id = env.zenith_cli.branch_timeline() + env.zenith_cli.create_branch('test_createdb', 'empty') - pg = env.postgres.create_start('test_createdb', timeline_id=test_createdb_timeline_id) + pg = env.postgres.create_start('test_createdb') log.info("postgres is running on 'test_createdb' branch") with closing(pg.connect()) as conn: @@ -27,9 +27,8 @@ def test_createdb(zenith_simple_env: ZenithEnv): lsn = cur.fetchone()[0] # Create a branch - test_createdb2_timeline_id = env.zenith_cli.branch_timeline( - ancestor_timeline_id=test_createdb_timeline_id, ancestor_start_lsn=lsn) - pg2 = env.postgres.create_start('test_createdb2', timeline_id=test_createdb2_timeline_id) + env.zenith_cli.create_branch('test_createdb2', 'test_createdb', ancestor_start_lsn=lsn) + pg2 = env.postgres.create_start('test_createdb2') # Test that you can connect to the new database on both branches for db in (pg, pg2): @@ -41,8 +40,8 @@ def test_createdb(zenith_simple_env: ZenithEnv): # def test_dropdb(zenith_simple_env: ZenithEnv, test_output_dir): env = zenith_simple_env - test_dropdb_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_dropdb', timeline_id=test_dropdb_timeline_id) + env.zenith_cli.create_branch('test_dropdb', 'empty') + pg = env.postgres.create_start('test_dropdb') log.info("postgres is running on 'test_dropdb' branch") with closing(pg.connect()) as conn: @@ -65,15 +64,15 @@ def test_dropdb(zenith_simple_env: ZenithEnv, test_output_dir): lsn_after_drop = cur.fetchone()[0] # Create two branches before and after database drop. - test_before_dropdb_timeline_db = env.zenith_cli.branch_timeline( - ancestor_timeline_id=test_dropdb_timeline_id, ancestor_start_lsn=lsn_before_drop) - pg_before = env.postgres.create_start('test_before_dropdb', - timeline_id=test_before_dropdb_timeline_db) + env.zenith_cli.create_branch('test_before_dropdb', + 'test_dropdb', + ancestor_start_lsn=lsn_before_drop) + pg_before = env.postgres.create_start('test_before_dropdb') - test_after_dropdb_timeline_id = env.zenith_cli.branch_timeline( - ancestor_timeline_id=test_dropdb_timeline_id, ancestor_start_lsn=lsn_after_drop) - pg_after = env.postgres.create_start('test_after_dropdb', - timeline_id=test_after_dropdb_timeline_id) + env.zenith_cli.create_branch('test_after_dropdb', + 'test_dropdb', + ancestor_start_lsn=lsn_after_drop) + pg_after = env.postgres.create_start('test_after_dropdb') # Test that database exists on the branch before drop pg_before.connect(dbname='foodb').close() diff --git a/test_runner/batch_others/test_createuser.py b/test_runner/batch_others/test_createuser.py index 8f825a0a1a..efb2af3f07 100644 --- a/test_runner/batch_others/test_createuser.py +++ b/test_runner/batch_others/test_createuser.py @@ -9,8 +9,8 @@ from fixtures.log_helper import log # def test_createuser(zenith_simple_env: ZenithEnv): env = zenith_simple_env - test_createuser_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_createuser', timeline_id=test_createuser_timeline_id) + env.zenith_cli.create_branch('test_createuser', 'empty') + pg = env.postgres.create_start('test_createuser') log.info("postgres is running on 'test_createuser' branch") with closing(pg.connect()) as conn: @@ -24,9 +24,8 @@ def test_createuser(zenith_simple_env: ZenithEnv): lsn = cur.fetchone()[0] # Create a branch - test_createuser2_timeline_id = env.zenith_cli.branch_timeline( - ancestor_timeline_id=test_createuser_timeline_id, ancestor_start_lsn=lsn) - pg2 = env.postgres.create_start('test_createuser2', timeline_id=test_createuser2_timeline_id) + env.zenith_cli.create_branch('test_createuser2', 'test_createuser', ancestor_start_lsn=lsn) + pg2 = env.postgres.create_start('test_createuser2') # Test that you can connect to new branch as a new user assert pg2.safe_psql('select current_user', username='testuser') == [('testuser', )] diff --git a/test_runner/batch_others/test_gc_aggressive.py b/test_runner/batch_others/test_gc_aggressive.py index 7dd38a5799..9de6ba9f59 100644 --- a/test_runner/batch_others/test_gc_aggressive.py +++ b/test_runner/batch_others/test_gc_aggressive.py @@ -1,6 +1,7 @@ from contextlib import closing import asyncio +import asyncpg import random from fixtures.zenith_fixtures import ZenithEnv, Postgres, Safekeeper @@ -54,8 +55,8 @@ async def update_and_gc(env: ZenithEnv, pg: Postgres, timeline: str): # def test_gc_aggressive(zenith_simple_env: ZenithEnv): env = zenith_simple_env - new_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_gc_aggressive', timeline_id=new_timeline_id) + env.zenith_cli.create_branch("test_gc_aggressive", "empty") + pg = env.postgres.create_start('test_gc_aggressive') log.info('postgres is running on test_gc_aggressive branch') conn = pg.connect() diff --git a/test_runner/batch_others/test_multixact.py b/test_runner/batch_others/test_multixact.py index 11f8000226..7a508a67fb 100644 --- a/test_runner/batch_others/test_multixact.py +++ b/test_runner/batch_others/test_multixact.py @@ -10,8 +10,8 @@ from fixtures.log_helper import log # def test_multixact(zenith_simple_env: ZenithEnv, test_output_dir): env = zenith_simple_env - test_multixact_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_multixact', timeline_id=test_multixact_timeline_id) + env.zenith_cli.create_branch('test_multixact', 'empty') + pg = env.postgres.create_start('test_multixact') log.info("postgres is running on 'test_multixact' branch") pg_conn = pg.connect() @@ -60,10 +60,8 @@ def test_multixact(zenith_simple_env: ZenithEnv, test_output_dir): assert int(next_multixact_id) > int(next_multixact_id_old) # Branch at this point - test_multixact_new_timeline_id = env.zenith_cli.branch_timeline( - ancestor_timeline_id=test_multixact_timeline_id, ancestor_start_lsn=lsn) - pg_new = env.postgres.create_start('test_multixact_new', - timeline_id=test_multixact_new_timeline_id) + env.zenith_cli.create_branch('test_multixact_new', 'test_multixact', ancestor_start_lsn=lsn) + pg_new = env.postgres.create_start('test_multixact_new') log.info("postgres is running on 'test_multixact_new' branch") pg_new_conn = pg_new.connect() diff --git a/test_runner/batch_others/test_old_request_lsn.py b/test_runner/batch_others/test_old_request_lsn.py index f0701dfe4f..d09fb24913 100644 --- a/test_runner/batch_others/test_old_request_lsn.py +++ b/test_runner/batch_others/test_old_request_lsn.py @@ -16,8 +16,8 @@ from fixtures.log_helper import log # def test_old_request_lsn(zenith_simple_env: ZenithEnv): env = zenith_simple_env - new_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_old_request_lsn', timeline_id=new_timeline_id) + env.zenith_cli.create_branch("test_old_request_lsn", "empty") + pg = env.postgres.create_start('test_old_request_lsn') log.info('postgres is running on test_old_request_lsn branch') pg_conn = pg.connect() diff --git a/test_runner/batch_others/test_pageserver_catchup.py b/test_runner/batch_others/test_pageserver_catchup.py index ba77a4a321..7093a1bdb3 100644 --- a/test_runner/batch_others/test_pageserver_catchup.py +++ b/test_runner/batch_others/test_pageserver_catchup.py @@ -16,9 +16,8 @@ def test_pageserver_catchup_while_compute_down(zenith_env_builder: ZenithEnvBuil zenith_env_builder.num_safekeepers = 3 env = zenith_env_builder.init_start() - new_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_pageserver_catchup_while_compute_down', - timeline_id=new_timeline_id) + env.zenith_cli.create_branch('test_pageserver_catchup_while_compute_down') + pg = env.postgres.create_start('test_pageserver_catchup_while_compute_down') pg_conn = pg.connect() cur = pg_conn.cursor() @@ -60,8 +59,7 @@ def test_pageserver_catchup_while_compute_down(zenith_env_builder: ZenithEnvBuil env.safekeepers[2].start() # restart compute node - pg.stop_and_destroy().create_start('test_pageserver_catchup_while_compute_down', - timeline_id=new_timeline_id) + pg.stop_and_destroy().create_start('test_pageserver_catchup_while_compute_down') # Ensure that basebackup went correct and pageserver returned all data pg_conn = pg.connect() diff --git a/test_runner/batch_others/test_pageserver_restart.py b/test_runner/batch_others/test_pageserver_restart.py index f1d154408c..57f9db8f96 100644 --- a/test_runner/batch_others/test_pageserver_restart.py +++ b/test_runner/batch_others/test_pageserver_restart.py @@ -15,8 +15,8 @@ def test_pageserver_restart(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 1 env = zenith_env_builder.init_start() - new_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_pageserver_restart', timeline_id=new_timeline_id) + env.zenith_cli.create_branch('test_pageserver_restart') + pg = env.postgres.create_start('test_pageserver_restart') pg_conn = pg.connect() cur = pg_conn.cursor() diff --git a/test_runner/batch_others/test_parallel_copy.py b/test_runner/batch_others/test_parallel_copy.py index 8e954a8e51..4b7cc58d42 100644 --- a/test_runner/batch_others/test_parallel_copy.py +++ b/test_runner/batch_others/test_parallel_copy.py @@ -35,8 +35,8 @@ async def parallel_load_same_table(pg: Postgres, n_parallel: int): # Load data into one table with COPY TO from 5 parallel connections def test_parallel_copy(zenith_simple_env: ZenithEnv, n_parallel=5): env = zenith_simple_env - new_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_parallel_copy', timeline_id=new_timeline_id) + env.zenith_cli.create_branch("test_parallel_copy", "empty") + pg = env.postgres.create_start('test_parallel_copy') log.info("postgres is running on 'test_parallel_copy' branch") # Create test table diff --git a/test_runner/batch_others/test_pgbench.py b/test_runner/batch_others/test_pgbench.py index 207f1e1e2c..09713023bc 100644 --- a/test_runner/batch_others/test_pgbench.py +++ b/test_runner/batch_others/test_pgbench.py @@ -4,8 +4,8 @@ from fixtures.log_helper import log def test_pgbench(zenith_simple_env: ZenithEnv, pg_bin): env = zenith_simple_env - new_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_pgbench', timeline_id=new_timeline_id) + env.zenith_cli.create_branch("test_pgbench", "empty") + pg = env.postgres.create_start('test_pgbench') log.info("postgres is running on 'test_pgbench' branch") connstr = pg.connstr() diff --git a/test_runner/batch_others/test_readonly_node.py b/test_runner/batch_others/test_readonly_node.py index 2998ea7528..5d5949add6 100644 --- a/test_runner/batch_others/test_readonly_node.py +++ b/test_runner/batch_others/test_readonly_node.py @@ -11,9 +11,8 @@ from fixtures.zenith_fixtures import ZenithEnv # def test_readonly_node(zenith_simple_env: ZenithEnv): env = zenith_simple_env - test_readonly_node_timeline_id = env.zenith_cli.branch_timeline() - pgmain = env.postgres.create_start('test_readonly_node', - timeline_id=test_readonly_node_timeline_id) + env.zenith_cli.create_branch('test_readonly_node', 'empty') + pgmain = env.postgres.create_start('test_readonly_node') log.info("postgres is running on 'test_readonly_node' branch") main_pg_conn = pgmain.connect() @@ -53,14 +52,10 @@ def test_readonly_node(zenith_simple_env: ZenithEnv): log.info('LSN after 400100 rows: ' + lsn_c) # Create first read-only node at the point where only 100 rows were inserted - pg_hundred = env.postgres.create_start("test_readonly_node_hundred", - timeline_id=test_readonly_node_timeline_id, - lsn=lsn_a) + pg_hundred = env.postgres.create_start("test_readonly_node_hundred", lsn=lsn_a) # And another at the point where 200100 rows were inserted - pg_more = env.postgres.create_start("test_readonly_node_more", - timeline_id=test_readonly_node_timeline_id, - lsn=lsn_b) + pg_more = env.postgres.create_start("test_readonly_node_more", lsn=lsn_b) # On the 'hundred' node, we should see only 100 rows hundred_pg_conn = pg_hundred.connect() @@ -79,9 +74,7 @@ def test_readonly_node(zenith_simple_env: ZenithEnv): assert main_cur.fetchone() == (400100, ) # Check creating a node at segment boundary - pg = env.postgres.create_start("test_branch_segment_boundary", - timeline_id=test_readonly_node_timeline_id, - lsn='0/3000000') + pg = env.postgres.create_start("test_branch_segment_boundary", lsn='0/3000000') cur = pg.connect().cursor() cur.execute('SELECT 1') assert cur.fetchone() == (1, ) @@ -89,6 +82,4 @@ def test_readonly_node(zenith_simple_env: ZenithEnv): # Create node at pre-initdb lsn with pytest.raises(Exception, match="invalid basebackup lsn"): # compute node startup with invalid LSN should fail - env.zenith_cli.pg_start("test_readonly_node_preinitdb", - timeline_id=test_readonly_node_timeline_id, - lsn="0/42") + env.zenith_cli.pg_start("test_readonly_node_preinitdb", lsn="0/42") diff --git a/test_runner/batch_others/test_restart_compute.py b/test_runner/batch_others/test_restart_compute.py index baa1f787df..fd06561c00 100644 --- a/test_runner/batch_others/test_restart_compute.py +++ b/test_runner/batch_others/test_restart_compute.py @@ -15,8 +15,8 @@ def test_restart_compute(zenith_env_builder: ZenithEnvBuilder, with_wal_acceptor zenith_env_builder.num_safekeepers = 3 env = zenith_env_builder.init_start() - new_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_restart_compute', timeline_id=new_timeline_id) + env.zenith_cli.create_branch('test_restart_compute') + pg = env.postgres.create_start('test_restart_compute') log.info("postgres is running on 'test_restart_compute' branch") with closing(pg.connect()) as conn: @@ -29,7 +29,7 @@ def test_restart_compute(zenith_env_builder: ZenithEnvBuilder, with_wal_acceptor log.info(f"res = {r}") # Remove data directory and restart - pg.stop_and_destroy().create_start('test_restart_compute', timeline_id=new_timeline_id) + pg.stop_and_destroy().create_start('test_restart_compute') with closing(pg.connect()) as conn: with conn.cursor() as cur: @@ -48,7 +48,7 @@ def test_restart_compute(zenith_env_builder: ZenithEnvBuilder, with_wal_acceptor log.info(f"res = {r}") # Again remove data directory and restart - pg.stop_and_destroy().create_start('test_restart_compute', timeline_id=new_timeline_id) + pg.stop_and_destroy().create_start('test_restart_compute') # That select causes lots of FPI's and increases probability of wakeepers # lagging behind after query completion @@ -62,7 +62,7 @@ def test_restart_compute(zenith_env_builder: ZenithEnvBuilder, with_wal_acceptor log.info(f"res = {r}") # And again remove data directory and restart - pg.stop_and_destroy().create_start('test_restart_compute', timeline_id=new_timeline_id) + pg.stop_and_destroy().create_start('test_restart_compute') with closing(pg.connect()) as conn: with conn.cursor() as cur: diff --git a/test_runner/batch_others/test_snapfiles_gc.py b/test_runner/batch_others/test_snapfiles_gc.py index fb02e54be2..c6d4512bc9 100644 --- a/test_runner/batch_others/test_snapfiles_gc.py +++ b/test_runner/batch_others/test_snapfiles_gc.py @@ -14,8 +14,8 @@ from fixtures.log_helper import log # def test_layerfiles_gc(zenith_simple_env: ZenithEnv): env = zenith_simple_env - new_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_layerfiles_gc', timeline_id=new_timeline_id) + env.zenith_cli.create_branch("test_layerfiles_gc", "empty") + pg = env.postgres.create_start('test_layerfiles_gc') with closing(pg.connect()) as conn: with conn.cursor() as cur: diff --git a/test_runner/batch_others/test_subxacts.py b/test_runner/batch_others/test_subxacts.py index 6153bd1fe2..bed1c4be63 100644 --- a/test_runner/batch_others/test_subxacts.py +++ b/test_runner/batch_others/test_subxacts.py @@ -10,8 +10,8 @@ from fixtures.log_helper import log # CLOG. def test_subxacts(zenith_simple_env: ZenithEnv, test_output_dir): env = zenith_simple_env - new_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_subxacts', timeline_id=new_timeline_id) + env.zenith_cli.create_branch("test_subxacts", "empty") + pg = env.postgres.create_start('test_subxacts') log.info("postgres is running on 'test_subxacts' branch") pg_conn = pg.connect() diff --git a/test_runner/batch_others/test_tenant_relocation.py b/test_runner/batch_others/test_tenant_relocation.py index 429aee8488..a1286adfb0 100644 --- a/test_runner/batch_others/test_tenant_relocation.py +++ b/test_runner/batch_others/test_tenant_relocation.py @@ -127,14 +127,12 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder, # create folder for remote storage mock remote_storage_mock_path = env.repo_dir / 'local_fs_remote_storage' - (tenant, _) = env.zenith_cli.create_tenant(UUID("74ee8b079a0e437eb0afea7d26a07209")) + tenant = env.zenith_cli.create_tenant(UUID("74ee8b079a0e437eb0afea7d26a07209")) log.info("tenant to relocate %s", tenant) - new_timeline_id = env.zenith_cli.branch_timeline(tenant_id=tenant) + env.zenith_cli.create_branch('test_tenant_relocation', tenant_id=tenant) - tenant_pg = env.postgres.create_start("test_tenant_relocation", - tenant_id=tenant, - timeline_id=new_timeline_id) + tenant_pg = env.postgres.create_start("test_tenant_relocation", tenant_id=tenant) # insert some data with closing(tenant_pg.connect()) as conn: diff --git a/test_runner/batch_others/test_tenants.py b/test_runner/batch_others/test_tenants.py index 20a910e9ce..87acf2086d 100644 --- a/test_runner/batch_others/test_tenants.py +++ b/test_runner/batch_others/test_tenants.py @@ -12,23 +12,21 @@ def test_tenants_normal_work(zenith_env_builder: ZenithEnvBuilder, with_wal_acce env = zenith_env_builder.init_start() """Tests tenants with and without wal acceptors""" - (tenant_1, initial_timeline_1) = env.zenith_cli.create_tenant() - (tenant_2, initial_timeline_2) = env.zenith_cli.create_tenant() + tenant_1 = env.zenith_cli.create_tenant() + tenant_2 = env.zenith_cli.create_tenant() - new_timeline_tenant_1 = env.zenith_cli.branch_timeline(tenant_id=tenant_1, - ancestor_timeline_id=initial_timeline_1) - new_timeline_tenant_2 = env.zenith_cli.branch_timeline(tenant_id=tenant_2, - ancestor_timeline_id=initial_timeline_2) + env.zenith_cli.create_branch(f'test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}', + tenant_id=tenant_1) + env.zenith_cli.create_branch(f'test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}', + tenant_id=tenant_2) pg_tenant1 = env.postgres.create_start( - f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}", + f'test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}', tenant_id=tenant_1, - timeline_id=new_timeline_tenant_1, ) pg_tenant2 = env.postgres.create_start( - f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}", + f'test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}', tenant_id=tenant_2, - timeline_id=new_timeline_tenant_2, ) for pg in [pg_tenant1, pg_tenant2]: diff --git a/test_runner/batch_others/test_timeline_size.py b/test_runner/batch_others/test_timeline_size.py index 49143d0000..7d8ab551b0 100644 --- a/test_runner/batch_others/test_timeline_size.py +++ b/test_runner/batch_others/test_timeline_size.py @@ -10,14 +10,13 @@ import time def test_timeline_size(zenith_simple_env: ZenithEnv): env = zenith_simple_env # Branch at the point where only 100 rows were inserted - new_timeline_id = env.zenith_cli.branch_timeline() + new_timeline_id = env.zenith_cli.create_branch('test_timeline_size', 'empty') client = env.pageserver.http_client() res = client.timeline_detail(tenant_id=env.initial_tenant, timeline_id=new_timeline_id) - print(f'@@@@@@@@@@\n{res}\n@@@@@@@@@@@') assert res["current_logical_size"] == res["current_logical_size_non_incremental"] - pgmain = env.postgres.create_start("test_timeline_size", timeline_id=new_timeline_id) + pgmain = env.postgres.create_start("test_timeline_size") log.info("postgres is running on 'test_timeline_size' branch") with closing(pgmain.connect()) as conn: @@ -69,7 +68,7 @@ def wait_for_pageserver_catchup(pgmain: Postgres, polling_interval=1, timeout=60 def test_timeline_size_quota(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 1 env = zenith_env_builder.init_start() - new_timeline_id = env.zenith_cli.branch_timeline() + new_timeline_id = env.zenith_cli.create_branch('test_timeline_size_quota') client = env.pageserver.http_client() res = client.timeline_detail(tenant_id=env.initial_tenant, timeline_id=new_timeline_id) @@ -78,8 +77,7 @@ def test_timeline_size_quota(zenith_env_builder: ZenithEnvBuilder): pgmain = env.postgres.create_start( "test_timeline_size_quota", # Set small limit for the test - config_lines=['zenith.max_cluster_size=30MB'], - timeline_id=new_timeline_id) + config_lines=['zenith.max_cluster_size=30MB']) log.info("postgres is running on 'test_timeline_size_quota' branch") with closing(pgmain.connect()) as conn: diff --git a/test_runner/batch_others/test_twophase.py b/test_runner/batch_others/test_twophase.py index b479e9de22..4afdc7e0be 100644 --- a/test_runner/batch_others/test_twophase.py +++ b/test_runner/batch_others/test_twophase.py @@ -9,10 +9,8 @@ from fixtures.log_helper import log # def test_twophase(zenith_simple_env: ZenithEnv): env = zenith_simple_env - test_twophase_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_twophase', - config_lines=['max_prepared_transactions=5'], - timeline_id=test_twophase_timeline_id) + env.zenith_cli.create_branch("test_twophase", "empty") + pg = env.postgres.create_start('test_twophase', config_lines=['max_prepared_transactions=5']) log.info("postgres is running on 'test_twophase' branch") conn = pg.connect() @@ -57,14 +55,12 @@ def test_twophase(zenith_simple_env: ZenithEnv): assert len(twophase_files) == 2 # Create a branch with the transaction in prepared state - test_twophase_prepared_timeline_id = env.zenith_cli.branch_timeline( - ancestor_timeline_id=test_twophase_timeline_id) + env.zenith_cli.create_branch("test_twophase_prepared", "test_twophase") # Start compute on the new branch pg2 = env.postgres.create_start( 'test_twophase_prepared', config_lines=['max_prepared_transactions=5'], - timeline_id=test_twophase_prepared_timeline_id, ) # Check that we restored only needed twophase files diff --git a/test_runner/batch_others/test_vm_bits.py b/test_runner/batch_others/test_vm_bits.py index a657b3e3fd..49e48dd450 100644 --- a/test_runner/batch_others/test_vm_bits.py +++ b/test_runner/batch_others/test_vm_bits.py @@ -9,8 +9,8 @@ from fixtures.log_helper import log def test_vm_bit_clear(zenith_simple_env: ZenithEnv): env = zenith_simple_env - test_vm_bit_clear_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_vm_bit_clear', timeline_id=test_vm_bit_clear_timeline_id) + env.zenith_cli.create_branch("test_vm_bit_clear", "empty") + pg = env.postgres.create_start('test_vm_bit_clear') log.info("postgres is running on 'test_vm_bit_clear' branch") pg_conn = pg.connect() @@ -33,8 +33,7 @@ def test_vm_bit_clear(zenith_simple_env: ZenithEnv): cur.execute('UPDATE vmtest_update SET id = 5000 WHERE id = 1') # Branch at this point, to test that later - test_vm_bit_clear_new_timeline_id = env.zenith_cli.branch_timeline( - ancestor_timeline_id=test_vm_bit_clear_timeline_id) + env.zenith_cli.create_branch("test_vm_bit_clear_new", "test_vm_bit_clear") # Clear the buffer cache, to force the VM page to be re-fetched from # the page server @@ -62,8 +61,7 @@ def test_vm_bit_clear(zenith_simple_env: ZenithEnv): # a dirty VM page is evicted. If the VM bit was not correctly cleared by the # earlier WAL record, the full-page image hides the problem. Starting a new # server at the right point-in-time avoids that full-page image. - pg_new = env.postgres.create_start('test_vm_bit_clear_new', - timeline_id=test_vm_bit_clear_new_timeline_id) + pg_new = env.postgres.create_start('test_vm_bit_clear_new') log.info("postgres is running on 'test_vm_bit_clear_new' branch") pg_new_conn = pg_new.connect() diff --git a/test_runner/batch_others/test_wal_acceptor.py b/test_runner/batch_others/test_wal_acceptor.py index 3e39228494..9518a14b75 100644 --- a/test_runner/batch_others/test_wal_acceptor.py +++ b/test_runner/batch_others/test_wal_acceptor.py @@ -24,8 +24,8 @@ def test_normal_work(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 3 env = zenith_env_builder.init_start() - new_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_wal_acceptors_normal_work', timeline_id=new_timeline_id) + env.zenith_cli.create_branch('test_wal_acceptors_normal_work') + pg = env.postgres.create_start('test_wal_acceptors_normal_work') with closing(pg.connect()) as conn: with conn.cursor() as cur: @@ -62,8 +62,8 @@ def test_many_timelines(zenith_env_builder: ZenithEnvBuilder): # start postgres on each timeline pgs = [] for branch_name in branch_names: - new_timeline_id = env.zenith_cli.branch_timeline() - pgs.append(env.postgres.create_start(branch_name, timeline_id=new_timeline_id)) + new_timeline_id = env.zenith_cli.create_branch(branch_name) + pgs.append(env.postgres.create_start(branch_name)) branch_names_to_timeline_ids[branch_name] = new_timeline_id tenant_id = env.initial_tenant @@ -87,7 +87,6 @@ def test_many_timelines(zenith_env_builder: ZenithEnvBuilder): timeline_metrics = [] with env.pageserver.http_client() as pageserver_http: for timeline_detail in timeline_details: - print(f"@@@@@@@@@@@\n{timeline_detail}\n@@@@@@@@@@@") timeline_id: str = timeline_detail["timeline_id"] m = TimelineMetrics( @@ -188,8 +187,8 @@ def test_restarts(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = n_acceptors env = zenith_env_builder.init_start() - new_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_wal_acceptors_restarts', timeline_id=new_timeline_id) + env.zenith_cli.create_branch('test_wal_acceptors_restarts') + pg = env.postgres.create_start('test_wal_acceptors_restarts') # we rely upon autocommit after each statement # as waiting for acceptors happens there @@ -225,8 +224,8 @@ def test_unavailability(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 2 env = zenith_env_builder.init_start() - new_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_wal_acceptors_unavailability', timeline_id=new_timeline_id) + env.zenith_cli.create_branch('test_wal_acceptors_unavailability') + pg = env.postgres.create_start('test_wal_acceptors_unavailability') # we rely upon autocommit after each statement # as waiting for acceptors happens there @@ -296,9 +295,8 @@ def test_race_conditions(zenith_env_builder: ZenithEnvBuilder, stop_value): zenith_env_builder.num_safekeepers = 3 env = zenith_env_builder.init_start() - new_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_wal_acceptors_race_conditions', - timeline_id=new_timeline_id) + env.zenith_cli.create_branch('test_wal_acceptors_race_conditions') + pg = env.postgres.create_start('test_wal_acceptors_race_conditions') # we rely upon autocommit after each statement # as waiting for acceptors happens there @@ -462,8 +460,8 @@ def test_timeline_status(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 1 env = zenith_env_builder.init_start() - new_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_timeline_status', timeline_id=new_timeline_id) + env.zenith_cli.create_branch('test_timeline_status') + pg = env.postgres.create_start('test_timeline_status') wa = env.safekeepers[0] wa_http_cli = wa.http_client() @@ -636,12 +634,12 @@ def test_replace_safekeeper(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 4 env = zenith_env_builder.init_start() - new_timeline_id = env.zenith_cli.branch_timeline() + env.zenith_cli.create_branch('test_replace_safekeeper') log.info("Use only first 3 safekeepers") env.safekeepers[3].stop() active_safekeepers = [1, 2, 3] - pg = env.postgres.create('test_replace_safekeeper', timeline_id=new_timeline_id) + pg = env.postgres.create('test_replace_safekeeper') pg.adjust_for_wal_acceptors(safekeepers_guc(env, active_safekeepers)) pg.start() @@ -679,7 +677,7 @@ def test_replace_safekeeper(zenith_env_builder: ZenithEnvBuilder): show_statuses(env.safekeepers, tenant_id, timeline_id) log.info("Recreate postgres to replace failed sk1 with new sk4") - pg.stop_and_destroy().create('test_replace_safekeeper', timeline_id=uuid.UUID(timeline_id)) + pg.stop_and_destroy().create('test_replace_safekeeper') active_safekeepers = [2, 3, 4] env.safekeepers[3].start() pg.adjust_for_wal_acceptors(safekeepers_guc(env, active_safekeepers)) diff --git a/test_runner/batch_others/test_wal_acceptor_async.py b/test_runner/batch_others/test_wal_acceptor_async.py index 719e8c163f..31ace7eab3 100644 --- a/test_runner/batch_others/test_wal_acceptor_async.py +++ b/test_runner/batch_others/test_wal_acceptor_async.py @@ -202,9 +202,8 @@ def test_restarts_under_load(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 3 env = zenith_env_builder.init_start() - new_timeline_id = env.zenith_cli.branch_timeline() - pg = env.postgres.create_start('test_wal_acceptors_restarts_under_load', - timeline_id=new_timeline_id) + env.zenith_cli.create_branch('test_wal_acceptors_restarts_under_load') + pg = env.postgres.create_start('test_wal_acceptors_restarts_under_load') asyncio.run(run_restarts_under_load(pg, env.safekeepers)) diff --git a/test_runner/batch_others/test_zenith_cli.py b/test_runner/batch_others/test_zenith_cli.py index 4f089d4354..8777a653b3 100644 --- a/test_runner/batch_others/test_zenith_cli.py +++ b/test_runner/batch_others/test_zenith_cli.py @@ -36,11 +36,12 @@ def test_cli_timeline_list(zenith_simple_env: ZenithEnv): helper_compare_timeline_list(pageserver_http_client, env, env.initial_tenant) # Create a branch for us - main_timeline_id = env.zenith_cli.branch_timeline() + main_timeline_id = env.zenith_cli.create_branch('test_cli_branch_list_main') helper_compare_timeline_list(pageserver_http_client, env, env.initial_tenant) # Create a nested branch - nested_timeline_id = env.zenith_cli.branch_timeline(ancestor_timeline_id=main_timeline_id) + nested_timeline_id = env.zenith_cli.create_branch('test_cli_branch_list_nested', + 'test_cli_branch_list_main') helper_compare_timeline_list(pageserver_http_client, env, env.initial_tenant) # Check that all new branches are visible via CLI @@ -67,15 +68,13 @@ def test_cli_tenant_list(zenith_simple_env: ZenithEnv): helper_compare_tenant_list(pageserver_http_client, env) # Create new tenant - tenant1 = uuid.uuid4() - env.zenith_cli.create_tenant(tenant_id=tenant1) + tenant1 = env.zenith_cli.create_tenant() # check tenant1 appeared helper_compare_tenant_list(pageserver_http_client, env) # Create new tenant - tenant2 = uuid.uuid4() - env.zenith_cli.create_tenant(tenant_id=tenant2) + tenant2 = env.zenith_cli.create_tenant() # check tenant2 appeared helper_compare_tenant_list(pageserver_http_client, env) diff --git a/test_runner/batch_pg_regress/test_isolation.py b/test_runner/batch_pg_regress/test_isolation.py index 8dce020dc0..ddafc3815b 100644 --- a/test_runner/batch_pg_regress/test_isolation.py +++ b/test_runner/batch_pg_regress/test_isolation.py @@ -7,12 +7,10 @@ from fixtures.zenith_fixtures import ZenithEnv, base_dir, pg_distrib_dir def test_isolation(zenith_simple_env: ZenithEnv, test_output_dir, pg_bin, capsys): env = zenith_simple_env - new_timeline_id = env.zenith_cli.branch_timeline() + env.zenith_cli.create_branch("test_isolation", "empty") # Connect to postgres and create a database called "regression". # isolation tests use prepared transactions, so enable them - pg = env.postgres.create_start('test_isolation', - config_lines=['max_prepared_transactions=100'], - timeline_id=new_timeline_id) + pg = env.postgres.create_start('test_isolation', config_lines=['max_prepared_transactions=100']) pg.safe_psql('CREATE DATABASE isolation_regression') # Create some local directories for pg_isolation_regress to run in. diff --git a/test_runner/batch_pg_regress/test_pg_regress.py b/test_runner/batch_pg_regress/test_pg_regress.py index efeb63fce3..5199f65216 100644 --- a/test_runner/batch_pg_regress/test_pg_regress.py +++ b/test_runner/batch_pg_regress/test_pg_regress.py @@ -7,9 +7,9 @@ from fixtures.zenith_fixtures import ZenithEnv, check_restored_datadir_content, def test_pg_regress(zenith_simple_env: ZenithEnv, test_output_dir: str, pg_bin, capsys): env = zenith_simple_env - new_timeline_id = env.zenith_cli.branch_timeline() + env.zenith_cli.create_branch("test_pg_regress", "empty") # Connect to postgres and create a database called "regression". - pg = env.postgres.create_start('test_pg_regress', timeline_id=new_timeline_id) + pg = env.postgres.create_start('test_pg_regress') pg.safe_psql('CREATE DATABASE regression') # Create some local directories for pg_regress to run in. diff --git a/test_runner/batch_pg_regress/test_zenith_regress.py b/test_runner/batch_pg_regress/test_zenith_regress.py index 2ccbafccfd..31d5b07093 100644 --- a/test_runner/batch_pg_regress/test_zenith_regress.py +++ b/test_runner/batch_pg_regress/test_zenith_regress.py @@ -11,9 +11,9 @@ from fixtures.log_helper import log def test_zenith_regress(zenith_simple_env: ZenithEnv, test_output_dir, pg_bin, capsys): env = zenith_simple_env - new_timeline_id = env.zenith_cli.branch_timeline() + env.zenith_cli.create_branch("test_zenith_regress", "empty") # Connect to postgres and create a database called "regression". - pg = env.postgres.create_start('test_zenith_regress', timeline_id=new_timeline_id) + pg = env.postgres.create_start('test_zenith_regress') pg.safe_psql('CREATE DATABASE regression') # Create some local directories for pg_regress to run in. diff --git a/test_runner/fixtures/compare_fixtures.py b/test_runner/fixtures/compare_fixtures.py index 66b9fe54ea..750b02c894 100644 --- a/test_runner/fixtures/compare_fixtures.py +++ b/test_runner/fixtures/compare_fixtures.py @@ -64,8 +64,8 @@ class ZenithCompare(PgCompare): self._pg_bin = pg_bin # We only use one branch and one timeline - timeline_id = self.env.zenith_cli.branch_timeline() - self._pg = self.env.postgres.create_start("branch", timeline_id=timeline_id) + self.env.zenith_cli.create_branch(branch_name, 'empty') + self._pg = self.env.postgres.create_start(branch_name) self.timeline = self.pg.safe_psql("SHOW zenith.zenith_timeline")[0][0] # Long-lived cursor, useful for flushing diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index 7c4d178a3f..9345c7f238 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -1,6 +1,6 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import field import textwrap from cached_property import cached_property import asyncpg @@ -29,7 +29,6 @@ from dataclasses import dataclass from psycopg2.extensions import connection as PgConnection from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, TypeVar, cast, Union, Tuple from typing_extensions import Literal -import pytest import requests import backoff # type: ignore @@ -219,7 +218,7 @@ def can_bind(host: str, port: int) -> bool: class PortDistributor: - def __init__(self, base_port: int, port_number: int) -> None: + def __init__(self, base_port: int, port_number: int): self.iterator = iter(range(base_port, base_port + port_number)) def get_port(self) -> int: @@ -424,7 +423,8 @@ class ZenithEnvBuilder: pageserver_config_override: Optional[str] = None, num_safekeepers: int = 0, pageserver_auth_enabled: bool = False, - rust_log_override: Optional[str] = None): + rust_log_override: Optional[str] = None, + default_branch_name='main'): self.repo_dir = repo_dir self.rust_log_override = rust_log_override self.port_distributor = port_distributor @@ -432,6 +432,7 @@ class ZenithEnvBuilder: self.pageserver_config_override = pageserver_config_override self.num_safekeepers = num_safekeepers self.pageserver_auth_enabled = pageserver_auth_enabled + self.default_branch_name = default_branch_name self.env: Optional[ZenithEnv] = None self.s3_mock_server: Optional[MockS3Server] = None @@ -536,7 +537,7 @@ class ZenithEnv: initial_tenant - tenant ID of the initial tenant created in the repository - zenith_cli() - zenith_cli() can be used to run the 'zenith' CLI tool + zenith_cli - can be used to run the 'zenith' CLI tool create_tenant() - initializes a new tenant in the page server, returns the tenant id @@ -546,9 +547,9 @@ class ZenithEnv: self.rust_log_override = config.rust_log_override self.port_distributor = config.port_distributor self.s3_mock_server = config.s3_mock_server + self.default_branch_name = config.default_branch_name self.zenith_cli = ZenithCli(env=self) - - self.zenith_cli = ZenithCli(env=self) + self.postgres = PostgresFactory(self) self.safekeepers: List[Safekeeper] = [] # generate initial tenant ID here instead of letting 'zenith init' generate it, @@ -599,9 +600,7 @@ class ZenithEnv: self.safekeepers.append(safekeeper) log.info(f"Config: {toml}") - # TODO kb is this a wrong concept? will break for multiple tenant tests - self.initial_timeline = self.zenith_cli.init(toml) - self.postgres = PostgresFactory(self) + self.zenith_cli.init(toml) def start(self): # Start up the page server and all the safekeepers @@ -637,7 +636,12 @@ def _shared_simple_env(request: Any, port_distributor) -> Iterator[ZenithEnv]: shutil.rmtree(repo_dir, ignore_errors=True) with ZenithEnvBuilder(Path(repo_dir), port_distributor) as builder: - yield builder.init_start() + env = builder.init_start() + + # For convenience in tests, create a branch from the freshly-initialized cluster. + env.zenith_cli.create_branch("empty") + + yield env @pytest.fixture(scope='function') @@ -685,7 +689,7 @@ class ZenithPageserverApiException(Exception): class ZenithPageserverHttpClient(requests.Session): - def __init__(self, port: int, auth_token: Optional[str] = None) -> None: + def __init__(self, port: int, auth_token: Optional[str] = None): super().__init__() self.port = port self.auth_token = auth_token @@ -804,59 +808,49 @@ class ZenithCli: A typed wrapper around the `zenith` CLI tool. Supports main commands via typed methods and a way to run arbitrary command directly via CLI. """ - def __init__(self, env: ZenithEnv) -> None: + def __init__(self, env: ZenithEnv): self.env = env pass - def create_tenant(self, tenant_id: Optional[uuid.UUID] = None) -> tuple[uuid.UUID, uuid.UUID]: + def create_tenant(self, tenant_id: Optional[uuid.UUID] = None) -> uuid.UUID: """ Creates a new tenant, returns its id and its initial timeline's id. """ if tenant_id is None: tenant_id = uuid.uuid4() res = self.raw_cli(['tenant', 'create', '--tenant-id', tenant_id.hex]) - - initial_timeline_id_extractor = re.compile(r"initial timeline: '(?P[^']+)'", - re.MULTILINE) - matches = initial_timeline_id_extractor.search(res.stdout) - - created_timeline_id = None - if matches is not None: - created_timeline_id = matches.group('timeline_id') - - if created_timeline_id is None: - raise Exception('could not find timeline id after `zenith tenant create` invocation') - else: - return (tenant_id, uuid.UUID(created_timeline_id)) + res.check_returncode() + return tenant_id def list_tenants(self) -> 'subprocess.CompletedProcess[str]': res = self.raw_cli(['tenant', 'list']) res.check_returncode() return res - def branch_timeline(self, - tenant_id: Optional[uuid.UUID] = None, - new_timeline_id: Optional[uuid.UUID] = None, - ancestor_timeline_id: Optional[uuid.UUID] = None, - ancestor_start_lsn: Optional[str] = None) -> uuid.UUID: + def create_branch(self, + new_branch_name: str, + ancestor_branch_name: Optional[str] = None, + tenant_id: Optional[uuid.UUID] = None, + ancestor_start_lsn: Optional[str] = None) -> uuid.UUID: cmd = [ 'timeline', 'branch', + '--name', + new_branch_name, '--tenant-id', (tenant_id or self.env.initial_tenant).hex, - '--ancestor-timeline-id', - (ancestor_timeline_id or self.env.initial_timeline).hex, + '--ancestor-branch-name', + ancestor_branch_name or self.env.default_branch_name, ] if ancestor_start_lsn is not None: cmd.extend(['--ancestor-start-lsn', ancestor_start_lsn]) - if new_timeline_id is not None: - cmd.extend(['--timeline-id', new_timeline_id.hex]) - completed_process = self.raw_cli(cmd) - completed_process.check_returncode() + res = self.raw_cli(cmd) + res.check_returncode() + create_timeline_id_extractor = re.compile(r"^Created timeline '(?P[^']+)'", re.MULTILINE) - matches = create_timeline_id_extractor.search(completed_process.stdout) + matches = create_timeline_id_extractor.search(res.stdout) created_timeline_id = None if matches is not None: @@ -875,9 +869,7 @@ class ZenithCli: res.stdout.strip().split("\n"))) return branches_cli - def init(self, config_toml: str) -> uuid.UUID: - initial_timeline = None - + def init(self, config_toml: str) -> 'subprocess.CompletedProcess[str]': with tempfile.NamedTemporaryFile(mode='w+') as tmp: tmp.write(config_toml) tmp.flush() @@ -887,18 +879,9 @@ class ZenithCli: self.env.pageserver.remote_storage, self.env.pageserver.config_override) - completed_process = self.raw_cli(cmd) - completed_process.check_returncode() - init_timeline_id_extractor = re.compile( - r'^created initial timeline (?P[^\s]+)\s', re.MULTILINE) - matches = init_timeline_id_extractor.search(completed_process.stdout) - if matches is not None: - initial_timeline = matches.group('timeline_id') - - if initial_timeline is None: - raise Exception('could not find timeline id after `zenith init` invocation') - else: - return uuid.UUID(initial_timeline) + res = self.raw_cli(cmd) + res.check_returncode() + return res def pageserver_start(self, overrides=()) -> 'subprocess.CompletedProcess[str]': start_args = ['pageserver', 'start', *overrides] @@ -930,9 +913,8 @@ class ZenithCli: def pg_create( self, - node_name: str, + branch_name: str, tenant_id: Optional[uuid.UUID] = None, - timeline_id: Optional[uuid.UUID] = None, lsn: Optional[str] = None, port: Optional[int] = None, ) -> 'subprocess.CompletedProcess[str]': @@ -940,22 +922,21 @@ class ZenithCli: 'pg', 'create', '--tenant-id', (tenant_id or self.env.initial_tenant).hex, - '--timeline-id', (timeline_id or self.env.initial_timeline).hex + '--name', + branch_name ] if lsn is not None: args.append(f'--lsn={lsn}') if port is not None: args.append(f'--port={port}') - args.append(node_name) res = self.raw_cli(args) res.check_returncode() return res def pg_start( self, - node_name: str, + branch_name: str, tenant_id: Optional[uuid.UUID] = None, - timeline_id: Optional[uuid.UUID] = None, lsn: Optional[str] = None, port: Optional[int] = None, ) -> 'subprocess.CompletedProcess[str]': @@ -964,14 +945,13 @@ class ZenithCli: 'start', '--tenant-id', (tenant_id or self.env.initial_tenant).hex, - '--timeline-id', - (timeline_id or self.env.initial_timeline).hex, + '--name', + branch_name, ] if lsn is not None: args.append(f'--lsn={lsn}') if port is not None: args.append(f'--port={port}') - args.append(node_name) res = self.raw_cli(args) res.check_returncode() @@ -979,14 +959,19 @@ class ZenithCli: def pg_stop( self, - node_name: str, + branch_name: str, tenant_id: Optional[uuid.UUID] = None, destroy=False, ) -> 'subprocess.CompletedProcess[str]': - args = ['pg', 'stop', f'--tenant-id={(tenant_id or self.env.initial_tenant).hex}'] + args = [ + 'pg', + 'stop', + f'--tenant-id={(tenant_id or self.env.initial_tenant).hex}', + '--name', + branch_name + ] if destroy: args.append('--destroy') - args.append(node_name) return self.raw_cli(args) @@ -1061,8 +1046,7 @@ class ZenithPageserver(PgProtocol): env: ZenithEnv, port: PageserverPort, remote_storage: Optional[RemoteStorage] = None, - config_override: Optional[str] = None, - enable_auth=False): + config_override: Optional[str] = None): super().__init__(host='localhost', port=port.pg, username='zenith_admin') self.env = env self.running = False @@ -1150,7 +1134,7 @@ class PgBin: self.env = os.environ.copy() self.env['LD_LIBRARY_PATH'] = os.path.join(str(pg_distrib_dir), 'lib') - def _fixpath(self, command: List[str]) -> None: + def _fixpath(self, command: List[str]): if '/' not in command[0]: command[0] = os.path.join(self.pg_bin_path, command[0]) @@ -1161,7 +1145,7 @@ class PgBin: env.update(env_add) return env - def run(self, command: List[str], env: Optional[Env] = None, cwd: Optional[str] = None) -> None: + def run(self, command: List[str], env: Optional[Env] = None, cwd: Optional[str] = None): """ Run one of the postgres binaries. @@ -1211,18 +1195,18 @@ class VanillaPostgres(PgProtocol): self.running = False self.pg_bin.run_capture(['initdb', '-D', pgdatadir]) - def configure(self, options: List[str]) -> None: + def configure(self, options: List[str]): """Append lines into postgresql.conf file.""" assert not self.running with open(os.path.join(self.pgdatadir, 'postgresql.conf'), 'a') as conf_file: conf_file.writelines(options) - def start(self) -> None: + def start(self): assert not self.running self.running = True self.pg_bin.run_capture(['pg_ctl', '-D', self.pgdatadir, 'start']) - def stop(self) -> None: + def stop(self): assert self.running self.running = False self.pg_bin.run_capture(['pg_ctl', '-D', self.pgdatadir, 'stop']) @@ -1298,15 +1282,14 @@ class Postgres(PgProtocol): self.env = env self.running = False - self.node_name: Optional[str] = None # dubious, see asserts below + self.branch_name: Optional[str] = None # dubious, see asserts below self.pgdata_dir: Optional[str] = None # Path to computenode PGDATA self.tenant_id = tenant_id - # path to conf is /pgdatadirs/tenants///postgresql.conf + # path to conf is /pgdatadirs/tenants///postgresql.conf def create( self, - node_name: str, - timeline_id: uuid.UUID, + branch_name: str, lsn: Optional[str] = None, config_lines: Optional[List[str]] = None, ) -> 'Postgres': @@ -1318,13 +1301,12 @@ class Postgres(PgProtocol): if not config_lines: config_lines = [] - self.env.zenith_cli.pg_create(node_name, - timeline_id=timeline_id, + self.env.zenith_cli.pg_create(branch_name, tenant_id=self.tenant_id, lsn=lsn, port=self.port) - self.node_name = node_name - path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id.hex / self.node_name + self.branch_name = branch_name + path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id.hex / self.branch_name self.pgdata_dir = os.path.join(self.env.repo_dir, path) if config_lines is None: @@ -1343,11 +1325,11 @@ class Postgres(PgProtocol): Returns self. """ - assert self.node_name is not None + assert self.branch_name is not None - log.info(f"Starting postgres node {self.node_name}") + log.info(f"Starting postgres node {self.branch_name}") - run_result = self.env.zenith_cli.pg_start(self.node_name, + run_result = self.env.zenith_cli.pg_start(self.branch_name, tenant_id=self.tenant_id, port=self.port) self.running = True @@ -1358,8 +1340,8 @@ class Postgres(PgProtocol): def pg_data_dir_path(self) -> str: """ Path to data directory """ - assert self.node_name - path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id.hex / self.node_name + assert self.branch_name + path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id.hex / self.branch_name return os.path.join(self.env.repo_dir, path) def pg_xact_dir_path(self) -> str: @@ -1418,8 +1400,8 @@ class Postgres(PgProtocol): """ if self.running: - assert self.node_name is not None - self.env.zenith_cli.pg_stop(self.node_name, self.tenant_id) + assert self.branch_name is not None + self.env.zenith_cli.pg_stop(self.branch_name, self.tenant_id) self.running = False return self @@ -1430,16 +1412,15 @@ class Postgres(PgProtocol): Returns self. """ - assert self.node_name is not None - self.env.zenith_cli.pg_stop(self.node_name, self.tenant_id, True) - self.node_name = None + assert self.branch_name is not None + self.env.zenith_cli.pg_stop(self.branch_name, self.tenant_id, True) + self.branch_name = None return self def create_start( self, - node_name: str, - timeline_id: uuid.UUID, + branch_name: str, lsn: Optional[str] = None, config_lines: Optional[List[str]] = None, ) -> 'Postgres': @@ -1450,8 +1431,7 @@ class Postgres(PgProtocol): """ self.create( - node_name=node_name, - timeline_id=timeline_id, + branch_name=branch_name, config_lines=config_lines, lsn=lsn, ).start() @@ -1473,9 +1453,8 @@ class PostgresFactory: self.instances: List[Postgres] = [] def create_start(self, - node_name: str = "main", + branch_name: Optional[str] = None, tenant_id: Optional[uuid.UUID] = None, - timeline_id: Optional[uuid.UUID] = None, lsn: Optional[str] = None, config_lines: Optional[List[str]] = None) -> Postgres: @@ -1488,16 +1467,14 @@ class PostgresFactory: self.instances.append(pg) return pg.create_start( - node_name=node_name, - timeline_id=timeline_id or self.env.initial_timeline, + branch_name=branch_name or self.env.default_branch_name, config_lines=config_lines, lsn=lsn, ) def create(self, - node_name: str = "main", + branch_name: Optional[str] = None, tenant_id: Optional[uuid.UUID] = None, - timeline_id: Optional[uuid.UUID] = None, lsn: Optional[str] = None, config_lines: Optional[List[str]] = None) -> Postgres: @@ -1511,8 +1488,7 @@ class PostgresFactory: self.instances.append(pg) return pg.create( - node_name=node_name, - timeline_id=timeline_id or self.env.initial_timeline, + branch_name=branch_name or self.env.default_branch_name, lsn=lsn, config_lines=config_lines, ) @@ -1616,7 +1592,7 @@ class SafekeeperMetrics: class SafekeeperHttpClient(requests.Session): - def __init__(self, port: int) -> None: + def __init__(self, port: int): super().__init__() self.port = port @@ -1743,7 +1719,7 @@ def check_restored_datadir_content(test_output_dir: str, env: ZenithEnv, pg: Pos pg.stop() # Take a basebackup from pageserver - restored_dir_path = os.path.join(env.repo_dir, f"{pg.node_name}_restored_datadir") + restored_dir_path = os.path.join(env.repo_dir, f"{pg.branch_name}_restored_datadir") mkdir_if_needed(restored_dir_path) pg_bin = PgBin(test_output_dir) diff --git a/test_runner/performance/test_bulk_tenant_create.py b/test_runner/performance/test_bulk_tenant_create.py index dda31ba692..def9753347 100644 --- a/test_runner/performance/test_bulk_tenant_create.py +++ b/test_runner/performance/test_bulk_tenant_create.py @@ -30,18 +30,16 @@ def test_bulk_tenant_create( for i in range(tenants_count): start = timeit.default_timer() - (tenant, tenant_initial_timeline_id) = env.zenith_cli.create_tenant() - new_timeline_id = env.zenith_cli.branch_timeline( - tenant_id=tenant, ancestor_timeline_id=tenant_initial_timeline_id) + tenant = env.zenith_cli.create_tenant() + env.zenith_cli.create_branch( + f'test_bulk_tenant_create_{tenants_count}_{i}_{use_wal_acceptors}', tenant_id=tenant) # FIXME: We used to start new safekeepers here. Did that make sense? Should we do it now? #if use_wal_acceptors == 'with_wa': # wa_factory.start_n_new(3) pg_tenant = env.postgres.create_start( - f"test_bulk_tenant_create_{tenants_count}_{i}_{use_wal_acceptors}", - tenant, - timeline_id=new_timeline_id) + f'test_bulk_tenant_create_{tenants_count}_{i}_{use_wal_acceptors}', tenant) end = timeit.default_timer() time_slices.append(end - start) diff --git a/test_runner/performance/test_parallel_copy_to.py b/test_runner/performance/test_parallel_copy_to.py index 0ee0a37ebb..e4388ce8e2 100644 --- a/test_runner/performance/test_parallel_copy_to.py +++ b/test_runner/performance/test_parallel_copy_to.py @@ -1,5 +1,6 @@ from io import BytesIO import asyncio +import asyncpg from fixtures.zenith_fixtures import ZenithEnv, Postgres, PgProtocol from fixtures.log_helper import log from fixtures.benchmark_fixture import MetricReport, ZenithBenchmarker diff --git a/test_runner/test_broken.py b/test_runner/test_broken.py index 994544666b..56c735e87c 100644 --- a/test_runner/test_broken.py +++ b/test_runner/test_broken.py @@ -21,8 +21,8 @@ run_broken = pytest.mark.skipif(os.environ.get('RUN_BROKEN') is None, def test_broken(zenith_simple_env: ZenithEnv, pg_bin): env = zenith_simple_env - new_timeline_id = env.zenith_cli.branch_timeline() - env.postgres.create_start("test_broken", timeline_id=new_timeline_id) + env.zenith_cli.create_branch("test_broken", "empty") + env.postgres.create_start("test_broken") log.info('postgres is running') log.info('THIS NEXT COMMAND WILL FAIL:') diff --git a/zenith/src/main.rs b/zenith/src/main.rs index dcfeb63309..fb0b230c2c 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -1,4 +1,4 @@ -use anyhow::{bail, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use clap::{App, AppSettings, Arg, ArgMatches}; use control_plane::compute::ComputeControlPlane; use control_plane::local_env; @@ -19,7 +19,7 @@ use walkeeper::defaults::{ use zenith_utils::auth::{Claims, Scope}; use zenith_utils::lsn::Lsn; use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId}; +use zenith_utils::zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId}; use zenith_utils::GIT_VERSION; use pageserver::timelines::TimelineInfo; @@ -27,6 +27,7 @@ use pageserver::timelines::TimelineInfo; // Default id of a safekeeper node, if not specified on the command line. const DEFAULT_SAFEKEEPER_ID: ZNodeId = ZNodeId(1); const DEFAULT_PAGESERVER_ID: ZNodeId = ZNodeId(1); +const DEFAULT_BRANCH_NAME: &str = "main"; fn default_conf() -> String { format!( @@ -57,7 +58,7 @@ http_port = {safekeeper_http_port} /// Timelines tree element used as a value in the HashMap. /// struct TimelineTreeEl { - /// `TimelineInfo` received from the `pageserver` via the `timeline_list` libpq API call. + /// `TimelineInfo` received from the `pageserver` via the `timeline_list` http API call. pub info: TimelineInfo, /// Holds all direct children of this timeline referenced using `timeline_id`. pub children: BTreeSet, @@ -71,16 +72,15 @@ struct TimelineTreeEl { // * Providing CLI api to the pageserver // * TODO: export/import to/from usual postgres fn main() -> Result<()> { - let pg_node_arg = Arg::new("node").help("Node name").required(true); + let branch_name_arg = Arg::new("name") + .long("name") + .short('n') + .takes_value(true) + .help("Name of the branch to be created or used as an alias for other services") + .required(false); let safekeeper_id_arg = Arg::new("id").help("safekeeper id").required(false); - let timeline_id_arg = Arg::new("timeline-id") - .long("timeline-id") - .help("Timeline id. Represented as a hexadecimal string 32 symbols length") - .takes_value(true) - .required(false); - let tenant_id_arg = Arg::new("tenant-id") .long("tenant-id") .help("Tenant id. Represented as a hexadecimal string 32 symbols length") @@ -137,15 +137,15 @@ fn main() -> Result<()> { .subcommand(App::new("branch") .about("Create a new timeline, using another timeline as a base, copying its data") .arg(tenant_id_arg.clone()) - .arg(timeline_id_arg.clone().help("Id of the new timeline, optional. If not specified, it will be generated randomly")) - .arg(Arg::new("ancestor-timeline-id").long("ancestor-timeline-id").takes_value(true) - .help("Use last Lsn of another timeline (and its data) as base when creating the new timeline").required(false)) + .arg(branch_name_arg.clone()) + .arg(Arg::new("ancestor-branch-name").long("ancestor-branch-name").takes_value(true) + .help("Use last Lsn of another timeline (and its data) as base when creating the new timeline. The timeline gets resolved by its branch name.").required(true)) .arg(Arg::new("ancestor-start-lsn").long("ancestor-start-lsn").takes_value(true) .help("When using another timeline as base, use a specific Lsn in it instead of the latest one").required(false))) .subcommand(App::new("create") .about("Create a new blank timeline") .arg(tenant_id_arg.clone()) - .arg(timeline_id_arg.clone().help("Id of the new timeline, optional. If not specified, it will be generated randomly"))) + .arg(branch_name_arg.clone())) ).subcommand( App::new("tenant") .setting(AppSettings::ArgRequiredElseHelp) @@ -189,8 +189,7 @@ fn main() -> Result<()> { .subcommand(App::new("list").arg(tenant_id_arg.clone())) .subcommand(App::new("create") .about("Create a postgres compute node") - .arg(pg_node_arg.clone()) - .arg(timeline_id_arg.clone()) + .arg(branch_name_arg.clone()) .arg(tenant_id_arg.clone()) .arg(lsn_arg.clone()) .arg(port_arg.clone()) @@ -202,14 +201,13 @@ fn main() -> Result<()> { )) .subcommand(App::new("start") .about("Start a postgres compute node.\n This command actually creates new node from scratch, but preserves existing config files") - .arg(pg_node_arg.clone()) - .arg(timeline_id_arg.clone()) + .arg(branch_name_arg.clone()) .arg(tenant_id_arg.clone()) .arg(lsn_arg.clone()) .arg(port_arg.clone())) .subcommand( App::new("stop") - .arg(pg_node_arg.clone()) + .arg(branch_name_arg.clone()) .arg(tenant_id_arg.clone()) .arg( Arg::new("destroy") @@ -242,24 +240,26 @@ fn main() -> Result<()> { handle_init(sub_args) } else { // all other commands need an existing config - let mut env = match LocalEnv::load_config() { - Ok(conf) => conf, - Err(e) => { - eprintln!("Error loading config: {}", e); - exit(1); - } - }; + let mut env = LocalEnv::load_config().context("Error loading config")?; + let original_env = env.clone(); - match sub_name { + let subcommand_result = match sub_name { "tenant" => handle_tenant(sub_args, &mut env), - "timeline" => handle_timeline(sub_args, &env), + "timeline" => handle_timeline(sub_args, &mut env), "start" => handle_start_all(sub_args, &env), "stop" => handle_stop_all(sub_args, &env), "pageserver" => handle_pageserver(sub_args, &env), "pg" => handle_pg(sub_args, &env), "safekeeper" => handle_safekeeper(sub_args, &env), _ => bail!("unexpected subcommand {}", sub_name), + }; + + if subcommand_result.is_ok() && original_env != env { + eprintln!("Subcommand had changed the config, updating"); + env.persist_config(&env.base_data_dir)?; } + + subcommand_result }; if let Err(e) = subcmd_result { eprintln!("command failed: {:#}", e); @@ -423,21 +423,6 @@ fn get_tenant_id(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow::R } } -fn get_timeline_id( - sub_match: &ArgMatches, - tenant_id: ZTenantId, - env: &local_env::LocalEnv, -) -> anyhow::Result { - if let Some(timeline_id) = sub_match.value_of("timeline-id") { - Ok(ZTimelineId::from_str(timeline_id) - .context("Failed to parse timeline id from arguments")?) - } else if let Some(&initial_timeline_id) = env.initial_timelines.get(&tenant_id) { - Ok(initial_timeline_id) - } else { - bail!("No timeline id, specify one in the subcommand's arguments"); - } -} - fn handle_init(init_match: &ArgMatches) -> Result<()> { // Create config file let toml_file: String = if let Some(config_path) = init_match.value_of("config") { @@ -491,7 +476,10 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> Re }; println!("using tenant id {}", tenant_id); let initial_timeline_id = pageserver.tenant_create(tenant_id)?; - env.initial_timelines.insert(tenant_id, initial_timeline_id); + env.branch_name_mappings.insert( + DEFAULT_BRANCH_NAME.to_owned(), + ZTenantTimelineId::new(tenant_id, initial_timeline_id), + ); println!( "tenant {} successfully created on the pageserver, initial timeline: '{}'", tenant_id, initial_timeline_id @@ -503,7 +491,7 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> Re Ok(()) } -fn handle_timeline(timeline_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { +fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) -> Result<()> { let pageserver = PageServerNode::from_env(env); match timeline_match.subcommand() { @@ -514,18 +502,28 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &local_env::LocalEnv) -> Re } Some(("create", create_match)) => { let tenant_id = get_tenant_id(create_match, env)?; - let timeline_id = get_timeline_id(create_match, tenant_id, env) - .unwrap_or_else(|_| ZTimelineId::generate()); - let timeline = pageserver.timeline_create(tenant_id, timeline_id, None, None)?; + let new_timeline_id = ZTimelineId::generate(); + let new_branch_name = create_match + .value_of("name") + .ok_or(anyhow!("No branch name provided"))?; + let timeline = pageserver.timeline_create(tenant_id, new_timeline_id, None, None)?; let last_record_lsn = match timeline { TimelineInfo::Local { last_record_lsn, .. } => last_record_lsn, TimelineInfo::Remote { .. } => { - bail!("Timeline {} was created as remote, not local", timeline_id) + bail!( + "Timeline {} was created as remote, not local", + new_timeline_id + ) } }; + env.branch_name_mappings.insert( + new_branch_name.to_string(), + ZTenantTimelineId::new(tenant_id, new_timeline_id), + ); + println!( "Created timeline '{}' at Lsn {} for tenant: {}", timeline.timeline_id(), @@ -535,18 +533,22 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &local_env::LocalEnv) -> Re } Some(("branch", branch_match)) => { let tenant_id = get_tenant_id(branch_match, env)?; - let timeline_id = get_timeline_id(branch_match, tenant_id, env) - .unwrap_or_else(|_| ZTimelineId::generate()); - let ancestor_timeline_id = match branch_match - .value_of("ancestor-timeline-id") - .map(ZTimelineId::from_str) - .transpose() - .context("Failed to parse ancestor timeline id from the request")? - .or_else(|| env.initial_timelines.get(&tenant_id).copied()) - { - Some(id) => id, - None => bail!("No ancestor timeline id provided"), - }; + let new_timeline_id = ZTimelineId::generate(); + let new_branch_name = branch_match + .value_of("name") + .ok_or(anyhow!("No branch name provided"))?; + let ancestor_branch_name = branch_match + .value_of("ancestor-branch-name") + .ok_or(anyhow!("No ancestor branch name provided"))?; + let ancestor_timeline_id = env + .branch_name_mappings + .get(ancestor_branch_name) + .ok_or(anyhow!( + "Found no timeline id for branch name '{}'", + ancestor_branch_name + ))? + .timeline_id; + let start_lsn = branch_match .value_of("ancestor-start-lsn") .map(Lsn::from_str) @@ -554,7 +556,7 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &local_env::LocalEnv) -> Re .context("Failed to parse ancestor start Lsn from the request")?; let timeline = pageserver.timeline_create( tenant_id, - timeline_id, + new_timeline_id, start_lsn, Some(ancestor_timeline_id), )?; @@ -563,16 +565,23 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &local_env::LocalEnv) -> Re TimelineInfo::Local { last_record_lsn, .. } => last_record_lsn, - TimelineInfo::Remote { .. } => { - bail!("Timeline {} was created as remote, not local", timeline_id) - } + TimelineInfo::Remote { .. } => bail!( + "Timeline {} was created as remote, not local", + new_timeline_id + ), }; + + env.branch_name_mappings.insert( + new_branch_name.to_string(), + ZTenantTimelineId::new(tenant_id, new_timeline_id), + ); + println!( "Created timeline '{}' at Lsn {} for tenant: {}. Ancestor timeline: '{}'", timeline.timeline_id(), last_record_lsn, tenant_id, - ancestor_timeline_id, + ancestor_branch_name, ); } Some((sub_name, _)) => bail!("Unexpected tenant subcommand '{}'", sub_name), @@ -592,6 +601,7 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { // All subcommands take an optional --tenant-id option let tenant_id = get_tenant_id(sub_args, env)?; + let node_name = sub_args.value_of("name").unwrap_or(DEFAULT_BRANCH_NAME); match sub_name { "list" => { @@ -630,13 +640,16 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { } } "create" => { - let node_name = sub_args.value_of("node").unwrap_or("main"); let lsn = sub_args .value_of("lsn") .map(Lsn::from_str) .transpose() .context("Failed to parse Lsn from the request")?; - let timeline_id = get_timeline_id(sub_args, tenant_id, env)?; + let timeline_id = env + .branch_name_mappings + .get(node_name) + .ok_or(anyhow!("Found no timeline id for node name {}", node_name))? + .timeline_id; let port: Option = match sub_args.value_of("port") { Some(p) => Some(p.parse()?), @@ -645,8 +658,6 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { cplane.new_node(tenant_id, node_name, timeline_id, lsn, port)?; } "start" => { - let node_name = sub_args.value_of("node").unwrap_or("main"); - let port: Option = match sub_args.value_of("port") { Some(p) => Some(p.parse()?), None => None, @@ -666,7 +677,11 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { println!("Starting existing postgres {}...", node_name); node.start(&auth_token)?; } else { - let timeline_id = get_timeline_id(sub_args, tenant_id, env)?; + let timeline_id = env + .branch_name_mappings + .get(node_name) + .ok_or(anyhow!("Found no timeline id for node name {}", node_name))? + .timeline_id; let lsn = sub_args .value_of("lsn") .map(Lsn::from_str) @@ -686,7 +701,6 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { } } "stop" => { - let node_name = sub_args.value_of("node").unwrap_or("main"); let destroy = sub_args.is_present("destroy"); let node = cplane diff --git a/zenith_utils/src/zid.rs b/zenith_utils/src/zid.rs index a740d4fb48..89708ee0df 100644 --- a/zenith_utils/src/zid.rs +++ b/zenith_utils/src/zid.rs @@ -317,7 +317,7 @@ zid_newtype!(ZTenantId); mutual_from!(ZTenantId, HexZTenantId); // A pair uniquely identifying Zenith instance. -#[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct ZTenantTimelineId { pub tenant_id: ZTenantId, pub timeline_id: ZTimelineId, From c7569dce472182016e7e2925c5fc8a9e93c407f0 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Thu, 24 Feb 2022 20:35:41 +0200 Subject: [PATCH 18/42] Allow passing initial timeline id into zenith CLI commands --- control_plane/src/storage.rs | 68 +++++++++++++--- pageserver/src/bin/pageserver.rs | 30 ++++++- pageserver/src/http/models.rs | 4 + pageserver/src/http/routes.rs | 8 +- pageserver/src/tenant_mgr.rs | 12 +-- pageserver/src/timelines.rs | 32 ++++---- test_runner/fixtures/zenith_fixtures.py | 6 +- zenith/src/main.rs | 101 ++++++++++++++++-------- 8 files changed, 192 insertions(+), 69 deletions(-) diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index 9d5a88784d..e18be05cea 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -5,7 +5,7 @@ use std::process::Command; use std::time::Duration; use std::{io, result, thread}; -use anyhow::bail; +use anyhow::{bail, Context}; use nix::errno::Errno; use nix::sys::signal::{kill, Signal}; use nix::unistd::Pid; @@ -99,9 +99,10 @@ impl PageServerNode { pub fn init( &self, - create_tenant: Option<&str>, + create_tenant: Option, + initial_timeline_id: Option, config_overrides: &[&str], - ) -> anyhow::Result<()> { + ) -> anyhow::Result { let mut cmd = Command::new(self.env.pageserver_bin()?); let id = format!("id={}", self.env.pageserver.id); @@ -138,19 +139,29 @@ impl PageServerNode { ]); } - if let Some(tenantid) = create_tenant { - args.extend(["--create-tenant", tenantid]) + let create_tenant = create_tenant.map(|id| id.to_string()); + if let Some(tenant_id) = create_tenant.as_deref() { + args.extend(["--create-tenant", tenant_id]) } - let status = fill_rust_env_vars(cmd.args(args)) - .status() - .expect("pageserver init failed"); + let initial_timeline_id_str = initial_timeline_id.map(|id| id.to_string()); + if let Some(timeline_id) = initial_timeline_id_str.as_deref() { + args.extend(["--initial-timeline-id", timeline_id]) + } - if !status.success() { + let init_output = fill_rust_env_vars(cmd.args(args)) + .output() + .context("pageserver init failed")?; + + if !init_output.status.success() { bail!("pageserver init failed"); } - Ok(()) + if let Some(initial_timeline_id) = initial_timeline_id { + Ok(initial_timeline_id) + } else { + extract_initial_timeline_id(init_output.stdout) + } } pub fn repo_path(&self) -> PathBuf { @@ -325,11 +336,16 @@ impl PageServerNode { .json()?) } - pub fn tenant_create(&self, tenantid: ZTenantId) -> Result { + pub fn tenant_create( + &self, + tenant_id: ZTenantId, + initial_timeline_id: Option, + ) -> Result { Ok(self .http_request(Method::POST, format!("{}/{}", self.http_base_url, "tenant")) .json(&TenantCreateRequest { - tenant_id: tenantid, + tenant_id, + initial_timeline_id, }) .send()? .error_from_body()? @@ -367,3 +383,31 @@ impl PageServerNode { .json()?) } } + +fn extract_initial_timeline_id(init_stdout: Vec) -> anyhow::Result { + let output_string = + String::from_utf8(init_stdout).context("Init stdout is not a valid unicode")?; + + let string_with_timeline_id = match output_string.split_once("created initial timeline ") { + Some((_, string_with_timeline_id)) => string_with_timeline_id, + None => bail!( + "Found no line with timeline id in the init output: '{}'", + output_string + ), + }; + + let timeline_id_str = match string_with_timeline_id.split_once(' ') { + Some((timeline_id_str, _)) => timeline_id_str, + None => bail!( + "Found no timeline id in the init output: '{}'", + output_string + ), + }; + + timeline_id_str.parse().with_context(|| { + format!( + "Failed to parse timeline id from string, extracted from the init output: '{}'", + timeline_id_str + ) + }) +} diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 2fa772af58..83b128dd74 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -2,7 +2,14 @@ use std::{env, path::Path, str::FromStr}; use tracing::*; -use zenith_utils::{auth::JwtAuth, logging, postgres_backend::AuthType, tcp_listener, GIT_VERSION}; +use zenith_utils::{ + auth::JwtAuth, + logging, + postgres_backend::AuthType, + tcp_listener, + zid::{ZTenantId, ZTimelineId}, + GIT_VERSION, +}; use anyhow::{bail, Context, Result}; @@ -52,6 +59,13 @@ fn main() -> Result<()> { .help("Create tenant during init") .requires("init"), ) + .arg( + Arg::new("initial-timeline-id") + .long("initial-timeline-id") + .takes_value(true) + .help("Use a specific timeline id during init and tenant creation") + .requires("create-tenant"), + ) // See `settings.md` for more details on the extra configuration patameters pageserver can process .arg( Arg::new("config-override") @@ -71,7 +85,16 @@ fn main() -> Result<()> { let cfg_file_path = workdir.join("pageserver.toml"); let init = arg_matches.is_present("init"); - let create_tenant = arg_matches.value_of("create-tenant"); + let create_tenant = arg_matches + .value_of("create-tenant") + .map(ZTenantId::from_str) + .transpose() + .context("Failed to parse tenant id from the arguments")?; + let initial_timeline_id = arg_matches + .value_of("initial-timeline-id") + .map(ZTimelineId::from_str) + .transpose() + .context("Failed to parse timeline id from the arguments")?; // Set CWD to workdir for non-daemon modes env::set_current_dir(&workdir).with_context(|| { @@ -142,7 +165,8 @@ fn main() -> Result<()> { // Create repo and exit if init was requested if init { - timelines::init_pageserver(conf, create_tenant).context("Failed to init pageserver")?; + timelines::init_pageserver(conf, create_tenant, initial_timeline_id) + .context("Failed to init pageserver")?; // write the config file std::fs::write(&cfg_file_path, toml.to_string()).with_context(|| { format!( diff --git a/pageserver/src/http/models.rs b/pageserver/src/http/models.rs index 7f95c64527..04ccb9708e 100644 --- a/pageserver/src/http/models.rs +++ b/pageserver/src/http/models.rs @@ -11,6 +11,7 @@ pub struct TimelineCreateRequest { pub tenant_id: ZTenantId, #[serde(with = "hex")] pub timeline_id: ZTimelineId, + #[serde(default)] #[serde(with = "opt_display_serde")] pub ancestor_timeline_id: Option, pub start_lsn: Option, @@ -20,6 +21,9 @@ pub struct TimelineCreateRequest { pub struct TenantCreateRequest { #[serde(with = "hex")] pub tenant_id: ZTenantId, + #[serde(default)] + #[serde(with = "opt_display_serde")] + pub initial_timeline_id: Option, } #[derive(Serialize)] diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index f332e59135..45b0c3d4be 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -214,8 +214,12 @@ async fn tenant_create_handler(mut request: Request) -> Result, ) -> Result { - let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenantid)); - let (initial_timeline_id, repo) = timelines::create_repo(conf, tenantid, wal_redo_manager)?; + let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenant_id)); + let (initial_timeline_id, repo) = + timelines::create_repo(conf, tenant_id, initial_timeline_id, wal_redo_manager)?; - match access_tenants().entry(tenantid) { - hash_map::Entry::Occupied(_) => bail!("tenant {} already exists", tenantid), + match access_tenants().entry(tenant_id) { + hash_map::Entry::Occupied(_) => bail!("tenant {} already exists", tenant_id), hash_map::Entry::Vacant(v) => { v.insert(Tenant { state: TenantState::Idle, diff --git a/pageserver/src/timelines.rs b/pageserver/src/timelines.rs index 8b4dc57342..b97ab045c7 100644 --- a/pageserver/src/timelines.rs +++ b/pageserver/src/timelines.rs @@ -9,7 +9,6 @@ use std::{ fs, path::Path, process::{Command, Stdio}, - str::FromStr, sync::Arc, }; use tracing::*; @@ -150,7 +149,11 @@ pub struct PointInTime { pub lsn: Lsn, } -pub fn init_pageserver(conf: &'static PageServerConf, create_tenant: Option<&str>) -> Result<()> { +pub fn init_pageserver( + conf: &'static PageServerConf, + create_tenant: Option, + initial_timeline_id: Option, +) -> Result<()> { // Initialize logger // use true as daemonize parameter because otherwise we pollute zenith cli output with a few pages long output of info messages let _log_file = logging::init(LOG_FILE_NAME, true)?; @@ -167,10 +170,10 @@ pub fn init_pageserver(conf: &'static PageServerConf, create_tenant: Option<&str // anymore, but I think that could still happen. let dummy_redo_mgr = Arc::new(crate::walredo::DummyRedoManager {}); - if let Some(tenantid) = create_tenant { - let tenantid = ZTenantId::from_str(tenantid)?; - println!("initializing tenantid {}", tenantid); - create_repo(conf, tenantid, dummy_redo_mgr).context("failed to create repo")?; + if let Some(tenant_id) = create_tenant { + println!("initializing tenantid {}", tenant_id); + create_repo(conf, tenant_id, initial_timeline_id, dummy_redo_mgr) + .context("failed to create repo")?; } crashsafe_dir::create_dir_all(conf.tenants_path())?; @@ -180,39 +183,40 @@ pub fn init_pageserver(conf: &'static PageServerConf, create_tenant: Option<&str pub fn create_repo( conf: &'static PageServerConf, - tenantid: ZTenantId, + tenant_id: ZTenantId, + init_timeline_id: Option, wal_redo_manager: Arc, ) -> Result<(ZTimelineId, Arc)> { - let repo_dir = conf.tenant_path(&tenantid); + let repo_dir = conf.tenant_path(&tenant_id); if repo_dir.exists() { - bail!("repo for {} already exists", tenantid) + bail!("repo for {} already exists", tenant_id) } // top-level dir may exist if we are creating it through CLI crashsafe_dir::create_dir_all(&repo_dir) .with_context(|| format!("could not create directory {}", repo_dir.display()))?; - crashsafe_dir::create_dir(conf.timelines_path(&tenantid))?; + crashsafe_dir::create_dir(conf.timelines_path(&tenant_id))?; info!("created directory structure in {}", repo_dir.display()); // create a new timeline directory - let timeline_id = ZTimelineId::generate(); - let timelinedir = conf.timeline_path(&timeline_id, &tenantid); + let timeline_id = init_timeline_id.unwrap_or_else(|| ZTimelineId::generate()); + let timelinedir = conf.timeline_path(&timeline_id, &tenant_id); crashsafe_dir::create_dir(&timelinedir)?; let repo = Arc::new(crate::layered_repository::LayeredRepository::new( conf, wal_redo_manager, - tenantid, + tenant_id, conf.remote_storage_config.is_some(), )); // Load data into pageserver // TODO To implement zenith import we need to // move data loading out of create_repo() - bootstrap_timeline(conf, tenantid, timeline_id, repo.as_ref())?; + bootstrap_timeline(conf, tenant_id, timeline_id, repo.as_ref())?; Ok((timeline_id, repo)) } diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index 9345c7f238..c283bea48e 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -869,12 +869,16 @@ class ZenithCli: res.stdout.strip().split("\n"))) return branches_cli - def init(self, config_toml: str) -> 'subprocess.CompletedProcess[str]': + def init(self, + config_toml: str, + initial_timeline_id: Optional[uuid.UUID] = None) -> 'subprocess.CompletedProcess[str]': with tempfile.NamedTemporaryFile(mode='w+') as tmp: tmp.write(config_toml) tmp.flush() cmd = ['init', f'--config={tmp.name}'] + if initial_timeline_id: + cmd.extend(['--timeline-id', initial_timeline_id.hex]) append_pageserver_param_overrides(cmd, self.env.pageserver.remote_storage, self.env.pageserver.config_override) diff --git a/zenith/src/main.rs b/zenith/src/main.rs index fb0b230c2c..34cab4b381 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -87,6 +87,12 @@ fn main() -> Result<()> { .takes_value(true) .required(false); + let timeline_id_arg = Arg::new("timeline-id") + .long("timeline-id") + .help("Timeline id. Represented as a hexadecimal string 32 symbols length") + .takes_value(true) + .required(false); + let port_arg = Arg::new("port") .long("port") .required(false) @@ -121,6 +127,7 @@ fn main() -> Result<()> { App::new("init") .about("Initialize a new Zenith repository") .arg(pageserver_config_args.clone()) + .arg(timeline_id_arg.clone().help("Use a specific timeline id when creating a tenant and its initial timeline")) .arg( Arg::new("config") .long("config") @@ -151,7 +158,10 @@ fn main() -> Result<()> { .setting(AppSettings::ArgRequiredElseHelp) .about("Manage tenants") .subcommand(App::new("list")) - .subcommand(App::new("create").arg(tenant_id_arg.clone())) + .subcommand(App::new("create") + .arg(tenant_id_arg.clone()) + .arg(timeline_id_arg.clone().help("Use a specific timeline id when creating a tenant and its initial timeline")) + ) ) .subcommand( App::new("pageserver") @@ -236,8 +246,8 @@ fn main() -> Result<()> { }; // Check for 'zenith init' command first. - let subcmd_result = if sub_name == "init" { - handle_init(sub_args) + let subcommand_result = if sub_name == "init" { + handle_init(sub_args).map(Some) } else { // all other commands need an existing config let mut env = LocalEnv::load_config().context("Error loading config")?; @@ -254,18 +264,21 @@ fn main() -> Result<()> { _ => bail!("unexpected subcommand {}", sub_name), }; - if subcommand_result.is_ok() && original_env != env { - eprintln!("Subcommand had changed the config, updating"); - env.persist_config(&env.base_data_dir)?; + if original_env != env { + subcommand_result.map(|()| Some(env)) + } else { + subcommand_result.map(|()| None) } - - subcommand_result }; - if let Err(e) = subcmd_result { - eprintln!("command failed: {:#}", e); - exit(1); - } + match subcommand_result { + Ok(Some(updated_env)) => updated_env.persist_config(&updated_env.base_data_dir)?, + Ok(None) => (), + Err(e) => { + eprintln!("command failed: {:?}", e); + exit(1); + } + } Ok(()) } @@ -411,11 +424,8 @@ fn get_timeline_infos( // Helper function to parse --tenant_id option, or get the default from config file fn get_tenant_id(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow::Result { - if let Some(tenantid_cmd) = sub_match.value_of("tenant-id") { - Ok( - ZTenantId::from_str(tenantid_cmd) - .context("Failed to parse tenant id from arguments")?, - ) + if let Some(tenant_id_from_arguments) = parse_tenant_id(sub_match).transpose() { + tenant_id_from_arguments } else if let Some(tenantid_conf) = env.default_tenant_id { Ok(ZTenantId::from(tenantid_conf)) } else { @@ -423,7 +433,25 @@ fn get_tenant_id(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow::R } } -fn handle_init(init_match: &ArgMatches) -> Result<()> { +fn parse_tenant_id(sub_match: &ArgMatches) -> anyhow::Result> { + sub_match + .value_of("tenant-id") + .map(ZTenantId::from_str) + .transpose() + .context("Failed to parse tenant id from the argument string") +} + +fn parse_timeline_id(sub_match: &ArgMatches) -> anyhow::Result> { + sub_match + .value_of("timeline-id") + .map(ZTimelineId::from_str) + .transpose() + .context("Failed to parse timeline id from the argument string") +} + +fn handle_init(init_match: &ArgMatches) -> Result { + let initial_timeline_id_arg = parse_timeline_id(init_match)?; + // Create config file let toml_file: String = if let Some(config_path) = init_match.value_of("config") { // load and parse the file @@ -439,18 +467,28 @@ fn handle_init(init_match: &ArgMatches) -> Result<()> { env.init() .context("Failed to initialize zenith repository")?; + // default_tenantid was generated by the `env.init()` call above + let initial_tenant_id = env.default_tenant_id.unwrap(); + // Call 'pageserver init'. let pageserver = PageServerNode::from_env(&env); - if let Err(e) = pageserver.init( - // default_tenantid was generated by the `env.init()` call above - Some(&ZTenantId::from(env.default_tenant_id.unwrap()).to_string()), - &pageserver_config_overrides(init_match), - ) { - eprintln!("pageserver init failed: {}", e); - exit(1); - } + let initial_timeline_id = pageserver + .init( + Some(initial_tenant_id), + initial_timeline_id_arg, + &pageserver_config_overrides(init_match), + ) + .unwrap_or_else(|e| { + eprintln!("pageserver init failed: {}", e); + exit(1); + }); - Ok(()) + env.branch_name_mappings.insert( + DEFAULT_BRANCH_NAME.to_owned(), + ZTenantTimelineId::new(initial_tenant_id, initial_timeline_id), + ); + + Ok(env) } fn pageserver_config_overrides(init_match: &ArgMatches) -> Vec<&str> { @@ -470,12 +508,11 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> Re } } Some(("create", create_match)) => { - let tenant_id = match create_match.value_of("tenant-id") { - Some(id) => ZTenantId::from_str(id)?, - None => ZTenantId::generate(), - }; + let tenant_id = parse_tenant_id(create_match)?.unwrap_or_else(|| ZTenantId::generate()); println!("using tenant id {}", tenant_id); - let initial_timeline_id = pageserver.tenant_create(tenant_id)?; + let initial_timeline_id_argument = parse_timeline_id(create_match)?; + let initial_timeline_id = + pageserver.tenant_create(tenant_id, initial_timeline_id_argument)?; env.branch_name_mappings.insert( DEFAULT_BRANCH_NAME.to_owned(), ZTenantTimelineId::new(tenant_id, initial_timeline_id), From 7b5482bac0f052913a056649d839792f457f1019 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Thu, 24 Feb 2022 23:28:30 +0200 Subject: [PATCH 19/42] Properly store the branch name mappings --- control_plane/src/compute.rs | 2 +- control_plane/src/local_env.rs | 32 +++++- pageserver/src/timelines.rs | 2 +- .../batch_others/test_readonly_node.py | 16 ++- .../batch_others/test_remote_storage.py | 4 +- .../batch_others/test_tenant_relocation.py | 4 +- test_runner/fixtures/zenith_fixtures.py | 79 +++++++------ .../performance/test_bulk_tenant_create.py | 2 +- zenith/src/main.rs | 105 ++++++++++-------- zenith_utils/src/zid.rs | 2 +- 10 files changed, 158 insertions(+), 90 deletions(-) diff --git a/control_plane/src/compute.rs b/control_plane/src/compute.rs index e8baffdc74..64cd46fef6 100644 --- a/control_plane/src/compute.rs +++ b/control_plane/src/compute.rs @@ -37,7 +37,7 @@ impl ComputeControlPlane { // pgdatadirs // |- tenants // | |- - // | | |- + // | | |- pub fn load(env: LocalEnv) -> Result { let pageserver = Arc::new(PageServerNode::from_env(&env)); diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 9b50a6b9e4..a9352bdfcc 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -12,7 +12,8 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use zenith_utils::auth::{encode_from_key_file, Claims, Scope}; use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{HexZTenantId, ZNodeId, ZTenantId, ZTenantTimelineId}; +use zenith_utils::zid::ZTimelineId; +use zenith_utils::zid::{HexZTenantId, ZNodeId, ZTenantId}; use crate::safekeeper::SafekeeperNode; @@ -62,7 +63,10 @@ pub struct LocalEnv { /// Every tenant has a first timeline created for it, currently the only one ancestor-less for this tenant. /// It is used as a default timeline for branching, if no ancestor timeline is specified. #[serde(default)] - pub branch_name_mappings: HashMap, + // A `HashMap>` would be more appropriate here, + // but deserialization into a generic toml object as `toml::Value::try_from` fails with an error. + // https://toml.io/en/v1.0.0 does not contain a concept of "a table inside another table". + branch_name_mappings: HashMap>, } #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] @@ -149,6 +153,30 @@ impl LocalEnv { self.base_data_dir.join("safekeepers").join(data_dir_name) } + pub fn register_branch_mapping( + &mut self, + branch_name: String, + tenant_id: ZTenantId, + timeline_id: ZTimelineId, + ) { + self.branch_name_mappings + .entry(branch_name) + .or_default() + .push((tenant_id, timeline_id)); + } + + pub fn get_branch_timeline_id( + &self, + branch_name: &str, + tenant_id: ZTenantId, + ) -> Option { + self.branch_name_mappings + .get(branch_name)? + .iter() + .find(|(mapped_tenant_id, _)| mapped_tenant_id == &tenant_id) + .map(|&(_, timeline_id)| timeline_id) + } + /// Create a LocalEnv from a config file. /// /// Unlike 'load_config', this function fills in any defaults that are missing diff --git a/pageserver/src/timelines.rs b/pageserver/src/timelines.rs index b97ab045c7..786e102747 100644 --- a/pageserver/src/timelines.rs +++ b/pageserver/src/timelines.rs @@ -201,7 +201,7 @@ pub fn create_repo( info!("created directory structure in {}", repo_dir.display()); // create a new timeline directory - let timeline_id = init_timeline_id.unwrap_or_else(|| ZTimelineId::generate()); + let timeline_id = init_timeline_id.unwrap_or_else(ZTimelineId::generate); let timelinedir = conf.timeline_path(&timeline_id, &tenant_id); crashsafe_dir::create_dir(&timelinedir)?; diff --git a/test_runner/batch_others/test_readonly_node.py b/test_runner/batch_others/test_readonly_node.py index 5d5949add6..808ee62def 100644 --- a/test_runner/batch_others/test_readonly_node.py +++ b/test_runner/batch_others/test_readonly_node.py @@ -52,10 +52,14 @@ def test_readonly_node(zenith_simple_env: ZenithEnv): log.info('LSN after 400100 rows: ' + lsn_c) # Create first read-only node at the point where only 100 rows were inserted - pg_hundred = env.postgres.create_start("test_readonly_node_hundred", lsn=lsn_a) + pg_hundred = env.postgres.create_start(branch_name='test_readonly_node', + node_name='test_readonly_node_hundred', + lsn=lsn_a) # And another at the point where 200100 rows were inserted - pg_more = env.postgres.create_start("test_readonly_node_more", lsn=lsn_b) + pg_more = env.postgres.create_start(branch_name='test_readonly_node', + node_name='test_readonly_node_more', + lsn=lsn_b) # On the 'hundred' node, we should see only 100 rows hundred_pg_conn = pg_hundred.connect() @@ -74,7 +78,9 @@ def test_readonly_node(zenith_simple_env: ZenithEnv): assert main_cur.fetchone() == (400100, ) # Check creating a node at segment boundary - pg = env.postgres.create_start("test_branch_segment_boundary", lsn='0/3000000') + pg = env.postgres.create_start(branch_name='test_readonly_node', + node_name='test_branch_segment_boundary', + lsn='0/3000000') cur = pg.connect().cursor() cur.execute('SELECT 1') assert cur.fetchone() == (1, ) @@ -82,4 +88,6 @@ def test_readonly_node(zenith_simple_env: ZenithEnv): # Create node at pre-initdb lsn with pytest.raises(Exception, match="invalid basebackup lsn"): # compute node startup with invalid LSN should fail - env.zenith_cli.pg_start("test_readonly_node_preinitdb", lsn="0/42") + env.postgres.create_start(branch_name='test_readonly_node', + node_name='test_readonly_node_preinitdb', + lsn='0/42') diff --git a/test_runner/batch_others/test_remote_storage.py b/test_runner/batch_others/test_remote_storage.py index 61feb1a5bd..abd06bf5e9 100644 --- a/test_runner/batch_others/test_remote_storage.py +++ b/test_runner/batch_others/test_remote_storage.py @@ -43,7 +43,7 @@ def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder, ##### First start, insert secret data and upload it to the remote storage env = zenith_env_builder.init_start() - pg = env.postgres.create_start() + pg = env.postgres.create_start('main') tenant_id = pg.safe_psql("show zenith.zenith_tenant")[0][0] timeline_id = pg.safe_psql("show zenith.zenith_timeline")[0][0] @@ -94,7 +94,7 @@ def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder, log.debug("still waiting") time.sleep(1) - pg = env.postgres.create_start() + pg = env.postgres.create_start('main') with closing(pg.connect()) as conn: with conn.cursor() as cur: cur.execute(f'SELECT secret FROM t1 WHERE id = {data_id};') diff --git a/test_runner/batch_others/test_tenant_relocation.py b/test_runner/batch_others/test_tenant_relocation.py index a1286adfb0..7a9d478f16 100644 --- a/test_runner/batch_others/test_tenant_relocation.py +++ b/test_runner/batch_others/test_tenant_relocation.py @@ -132,7 +132,9 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder, env.zenith_cli.create_branch('test_tenant_relocation', tenant_id=tenant) - tenant_pg = env.postgres.create_start("test_tenant_relocation", tenant_id=tenant) + tenant_pg = env.postgres.create_start(branch_name='main', + node_name='test_tenant_relocation', + tenant_id=tenant) # insert some data with closing(tenant_pg.connect()) as conn: diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index c283bea48e..04b795b244 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -835,7 +835,7 @@ class ZenithCli: cmd = [ 'timeline', 'branch', - '--name', + '--branch-name', new_branch_name, '--tenant-id', (tenant_id or self.env.initial_tenant).hex, @@ -918,6 +918,7 @@ class ZenithCli: def pg_create( self, branch_name: str, + node_name: Optional[str] = None, tenant_id: Optional[uuid.UUID] = None, lsn: Optional[str] = None, port: Optional[int] = None, @@ -925,21 +926,25 @@ class ZenithCli: args = [ 'pg', 'create', - '--tenant-id', (tenant_id or self.env.initial_tenant).hex, - '--name', - branch_name + '--tenant-id', + (tenant_id or self.env.initial_tenant).hex, + '--branch-name', + branch_name, ] if lsn is not None: - args.append(f'--lsn={lsn}') + args.extend(['--lsn', lsn]) if port is not None: - args.append(f'--port={port}') + args.extend(['--port', str(port)]) + if node_name is not None: + args.append(node_name) + res = self.raw_cli(args) res.check_returncode() return res def pg_start( self, - branch_name: str, + node_name: str, tenant_id: Optional[uuid.UUID] = None, lsn: Optional[str] = None, port: Optional[int] = None, @@ -949,13 +954,13 @@ class ZenithCli: 'start', '--tenant-id', (tenant_id or self.env.initial_tenant).hex, - '--name', - branch_name, ] if lsn is not None: args.append(f'--lsn={lsn}') if port is not None: args.append(f'--port={port}') + if node_name is not None: + args.append(node_name) res = self.raw_cli(args) res.check_returncode() @@ -963,19 +968,20 @@ class ZenithCli: def pg_stop( self, - branch_name: str, + node_name: str, tenant_id: Optional[uuid.UUID] = None, destroy=False, ) -> 'subprocess.CompletedProcess[str]': args = [ 'pg', 'stop', - f'--tenant-id={(tenant_id or self.env.initial_tenant).hex}', - '--name', - branch_name + '--tenant-id', + (tenant_id or self.env.initial_tenant).hex, ] if destroy: args.append('--destroy') + if node_name is not None: + args.append(node_name) return self.raw_cli(args) @@ -1286,14 +1292,15 @@ class Postgres(PgProtocol): self.env = env self.running = False - self.branch_name: Optional[str] = None # dubious, see asserts below + self.node_name: Optional[str] = None # dubious, see asserts below self.pgdata_dir: Optional[str] = None # Path to computenode PGDATA self.tenant_id = tenant_id - # path to conf is /pgdatadirs/tenants///postgresql.conf + # path to conf is /pgdatadirs/tenants///postgresql.conf def create( self, branch_name: str, + node_name: Optional[str] = None, lsn: Optional[str] = None, config_lines: Optional[List[str]] = None, ) -> 'Postgres': @@ -1305,12 +1312,13 @@ class Postgres(PgProtocol): if not config_lines: config_lines = [] + self.node_name = node_name or f'{branch_name}_pg_node' self.env.zenith_cli.pg_create(branch_name, + node_name=self.node_name, tenant_id=self.tenant_id, lsn=lsn, port=self.port) - self.branch_name = branch_name - path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id.hex / self.branch_name + path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id.hex / self.node_name self.pgdata_dir = os.path.join(self.env.repo_dir, path) if config_lines is None: @@ -1329,11 +1337,11 @@ class Postgres(PgProtocol): Returns self. """ - assert self.branch_name is not None + assert self.node_name is not None - log.info(f"Starting postgres node {self.branch_name}") + log.info(f"Starting postgres node {self.node_name}") - run_result = self.env.zenith_cli.pg_start(self.branch_name, + run_result = self.env.zenith_cli.pg_start(self.node_name, tenant_id=self.tenant_id, port=self.port) self.running = True @@ -1344,8 +1352,8 @@ class Postgres(PgProtocol): def pg_data_dir_path(self) -> str: """ Path to data directory """ - assert self.branch_name - path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id.hex / self.branch_name + assert self.node_name + path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id.hex / self.node_name return os.path.join(self.env.repo_dir, path) def pg_xact_dir_path(self) -> str: @@ -1404,8 +1412,8 @@ class Postgres(PgProtocol): """ if self.running: - assert self.branch_name is not None - self.env.zenith_cli.pg_stop(self.branch_name, self.tenant_id) + assert self.node_name is not None + self.env.zenith_cli.pg_stop(self.node_name, self.tenant_id) self.running = False return self @@ -1416,15 +1424,16 @@ class Postgres(PgProtocol): Returns self. """ - assert self.branch_name is not None - self.env.zenith_cli.pg_stop(self.branch_name, self.tenant_id, True) - self.branch_name = None + assert self.node_name is not None + self.env.zenith_cli.pg_stop(self.node_name, self.tenant_id, True) + self.node_name = None return self def create_start( self, branch_name: str, + node_name: Optional[str] = None, lsn: Optional[str] = None, config_lines: Optional[List[str]] = None, ) -> 'Postgres': @@ -1436,6 +1445,7 @@ class Postgres(PgProtocol): self.create( branch_name=branch_name, + node_name=node_name, config_lines=config_lines, lsn=lsn, ).start() @@ -1457,7 +1467,8 @@ class PostgresFactory: self.instances: List[Postgres] = [] def create_start(self, - branch_name: Optional[str] = None, + branch_name: str, + node_name: Optional[str] = None, tenant_id: Optional[uuid.UUID] = None, lsn: Optional[str] = None, config_lines: Optional[List[str]] = None) -> Postgres: @@ -1471,13 +1482,15 @@ class PostgresFactory: self.instances.append(pg) return pg.create_start( - branch_name=branch_name or self.env.default_branch_name, + branch_name=branch_name, + node_name=node_name, config_lines=config_lines, lsn=lsn, ) def create(self, - branch_name: Optional[str] = None, + branch_name: str, + node_name: Optional[str] = None, tenant_id: Optional[uuid.UUID] = None, lsn: Optional[str] = None, config_lines: Optional[List[str]] = None) -> Postgres: @@ -1492,7 +1505,8 @@ class PostgresFactory: self.instances.append(pg) return pg.create( - branch_name=branch_name or self.env.default_branch_name, + branch_name=branch_name, + node_name=node_name, lsn=lsn, config_lines=config_lines, ) @@ -1713,6 +1727,7 @@ def list_files_to_compare(pgdata_dir: str): # pg is the existing and running compute node, that we want to compare with a basebackup def check_restored_datadir_content(test_output_dir: str, env: ZenithEnv, pg: Postgres): + # Get the timeline ID. We need it for the 'basebackup' command with closing(pg.connect()) as conn: with conn.cursor() as cur: @@ -1723,7 +1738,7 @@ def check_restored_datadir_content(test_output_dir: str, env: ZenithEnv, pg: Pos pg.stop() # Take a basebackup from pageserver - restored_dir_path = os.path.join(env.repo_dir, f"{pg.branch_name}_restored_datadir") + restored_dir_path = os.path.join(env.repo_dir, f"{pg.node_name}_restored_datadir") mkdir_if_needed(restored_dir_path) pg_bin = PgBin(test_output_dir) diff --git a/test_runner/performance/test_bulk_tenant_create.py b/test_runner/performance/test_bulk_tenant_create.py index def9753347..2430eec33e 100644 --- a/test_runner/performance/test_bulk_tenant_create.py +++ b/test_runner/performance/test_bulk_tenant_create.py @@ -39,7 +39,7 @@ def test_bulk_tenant_create( # wa_factory.start_n_new(3) pg_tenant = env.postgres.create_start( - f'test_bulk_tenant_create_{tenants_count}_{i}_{use_wal_acceptors}', tenant) + f'test_bulk_tenant_create_{tenants_count}_{i}_{use_wal_acceptors}', tenant_id=tenant) end = timeit.default_timer() time_slices.append(end - start) diff --git a/zenith/src/main.rs b/zenith/src/main.rs index 34cab4b381..9a578f79f1 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -19,7 +19,7 @@ use walkeeper::defaults::{ use zenith_utils::auth::{Claims, Scope}; use zenith_utils::lsn::Lsn; use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId}; +use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId}; use zenith_utils::GIT_VERSION; use pageserver::timelines::TimelineInfo; @@ -72,13 +72,17 @@ struct TimelineTreeEl { // * Providing CLI api to the pageserver // * TODO: export/import to/from usual postgres fn main() -> Result<()> { - let branch_name_arg = Arg::new("name") - .long("name") - .short('n') + let branch_name_arg = Arg::new("branch-name") + .long("branch-name") .takes_value(true) .help("Name of the branch to be created or used as an alias for other services") .required(false); + let pg_node_arg = Arg::new("node").help("Postgres node name").required(false); + let safekeeper_node_arg = Arg::new("node") + .help("Safekeeper node name") + .required(false); + let safekeeper_id_arg = Arg::new("id").help("safekeeper id").required(false); let tenant_id_arg = Arg::new("tenant-id") @@ -199,6 +203,7 @@ fn main() -> Result<()> { .subcommand(App::new("list").arg(tenant_id_arg.clone())) .subcommand(App::new("create") .about("Create a postgres compute node") + .arg(pg_node_arg.clone()) .arg(branch_name_arg.clone()) .arg(tenant_id_arg.clone()) .arg(lsn_arg.clone()) @@ -211,20 +216,20 @@ fn main() -> Result<()> { )) .subcommand(App::new("start") .about("Start a postgres compute node.\n This command actually creates new node from scratch, but preserves existing config files") - .arg(branch_name_arg.clone()) + .arg(pg_node_arg.clone()) .arg(tenant_id_arg.clone()) .arg(lsn_arg.clone()) .arg(port_arg.clone())) .subcommand( App::new("stop") - .arg(branch_name_arg.clone()) - .arg(tenant_id_arg.clone()) - .arg( - Arg::new("destroy") - .help("Also delete data directory (now optional, should be default in future)") - .long("destroy") - .required(false) - ) + .arg(pg_node_arg.clone()) + .arg(tenant_id_arg.clone()) + .arg( + Arg::new("destroy") + .help("Also delete data directory (now optional, should be default in future)") + .long("destroy") + .required(false) + ) ) ) @@ -483,9 +488,10 @@ fn handle_init(init_match: &ArgMatches) -> Result { exit(1); }); - env.branch_name_mappings.insert( + env.register_branch_mapping( DEFAULT_BRANCH_NAME.to_owned(), - ZTenantTimelineId::new(initial_tenant_id, initial_timeline_id), + initial_tenant_id, + initial_timeline_id, ); Ok(env) @@ -508,14 +514,15 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> Re } } Some(("create", create_match)) => { - let tenant_id = parse_tenant_id(create_match)?.unwrap_or_else(|| ZTenantId::generate()); + let tenant_id = parse_tenant_id(create_match)?.unwrap_or_else(ZTenantId::generate); println!("using tenant id {}", tenant_id); let initial_timeline_id_argument = parse_timeline_id(create_match)?; let initial_timeline_id = pageserver.tenant_create(tenant_id, initial_timeline_id_argument)?; - env.branch_name_mappings.insert( + env.register_branch_mapping( DEFAULT_BRANCH_NAME.to_owned(), - ZTenantTimelineId::new(tenant_id, initial_timeline_id), + tenant_id, + initial_timeline_id, ); println!( "tenant {} successfully created on the pageserver, initial timeline: '{}'", @@ -541,7 +548,7 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - let tenant_id = get_tenant_id(create_match, env)?; let new_timeline_id = ZTimelineId::generate(); let new_branch_name = create_match - .value_of("name") + .value_of("branch-name") .ok_or(anyhow!("No branch name provided"))?; let timeline = pageserver.timeline_create(tenant_id, new_timeline_id, None, None)?; @@ -556,10 +563,7 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - ) } }; - env.branch_name_mappings.insert( - new_branch_name.to_string(), - ZTenantTimelineId::new(tenant_id, new_timeline_id), - ); + env.register_branch_mapping(new_branch_name.to_string(), tenant_id, new_timeline_id); println!( "Created timeline '{}' at Lsn {} for tenant: {}", @@ -572,19 +576,19 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - let tenant_id = get_tenant_id(branch_match, env)?; let new_timeline_id = ZTimelineId::generate(); let new_branch_name = branch_match - .value_of("name") + .value_of("branch-name") .ok_or(anyhow!("No branch name provided"))?; let ancestor_branch_name = branch_match .value_of("ancestor-branch-name") .ok_or(anyhow!("No ancestor branch name provided"))?; let ancestor_timeline_id = env - .branch_name_mappings - .get(ancestor_branch_name) - .ok_or(anyhow!( - "Found no timeline id for branch name '{}'", - ancestor_branch_name - ))? - .timeline_id; + .get_branch_timeline_id(ancestor_branch_name, tenant_id) + .ok_or_else(|| { + anyhow!( + "Found no timeline id for branch name '{}'", + ancestor_branch_name + ) + })?; let start_lsn = branch_match .value_of("ancestor-start-lsn") @@ -608,10 +612,7 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - ), }; - env.branch_name_mappings.insert( - new_branch_name.to_string(), - ZTenantTimelineId::new(tenant_id, new_timeline_id), - ); + env.register_branch_mapping(new_branch_name.to_string(), tenant_id, new_timeline_id); println!( "Created timeline '{}' at Lsn {} for tenant: {}. Ancestor timeline: '{}'", @@ -638,7 +639,6 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { // All subcommands take an optional --tenant-id option let tenant_id = get_tenant_id(sub_args, env)?; - let node_name = sub_args.value_of("name").unwrap_or(DEFAULT_BRANCH_NAME); match sub_name { "list" => { @@ -677,28 +677,37 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { } } "create" => { + let branch_name = sub_args + .value_of("branch-name") + .unwrap_or(DEFAULT_BRANCH_NAME); + let node_name = sub_args + .value_of("node") + .map(ToString::to_string) + .unwrap_or_else(|| format!("{}_node", branch_name)); + let lsn = sub_args .value_of("lsn") .map(Lsn::from_str) .transpose() .context("Failed to parse Lsn from the request")?; let timeline_id = env - .branch_name_mappings - .get(node_name) - .ok_or(anyhow!("Found no timeline id for node name {}", node_name))? - .timeline_id; + .get_branch_timeline_id(branch_name, tenant_id) + .ok_or_else(|| anyhow!("Found no timeline id for branch name '{}'", branch_name))?; let port: Option = match sub_args.value_of("port") { Some(p) => Some(p.parse()?), None => None, }; - cplane.new_node(tenant_id, node_name, timeline_id, lsn, port)?; + cplane.new_node(tenant_id, &node_name, timeline_id, lsn, port)?; } "start" => { let port: Option = match sub_args.value_of("port") { Some(p) => Some(p.parse()?), None => None, }; + let node_name = sub_args + .value_of("node") + .ok_or_else(|| anyhow!("No node name was provided to start"))?; let node = cplane.nodes.get(&(tenant_id, node_name.to_owned())); @@ -714,11 +723,14 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { println!("Starting existing postgres {}...", node_name); node.start(&auth_token)?; } else { + let branch_name = sub_args + .value_of("branch-name") + .unwrap_or(DEFAULT_BRANCH_NAME); let timeline_id = env - .branch_name_mappings - .get(node_name) - .ok_or(anyhow!("Found no timeline id for node name {}", node_name))? - .timeline_id; + .get_branch_timeline_id(branch_name, tenant_id) + .ok_or_else(|| { + anyhow!("Found no timeline id for branch name '{}'", branch_name) + })?; let lsn = sub_args .value_of("lsn") .map(Lsn::from_str) @@ -738,6 +750,9 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { } } "stop" => { + let node_name = sub_args + .value_of("node") + .ok_or_else(|| anyhow!("No node name was provided to stop"))?; let destroy = sub_args.is_present("destroy"); let node = cplane diff --git a/zenith_utils/src/zid.rs b/zenith_utils/src/zid.rs index 89708ee0df..a740d4fb48 100644 --- a/zenith_utils/src/zid.rs +++ b/zenith_utils/src/zid.rs @@ -317,7 +317,7 @@ zid_newtype!(ZTenantId); mutual_from!(ZTenantId, HexZTenantId); // A pair uniquely identifying Zenith instance. -#[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash)] pub struct ZTenantTimelineId { pub tenant_id: ZTenantId, pub timeline_id: ZTimelineId, From a5e10c4f64d87d286e46c2425699e90f2a5d0baa Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Sun, 27 Feb 2022 23:57:00 +0200 Subject: [PATCH 20/42] Tidy up pageserver's endpoints --- README.md | 18 ++--- control_plane/src/local_env.rs | 46 ++++++++++-- control_plane/src/storage.rs | 32 ++++---- pageserver/src/http/models.rs | 22 ++++-- pageserver/src/http/openapi_spec.yml | 83 +++++++++------------ pageserver/src/http/routes.rs | 35 +++++---- pageserver/src/tenant_mgr.rs | 9 ++- pageserver/src/timelines.rs | 11 ++- test_runner/batch_others/test_auth.py | 13 ++-- test_runner/batch_others/test_zenith_cli.py | 18 ++--- test_runner/fixtures/zenith_fixtures.py | 63 ++++++++++------ zenith/src/main.rs | 68 +++++++++++------ 12 files changed, 246 insertions(+), 172 deletions(-) diff --git a/README.md b/README.md index 8dd407f41a..c8acf526b9 100644 --- a/README.md +++ b/README.md @@ -57,12 +57,12 @@ pageserver init succeeded Starting pageserver at 'localhost:64000' in '.zenith' Pageserver started initializing for single for 7676 -Starting safekeeper at 'localhost:5454' in '.zenith/safekeepers/single' +Starting safekeeper at '127.0.0.1:5454' in '.zenith/safekeepers/single' Safekeeper started # start postgres compute node > ./target/debug/zenith pg start main -Starting new postgres main on main... +Starting new postgres main on timeline 5b014a9e41b4b63ce1a1febc04503636 ... Extracting base backup to create postgres instance: path=.zenith/pgdatadirs/tenants/c03ba6b7ad4c5e9cf556f059ade44229/main port=55432 Starting postgres node at 'host=127.0.0.1 port=55432 user=zenith_admin dbname=postgres' waiting for server to start.... done @@ -70,8 +70,8 @@ server started # check list of running postgres instances > ./target/debug/zenith pg list -BRANCH ADDRESS LSN STATUS -main 127.0.0.1:55432 0/1609610 running +NODE ADDRESS TIMELINES BRANCH NAME LSN STATUS +main 127.0.0.1:55432 5b014a9e41b4b63ce1a1febc04503636 main 0/1609610 running ``` 4. Now it is possible to connect to postgres and run some queries: @@ -91,13 +91,13 @@ postgres=# select * from t; 5. And create branches and run postgres on them: ```sh # create branch named migration_check -> ./target/debug/zenith branch migration_check main -Created branch 'migration_check' at 0/1609610 +> ./target/debug/zenith timeline branch --branch-name migration_check +Created timeline '0e9331cad6efbafe6a88dd73ae21a5c9' at Lsn 0/16F5830 for tenant: c03ba6b7ad4c5e9cf556f059ade44229. Ancestor timeline: 'main' # check branches tree -> ./target/debug/zenith branch - main - ┗━ @0/1609610: migration_check +> ./target/debug/zenith timeline list + main [5b014a9e41b4b63ce1a1febc04503636] + ┗━ @0/1609610: migration_check [0e9331cad6efbafe6a88dd73ae21a5c9] # start postgres on that branch > ./target/debug/zenith pg start migration_check diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index a9352bdfcc..9278a9df5a 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -12,6 +12,7 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use zenith_utils::auth::{encode_from_key_file, Claims, Scope}; use zenith_utils::postgres_backend::AuthType; +use zenith_utils::zid::ZTenantTimelineId; use zenith_utils::zid::ZTimelineId; use zenith_utils::zid::{HexZTenantId, ZNodeId, ZTenantId}; @@ -60,8 +61,7 @@ pub struct LocalEnv { #[serde(default)] pub safekeepers: Vec, - /// Every tenant has a first timeline created for it, currently the only one ancestor-less for this tenant. - /// It is used as a default timeline for branching, if no ancestor timeline is specified. + /// Keep human-readable aliases in memory (and persist them to config), to hind ZId hex strings from the user. #[serde(default)] // A `HashMap>` would be more appropriate here, // but deserialization into a generic toml object as `toml::Value::try_from` fails with an error. @@ -158,11 +158,31 @@ impl LocalEnv { branch_name: String, tenant_id: ZTenantId, timeline_id: ZTimelineId, - ) { - self.branch_name_mappings - .entry(branch_name) - .or_default() - .push((tenant_id, timeline_id)); + ) -> anyhow::Result<()> { + let existing_values = self + .branch_name_mappings + .entry(branch_name.clone()) + .or_default(); + + let existing_ids = existing_values + .iter() + .find(|(existing_tenant_id, _)| existing_tenant_id == &tenant_id); + + if let Some((_, old_timeline_id)) = existing_ids { + if old_timeline_id == &timeline_id { + Ok(()) + } else { + bail!( + "branch '{}' is already mapped to timeline {}, cannot map to another timeline {}", + branch_name, + old_timeline_id, + timeline_id + ); + } + } else { + existing_values.push((tenant_id, timeline_id)); + Ok(()) + } } pub fn get_branch_timeline_id( @@ -177,6 +197,18 @@ impl LocalEnv { .map(|&(_, timeline_id)| timeline_id) } + pub fn timeline_name_mappings(&self) -> HashMap { + self.branch_name_mappings + .iter() + .map(|(name, tenant_timelines)| { + tenant_timelines.iter().map(|&(tenant_id, timeline_id)| { + (ZTenantTimelineId::new(tenant_id, timeline_id), name.clone()) + }) + }) + .flatten() + .collect() + } + /// Create a LocalEnv from a config file. /// /// Unlike 'load_config', this function fills in any defaults that are missing diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index e18be05cea..c2b99972e7 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -9,7 +9,7 @@ use anyhow::{bail, Context}; use nix::errno::Errno; use nix::sys::signal::{kill, Signal}; use nix::unistd::Pid; -use pageserver::http::models::{TenantCreateRequest, TimelineCreateRequest}; +use pageserver::http::models::{TenantCreateRequest, TenantCreateResponse, TimelineCreateRequest}; use pageserver::timelines::TimelineInfo; use postgres::{Config, NoTls}; use reqwest::blocking::{Client, RequestBuilder, Response}; @@ -322,7 +322,7 @@ impl PageServerNode { } pub fn check_status(&self) -> Result<()> { - self.http_request(Method::GET, format!("{}/{}", self.http_base_url, "status")) + self.http_request(Method::GET, format!("{}/status", self.http_base_url)) .send()? .error_from_body()?; Ok(()) @@ -330,7 +330,7 @@ impl PageServerNode { pub fn tenant_list(&self) -> Result> { Ok(self - .http_request(Method::GET, format!("{}/{}", self.http_base_url, "tenant")) + .http_request(Method::GET, format!("{}/tenant", self.http_base_url)) .send()? .error_from_body()? .json()?) @@ -338,13 +338,13 @@ impl PageServerNode { pub fn tenant_create( &self, - tenant_id: ZTenantId, + new_tenant_id: Option, initial_timeline_id: Option, - ) -> Result { + ) -> Result { Ok(self - .http_request(Method::POST, format!("{}/{}", self.http_base_url, "tenant")) + .http_request(Method::POST, format!("{}/tenant", self.http_base_url)) .json(&TenantCreateRequest { - tenant_id, + new_tenant_id, initial_timeline_id, }) .send()? @@ -352,11 +352,11 @@ impl PageServerNode { .json()?) } - pub fn timeline_list(&self, tenantid: &ZTenantId) -> Result> { + pub fn timeline_list(&self, tenant_id: &ZTenantId) -> Result> { Ok(self .http_request( Method::GET, - format!("{}/timeline/{}", self.http_base_url, tenantid), + format!("{}/tenant/{}/timeline", self.http_base_url, tenant_id), ) .send()? .error_from_body()? @@ -366,16 +366,18 @@ impl PageServerNode { pub fn timeline_create( &self, tenant_id: ZTenantId, - timeline_id: ZTimelineId, - start_lsn: Option, + new_timeline_id: Option, + ancestor_start_lsn: Option, ancestor_timeline_id: Option, ) -> Result { Ok(self - .http_request(Method::POST, format!("{}/timeline", self.http_base_url)) + .http_request( + Method::POST, + format!("{}/tenant/{}/timeline", self.http_base_url, tenant_id), + ) .json(&TimelineCreateRequest { - tenant_id, - timeline_id, - start_lsn, + new_timeline_id, + ancestor_start_lsn, ancestor_timeline_id, }) .send()? diff --git a/pageserver/src/http/models.rs b/pageserver/src/http/models.rs index 04ccb9708e..9b321744eb 100644 --- a/pageserver/src/http/models.rs +++ b/pageserver/src/http/models.rs @@ -7,25 +7,33 @@ use zenith_utils::{ #[derive(Serialize, Deserialize)] pub struct TimelineCreateRequest { - #[serde(with = "hex")] - pub tenant_id: ZTenantId, - #[serde(with = "hex")] - pub timeline_id: ZTimelineId, + #[serde(default)] + #[serde(with = "opt_display_serde")] + pub new_timeline_id: Option, #[serde(default)] #[serde(with = "opt_display_serde")] pub ancestor_timeline_id: Option, - pub start_lsn: Option, + pub ancestor_start_lsn: Option, } #[derive(Serialize, Deserialize)] pub struct TenantCreateRequest { - #[serde(with = "hex")] - pub tenant_id: ZTenantId, + #[serde(default)] + #[serde(with = "opt_display_serde")] + pub new_tenant_id: Option, #[serde(default)] #[serde(with = "opt_display_serde")] pub initial_timeline_id: Option, } +#[derive(Deserialize, Serialize)] +pub struct TenantCreateResponse { + #[serde(with = "hex")] + pub tenant_id: ZTenantId, + #[serde(with = "hex")] + pub timeline_id: ZTimelineId, +} + #[derive(Serialize)] pub struct StatusResponse { pub id: ZNodeId, diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 7f3bf97bfe..f276e01227 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -22,7 +22,7 @@ paths: properties: id: type: integer - /v1/timeline/{tenant_id}: + /v1/tenant/{tenant_id}/timeline: parameters: - name: tenant_id in: path @@ -70,7 +70,7 @@ paths: application/json: schema: $ref: "#/components/schemas/Error" - /v1/timeline/{tenant_id}/{timeline_id}: + /v1/tenant/{tenant_id}/timeline/{timeline_id}: parameters: - name: tenant_id in: path @@ -90,7 +90,7 @@ paths: type: string description: Controls calculation of current_logical_size_non_incremental get: - description: Get timelines for tenant + description: Get info about the timeline responses: "200": description: TimelineInfo @@ -122,7 +122,14 @@ paths: application/json: schema: $ref: "#/components/schemas/Error" - /v1/timeline/: + /v1/tenant/{tenant_id}/timeline/: + parameters: + - name: tenant_id + in: path + required: true + schema: + type: string + format: hex post: description: Create timeline requestBody: @@ -130,18 +137,14 @@ paths: application/json: schema: type: object - required: - - "tenant_id" - - "timeline_id" - - "start_point" properties: - tenant_id: + new_timeline_id: type: string format: hex - timeline_id: + ancestor_timeline_id: type: string format: hex - start_point: + ancestor_start_lsn: type: string responses: "201": @@ -149,7 +152,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/TImelineInfo" + $ref: "#/components/schemas/TimelineInfo" "400": description: Malformed timeline create request content: @@ -211,10 +214,11 @@ paths: application/json: schema: type: object - required: - - "tenant_id" properties: - tenant_id: + new_tenant_id: + type: string + format: hex + initial_timeline_id: type: string format: hex responses: @@ -223,9 +227,14 @@ paths: content: application/json: schema: - type: array - items: - type: string + type: object + properties: + tenant_id: + type: string + format: hex + timeline_id: + type: string + format: hex "400": description: Malformed tenant create request content: @@ -268,35 +277,11 @@ components: type: string state: type: string - TimelineInfo: - type: object - required: - - timeline_id - - latest_valid_lsn - - current_logical_size - properties: - timeline_id: - type: string - format: hex - ancestor_id: - type: string - format: hex - ancestor_lsn: - type: string - current_logical_size: - type: integer - current_logical_size_non_incremental: - type: integer - latest_valid_lsn: - type: integer TimelineInfo: type: object required: - timeline_id - tenant_id - - last_record_lsn - - prev_record_lsn - - start_lsn - disk_consistent_lsn properties: timeline_id: @@ -305,19 +290,21 @@ components: tenant_id: type: string format: hex - ancestor_timeline_id: - type: string - format: hex last_record_lsn: type: string prev_record_lsn: type: string - start_lsn: + ancestor_timeline_id: + type: string + format: hex + ancestor_lsn: type: string disk_consistent_lsn: type: string - timeline_state: - type: string + current_logical_size: + type: integer + current_logical_size_non_incremental: + type: integer Error: type: object diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 45b0c3d4be..ddb52e209b 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -20,6 +20,7 @@ use zenith_utils::zid::{HexZTimelineId, ZTimelineId}; use super::models::StatusResponse; use super::models::TenantCreateRequest; +use super::models::TenantCreateResponse; use super::models::TimelineCreateRequest; use crate::repository::RepositoryTimeline; use crate::timelines::TimelineInfo; @@ -69,18 +70,19 @@ async fn status_handler(request: Request) -> Result, ApiErr } async fn timeline_create_handler(mut request: Request) -> Result, ApiError> { + let tenant_id: ZTenantId = parse_request_param(&request, "tenant_id")?; let request_data: TimelineCreateRequest = json_request(&mut request).await?; - check_permission(&request, Some(request_data.tenant_id))?; + check_permission(&request, Some(tenant_id))?; let response_data = tokio::task::spawn_blocking(move || { - let _enter = info_span!("/timeline_create", timeline = %request_data.timeline_id, tenant = %request_data.tenant_id, lsn=?request_data.start_lsn).entered(); + let _enter = info_span!("/timeline_create", tenant = %tenant_id, new_timeline = ?request_data.new_timeline_id, lsn=?request_data.ancestor_start_lsn).entered(); timelines::create_timeline( get_config(&request), - request_data.tenant_id, - request_data.timeline_id, + tenant_id, + request_data.new_timeline_id, request_data.ancestor_timeline_id, - request_data.start_lsn, + request_data.ancestor_start_lsn, ) }) .await @@ -214,12 +216,15 @@ async fn tenant_create_handler(mut request: Request) -> Result> = Mutex::new(HashMap::new()); @@ -179,9 +179,10 @@ pub fn shutdown_all_tenants() { pub fn create_repository_for_tenant( conf: &'static PageServerConf, - tenant_id: ZTenantId, + new_tenant_id: Option, initial_timeline_id: Option, -) -> Result { +) -> Result { + let tenant_id = new_tenant_id.unwrap_or_else(ZTenantId::generate); let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenant_id)); let (initial_timeline_id, repo) = timelines::create_repo(conf, tenant_id, initial_timeline_id, wal_redo_manager)?; @@ -196,7 +197,7 @@ pub fn create_repository_for_tenant( } } - Ok(initial_timeline_id) + Ok(ZTenantTimelineId::new(tenant_id, initial_timeline_id)) } pub fn get_tenant_state(tenantid: ZTenantId) -> Option { diff --git a/pageserver/src/timelines.rs b/pageserver/src/timelines.rs index 786e102747..c6b2e81abc 100644 --- a/pageserver/src/timelines.rs +++ b/pageserver/src/timelines.rs @@ -125,6 +125,13 @@ impl TimelineInfo { TimelineInfo::Remote { timeline_id, .. } => timeline_id, } } + + pub fn tenant_id(&self) -> ZTenantId { + match *self { + TimelineInfo::Local { tenant_id, .. } => tenant_id, + TimelineInfo::Remote { tenant_id, .. } => tenant_id, + } + } } fn get_current_logical_size_non_incremental( @@ -335,10 +342,12 @@ pub(crate) fn get_timelines( pub(crate) fn create_timeline( conf: &'static PageServerConf, tenant_id: ZTenantId, - new_timeline_id: ZTimelineId, + new_timeline_id: Option, ancestor_timeline_id: Option, ancestor_start_lsn: Option, ) -> Result { + let new_timeline_id = new_timeline_id.unwrap_or_else(ZTimelineId::generate); + if conf.timeline_path(&new_timeline_id, &tenant_id).exists() { bail!("timeline {} already exists", new_timeline_id); } diff --git a/test_runner/batch_others/test_auth.py b/test_runner/batch_others/test_auth.py index e92eb2e044..bda6349ef9 100644 --- a/test_runner/batch_others/test_auth.py +++ b/test_runner/batch_others/test_auth.py @@ -29,30 +29,27 @@ def test_pageserver_auth(zenith_env_builder: ZenithEnvBuilder): tenant_id=env.initial_tenant) # tenant can create branches - tenant_http_client.timeline_create(timeline_id=uuid4(), - tenant_id=env.initial_tenant, + tenant_http_client.timeline_create(tenant_id=env.initial_tenant, ancestor_timeline_id=new_timeline_id) # console can create branches for tenant - management_http_client.timeline_create(timeline_id=uuid4(), - tenant_id=env.initial_tenant, + management_http_client.timeline_create(tenant_id=env.initial_tenant, ancestor_timeline_id=new_timeline_id) # fail to create branch using token with different tenant_id with pytest.raises(ZenithPageserverApiException, match='Forbidden: Tenant id mismatch. Permission denied'): - invalid_tenant_http_client.timeline_create(timeline_id=uuid4(), - tenant_id=env.initial_tenant, + invalid_tenant_http_client.timeline_create(tenant_id=env.initial_tenant, ancestor_timeline_id=new_timeline_id) # create tenant using management token - management_http_client.tenant_create(uuid4()) + management_http_client.tenant_create() # fail to create tenant using tenant token with pytest.raises( ZenithPageserverApiException, match='Forbidden: Attempt to access management api with tenant scope. Permission denied' ): - tenant_http_client.tenant_create(uuid4()) + tenant_http_client.tenant_create() @pytest.mark.parametrize('with_wal_acceptors', [False, True]) diff --git a/test_runner/batch_others/test_zenith_cli.py b/test_runner/batch_others/test_zenith_cli.py index 8777a653b3..4a62a1430a 100644 --- a/test_runner/batch_others/test_zenith_cli.py +++ b/test_runner/batch_others/test_zenith_cli.py @@ -15,17 +15,15 @@ def helper_compare_timeline_list(pageserver_http_client: ZenithPageserverHttpCli Filters out timelines created by other tests. """ + timelines_api = sorted( + map(lambda t: cast(str, t['timeline_id']), + pageserver_http_client.timeline_list(initial_tenant))) + timelines_cli = env.zenith_cli.list_timelines() - timelines_cli = [ - b for b in timelines_cli if b.startswith('test_cli_') or b in ('empty', 'main') - ] + assert timelines_cli == env.zenith_cli.list_timelines(initial_tenant) - timelines_cli_with_tenant_arg = env.zenith_cli.list_timelines(initial_tenant) - timelines_cli_with_tenant_arg = [ - b for b in timelines_cli if b.startswith('test_cli_') or b in ('empty', 'main') - ] - - assert timelines_cli == timelines_cli_with_tenant_arg + cli_timeline_ids = sorted([timeline_id for (_, timeline_id) in timelines_cli]) + assert timelines_api == cli_timeline_ids def test_cli_timeline_list(zenith_simple_env: ZenithEnv): @@ -45,7 +43,7 @@ def test_cli_timeline_list(zenith_simple_env: ZenithEnv): helper_compare_timeline_list(pageserver_http_client, env, env.initial_tenant) # Check that all new branches are visible via CLI - timelines_cli = env.zenith_cli.list_timelines() + timelines_cli = [timeline_id for (_, timeline_id) in env.zenith_cli.list_timelines()] assert main_timeline_id.hex in timelines_cli assert nested_timeline_id.hex in timelines_cli diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index 04b795b244..bb2e690cb3 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -712,27 +712,29 @@ class ZenithPageserverHttpClient(requests.Session): def timeline_attach(self, tenant_id: uuid.UUID, timeline_id: uuid.UUID): res = self.post( - f"http://localhost:{self.port}/v1/timeline/{tenant_id.hex}/{timeline_id.hex}/attach", ) + f"http://localhost:{self.port}/v1/tenant/{tenant_id.hex}/timeline/{timeline_id.hex}/attach", + ) self.verbose_error(res) def timeline_detach(self, tenant_id: uuid.UUID, timeline_id: uuid.UUID): res = self.post( - f"http://localhost:{self.port}/v1/timeline/{tenant_id.hex}/{timeline_id.hex}/detach", ) + f"http://localhost:{self.port}/v1/tenant/{tenant_id.hex}/timeline/{timeline_id.hex}/detach", + ) self.verbose_error(res) - def timeline_create(self, - tenant_id: uuid.UUID, - timeline_id: uuid.UUID, - start_lsn: Optional[str] = None, - ancestor_timeline_id: Optional[uuid.UUID] = None) -> Dict[Any, Any]: - res = self.post(f"http://localhost:{self.port}/v1/timeline", + def timeline_create( + self, + tenant_id: uuid.UUID, + timeline_id: Optional[uuid.UUID] = None, + ancestor_timeline_id: Optional[uuid.UUID] = None, + ancestor_start_lsn: Optional[str] = None, + ) -> Dict[Any, Any]: + res = self.post(f"http://localhost:{self.port}/v1/tenant/{tenant_id.hex}/timeline", json={ - 'tenant_id': - tenant_id.hex, - 'timeline_id': - timeline_id.hex, - 'start_lsn': - start_lsn, + 'new_timeline_id': + timeline_id.hex if timeline_id else None, + 'ancestor_start_lsn': + ancestor_start_lsn, 'ancestor_timeline_id': ancestor_timeline_id.hex if ancestor_timeline_id else None, }) @@ -748,18 +750,23 @@ class ZenithPageserverHttpClient(requests.Session): assert isinstance(res_json, list) return res_json - def tenant_create(self, tenant_id: uuid.UUID): + def tenant_create(self, + tenant_id: Optional[uuid.UUID] = None, + new_timeline_id: Optional[uuid.UUID] = None) -> Dict[Any, Any]: res = self.post( f"http://localhost:{self.port}/v1/tenant", json={ - 'tenant_id': tenant_id.hex, + 'new_tenant_id': tenant_id.hex if tenant_id else None, + 'initial_timeline_id': new_timeline_id.hex if new_timeline_id else None, }, ) self.verbose_error(res) - return res.json() + res_json = res.json() + assert isinstance(res_json, dict) + return res_json def timeline_list(self, tenant_id: uuid.UUID) -> List[Dict[Any, Any]]: - res = self.get(f"http://localhost:{self.port}/v1/timeline/{tenant_id.hex}") + res = self.get(f"http://localhost:{self.port}/v1/tenant/{tenant_id.hex}/timeline") self.verbose_error(res) res_json = res.json() assert isinstance(res_json, list) @@ -767,7 +774,7 @@ class ZenithPageserverHttpClient(requests.Session): def timeline_detail(self, tenant_id: uuid.UUID, timeline_id: uuid.UUID) -> Dict[Any, Any]: res = self.get( - f"http://localhost:{self.port}/v1/timeline/{tenant_id.hex}/{timeline_id.hex}?include-non-incremental-logical-size=1" + f"http://localhost:{self.port}/v1/tenant/{tenant_id.hex}/timeline/{timeline_id.hex}?include-non-incremental-logical-size=1" ) self.verbose_error(res) res_json = res.json() @@ -861,13 +868,21 @@ class ZenithCli: else: return uuid.UUID(created_timeline_id) - def list_timelines(self, tenant_id: Optional[uuid.UUID] = None) -> List[str]: + def list_timelines(self, tenant_id: Optional[uuid.UUID] = None) -> List[Tuple[str, str]]: + """ + Returns a list of (branch_name, timeline_id) tuples out of parsed `zenith timeline list` CLI output. + """ + + # (L) main [b49f7954224a0ad25cc0013ea107b54b] + # (L) ┣━ @0/16B5A50: test_cli_branch_list_main [20f98c79111b9015d84452258b7d5540] + timeline_data_extractor = re.compile( + r"\s(?P[^\s]+)\s\[(?P[^\]]+)\]", re.MULTILINE) res = self.raw_cli( ['timeline', 'list', '--tenant-id', (tenant_id or self.env.initial_tenant).hex]) - branches_cli = sorted( - map(lambda b: b.split(') ')[-1].strip().split(':')[-1].strip(), - res.stdout.strip().split("\n"))) - return branches_cli + timelines_cli = sorted( + map(lambda branch_and_id: (branch_and_id[0], branch_and_id[1]), + timeline_data_extractor.findall(res.stdout))) + return timelines_cli def init(self, config_toml: str, diff --git a/zenith/src/main.rs b/zenith/src/main.rs index 9a578f79f1..5f2489a41d 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -19,7 +19,7 @@ use walkeeper::defaults::{ use zenith_utils::auth::{Claims, Scope}; use zenith_utils::lsn::Lsn; use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId}; +use zenith_utils::zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId}; use zenith_utils::GIT_VERSION; use pageserver::timelines::TimelineInfo; @@ -60,6 +60,8 @@ http_port = {safekeeper_http_port} struct TimelineTreeEl { /// `TimelineInfo` received from the `pageserver` via the `timeline_list` http API call. pub info: TimelineInfo, + /// Name, recovered from zenith config mappings + pub name: Option, /// Holds all direct children of this timeline referenced using `timeline_id`. pub children: BTreeSet, } @@ -150,7 +152,7 @@ fn main() -> Result<()> { .arg(tenant_id_arg.clone()) .arg(branch_name_arg.clone()) .arg(Arg::new("ancestor-branch-name").long("ancestor-branch-name").takes_value(true) - .help("Use last Lsn of another timeline (and its data) as base when creating the new timeline. The timeline gets resolved by its branch name.").required(true)) + .help("Use last Lsn of another timeline (and its data) as base when creating the new timeline. The timeline gets resolved by its branch name.").required(false)) .arg(Arg::new("ancestor-start-lsn").long("ancestor-start-lsn").takes_value(true) .help("When using another timeline as base, use a specific Lsn in it instead of the latest one").required(false))) .subcommand(App::new("create") @@ -218,6 +220,8 @@ fn main() -> Result<()> { .about("Start a postgres compute node.\n This command actually creates new node from scratch, but preserves existing config files") .arg(pg_node_arg.clone()) .arg(tenant_id_arg.clone()) + .arg(branch_name_arg.clone()) + .arg(timeline_id_arg.clone()) .arg(lsn_arg.clone()) .arg(port_arg.clone())) .subcommand( @@ -290,7 +294,10 @@ fn main() -> Result<()> { /// /// Prints timelines list as a tree-like structure. /// -fn print_timelines_tree(timelines: Vec) -> Result<()> { +fn print_timelines_tree( + timelines: Vec, + mut timeline_name_mappings: HashMap, +) -> Result<()> { let mut timelines_hash = timelines .iter() .map(|t| { @@ -299,6 +306,8 @@ fn print_timelines_tree(timelines: Vec) -> Result<()> { TimelineTreeEl { info: t.clone(), children: BTreeSet::new(), + name: timeline_name_mappings + .remove(&ZTenantTimelineId::new(t.tenant_id(), t.timeline_id())), }, ) }) @@ -380,8 +389,12 @@ fn print_timeline( print!("{} @{}: ", br_sym, lsn_string); } - // Finally print a timeline name with new line - println!("{}", timeline.info.timeline_id()); + // Finally print a timeline id and name with new line + println!( + "{} [{}]", + timeline.name.as_deref().unwrap_or("_no_name_"), + timeline.info.timeline_id() + ); let len = timeline.children.len(); let mut i: usize = 0; @@ -492,7 +505,7 @@ fn handle_init(init_match: &ArgMatches) -> Result { DEFAULT_BRANCH_NAME.to_owned(), initial_tenant_id, initial_timeline_id, - ); + )?; Ok(env) } @@ -514,19 +527,18 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> Re } } Some(("create", create_match)) => { - let tenant_id = parse_tenant_id(create_match)?.unwrap_or_else(ZTenantId::generate); - println!("using tenant id {}", tenant_id); + let initial_tenant_id = parse_tenant_id(create_match)?; let initial_timeline_id_argument = parse_timeline_id(create_match)?; - let initial_timeline_id = - pageserver.tenant_create(tenant_id, initial_timeline_id_argument)?; + let new_ds = + pageserver.tenant_create(initial_tenant_id, initial_timeline_id_argument)?; env.register_branch_mapping( DEFAULT_BRANCH_NAME.to_owned(), - tenant_id, - initial_timeline_id, - ); + new_ds.tenant_id, + new_ds.timeline_id, + )?; println!( "tenant {} successfully created on the pageserver, initial timeline: '{}'", - tenant_id, initial_timeline_id + new_ds.tenant_id, new_ds.timeline_id ); } Some((sub_name, _)) => bail!("Unexpected tenant subcommand '{}'", sub_name), @@ -542,15 +554,15 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - Some(("list", list_match)) => { let tenant_id = get_tenant_id(list_match, env)?; let timelines = pageserver.timeline_list(&tenant_id)?; - print_timelines_tree(timelines)?; + print_timelines_tree(timelines, env.timeline_name_mappings())?; } Some(("create", create_match)) => { let tenant_id = get_tenant_id(create_match, env)?; - let new_timeline_id = ZTimelineId::generate(); let new_branch_name = create_match .value_of("branch-name") .ok_or(anyhow!("No branch name provided"))?; - let timeline = pageserver.timeline_create(tenant_id, new_timeline_id, None, None)?; + let timeline = pageserver.timeline_create(tenant_id, None, None, None)?; + let new_timeline_id = timeline.timeline_id(); let last_record_lsn = match timeline { TimelineInfo::Local { @@ -563,7 +575,7 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - ) } }; - env.register_branch_mapping(new_branch_name.to_string(), tenant_id, new_timeline_id); + env.register_branch_mapping(new_branch_name.to_string(), tenant_id, new_timeline_id)?; println!( "Created timeline '{}' at Lsn {} for tenant: {}", @@ -574,13 +586,12 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - } Some(("branch", branch_match)) => { let tenant_id = get_tenant_id(branch_match, env)?; - let new_timeline_id = ZTimelineId::generate(); let new_branch_name = branch_match .value_of("branch-name") .ok_or(anyhow!("No branch name provided"))?; let ancestor_branch_name = branch_match .value_of("ancestor-branch-name") - .ok_or(anyhow!("No ancestor branch name provided"))?; + .unwrap_or(DEFAULT_BRANCH_NAME); let ancestor_timeline_id = env .get_branch_timeline_id(ancestor_branch_name, tenant_id) .ok_or_else(|| { @@ -597,10 +608,11 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - .context("Failed to parse ancestor start Lsn from the request")?; let timeline = pageserver.timeline_create( tenant_id, - new_timeline_id, + None, start_lsn, Some(ancestor_timeline_id), )?; + let new_timeline_id = timeline.timeline_id(); let last_record_lsn = match timeline { TimelineInfo::Local { @@ -612,7 +624,7 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - ), }; - env.register_branch_mapping(new_branch_name.to_string(), tenant_id, new_timeline_id); + env.register_branch_mapping(new_branch_name.to_string(), tenant_id, new_timeline_id)?; println!( "Created timeline '{}' at Lsn {} for tenant: {}. Ancestor timeline: '{}'", @@ -647,7 +659,9 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { HashMap::new() }); - println!("NODE\tADDRESS\t\tTIMELINE\tLSN\t\tSTATUS"); + let timeline_name_mappings = env.timeline_name_mappings(); + + println!("NODE\tADDRESS\tTIMELINE\tBRANCH NAME\tLSN\t\tSTATUS"); for ((_, node_name), node) in cplane .nodes .iter() @@ -666,11 +680,17 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { }) .unwrap_or_else(|| '?'.to_string()); + let branch_name = timeline_name_mappings + .get(&ZTenantTimelineId::new(tenant_id, node.timeline_id)) + .map(|name| name.as_str()) + .unwrap_or("?"); + println!( - "{}\t{}\t{}\t{}\t{}", + "{}\t{}\t{}\t{}\t{}\t{}", node_name, node.address, node.timeline_id, + branch_name, lsn_str, node.status(), ); From dd74c66ef0d8b3bd2c5636549a04f17823748cb5 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Thu, 3 Mar 2022 19:21:35 +0200 Subject: [PATCH 21/42] Do not create timeline along with tenant --- control_plane/src/storage.rs | 23 ++++----- pageserver/src/bin/pageserver.rs | 2 +- pageserver/src/http/models.rs | 11 ---- pageserver/src/http/openapi_spec.yml | 13 +---- pageserver/src/http/routes.rs | 20 +++----- pageserver/src/tenant_mgr.rs | 18 +++---- pageserver/src/timelines.rs | 31 ++++------- .../batch_others/test_pageserver_api.py | 18 ++++--- test_runner/batch_others/test_tenants.py | 8 +-- test_runner/fixtures/zenith_fixtures.py | 51 ++++++++++++++----- .../performance/test_bulk_tenant_create.py | 2 +- zenith/src/main.rs | 13 ++--- 12 files changed, 97 insertions(+), 113 deletions(-) diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index c2b99972e7..3c68823760 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -9,7 +9,7 @@ use anyhow::{bail, Context}; use nix::errno::Errno; use nix::sys::signal::{kill, Signal}; use nix::unistd::Pid; -use pageserver::http::models::{TenantCreateRequest, TenantCreateResponse, TimelineCreateRequest}; +use pageserver::http::models::{TenantCreateRequest, TimelineCreateRequest}; use pageserver::timelines::TimelineInfo; use postgres::{Config, NoTls}; use reqwest::blocking::{Client, RequestBuilder, Response}; @@ -336,20 +336,19 @@ impl PageServerNode { .json()?) } - pub fn tenant_create( - &self, - new_tenant_id: Option, - initial_timeline_id: Option, - ) -> Result { - Ok(self + pub fn tenant_create(&self, new_tenant_id: Option) -> anyhow::Result { + let tenant_id_string = self .http_request(Method::POST, format!("{}/tenant", self.http_base_url)) - .json(&TenantCreateRequest { - new_tenant_id, - initial_timeline_id, - }) + .json(&TenantCreateRequest { new_tenant_id }) .send()? .error_from_body()? - .json()?) + .json::()?; + tenant_id_string.parse().with_context(|| { + format!( + "Failed to parse tennat creation response as tenant id: {}", + tenant_id_string + ) + }) } pub fn timeline_list(&self, tenant_id: &ZTenantId) -> Result> { diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 83b128dd74..d37ba0cece 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -43,7 +43,7 @@ fn main() -> Result<()> { Arg::new("init") .long("init") .takes_value(false) - .help("Initialize pageserver repo"), + .help("Initialize pageserver service: creates an initial config, tenant and timeline, if specified"), ) .arg( Arg::new("workdir") diff --git a/pageserver/src/http/models.rs b/pageserver/src/http/models.rs index 9b321744eb..28d9791438 100644 --- a/pageserver/src/http/models.rs +++ b/pageserver/src/http/models.rs @@ -21,17 +21,6 @@ pub struct TenantCreateRequest { #[serde(default)] #[serde(with = "opt_display_serde")] pub new_tenant_id: Option, - #[serde(default)] - #[serde(with = "opt_display_serde")] - pub initial_timeline_id: Option, -} - -#[derive(Deserialize, Serialize)] -pub struct TenantCreateResponse { - #[serde(with = "hex")] - pub tenant_id: ZTenantId, - #[serde(with = "hex")] - pub timeline_id: ZTimelineId, } #[derive(Serialize)] diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index f276e01227..823f927796 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -218,23 +218,14 @@ paths: new_tenant_id: type: string format: hex - initial_timeline_id: - type: string - format: hex responses: "201": description: CREATED content: application/json: schema: - type: object - properties: - tenant_id: - type: string - format: hex - timeline_id: - type: string - format: hex + type: string + format: hex "400": description: Malformed tenant create request content: diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index ddb52e209b..efcc7ae2f3 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -20,7 +20,6 @@ use zenith_utils::zid::{HexZTimelineId, ZTimelineId}; use super::models::StatusResponse; use super::models::TenantCreateRequest; -use super::models::TenantCreateResponse; use super::models::TimelineCreateRequest; use crate::repository::RepositoryTimeline; use crate::timelines::TimelineInfo; @@ -215,20 +214,17 @@ async fn tenant_create_handler(mut request: Request) -> Result) -> Result, ApiError> { diff --git a/pageserver/src/tenant_mgr.rs b/pageserver/src/tenant_mgr.rs index b40c1c6f2c..7076962830 100644 --- a/pageserver/src/tenant_mgr.rs +++ b/pageserver/src/tenant_mgr.rs @@ -16,7 +16,7 @@ use serde::{Deserialize, Serialize}; use std::collections::{hash_map, HashMap}; use std::fmt; use std::sync::{Arc, Mutex, MutexGuard}; -use zenith_utils::zid::{ZTenantId, ZTenantTimelineId, ZTimelineId}; +use zenith_utils::zid::{ZTenantId, ZTimelineId}; lazy_static! { static ref TENANTS: Mutex> = Mutex::new(HashMap::new()); @@ -180,15 +180,13 @@ pub fn shutdown_all_tenants() { pub fn create_repository_for_tenant( conf: &'static PageServerConf, new_tenant_id: Option, - initial_timeline_id: Option, -) -> Result { - let tenant_id = new_tenant_id.unwrap_or_else(ZTenantId::generate); - let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenant_id)); - let (initial_timeline_id, repo) = - timelines::create_repo(conf, tenant_id, initial_timeline_id, wal_redo_manager)?; +) -> Result { + let new_tenant_id = new_tenant_id.unwrap_or_else(ZTenantId::generate); + let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, new_tenant_id)); + let repo = timelines::create_repo(conf, new_tenant_id, wal_redo_manager)?; - match access_tenants().entry(tenant_id) { - hash_map::Entry::Occupied(_) => bail!("tenant {} already exists", tenant_id), + match access_tenants().entry(new_tenant_id) { + hash_map::Entry::Occupied(_) => bail!("tenant {} already exists", new_tenant_id), hash_map::Entry::Vacant(v) => { v.insert(Tenant { state: TenantState::Idle, @@ -197,7 +195,7 @@ pub fn create_repository_for_tenant( } } - Ok(ZTenantTimelineId::new(tenant_id, initial_timeline_id)) + Ok(new_tenant_id) } pub fn get_tenant_state(tenantid: ZTenantId) -> Option { diff --git a/pageserver/src/timelines.rs b/pageserver/src/timelines.rs index c6b2e81abc..54f0a302f4 100644 --- a/pageserver/src/timelines.rs +++ b/pageserver/src/timelines.rs @@ -17,9 +17,9 @@ use zenith_utils::lsn::Lsn; use zenith_utils::zid::{opt_display_serde, ZTenantId, ZTimelineId}; use zenith_utils::{crashsafe_dir, logging}; -use crate::walredo::WalRedoManager; use crate::{config::PageServerConf, repository::Repository}; use crate::{import_datadir, LOG_FILE_NAME}; +use crate::{layered_repository::LayeredRepository, walredo::WalRedoManager}; use crate::{repository::RepositoryTimeline, tenant_mgr}; use crate::{repository::Timeline, CheckpointConfig}; @@ -179,8 +179,13 @@ pub fn init_pageserver( if let Some(tenant_id) = create_tenant { println!("initializing tenantid {}", tenant_id); - create_repo(conf, tenant_id, initial_timeline_id, dummy_redo_mgr) - .context("failed to create repo")?; + let repo = create_repo(conf, tenant_id, dummy_redo_mgr).context("failed to create repo")?; + let new_timeline_id = initial_timeline_id.unwrap_or_else(ZTimelineId::generate); + bootstrap_timeline(conf, tenant_id, new_timeline_id, repo.as_ref()) + .context("failed to create initial timeline")?; + println!("initial timeline {} created", new_timeline_id) + } else if initial_timeline_id.is_some() { + println!("Ignoring initial timeline parameter, due to no tenant id to create given"); } crashsafe_dir::create_dir_all(conf.tenants_path())?; @@ -191,9 +196,8 @@ pub fn init_pageserver( pub fn create_repo( conf: &'static PageServerConf, tenant_id: ZTenantId, - init_timeline_id: Option, wal_redo_manager: Arc, -) -> Result<(ZTimelineId, Arc)> { +) -> Result> { let repo_dir = conf.tenant_path(&tenant_id); if repo_dir.exists() { bail!("repo for {} already exists", tenant_id) @@ -207,25 +211,12 @@ pub fn create_repo( info!("created directory structure in {}", repo_dir.display()); - // create a new timeline directory - let timeline_id = init_timeline_id.unwrap_or_else(ZTimelineId::generate); - let timelinedir = conf.timeline_path(&timeline_id, &tenant_id); - - crashsafe_dir::create_dir(&timelinedir)?; - - let repo = Arc::new(crate::layered_repository::LayeredRepository::new( + Ok(Arc::new(LayeredRepository::new( conf, wal_redo_manager, tenant_id, conf.remote_storage_config.is_some(), - )); - - // Load data into pageserver - // TODO To implement zenith import we need to - // move data loading out of create_repo() - bootstrap_timeline(conf, tenant_id, timeline_id, repo.as_ref())?; - - Ok((timeline_id, repo)) + ))) } // Returns checkpoint LSN from controlfile diff --git a/test_runner/batch_others/test_pageserver_api.py b/test_runner/batch_others/test_pageserver_api.py index 4c3b98e838..7d2c0800a2 100644 --- a/test_runner/batch_others/test_pageserver_api.py +++ b/test_runner/batch_others/test_pageserver_api.py @@ -23,9 +23,18 @@ def check_client(client: ZenithPageserverHttpClient, initial_tenant: UUID): client.tenant_create(tenant_id) assert tenant_id.hex in {t['id'] for t in client.tenant_list()} - # check its timelines + timelines = client.timeline_list(tenant_id) + assert len(timelines) == 0, "initial tenant should not have any timelines" + + # create timeline + timeline_id = uuid4() + client.timeline_create(tenant_id=tenant_id, timeline_id=timeline_id) + timelines = client.timeline_list(tenant_id) assert len(timelines) > 0 + + # check it is there + assert timeline_id.hex in {b['timeline_id'] for b in client.timeline_list(tenant_id)} for timeline in timelines: timeline_id_str = str(timeline['timeline_id']) timeline_details = client.timeline_detail(tenant_id=tenant_id, @@ -34,13 +43,6 @@ def check_client(client: ZenithPageserverHttpClient, initial_tenant: UUID): assert timeline_details['tenant_id'] == tenant_id.hex assert timeline_details['timeline_id'] == timeline_id_str - # create timeline - timeline_id = uuid4() - client.timeline_create(tenant_id=tenant_id, timeline_id=timeline_id) - - # check it is there - assert timeline_id.hex in {b['timeline_id'] for b in client.timeline_list(tenant_id)} - def test_pageserver_http_api_client(zenith_simple_env: ZenithEnv): env = zenith_simple_env diff --git a/test_runner/batch_others/test_tenants.py b/test_runner/batch_others/test_tenants.py index 87acf2086d..e883018628 100644 --- a/test_runner/batch_others/test_tenants.py +++ b/test_runner/batch_others/test_tenants.py @@ -15,10 +15,10 @@ def test_tenants_normal_work(zenith_env_builder: ZenithEnvBuilder, with_wal_acce tenant_1 = env.zenith_cli.create_tenant() tenant_2 = env.zenith_cli.create_tenant() - env.zenith_cli.create_branch(f'test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}', - tenant_id=tenant_1) - env.zenith_cli.create_branch(f'test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}', - tenant_id=tenant_2) + env.zenith_cli.create_timeline( + f'test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}', tenant_id=tenant_1) + env.zenith_cli.create_timeline( + f'test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}', tenant_id=tenant_2) pg_tenant1 = env.postgres.create_start( f'test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}', diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index bb2e690cb3..e2c9f16630 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -57,6 +57,7 @@ Fn = TypeVar('Fn', bound=Callable[..., Any]) DEFAULT_OUTPUT_DIR = 'test_output' DEFAULT_POSTGRES_DIR = 'tmp_install' +DEFAULT_BRANCH_NAME = 'main' BASE_PORT = 15000 WORKER_PORT_NUM = 100 @@ -424,7 +425,7 @@ class ZenithEnvBuilder: num_safekeepers: int = 0, pageserver_auth_enabled: bool = False, rust_log_override: Optional[str] = None, - default_branch_name='main'): + default_branch_name=DEFAULT_BRANCH_NAME): self.repo_dir = repo_dir self.rust_log_override = rust_log_override self.port_distributor = port_distributor @@ -547,7 +548,6 @@ class ZenithEnv: self.rust_log_override = config.rust_log_override self.port_distributor = config.port_distributor self.s3_mock_server = config.s3_mock_server - self.default_branch_name = config.default_branch_name self.zenith_cli = ZenithCli(env=self) self.postgres = PostgresFactory(self) self.safekeepers: List[Safekeeper] = [] @@ -639,7 +639,7 @@ def _shared_simple_env(request: Any, port_distributor) -> Iterator[ZenithEnv]: env = builder.init_start() # For convenience in tests, create a branch from the freshly-initialized cluster. - env.zenith_cli.create_branch("empty") + env.zenith_cli.create_branch('empty', ancestor_branch_name=DEFAULT_BRANCH_NAME) yield env @@ -750,20 +750,17 @@ class ZenithPageserverHttpClient(requests.Session): assert isinstance(res_json, list) return res_json - def tenant_create(self, - tenant_id: Optional[uuid.UUID] = None, - new_timeline_id: Optional[uuid.UUID] = None) -> Dict[Any, Any]: + def tenant_create(self, tenant_id: Optional[uuid.UUID] = None) -> uuid.UUID: res = self.post( f"http://localhost:{self.port}/v1/tenant", json={ 'new_tenant_id': tenant_id.hex if tenant_id else None, - 'initial_timeline_id': new_timeline_id.hex if new_timeline_id else None, }, ) self.verbose_error(res) - res_json = res.json() - assert isinstance(res_json, dict) - return res_json + new_tenant_id = res.json() + assert isinstance(new_tenant_id, str) + return uuid.UUID(new_tenant_id) def timeline_list(self, tenant_id: uuid.UUID) -> List[Dict[Any, Any]]: res = self.get(f"http://localhost:{self.port}/v1/tenant/{tenant_id.hex}/timeline") @@ -834,8 +831,36 @@ class ZenithCli: res.check_returncode() return res + def create_timeline(self, + new_branch_name: str, + tenant_id: Optional[uuid.UUID] = None) -> uuid.UUID: + cmd = [ + 'timeline', + 'create', + '--branch-name', + new_branch_name, + '--tenant-id', + (tenant_id or self.env.initial_tenant).hex, + ] + + res = self.raw_cli(cmd) + res.check_returncode() + + create_timeline_id_extractor = re.compile(r"^Created timeline '(?P[^']+)'", + re.MULTILINE) + matches = create_timeline_id_extractor.search(res.stdout) + + created_timeline_id = None + if matches is not None: + created_timeline_id = matches.group('timeline_id') + + if created_timeline_id is None: + raise Exception('could not find timeline id after `zenith timeline create` invocation') + else: + return uuid.UUID(created_timeline_id) + def create_branch(self, - new_branch_name: str, + new_branch_name: str = DEFAULT_BRANCH_NAME, ancestor_branch_name: Optional[str] = None, tenant_id: Optional[uuid.UUID] = None, ancestor_start_lsn: Optional[str] = None) -> uuid.UUID: @@ -846,9 +871,9 @@ class ZenithCli: new_branch_name, '--tenant-id', (tenant_id or self.env.initial_tenant).hex, - '--ancestor-branch-name', - ancestor_branch_name or self.env.default_branch_name, ] + if ancestor_branch_name is not None: + cmd.extend(['--ancestor-branch-name', ancestor_branch_name]) if ancestor_start_lsn is not None: cmd.extend(['--ancestor-start-lsn', ancestor_start_lsn]) diff --git a/test_runner/performance/test_bulk_tenant_create.py b/test_runner/performance/test_bulk_tenant_create.py index 2430eec33e..fbef131ffd 100644 --- a/test_runner/performance/test_bulk_tenant_create.py +++ b/test_runner/performance/test_bulk_tenant_create.py @@ -31,7 +31,7 @@ def test_bulk_tenant_create( start = timeit.default_timer() tenant = env.zenith_cli.create_tenant() - env.zenith_cli.create_branch( + env.zenith_cli.create_timeline( f'test_bulk_tenant_create_{tenants_count}_{i}_{use_wal_acceptors}', tenant_id=tenant) # FIXME: We used to start new safekeepers here. Did that make sense? Should we do it now? diff --git a/zenith/src/main.rs b/zenith/src/main.rs index 5f2489a41d..c4636fa1a6 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -528,17 +528,10 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> Re } Some(("create", create_match)) => { let initial_tenant_id = parse_tenant_id(create_match)?; - let initial_timeline_id_argument = parse_timeline_id(create_match)?; - let new_ds = - pageserver.tenant_create(initial_tenant_id, initial_timeline_id_argument)?; - env.register_branch_mapping( - DEFAULT_BRANCH_NAME.to_owned(), - new_ds.tenant_id, - new_ds.timeline_id, - )?; + let new_tenant_id = pageserver.tenant_create(initial_tenant_id)?; println!( - "tenant {} successfully created on the pageserver, initial timeline: '{}'", - new_ds.tenant_id, new_ds.timeline_id + "tenant {} successfully created on the pageserver", + new_tenant_id ); } Some((sub_name, _)) => bail!("Unexpected tenant subcommand '{}'", sub_name), From fe6fccfdae3e968cf207c41ae7218adced458764 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Thu, 3 Mar 2022 23:16:54 +0200 Subject: [PATCH 22/42] Allow already existing repo when creating a tenant --- pageserver/src/http/openapi_spec.yml | 2 +- pageserver/src/tenant_mgr.rs | 22 +++++++++++----------- pageserver/src/timelines.rs | 22 ++++++++++++---------- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 823f927796..25d5ceae4e 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -220,7 +220,7 @@ paths: format: hex responses: "201": - description: CREATED + description: Already exists or created content: application/json: schema: diff --git a/pageserver/src/tenant_mgr.rs b/pageserver/src/tenant_mgr.rs index 7076962830..77ef865ec5 100644 --- a/pageserver/src/tenant_mgr.rs +++ b/pageserver/src/tenant_mgr.rs @@ -9,11 +9,11 @@ use crate::thread_mgr::ThreadKind; use crate::timelines; use crate::walredo::PostgresRedoManager; use crate::CheckpointConfig; -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result}; use lazy_static::lazy_static; use log::*; use serde::{Deserialize, Serialize}; -use std::collections::{hash_map, HashMap}; +use std::collections::HashMap; use std::fmt; use std::sync::{Arc, Mutex, MutexGuard}; use zenith_utils::zid::{ZTenantId, ZTimelineId}; @@ -183,16 +183,16 @@ pub fn create_repository_for_tenant( ) -> Result { let new_tenant_id = new_tenant_id.unwrap_or_else(ZTenantId::generate); let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, new_tenant_id)); - let repo = timelines::create_repo(conf, new_tenant_id, wal_redo_manager)?; - - match access_tenants().entry(new_tenant_id) { - hash_map::Entry::Occupied(_) => bail!("tenant {} already exists", new_tenant_id), - hash_map::Entry::Vacant(v) => { - v.insert(Tenant { - state: TenantState::Idle, - repo, - }); + match timelines::create_repo(conf, new_tenant_id, wal_redo_manager)? { + Some(repo) => { + access_tenants() + .entry(new_tenant_id) + .or_insert_with(|| Tenant { + state: TenantState::Idle, + repo, + }); } + None => debug!("repository already exists for tenant {}", new_tenant_id), } Ok(new_tenant_id) diff --git a/pageserver/src/timelines.rs b/pageserver/src/timelines.rs index 54f0a302f4..587b9a2cf2 100644 --- a/pageserver/src/timelines.rs +++ b/pageserver/src/timelines.rs @@ -2,7 +2,7 @@ //! Timeline management code // -use anyhow::{bail, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use postgres_ffi::ControlFileData; use serde::{Deserialize, Serialize}; use std::{ @@ -160,7 +160,7 @@ pub fn init_pageserver( conf: &'static PageServerConf, create_tenant: Option, initial_timeline_id: Option, -) -> Result<()> { +) -> anyhow::Result<()> { // Initialize logger // use true as daemonize parameter because otherwise we pollute zenith cli output with a few pages long output of info messages let _log_file = logging::init(LOG_FILE_NAME, true)?; @@ -177,9 +177,13 @@ pub fn init_pageserver( // anymore, but I think that could still happen. let dummy_redo_mgr = Arc::new(crate::walredo::DummyRedoManager {}); + crashsafe_dir::create_dir_all(conf.tenants_path())?; + if let Some(tenant_id) = create_tenant { println!("initializing tenantid {}", tenant_id); - let repo = create_repo(conf, tenant_id, dummy_redo_mgr).context("failed to create repo")?; + let repo = create_repo(conf, tenant_id, dummy_redo_mgr) + .context("failed to create repo")? + .ok_or_else(|| anyhow!("For newely created pageserver, found already existing repository for tenant {}", tenant_id))?; let new_timeline_id = initial_timeline_id.unwrap_or_else(ZTimelineId::generate); bootstrap_timeline(conf, tenant_id, new_timeline_id, repo.as_ref()) .context("failed to create initial timeline")?; @@ -187,7 +191,6 @@ pub fn init_pageserver( } else if initial_timeline_id.is_some() { println!("Ignoring initial timeline parameter, due to no tenant id to create given"); } - crashsafe_dir::create_dir_all(conf.tenants_path())?; println!("pageserver init succeeded"); Ok(()) @@ -197,26 +200,25 @@ pub fn create_repo( conf: &'static PageServerConf, tenant_id: ZTenantId, wal_redo_manager: Arc, -) -> Result> { +) -> Result>> { let repo_dir = conf.tenant_path(&tenant_id); if repo_dir.exists() { - bail!("repo for {} already exists", tenant_id) + debug!("repo for {} already exists", tenant_id); + return Ok(None); } // top-level dir may exist if we are creating it through CLI crashsafe_dir::create_dir_all(&repo_dir) .with_context(|| format!("could not create directory {}", repo_dir.display()))?; - crashsafe_dir::create_dir(conf.timelines_path(&tenant_id))?; - info!("created directory structure in {}", repo_dir.display()); - Ok(Arc::new(LayeredRepository::new( + Ok(Some(Arc::new(LayeredRepository::new( conf, wal_redo_manager, tenant_id, conf.remote_storage_config.is_some(), - ))) + )))) } // Returns checkpoint LSN from controlfile From c51d545fd974385c104799b9e18d67d6a8047afa Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Fri, 4 Mar 2022 11:11:42 +0200 Subject: [PATCH 23/42] Serialize Lsn as strings in http api --- control_plane/src/local_env.rs | 26 ++-- control_plane/src/storage.rs | 32 +++-- pageserver/src/http/models.rs | 117 ++++++++++++++++-- pageserver/src/http/routes.rs | 32 +++-- pageserver/src/timelines.rs | 11 +- .../batch_others/test_pageserver_api.py | 2 +- .../batch_others/test_remote_storage.py | 2 +- test_runner/batch_others/test_wal_acceptor.py | 4 +- zenith/src/main.rs | 16 +-- 9 files changed, 175 insertions(+), 67 deletions(-) diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 9278a9df5a..2a1d51fe08 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -12,9 +12,9 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use zenith_utils::auth::{encode_from_key_file, Claims, Scope}; use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::ZTenantTimelineId; -use zenith_utils::zid::ZTimelineId; -use zenith_utils::zid::{HexZTenantId, ZNodeId, ZTenantId}; +use zenith_utils::zid::{ + HexZTenantId, HexZTimelineId, ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId, +}; use crate::safekeeper::SafekeeperNode; @@ -61,12 +61,12 @@ pub struct LocalEnv { #[serde(default)] pub safekeepers: Vec, - /// Keep human-readable aliases in memory (and persist them to config), to hind ZId hex strings from the user. + /// Keep human-readable aliases in memory (and persist them to config), to hide ZId hex strings from the user. #[serde(default)] // A `HashMap>` would be more appropriate here, // but deserialization into a generic toml object as `toml::Value::try_from` fails with an error. // https://toml.io/en/v1.0.0 does not contain a concept of "a table inside another table". - branch_name_mappings: HashMap>, + branch_name_mappings: HashMap>, } #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] @@ -164,6 +164,9 @@ impl LocalEnv { .entry(branch_name.clone()) .or_default(); + let tenant_id = HexZTenantId::from(tenant_id); + let timeline_id = HexZTimelineId::from(timeline_id); + let existing_ids = existing_values .iter() .find(|(existing_tenant_id, _)| existing_tenant_id == &tenant_id); @@ -190,22 +193,29 @@ impl LocalEnv { branch_name: &str, tenant_id: ZTenantId, ) -> Option { + let tenant_id = HexZTenantId::from(tenant_id); self.branch_name_mappings .get(branch_name)? .iter() .find(|(mapped_tenant_id, _)| mapped_tenant_id == &tenant_id) .map(|&(_, timeline_id)| timeline_id) + .map(ZTimelineId::from) } pub fn timeline_name_mappings(&self) -> HashMap { self.branch_name_mappings .iter() - .map(|(name, tenant_timelines)| { + .flat_map(|(name, tenant_timelines)| { tenant_timelines.iter().map(|&(tenant_id, timeline_id)| { - (ZTenantTimelineId::new(tenant_id, timeline_id), name.clone()) + ( + ZTenantTimelineId::new( + ZTenantId::from(tenant_id), + ZTimelineId::from(timeline_id), + ), + name.clone(), + ) }) }) - .flatten() .collect() } diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index 3c68823760..259fc79708 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -1,3 +1,4 @@ +use std::convert::TryFrom; use std::io::Write; use std::net::TcpStream; use std::path::PathBuf; @@ -9,7 +10,7 @@ use anyhow::{bail, Context}; use nix::errno::Errno; use nix::sys::signal::{kill, Signal}; use nix::unistd::Pid; -use pageserver::http::models::{TenantCreateRequest, TimelineCreateRequest}; +use pageserver::http::models::{TenantCreateRequest, TimelineCreateRequest, TimelineInfoResponse}; use pageserver::timelines::TimelineInfo; use postgres::{Config, NoTls}; use reqwest::blocking::{Client, RequestBuilder, Response}; @@ -18,7 +19,7 @@ use thiserror::Error; use zenith_utils::http::error::HttpErrorBody; use zenith_utils::lsn::Lsn; use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{ZTenantId, ZTimelineId}; +use zenith_utils::zid::{HexZTenantId, HexZTimelineId, ZTenantId, ZTimelineId}; use crate::local_env::LocalEnv; use crate::{fill_rust_env_vars, read_pidfile}; @@ -339,7 +340,9 @@ impl PageServerNode { pub fn tenant_create(&self, new_tenant_id: Option) -> anyhow::Result { let tenant_id_string = self .http_request(Method::POST, format!("{}/tenant", self.http_base_url)) - .json(&TenantCreateRequest { new_tenant_id }) + .json(&TenantCreateRequest { + new_tenant_id: new_tenant_id.map(HexZTenantId::from), + }) .send()? .error_from_body()? .json::()?; @@ -351,15 +354,20 @@ impl PageServerNode { }) } - pub fn timeline_list(&self, tenant_id: &ZTenantId) -> Result> { - Ok(self + pub fn timeline_list(&self, tenant_id: &ZTenantId) -> anyhow::Result> { + let timeline_infos: Vec = self .http_request( Method::GET, format!("{}/tenant/{}/timeline", self.http_base_url, tenant_id), ) .send()? .error_from_body()? - .json()?) + .json()?; + + timeline_infos + .into_iter() + .map(TimelineInfo::try_from) + .collect() } pub fn timeline_create( @@ -368,20 +376,22 @@ impl PageServerNode { new_timeline_id: Option, ancestor_start_lsn: Option, ancestor_timeline_id: Option, - ) -> Result { - Ok(self + ) -> anyhow::Result { + let timeline_info_response = self .http_request( Method::POST, format!("{}/tenant/{}/timeline", self.http_base_url, tenant_id), ) .json(&TimelineCreateRequest { - new_timeline_id, + new_timeline_id: new_timeline_id.map(HexZTimelineId::from), ancestor_start_lsn, - ancestor_timeline_id, + ancestor_timeline_id: ancestor_timeline_id.map(HexZTimelineId::from), }) .send()? .error_from_body()? - .json()?) + .json::()?; + + TimelineInfo::try_from(timeline_info_response) } } diff --git a/pageserver/src/http/models.rs b/pageserver/src/http/models.rs index 28d9791438..9844e7ea82 100644 --- a/pageserver/src/http/models.rs +++ b/pageserver/src/http/models.rs @@ -1,26 +1,121 @@ +use crate::timelines::TimelineInfo; +use anyhow::{anyhow, bail, Context}; use serde::{Deserialize, Serialize}; -use zenith_utils::zid::ZNodeId; use zenith_utils::{ lsn::Lsn, - zid::{opt_display_serde, ZTenantId, ZTimelineId}, + zid::{HexZTenantId, HexZTimelineId, ZNodeId, ZTenantId, ZTimelineId}, }; #[derive(Serialize, Deserialize)] pub struct TimelineCreateRequest { - #[serde(default)] - #[serde(with = "opt_display_serde")] - pub new_timeline_id: Option, - #[serde(default)] - #[serde(with = "opt_display_serde")] - pub ancestor_timeline_id: Option, + pub new_timeline_id: Option, + pub ancestor_timeline_id: Option, pub ancestor_start_lsn: Option, } #[derive(Serialize, Deserialize)] pub struct TenantCreateRequest { - #[serde(default)] - #[serde(with = "opt_display_serde")] - pub new_tenant_id: Option, + pub new_tenant_id: Option, +} + +#[derive(Serialize, Deserialize)] +pub struct TimelineInfoResponse { + pub kind: String, + #[serde(with = "hex")] + timeline_id: ZTimelineId, + #[serde(with = "hex")] + tenant_id: ZTenantId, + disk_consistent_lsn: String, + last_record_lsn: Option, + prev_record_lsn: Option, + ancestor_timeline_id: Option, + ancestor_lsn: Option, + current_logical_size: Option, + current_logical_size_non_incremental: Option, +} + +impl From for TimelineInfoResponse { + fn from(other: TimelineInfo) -> Self { + match other { + TimelineInfo::Local { + timeline_id, + tenant_id, + last_record_lsn, + prev_record_lsn, + ancestor_timeline_id, + ancestor_lsn, + disk_consistent_lsn, + current_logical_size, + current_logical_size_non_incremental, + } => TimelineInfoResponse { + kind: "Local".to_owned(), + timeline_id, + tenant_id, + disk_consistent_lsn: disk_consistent_lsn.to_string(), + last_record_lsn: Some(last_record_lsn.to_string()), + prev_record_lsn: Some(prev_record_lsn.to_string()), + ancestor_timeline_id: ancestor_timeline_id.map(HexZTimelineId::from), + ancestor_lsn: ancestor_lsn.map(|lsn| lsn.to_string()), + current_logical_size: Some(current_logical_size), + current_logical_size_non_incremental, + }, + TimelineInfo::Remote { + timeline_id, + tenant_id, + disk_consistent_lsn, + } => TimelineInfoResponse { + kind: "Remote".to_owned(), + timeline_id, + tenant_id, + disk_consistent_lsn: disk_consistent_lsn.to_string(), + last_record_lsn: None, + prev_record_lsn: None, + ancestor_timeline_id: None, + ancestor_lsn: None, + current_logical_size: None, + current_logical_size_non_incremental: None, + }, + } + } +} + +impl TryFrom for TimelineInfo { + type Error = anyhow::Error; + + fn try_from(other: TimelineInfoResponse) -> anyhow::Result { + let parse_lsn_hex_string = |lsn_string: String| { + lsn_string + .parse::() + .with_context(|| format!("Failed to parse Lsn as hex string from '{}'", lsn_string)) + }; + + let disk_consistent_lsn = parse_lsn_hex_string(other.disk_consistent_lsn)?; + Ok(match other.kind.as_str() { + "Local" => TimelineInfo::Local { + timeline_id: other.timeline_id, + tenant_id: other.tenant_id, + last_record_lsn: other + .last_record_lsn + .ok_or(anyhow!("Local timeline should have last_record_lsn")) + .and_then(parse_lsn_hex_string)?, + prev_record_lsn: other + .prev_record_lsn + .ok_or(anyhow!("Local timeline should have prev_record_lsn")) + .and_then(parse_lsn_hex_string)?, + ancestor_timeline_id: other.ancestor_timeline_id.map(ZTimelineId::from), + ancestor_lsn: other.ancestor_lsn.map(parse_lsn_hex_string).transpose()?, + disk_consistent_lsn, + current_logical_size: other.current_logical_size.ok_or(anyhow!("No "))?, + current_logical_size_non_incremental: other.current_logical_size_non_incremental, + }, + "Remote" => TimelineInfo::Remote { + timeline_id: other.timeline_id, + tenant_id: other.tenant_id, + disk_consistent_lsn, + }, + unknown => bail!("Unknown timeline kind: {}", unknown), + }) + } } #[derive(Serialize)] diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index efcc7ae2f3..abc4043bdd 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -16,11 +16,11 @@ use zenith_utils::http::{ request::parse_request_param, }; use zenith_utils::http::{RequestExt, RouterBuilder}; -use zenith_utils::zid::{HexZTimelineId, ZTimelineId}; +use zenith_utils::zid::{HexZTenantId, ZTimelineId}; -use super::models::StatusResponse; -use super::models::TenantCreateRequest; -use super::models::TimelineCreateRequest; +use super::models::{ + StatusResponse, TenantCreateRequest, TimelineCreateRequest, TimelineInfoResponse, +}; use crate::repository::RepositoryTimeline; use crate::timelines::TimelineInfo; use crate::{config::PageServerConf, tenant_mgr, timelines, ZTenantId}; @@ -79,13 +79,13 @@ async fn timeline_create_handler(mut request: Request) -> Result) -> Result, let tenant_id: ZTenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; let include_non_incremental_logical_size = get_include_non_incremental_logical_size(&request); - let response_data = tokio::task::spawn_blocking(move || { + let response_data: Vec = tokio::task::spawn_blocking(move || { let _enter = info_span!("timeline_list", tenant = %tenant_id).entered(); crate::timelines::get_timelines(tenant_id, include_non_incremental_logical_size) }) .await - .map_err(ApiError::from_err)??; + .map_err(ApiError::from_err)?? + .into_iter() + .map(TimelineInfoResponse::from) + .collect(); Ok(json_response(StatusCode::OK, response_data)?) } @@ -137,7 +140,8 @@ async fn timeline_detail_handler(request: Request) -> Result) -> Result, ancestor_lsn: Option, disk_consistent_lsn: Lsn, @@ -41,9 +36,7 @@ pub enum TimelineInfo { current_logical_size_non_incremental: Option, }, Remote { - #[serde(with = "hex")] timeline_id: ZTimelineId, - #[serde(with = "hex")] tenant_id: ZTenantId, disk_consistent_lsn: Lsn, }, diff --git a/test_runner/batch_others/test_pageserver_api.py b/test_runner/batch_others/test_pageserver_api.py index 7d2c0800a2..41b1899882 100644 --- a/test_runner/batch_others/test_pageserver_api.py +++ b/test_runner/batch_others/test_pageserver_api.py @@ -39,7 +39,7 @@ def check_client(client: ZenithPageserverHttpClient, initial_tenant: UUID): timeline_id_str = str(timeline['timeline_id']) timeline_details = client.timeline_detail(tenant_id=tenant_id, timeline_id=UUID(timeline_id_str)) - assert timeline_details['type'] == 'Local' + assert timeline_details['kind'] == 'Local' assert timeline_details['tenant_id'] == tenant_id.hex assert timeline_details['timeline_id'] == timeline_id_str diff --git a/test_runner/batch_others/test_remote_storage.py b/test_runner/batch_others/test_remote_storage.py index abd06bf5e9..edcc768819 100644 --- a/test_runner/batch_others/test_remote_storage.py +++ b/test_runner/batch_others/test_remote_storage.py @@ -85,7 +85,7 @@ def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder, timeline_details = client.timeline_detail(UUID(tenant_id), UUID(timeline_id)) assert timeline_details['timeline_id'] == timeline_id assert timeline_details['tenant_id'] == tenant_id - if timeline_details['type'] == 'Local': + if timeline_details['kind'] == 'Local': log.info("timeline downloaded, checking its data") break attempts += 1 diff --git a/test_runner/batch_others/test_wal_acceptor.py b/test_runner/batch_others/test_wal_acceptor.py index 9518a14b75..02da7ee749 100644 --- a/test_runner/batch_others/test_wal_acceptor.py +++ b/test_runner/batch_others/test_wal_acceptor.py @@ -13,7 +13,7 @@ from dataclasses import dataclass, field from multiprocessing import Process, Value from pathlib import Path from fixtures.zenith_fixtures import PgBin, Postgres, Safekeeper, ZenithEnv, ZenithEnvBuilder, PortDistributor, SafekeeperPort, zenith_binpath, PgProtocol -from fixtures.utils import lsn_to_hex, mkdir_if_needed +from fixtures.utils import lsn_to_hex, mkdir_if_needed, lsn_from_hex from fixtures.log_helper import log from typing import List, Optional, Any @@ -91,7 +91,7 @@ def test_many_timelines(zenith_env_builder: ZenithEnvBuilder): m = TimelineMetrics( timeline_id=timeline_id, - last_record_lsn=timeline_detail["last_record_lsn"], + last_record_lsn=lsn_from_hex(timeline_detail["last_record_lsn"]), ) for sk_m in sk_metrics: m.flush_lsns.append(sk_m.flush_lsn_inexact[(tenant_id.hex, timeline_id)]) diff --git a/zenith/src/main.rs b/zenith/src/main.rs index c4636fa1a6..165a7d7950 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -81,9 +81,6 @@ fn main() -> Result<()> { .required(false); let pg_node_arg = Arg::new("node").help("Postgres node name").required(false); - let safekeeper_node_arg = Arg::new("node") - .help("Safekeeper node name") - .required(false); let safekeeper_id_arg = Arg::new("id").help("safekeeper id").required(false); @@ -361,7 +358,7 @@ fn print_timeline( print!("{} ", local_or_remote); if nesting_level > 0 { - let lsn_string = match timeline.info { + let lsn_string = match &timeline.info { TimelineInfo::Local { ancestor_lsn, .. } => ancestor_lsn .map(|lsn| lsn.to_string()) .unwrap_or_else(|| "Unknown local Lsn".to_string()), @@ -430,14 +427,11 @@ fn get_timeline_infos( env: &local_env::LocalEnv, tenant_id: &ZTenantId, ) -> Result> { - let page_server = PageServerNode::from_env(env); - let timeline_infos: Vec = page_server.timeline_list(tenant_id)?; - let timeline_infos: HashMap = timeline_infos + Ok(PageServerNode::from_env(env) + .timeline_list(tenant_id)? .into_iter() .map(|timeline_info| (timeline_info.timeline_id(), timeline_info)) - .collect(); - - Ok(timeline_infos) + .collect()) } // Helper function to parse --tenant_id option, or get the default from config file @@ -486,7 +480,7 @@ fn handle_init(init_match: &ArgMatches) -> Result { .context("Failed to initialize zenith repository")?; // default_tenantid was generated by the `env.init()` call above - let initial_tenant_id = env.default_tenant_id.unwrap(); + let initial_tenant_id = ZTenantId::from(env.default_tenant_id.unwrap()); // Call 'pageserver init'. let pageserver = PageServerNode::from_env(&env); From 093ad8ab59c9a385183df31b595abbbe037fd5bd Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Mon, 7 Mar 2022 23:12:36 +0200 Subject: [PATCH 24/42] Send 409 HTTP responses on timeline and tenant creation for existing entity --- control_plane/src/storage.rs | 73 +++++++------------ pageserver/src/http/openapi_spec.yml | 29 +++++++- pageserver/src/http/routes.rs | 21 ++++-- pageserver/src/tenant_mgr.rs | 12 +-- pageserver/src/timelines.rs | 30 +++++--- .../batch_others/test_pageserver_api.py | 2 +- test_runner/batch_others/test_wal_acceptor.py | 6 +- test_runner/fixtures/zenith_fixtures.py | 37 +++++----- zenith/src/main.rs | 19 +++-- 9 files changed, 126 insertions(+), 103 deletions(-) diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index 259fc79708..f6b7173067 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -145,10 +145,9 @@ impl PageServerNode { args.extend(["--create-tenant", tenant_id]) } - let initial_timeline_id_str = initial_timeline_id.map(|id| id.to_string()); - if let Some(timeline_id) = initial_timeline_id_str.as_deref() { - args.extend(["--initial-timeline-id", timeline_id]) - } + let initial_timeline_id = initial_timeline_id.unwrap_or_else(ZTimelineId::generate); + let initial_timeline_id_string = initial_timeline_id.to_string(); + args.extend(["--initial-timeline-id", &initial_timeline_id_string]); let init_output = fill_rust_env_vars(cmd.args(args)) .output() @@ -158,11 +157,7 @@ impl PageServerNode { bail!("pageserver init failed"); } - if let Some(initial_timeline_id) = initial_timeline_id { - Ok(initial_timeline_id) - } else { - extract_initial_timeline_id(init_output.stdout) - } + Ok(initial_timeline_id) } pub fn repo_path(&self) -> PathBuf { @@ -337,7 +332,10 @@ impl PageServerNode { .json()?) } - pub fn tenant_create(&self, new_tenant_id: Option) -> anyhow::Result { + pub fn tenant_create( + &self, + new_tenant_id: Option, + ) -> anyhow::Result> { let tenant_id_string = self .http_request(Method::POST, format!("{}/tenant", self.http_base_url)) .json(&TenantCreateRequest { @@ -345,13 +343,18 @@ impl PageServerNode { }) .send()? .error_from_body()? - .json::()?; - tenant_id_string.parse().with_context(|| { - format!( - "Failed to parse tennat creation response as tenant id: {}", - tenant_id_string - ) - }) + .json::>()?; + + tenant_id_string + .map(|id| { + id.parse().with_context(|| { + format!( + "Failed to parse tennat creation response as tenant id: {}", + id + ) + }) + }) + .transpose() } pub fn timeline_list(&self, tenant_id: &ZTenantId) -> anyhow::Result> { @@ -376,7 +379,7 @@ impl PageServerNode { new_timeline_id: Option, ancestor_start_lsn: Option, ancestor_timeline_id: Option, - ) -> anyhow::Result { + ) -> anyhow::Result> { let timeline_info_response = self .http_request( Method::POST, @@ -389,36 +392,10 @@ impl PageServerNode { }) .send()? .error_from_body()? - .json::()?; + .json::>()?; - TimelineInfo::try_from(timeline_info_response) + timeline_info_response + .map(TimelineInfo::try_from) + .transpose() } } - -fn extract_initial_timeline_id(init_stdout: Vec) -> anyhow::Result { - let output_string = - String::from_utf8(init_stdout).context("Init stdout is not a valid unicode")?; - - let string_with_timeline_id = match output_string.split_once("created initial timeline ") { - Some((_, string_with_timeline_id)) => string_with_timeline_id, - None => bail!( - "Found no line with timeline id in the init output: '{}'", - output_string - ), - }; - - let timeline_id_str = match string_with_timeline_id.split_once(' ') { - Some((timeline_id_str, _)) => timeline_id_str, - None => bail!( - "Found no timeline id in the init output: '{}'", - output_string - ), - }; - - timeline_id_str.parse().with_context(|| { - format!( - "Failed to parse timeline id from string, extracted from the init output: '{}'", - timeline_id_str - ) - }) -} diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 25d5ceae4e..d322b051a6 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -131,7 +131,9 @@ paths: type: string format: hex post: - description: Create timeline + description: | + Create a timeline. Returns new timeline id on success.\ + If no new timeline id is specified in parameters, it would be generated. It's an error to recreate the same timeline. requestBody: content: application/json: @@ -171,6 +173,12 @@ paths: application/json: schema: $ref: "#/components/schemas/ForbiddenError" + "409": + description: Timeline already exists, creation skipped + content: + application/json: + schema: + $ref: "#/components/schemas/AlreadyExistsError" "500": description: Generic operation error content: @@ -208,7 +216,9 @@ paths: schema: $ref: "#/components/schemas/Error" post: - description: Create tenant + description: | + Create a tenant. Returns new tenant id on success.\ + If no new tenant id is specified in parameters, it would be generated. It's an error to recreate the same tenant. requestBody: content: application/json: @@ -220,7 +230,7 @@ paths: format: hex responses: "201": - description: Already exists or created + description: New tenant created successfully content: application/json: schema: @@ -244,6 +254,12 @@ paths: application/json: schema: $ref: "#/components/schemas/ForbiddenError" + "409": + description: Tenant already exists, creation skipped + content: + application/json: + schema: + $ref: "#/components/schemas/AlreadyExistsError" "500": description: Generic operation error content: @@ -311,6 +327,13 @@ components: properties: msg: type: string + AlreadyExistsError: + type: object + required: + - msg + properties: + msg: + type: string ForbiddenError: type: object required: diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index abc4043bdd..8365601042 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -74,7 +74,7 @@ async fn timeline_create_handler(mut request: Request) -> Result) -> Result json_response(StatusCode::CREATED, TimelineInfoResponse::from(info))?, + None => json_response(StatusCode::CONFLICT, ())?, + }) } async fn timeline_list_handler(request: Request) -> Result, ApiError> { @@ -220,17 +224,18 @@ async fn tenant_create_handler(mut request: Request) -> Result json_response(StatusCode::CREATED, HexZTenantId::from(id))?, + None => json_response(StatusCode::CONFLICT, ())?, + }) } async fn handler_404(_: Request) -> Result, ApiError> { diff --git a/pageserver/src/tenant_mgr.rs b/pageserver/src/tenant_mgr.rs index 77ef865ec5..568088fc1d 100644 --- a/pageserver/src/tenant_mgr.rs +++ b/pageserver/src/tenant_mgr.rs @@ -177,10 +177,10 @@ pub fn shutdown_all_tenants() { } } -pub fn create_repository_for_tenant( +pub fn create_tenant_repository( conf: &'static PageServerConf, new_tenant_id: Option, -) -> Result { +) -> Result> { let new_tenant_id = new_tenant_id.unwrap_or_else(ZTenantId::generate); let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, new_tenant_id)); match timelines::create_repo(conf, new_tenant_id, wal_redo_manager)? { @@ -191,11 +191,13 @@ pub fn create_repository_for_tenant( state: TenantState::Idle, repo, }); + Ok(Some(new_tenant_id)) + } + None => { + debug!("repository already exists for tenant {}", new_tenant_id); + Ok(None) } - None => debug!("repository already exists for tenant {}", new_tenant_id), } - - Ok(new_tenant_id) } pub fn get_tenant_state(tenantid: ZTenantId) -> Option { diff --git a/pageserver/src/timelines.rs b/pageserver/src/timelines.rs index 924c25804a..4de131ef70 100644 --- a/pageserver/src/timelines.rs +++ b/pageserver/src/timelines.rs @@ -331,17 +331,26 @@ pub(crate) fn create_timeline( new_timeline_id: Option, ancestor_timeline_id: Option, ancestor_start_lsn: Option, -) -> Result { +) -> Result> { let new_timeline_id = new_timeline_id.unwrap_or_else(ZTimelineId::generate); + let repo = tenant_mgr::get_repository_for_tenant(tenant_id)?; if conf.timeline_path(&new_timeline_id, &tenant_id).exists() { - bail!("timeline {} already exists", new_timeline_id); + match repo.get_timeline(new_timeline_id)? { + RepositoryTimeline::Local { id, .. } => { + debug!("timeline {} already exists", id); + return Ok(None); + } + RepositoryTimeline::Remote { id, .. } => bail!( + "timeline {} already exists in pageserver's remote storage", + id + ), + } } - let repo = tenant_mgr::get_repository_for_tenant(tenant_id)?; let mut start_lsn = ancestor_start_lsn.unwrap_or(Lsn(0)); - match ancestor_timeline_id { + let new_timeline_info = match ancestor_timeline_id { Some(ancestor_timeline_id) => { let ancestor_timeline = repo .get_timeline(ancestor_timeline_id) @@ -383,20 +392,17 @@ pub(crate) fn create_timeline( repo.branch_timeline(ancestor_timeline_id, new_timeline_id, start_lsn)?; // load the timeline into memory let loaded_timeline = repo.get_timeline(new_timeline_id)?; - Ok(TimelineInfo::from_repo_timeline( - tenant_id, - loaded_timeline, - false, - )) + TimelineInfo::from_repo_timeline(tenant_id, loaded_timeline, false) } None => { let new_timeline = bootstrap_timeline(conf, tenant_id, new_timeline_id, repo.as_ref())?; - Ok(TimelineInfo::from_dyn_timeline( + TimelineInfo::from_dyn_timeline( tenant_id, new_timeline_id, new_timeline.as_ref(), false, - )) + ) } - } + }; + Ok(Some(new_timeline_info)) } diff --git a/test_runner/batch_others/test_pageserver_api.py b/test_runner/batch_others/test_pageserver_api.py index 41b1899882..2aa3686904 100644 --- a/test_runner/batch_others/test_pageserver_api.py +++ b/test_runner/batch_others/test_pageserver_api.py @@ -28,7 +28,7 @@ def check_client(client: ZenithPageserverHttpClient, initial_tenant: UUID): # create timeline timeline_id = uuid4() - client.timeline_create(tenant_id=tenant_id, timeline_id=timeline_id) + client.timeline_create(tenant_id=tenant_id, new_timeline_id=timeline_id) timelines = client.timeline_list(tenant_id) assert len(timelines) > 0 diff --git a/test_runner/batch_others/test_wal_acceptor.py b/test_runner/batch_others/test_wal_acceptor.py index 02da7ee749..bdc4c4f63c 100644 --- a/test_runner/batch_others/test_wal_acceptor.py +++ b/test_runner/batch_others/test_wal_acceptor.py @@ -57,6 +57,10 @@ def test_many_timelines(zenith_env_builder: ZenithEnvBuilder): branch_names = [ "test_wal_acceptors_many_timelines_{}".format(tlin) for tlin in range(n_timelines) ] + # pageserver, safekeeper operate timelines via their ids (can be represented in hex as 'ad50847381e248feaac9876cc71ae418') + # that's not really human readable, so the branch names are introduced in Zenith CLI. + # Zenith CLI stores its branch <-> timeline mapping in its internals, + # but we need this to collect metrics from other servers, related to the timeline. branch_names_to_timeline_ids = {} # start postgres on each timeline @@ -75,7 +79,7 @@ def test_many_timelines(zenith_env_builder: ZenithEnvBuilder): tenant_id=tenant_id, timeline_id=branch_names_to_timeline_ids[branch_name]) for branch_name in branch_names ] - # All changes visible to pageserver (latest_valid_lsn) should be + # All changes visible to pageserver (last_record_lsn) should be # confirmed by safekeepers first. As we cannot atomically get # state of both pageserver and safekeepers, we should start with # pageserver. Looking at outdated data from pageserver is ok. diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index e2c9f16630..ec570a7dac 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -725,20 +725,23 @@ class ZenithPageserverHttpClient(requests.Session): def timeline_create( self, tenant_id: uuid.UUID, - timeline_id: Optional[uuid.UUID] = None, + new_timeline_id: Optional[uuid.UUID] = None, ancestor_timeline_id: Optional[uuid.UUID] = None, ancestor_start_lsn: Optional[str] = None, ) -> Dict[Any, Any]: res = self.post(f"http://localhost:{self.port}/v1/tenant/{tenant_id.hex}/timeline", json={ 'new_timeline_id': - timeline_id.hex if timeline_id else None, + new_timeline_id.hex if new_timeline_id else None, 'ancestor_start_lsn': ancestor_start_lsn, 'ancestor_timeline_id': ancestor_timeline_id.hex if ancestor_timeline_id else None, }) self.verbose_error(res) + if res.status_code == 409: + raise Exception(f'could not create timeline: already exists for id {new_timeline_id}') + res_json = res.json() assert isinstance(res_json, dict) return res_json @@ -750,14 +753,16 @@ class ZenithPageserverHttpClient(requests.Session): assert isinstance(res_json, list) return res_json - def tenant_create(self, tenant_id: Optional[uuid.UUID] = None) -> uuid.UUID: + def tenant_create(self, new_tenant_id: Optional[uuid.UUID] = None) -> uuid.UUID: res = self.post( f"http://localhost:{self.port}/v1/tenant", json={ - 'new_tenant_id': tenant_id.hex if tenant_id else None, + 'new_tenant_id': new_tenant_id.hex if new_tenant_id else None, }, ) self.verbose_error(res) + if res.status_code == 409: + raise Exception(f'could not create tenant: already exists for id {new_tenant_id}') new_tenant_id = res.json() assert isinstance(new_tenant_id, str) return uuid.UUID(new_tenant_id) @@ -806,6 +811,13 @@ class S3Storage: RemoteStorage = Union[LocalFsStorage, S3Storage] +CREATE_TIMELINE_ID_EXTRACTOR = re.compile(r"^Created timeline '(?P[^']+)'", + re.MULTILINE) +CREATE_TIMELINE_ID_EXTRACTOR = re.compile(r"^Created timeline '(?P[^']+)'", + re.MULTILINE) +TIMELINE_DATA_EXTRACTOR = re.compile(r"\s(?P[^\s]+)\s\[(?P[^\]]+)\]", + re.MULTILINE) + class ZenithCli: """ @@ -846,18 +858,13 @@ class ZenithCli: res = self.raw_cli(cmd) res.check_returncode() - create_timeline_id_extractor = re.compile(r"^Created timeline '(?P[^']+)'", - re.MULTILINE) - matches = create_timeline_id_extractor.search(res.stdout) + matches = CREATE_TIMELINE_ID_EXTRACTOR.search(res.stdout) created_timeline_id = None if matches is not None: created_timeline_id = matches.group('timeline_id') - if created_timeline_id is None: - raise Exception('could not find timeline id after `zenith timeline create` invocation') - else: - return uuid.UUID(created_timeline_id) + return uuid.UUID(created_timeline_id) def create_branch(self, new_branch_name: str = DEFAULT_BRANCH_NAME, @@ -880,9 +887,7 @@ class ZenithCli: res = self.raw_cli(cmd) res.check_returncode() - create_timeline_id_extractor = re.compile(r"^Created timeline '(?P[^']+)'", - re.MULTILINE) - matches = create_timeline_id_extractor.search(res.stdout) + matches = CREATE_TIMELINE_ID_EXTRACTOR.search(res.stdout) created_timeline_id = None if matches is not None: @@ -900,13 +905,11 @@ class ZenithCli: # (L) main [b49f7954224a0ad25cc0013ea107b54b] # (L) ┣━ @0/16B5A50: test_cli_branch_list_main [20f98c79111b9015d84452258b7d5540] - timeline_data_extractor = re.compile( - r"\s(?P[^\s]+)\s\[(?P[^\]]+)\]", re.MULTILINE) res = self.raw_cli( ['timeline', 'list', '--tenant-id', (tenant_id or self.env.initial_tenant).hex]) timelines_cli = sorted( map(lambda branch_and_id: (branch_and_id[0], branch_and_id[1]), - timeline_data_extractor.findall(res.stdout))) + TIMELINE_DATA_EXTRACTOR.findall(res.stdout))) return timelines_cli def init(self, diff --git a/zenith/src/main.rs b/zenith/src/main.rs index 165a7d7950..dd35427d5d 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -522,7 +522,11 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> Re } Some(("create", create_match)) => { let initial_tenant_id = parse_tenant_id(create_match)?; - let new_tenant_id = pageserver.tenant_create(initial_tenant_id)?; + let new_tenant_id = pageserver + .tenant_create(initial_tenant_id)? + .ok_or_else(|| { + anyhow!("Tenant with id {:?} was already created", initial_tenant_id) + })?; println!( "tenant {} successfully created on the pageserver", new_tenant_id @@ -548,7 +552,9 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - let new_branch_name = create_match .value_of("branch-name") .ok_or(anyhow!("No branch name provided"))?; - let timeline = pageserver.timeline_create(tenant_id, None, None, None)?; + let timeline = pageserver + .timeline_create(tenant_id, None, None, None)? + .ok_or_else(|| anyhow!("Failed to create new timeline for tenant {}", tenant_id))?; let new_timeline_id = timeline.timeline_id(); let last_record_lsn = match timeline { @@ -593,12 +599,9 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - .map(Lsn::from_str) .transpose() .context("Failed to parse ancestor start Lsn from the request")?; - let timeline = pageserver.timeline_create( - tenant_id, - None, - start_lsn, - Some(ancestor_timeline_id), - )?; + let timeline = pageserver + .timeline_create(tenant_id, None, start_lsn, Some(ancestor_timeline_id))? + .ok_or_else(|| anyhow!("Failed to create new timeline for tenant {}", tenant_id))?; let new_timeline_id = timeline.timeline_id(); let last_record_lsn = match timeline { From f67d010d1bf3678eb8a287d47bac3fe1eed3e8cc Mon Sep 17 00:00:00 2001 From: Dhammika Pathirana Date: Mon, 21 Feb 2022 13:40:25 -0800 Subject: [PATCH 25/42] Add ps smgr/storage metrics tenant tags Signed-off-by: Dhammika Pathirana Add tenant_id,timeline_id in smgr/storage metrics (#1234) --- pageserver/src/layered_repository.rs | 42 ++++++++++++++++++---------- pageserver/src/page_service.rs | 10 ++++--- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index c3d42d1829..63ade9bb37 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -47,10 +47,8 @@ use crate::walredo::WalRedoManager; use crate::CheckpointConfig; use crate::{ZTenantId, ZTimelineId}; -use zenith_metrics::{ - register_histogram, register_int_gauge_vec, Histogram, IntGauge, IntGaugeVec, -}; use zenith_metrics::{register_histogram_vec, HistogramVec}; +use zenith_metrics::{register_int_gauge_vec, IntGauge, IntGaugeVec}; use zenith_utils::crashsafe_dir; use zenith_utils::lsn::{AtomicLsn, Lsn, RecordLsn}; use zenith_utils::seqwait::SeqWait; @@ -87,16 +85,17 @@ lazy_static! { static ref STORAGE_TIME: HistogramVec = register_histogram_vec!( "pageserver_storage_time", "Time spent on storage operations", - &["operation"] + &["operation", "tenant_id", "timeline_id"] ) .expect("failed to define a metric"); } // Metrics collected on operations on the storage repository. lazy_static! { - static ref RECONSTRUCT_TIME: Histogram = register_histogram!( + static ref RECONSTRUCT_TIME: HistogramVec = register_histogram_vec!( "pageserver_getpage_reconstruct_time", - "FIXME Time spent on storage operations" + "FIXME Time spent on storage operations", + &["tenant_id", "timeline_id"] ) .expect("failed to define a metric"); } @@ -248,11 +247,19 @@ impl Repository for LayeredRepository { horizon: u64, checkpoint_before_gc: bool, ) -> Result { - STORAGE_TIME - .with_label_values(&["gc"]) - .observe_closure_duration(|| { - self.gc_iteration_internal(target_timelineid, horizon, checkpoint_before_gc) - }) + if let Some(timeline_id) = target_timelineid { + STORAGE_TIME + .with_label_values(&["gc", &self.tenantid.to_string(), &timeline_id.to_string()]) + .observe_closure_duration(|| { + self.gc_iteration_internal(target_timelineid, horizon, checkpoint_before_gc) + }) + } else { + STORAGE_TIME + .with_label_values(&["gc", &self.tenantid.to_string(), "-"]) + .observe_closure_duration(|| { + self.gc_iteration_internal(target_timelineid, horizon, checkpoint_before_gc) + }) + } } fn checkpoint_iteration(&self, cconf: CheckpointConfig) -> Result<()> { @@ -859,7 +866,11 @@ impl Timeline for LayeredTimeline { let (seg, seg_blknum) = SegmentTag::from_blknum(rel, rel_blknum); if let Some((layer, lsn)) = self.get_layer_for_read(seg, lsn)? { + let tenant_id = self.tenantid.to_string(); + let timeline_id = self.timelineid.to_string(); + RECONSTRUCT_TIME + .with_label_values(&[&tenant_id, &timeline_id]) .observe_closure_duration(|| self.materialize_page(seg, seg_blknum, lsn, &*layer)) } else { // FIXME: This can happen if PostgreSQL extends a relation but never writes @@ -1009,15 +1020,18 @@ impl Timeline for LayeredTimeline { /// checkpoint_internal function, this public facade just wraps it for /// metrics collection. fn checkpoint(&self, cconf: CheckpointConfig) -> Result<()> { + let tenant_id = self.tenantid.to_string(); + let timeline_id = self.timelineid.to_string(); + match cconf { CheckpointConfig::Flush => STORAGE_TIME - .with_label_values(&["flush checkpoint"]) + .with_label_values(&["flush checkpoint", &tenant_id, &timeline_id]) .observe_closure_duration(|| self.checkpoint_internal(0, false)), CheckpointConfig::Forced => STORAGE_TIME - .with_label_values(&["forced checkpoint"]) + .with_label_values(&["forced checkpoint", &tenant_id, &timeline_id]) .observe_closure_duration(|| self.checkpoint_internal(0, true)), CheckpointConfig::Distance(distance) => STORAGE_TIME - .with_label_values(&["checkpoint"]) + .with_label_values(&["checkpoint", &tenant_id, &timeline_id]) .observe_closure_duration(|| self.checkpoint_internal(distance, true)), } } diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 7dc3c8c752..42a099cca5 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -298,7 +298,7 @@ lazy_static! { static ref SMGR_QUERY_TIME: HistogramVec = register_histogram_vec!( "pageserver_smgr_query_time", "Time spent on smgr query handling", - &["smgr_query_type"], + &["smgr_query_type", "tenant_id", "timeline_id"], TIME_BUCKETS.into() ) .expect("failed to define a metric"); @@ -340,20 +340,22 @@ impl PageServerHandler { }; let zenith_fe_msg = PagestreamFeMessage::parse(copy_data_bytes)?; + let tenant_id = tenantid.to_string(); + let timeline_id = timelineid.to_string(); let response = match zenith_fe_msg { PagestreamFeMessage::Exists(req) => SMGR_QUERY_TIME - .with_label_values(&["get_rel_exists"]) + .with_label_values(&["get_rel_exists", &tenant_id, &timeline_id]) .observe_closure_duration(|| { self.handle_get_rel_exists_request(timeline.as_ref(), &req) }), PagestreamFeMessage::Nblocks(req) => SMGR_QUERY_TIME - .with_label_values(&["get_rel_size"]) + .with_label_values(&["get_rel_size", &tenant_id, &timeline_id]) .observe_closure_duration(|| { self.handle_get_nblocks_request(timeline.as_ref(), &req) }), PagestreamFeMessage::GetPage(req) => SMGR_QUERY_TIME - .with_label_values(&["get_page_at_lsn"]) + .with_label_values(&["get_page_at_lsn", &tenant_id, &timeline_id]) .observe_closure_duration(|| { self.handle_get_page_at_lsn_request(timeline.as_ref(), &req) }), From 27dadba52c7543b9bd49b8c506fa74a1587df543 Mon Sep 17 00:00:00 2001 From: Dhammika Pathirana Date: Fri, 25 Feb 2022 14:22:48 -0800 Subject: [PATCH 26/42] Fix retain references to layer histograms Signed-off-by: Dhammika Pathirana --- pageserver/src/layered_repository.rs | 80 ++++++++++++++++++---------- 1 file changed, 52 insertions(+), 28 deletions(-) diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index 63ade9bb37..a6e61cb9e0 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -47,7 +47,7 @@ use crate::walredo::WalRedoManager; use crate::CheckpointConfig; use crate::{ZTenantId, ZTimelineId}; -use zenith_metrics::{register_histogram_vec, HistogramVec}; +use zenith_metrics::{register_histogram_vec, Histogram, HistogramVec}; use zenith_metrics::{register_int_gauge_vec, IntGauge, IntGaugeVec}; use zenith_utils::crashsafe_dir; use zenith_utils::lsn::{AtomicLsn, Lsn, RecordLsn}; @@ -247,19 +247,15 @@ impl Repository for LayeredRepository { horizon: u64, checkpoint_before_gc: bool, ) -> Result { - if let Some(timeline_id) = target_timelineid { - STORAGE_TIME - .with_label_values(&["gc", &self.tenantid.to_string(), &timeline_id.to_string()]) - .observe_closure_duration(|| { - self.gc_iteration_internal(target_timelineid, horizon, checkpoint_before_gc) - }) - } else { - STORAGE_TIME - .with_label_values(&["gc", &self.tenantid.to_string(), "-"]) - .observe_closure_duration(|| { - self.gc_iteration_internal(target_timelineid, horizon, checkpoint_before_gc) - }) - } + let timeline_str = target_timelineid + .map(|x| x.to_string()) + .unwrap_or_else(|| "-".to_string()); + + STORAGE_TIME + .with_label_values(&["gc", &self.tenantid.to_string(), &timeline_str]) + .observe_closure_duration(|| { + self.gc_iteration_internal(target_timelineid, horizon, checkpoint_before_gc) + }) } fn checkpoint_iteration(&self, cconf: CheckpointConfig) -> Result<()> { @@ -788,6 +784,12 @@ pub struct LayeredTimeline { // ordering for its operations, but involves private modules, and macro trickery current_logical_size_gauge: IntGauge, + // Metrics histograms + reconstruct_time_histo: Histogram, + checkpoint_time_histo: Histogram, + flush_checkpoint_time_histo: Histogram, + forced_checkpoint_time_histo: Histogram, + /// If `true`, will backup its files that appear after each checkpointing to the remote storage. upload_relishes: AtomicBool, @@ -866,11 +868,7 @@ impl Timeline for LayeredTimeline { let (seg, seg_blknum) = SegmentTag::from_blknum(rel, rel_blknum); if let Some((layer, lsn)) = self.get_layer_for_read(seg, lsn)? { - let tenant_id = self.tenantid.to_string(); - let timeline_id = self.timelineid.to_string(); - - RECONSTRUCT_TIME - .with_label_values(&[&tenant_id, &timeline_id]) + self.reconstruct_time_histo .observe_closure_duration(|| self.materialize_page(seg, seg_blknum, lsn, &*layer)) } else { // FIXME: This can happen if PostgreSQL extends a relation but never writes @@ -1020,18 +1018,15 @@ impl Timeline for LayeredTimeline { /// checkpoint_internal function, this public facade just wraps it for /// metrics collection. fn checkpoint(&self, cconf: CheckpointConfig) -> Result<()> { - let tenant_id = self.tenantid.to_string(); - let timeline_id = self.timelineid.to_string(); - match cconf { - CheckpointConfig::Flush => STORAGE_TIME - .with_label_values(&["flush checkpoint", &tenant_id, &timeline_id]) + CheckpointConfig::Flush => self + .flush_checkpoint_time_histo .observe_closure_duration(|| self.checkpoint_internal(0, false)), - CheckpointConfig::Forced => STORAGE_TIME - .with_label_values(&["forced checkpoint", &tenant_id, &timeline_id]) + CheckpointConfig::Forced => self + .forced_checkpoint_time_histo .observe_closure_duration(|| self.checkpoint_internal(0, true)), - CheckpointConfig::Distance(distance) => STORAGE_TIME - .with_label_values(&["checkpoint", &tenant_id, &timeline_id]) + CheckpointConfig::Distance(distance) => self + .checkpoint_time_histo .observe_closure_duration(|| self.checkpoint_internal(distance, true)), } } @@ -1130,6 +1125,31 @@ impl LayeredTimeline { let current_logical_size_gauge = LOGICAL_TIMELINE_SIZE .get_metric_with_label_values(&[&tenantid.to_string(), &timelineid.to_string()]) .unwrap(); + let reconstruct_time_histo = RECONSTRUCT_TIME + .get_metric_with_label_values(&[&tenantid.to_string(), &timelineid.to_string()]) + .unwrap(); + let checkpoint_time_histo = STORAGE_TIME + .get_metric_with_label_values(&[ + "checkpoint", + &tenantid.to_string(), + &timelineid.to_string(), + ]) + .unwrap(); + let flush_checkpoint_time_histo = STORAGE_TIME + .get_metric_with_label_values(&[ + "flush checkpoint", + &tenantid.to_string(), + &timelineid.to_string(), + ]) + .unwrap(); + let forced_checkpoint_time_histo = STORAGE_TIME + .get_metric_with_label_values(&[ + "forced checkpoint", + &tenantid.to_string(), + &timelineid.to_string(), + ]) + .unwrap(); + LayeredTimeline { conf, timelineid, @@ -1149,6 +1169,10 @@ impl LayeredTimeline { ancestor_lsn: metadata.ancestor_lsn(), current_logical_size: AtomicUsize::new(current_logical_size), current_logical_size_gauge, + reconstruct_time_histo, + checkpoint_time_histo, + flush_checkpoint_time_histo, + forced_checkpoint_time_histo, upload_relishes: AtomicBool::new(upload_relishes), write_lock: Mutex::new(()), From b2ad8342d21521226160416efe8e330cf1655852 Mon Sep 17 00:00:00 2001 From: Dhammika Pathirana Date: Mon, 28 Feb 2022 16:37:09 -0800 Subject: [PATCH 27/42] Add zid stringify bench test Signed-off-by: Dhammika Pathirana --- zenith_utils/Cargo.toml | 5 +++++ zenith_utils/benches/benchmarks.rs | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 zenith_utils/benches/benchmarks.rs diff --git a/zenith_utils/Cargo.toml b/zenith_utils/Cargo.toml index b22fcbf748..daaf345f8f 100644 --- a/zenith_utils/Cargo.toml +++ b/zenith_utils/Cargo.toml @@ -37,3 +37,8 @@ bytes = "1.0.1" hex-literal = "0.3" tempfile = "3.2" webpki = "0.21" +criterion = "0.3" + +[[bench]] +name = "benchmarks" +harness = false diff --git a/zenith_utils/benches/benchmarks.rs b/zenith_utils/benches/benchmarks.rs new file mode 100644 index 0000000000..c945d5021c --- /dev/null +++ b/zenith_utils/benches/benchmarks.rs @@ -0,0 +1,22 @@ +#![allow(unused)] + +use criterion::{criterion_group, criterion_main, Criterion}; +use zenith_utils::zid; + +pub fn bench_zid_stringify(c: &mut Criterion) { + // Can only use public methods. + let ztl = zid::ZTenantTimelineId::generate(); + + c.bench_function("zid.to_string", |b| { + b.iter(|| { + // FIXME measurement overhead? + //for _ in 0..1000 { + // ztl.tenant_id.to_string(); + //} + ztl.tenant_id.to_string(); + }) + }); +} + +criterion_group!(benches, bench_zid_stringify); +criterion_main!(benches); From a8a7dc9ca65352ad738e55a3a26a7171a89db17b Mon Sep 17 00:00:00 2001 From: Dhammika Pathirana Date: Tue, 1 Mar 2022 14:28:25 -0800 Subject: [PATCH 28/42] Fix zid encoding Signed-off-by: Dhammika Pathirana --- zenith_utils/src/zid.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/zenith_utils/src/zid.rs b/zenith_utils/src/zid.rs index a740d4fb48..e047e38da7 100644 --- a/zenith_utils/src/zid.rs +++ b/zenith_utils/src/zid.rs @@ -112,6 +112,17 @@ impl ZId { rand::thread_rng().fill(&mut tli_buf); ZId::from(tli_buf) } + + fn hex_encode(&self) -> String { + static HEX: &[u8] = b"0123456789abcdef"; + + let mut buf = vec![0u8; self.0.len() * 2]; + for (&b, chunk) in self.0.as_ref().iter().zip(buf.chunks_exact_mut(2)) { + chunk[0] = HEX[((b >> 4) & 0xf) as usize]; + chunk[1] = HEX[(b & 0xf) as usize]; + } + unsafe { String::from_utf8_unchecked(buf) } + } } impl FromStr for ZId { @@ -147,13 +158,13 @@ impl From<[u8; 16]> for ZId { impl fmt::Display for ZId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&hex::encode(self.0)) + f.write_str(&self.hex_encode()) } } impl fmt::Debug for ZId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&hex::encode(self.0)) + f.write_str(&self.hex_encode()) } } From 5d7bd8643ade07d0e8a1f2ee8c9b535336b65e90 Mon Sep 17 00:00:00 2001 From: Dhammika Pathirana Date: Wed, 2 Mar 2022 14:50:22 -0800 Subject: [PATCH 29/42] Fix page reconstruct time histo Signed-off-by: Dhammika Pathirana --- pageserver/src/layered_repository.rs | 29 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index a6e61cb9e0..9e0df5dab2 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -94,7 +94,7 @@ lazy_static! { lazy_static! { static ref RECONSTRUCT_TIME: HistogramVec = register_histogram_vec!( "pageserver_getpage_reconstruct_time", - "FIXME Time spent on storage operations", + "Time spent on storage operations", &["tenant_id", "timeline_id"] ) .expect("failed to define a metric"); @@ -868,8 +868,7 @@ impl Timeline for LayeredTimeline { let (seg, seg_blknum) = SegmentTag::from_blknum(rel, rel_blknum); if let Some((layer, lsn)) = self.get_layer_for_read(seg, lsn)? { - self.reconstruct_time_histo - .observe_closure_duration(|| self.materialize_page(seg, seg_blknum, lsn, &*layer)) + self.materialize_page(seg, seg_blknum, lsn, &*layer) } else { // FIXME: This can happen if PostgreSQL extends a relation but never writes // the page. See https://github.com/zenithdb/zenith/issues/841 @@ -2022,17 +2021,19 @@ impl LayeredTimeline { let mut layer_ref = layer; let mut curr_lsn = lsn; loop { - let result = layer_ref - .get_page_reconstruct_data(seg_blknum, curr_lsn, &mut data) - .with_context(|| { - format!( - "Failed to get reconstruct data {} {:?} {} {}", - layer_ref.get_seg_tag(), - layer_ref.filename(), - seg_blknum, - curr_lsn, - ) - })?; + let result = self.reconstruct_time_histo.observe_closure_duration(|| { + layer_ref + .get_page_reconstruct_data(seg_blknum, curr_lsn, &mut data) + .with_context(|| { + format!( + "Failed to get reconstruct data {} {:?} {} {}", + layer_ref.get_seg_tag(), + layer_ref.filename(), + seg_blknum, + curr_lsn, + ) + }) + })?; match result { PageReconstructResult::Complete => break, PageReconstructResult::Continue(cont_lsn) => { From d93fc371f348919ed728fd8539f34e4d0a270f9d Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 11 Mar 2022 18:49:36 +0200 Subject: [PATCH 30/42] Import all existing RFCs documents from the separate 'rfcs' repository. --- docs/rfcs/002-storage.md | 186 ++++++++++++ docs/rfcs/003-laptop-cli.md | 267 ++++++++++++++++++ docs/rfcs/004-durability.md | 218 ++++++++++++++ docs/rfcs/005-zenith_local.md | 103 +++++++ docs/rfcs/006-laptop-cli-v2-CLI.md | 64 +++++ .../006-laptop-cli-v2-repository-structure.md | 140 +++++++++ docs/rfcs/007-serverless-on-laptop.md | 93 ++++++ docs/rfcs/008-push-pull.md | 66 +++++ docs/rfcs/009-snapshot-first-storage-cli.md | 56 ++++ docs/rfcs/009-snapshot-first-storage-pitr.md | 227 +++++++++++++++ docs/rfcs/009-snapshot-first-storage.md | 148 ++++++++++ docs/rfcs/010-storage_details.md | 144 ++++++++++ docs/rfcs/011-retention-policy.md | 91 ++++++ docs/rfcs/012-background-tasks.md | 38 +++ docs/rfcs/013-term-history.md | 147 ++++++++++ docs/rfcs/README.md | 95 +++++++ docs/rfcs/images/storage.jpeg | Bin 0 -> 431075 bytes 17 files changed, 2083 insertions(+) create mode 100644 docs/rfcs/002-storage.md create mode 100644 docs/rfcs/003-laptop-cli.md create mode 100644 docs/rfcs/004-durability.md create mode 100644 docs/rfcs/005-zenith_local.md create mode 100644 docs/rfcs/006-laptop-cli-v2-CLI.md create mode 100644 docs/rfcs/006-laptop-cli-v2-repository-structure.md create mode 100644 docs/rfcs/007-serverless-on-laptop.md create mode 100644 docs/rfcs/008-push-pull.md create mode 100644 docs/rfcs/009-snapshot-first-storage-cli.md create mode 100644 docs/rfcs/009-snapshot-first-storage-pitr.md create mode 100644 docs/rfcs/009-snapshot-first-storage.md create mode 100644 docs/rfcs/010-storage_details.md create mode 100644 docs/rfcs/011-retention-policy.md create mode 100644 docs/rfcs/012-background-tasks.md create mode 100644 docs/rfcs/013-term-history.md create mode 100644 docs/rfcs/README.md create mode 100644 docs/rfcs/images/storage.jpeg diff --git a/docs/rfcs/002-storage.md b/docs/rfcs/002-storage.md new file mode 100644 index 0000000000..5cac377272 --- /dev/null +++ b/docs/rfcs/002-storage.md @@ -0,0 +1,186 @@ +# Zenith storage node — alternative + +## **Design considerations** + +Simplify storage operations for people => Gain adoption/installs on laptops and small private installation => Attract customers to DBaaS by seamless integration between our tooling and cloud. + +Proposed architecture addresses: + +- High availability -- tolerates n/2 - 1 failures +- Multi-tenancy -- one storage for all databases +- Elasticity -- increase storage size on the go by adding nodes +- Snapshots / backups / PITR with S3 offload +- Compression + +Minuses are: + +- Quite a lot of work +- Single page access may touch few disk pages +- Some bloat in data — may slowdown sequential scans + +## **Summary** + +Storage cluster is sharded key-value store with ordered keys. Key (****page_key****) is a tuple of `(pg_id, db_id, timeline_id, rel_id, forkno, segno, pageno, lsn)`. Value is either page or page diff/wal. Each chunk (chunk == shard) stores approx 50-100GB ~~and automatically splits in half when grows bigger then soft 100GB limit~~. by having a fixed range of pageno's it is responsible for. Chunks placement on storage nodes is stored in a separate metadata service, so chunk can be freely moved around the cluster if it is need. Chunk itself is a filesystem directory with following sub directories: + +``` + +|-chunk_42/ + |-store/ -- contains lsm with pages/pagediffs ranging from + | page_key_lo to page_key_hi + |-wal/ + | |- db_1234/ db-specific wal files with pages from page_key_lo + | to page_key_hi + | + |-chunk.meta -- small file with snapshot references + (page_key_prefix+lsn+name) + and PITR regions (page_key_start, page_key_end) +``` + +## **Chunk** + +Chunk is responsible for storing pages potentially from different databases and relations. Each page is addressed by a lexicographically ordered tuple (****page_key****) with following fields: + +- `pg_id` -- unique id of given postgres instance (or postgres cluster as it is called in postgres docs) +- `db_id` -- database that was created by 'CREATE DATABASE' in a given postgres instance +- `db_timeline` -- used to create Copy-on-Write instances from snapshots, described later +- `rel_id` -- tuple of (relation_id, 0) for tables and (indexed_relation_id, rel_id) for indices. Done this way so table indices were closer to table itself on our global key space. +- `(forkno, segno, pageno)` -- page coordinates in postgres data files +- `lsn_timeline` -- postgres feature, increments when PITR was done. +- `lsn` -- lsn of current page version. + +Chunk stores pages and page diffs ranging from page_key_lo to page_key_hi. Processing node looks at page in wal record and sends record to a chunk responsible for this page range. When wal record arrives to a chunk it is initially stored in `chunk_id/wal/db_id/wal_segno.wal`. Then background process moves records from that wal files to the lsm tree in `chunk_id/store`. Or, more precisely, wal records would be materialized into lsm memtable and when that memtable is flushed to SSTable on disk we may trim the wal. That way some not durably (in the distributed sense) committed pages may enter the tree -- here we rely on processing node behavior: page request from processing node should contain proper lsm horizons so that storage node may respond with proper page version. + +LSM here is a usual LSM for variable-length values: at first data is stored in memory (we hold incoming wal records to be able to regenerate it after restart) at some balanced tree. When this tree grows big enough we dump it into disk file (SSTable) sorting records by key. Then SStables are mergesorted in the background to a different files. All file operation are sequential and do not require WAL for durability. + +Content of SSTable can be following: + +```jsx +(pg_id, db_id, ... , pageno=42, lsn=100) (full 8k page data) +(pg_id, db_id, ... , pageno=42, lsn=150) (per-page diff) +(pg_id, db_id, ... , pageno=42, lsn=180) (per-page diff) +(pg_id, db_id, ... , pageno=42, lsn=200) (per-page diff) +(pg_id, db_id, ... , pageno=42, lsn=220) (full 8k page data) +(pg_id, db_id, ... , pageno=42, lsn=250) (per-page diff) +(pg_id, db_id, ... , pageno=42, lsn=270) (per-page diff) +(pg_id, db_id, ... , pageno=5000, lsn=100) (full 8k page data) +``` + +So query for `pageno=42 up to lsn=260` would need to find closest entry less then this key, iterate back to the latest full page and iterate forward to apply diffs. How often page is materialized in lsn-version sequence is up to us -- let's say each 5th version should be a full page. + +### **Page deletion** + +To delete old pages we insert blind deletion marker `(pg_id, db_id, #trim_lsn < 150)` into a lsm tree. During merges such marker would indicate that all pages with smaller lsn should be discarded. Delete marker will travel down the tree levels hierarchy until it reaches last level. In non-PITR scenario where old page version are not needed at all such deletion marker would (in average) prevent old page versions propagation down the tree -- so all bloat would concentrate at higher tree layers without affecting bigger bottom layers. + +### **Recovery** + +Upon storage node restart recent WAL files are applied to appropriate pages and resulting pages stored in lsm memtable. So this should be fast since we are not writing anything to disk. + +### **Checkpointing** + +No such mechanism is needed. Or we may look at the storage node as at kind of continuous chekpointer. + +### **Full page writes (torn page protection)** + +Storage node never updates individual pages, only merges SSTable, so torn pages is not an issue. + +### **Snapshot** + +That is the part that I like about this design -- snapshot creation is instant and cheap operation that can have flexible granularity level: whole instance, database, table. Snapshot creation inserts a record in `chunk.meta` file with lsn of this snapshot and key prefix `(pg_id, db_id, db_timeline, rel_id, *)` that prohibits pages deletion within this range. Storage node may not know anything about page internals, but by changing number of fields in our prefix we may change snapshot granularity. + +It is again useful to remap `rel_id` to `(indexed_relation_id, rel_id)` so that snapshot of relation would include it's indices. Also table snapshot would trickily interact with catalog. Probably all table snapshots should hold also a catalog snapshot. And when node is started with such snapshot it should check that only tables from snapshot are queried. I assume here that for snapshot reading one need to start a new postgres instance. + +Storage consumed by snapshot is proportional to the amount of data changed. We may have some heuristic (calculated based on cost of different storages) about when to offload old snapshot to s3. For example, if current database has more then 40% of changed pages with respect to previous snapshot then we may offload that snapshot to s3, and release this space. + +**Starting db from snapshot** + +When we are starting database from snapshot it can be done in two ways. First, we may create new db_id, move all the data from snapshot to a new db and start a database. Second option is to create Copy-on-Write (CoW) instance out of snapshot and read old pages from old snapshot and store new pages separately. That is why there is `db_timeline` key field near `db_id` -- CoW (🐮) database should create new `db_timeline` and remember old `db_timeline`. Such a database can have hashmap of pages that it is changed to query pages from proper snapshot on the first try. `db_timeline` is located near `db_id` so that new page versions generated by new instance would not bloat data of initial snapshot. It is not clear for whether it is possibly to effectively support "stacked" CoW snapshot, so we may disallow them. (Well, one way to support them is to move `db_timeline` close to `lsn` -- so we may scan neighboring pages and find right one. But again that way we bloat snapshot with unrelated data and may slowdown full scans that are happening in different database). + +**Snapshot export/import** + +Once we may start CoW instances it is easy to run auxiliary postgres instance on this snapshot and run `COPY FROM (...) TO stdout` or `pg_dump` and export data from the snapshot to some portable formats. Also we may start postgres on a new empty database and run `COPY FROM stdin`. This way we can initialize new non-CoW databases and transfer snapshots via network. + +### **PITR area** + +In described scheme PITR is just a prohibition to delete any versions within some key prefix, either it is a database or a table key prefix. So PITR may have different settings for different tables, databases, etc. + +PITR is quite bloaty, so we may aggressively offload it to s3 -- we may push same (or bigger) SSTables to s3 and maintain lsm structure there. + +### **Compression** + +Since we are storing page diffs of variable sizes there is no structural dependency on a page size and we may compress it. Again that could be enabled only on pages with some key prefixes, so we may have this with db/table granularity. + +### **Chunk metadata** + +Chunk metadata is a file lies in chunk directory that stores info about current snapshots and PITR regions. Chunck should always consult this data when merging SSTables and applying delete markers. + +### **Chunk splitting** + +*(NB: following paragraph is about how to avoid page splitting)* + +When chunks hits some soft storage limit (let's say 100Gb) it should be split in half and global matadata about chunk boundaries should be updated. Here i assume that chunk split is a local operation happening on single node. Process of chink splitting should look like following: + +1. Find separation key and spawn two new chunks with [lo, mid) [mid, hi) boundaries. + +2. Prohibit WAL deletion and old SSTables deletion on original chunk. + +3. On each lsm layer we would need to split only one SSTable, all other would fit within left or right range. Symlink/split that files to new chunks. + +4. Start WAL replay on new chunks. + +5. Update global metadata about new chunk boundaries. + +6. Eventually (metadata update should be pushed to processing node by metadata service) storage node will start sending WAL and page requests to the new nodes. + +7. New chunk may start serving read queries when following conditions are met: + +a) it receives at least on WAL record from processing node + +b) it replayed all WAL up to the new received one + +c) checked by downlinks that there were no WAL gaps. + +Chunk split as it is described here is quite fast operation when it is happening on the local disk -- vast majority of files will be just moved without copying anything. I suggest to keep split always local and not to mix it with chunk moving around cluster. So if we want to split some chunk but there is small amount of free space left on the device, we should first move some chunks away from the node and then proceed with splitting. + +### Fixed chunks + +Alternative strategy is to not to split at all and have pageno-fixed chunk boundaries. When table is created we first materialize this chunk by storing first new pages only and chunks is small. Then chunk is growing while table is filled, but it can't grow substantially bigger then allowed pageno range, so at max it would be 1GB or whatever limit we want + some bloat due to snapshots and old page versions. + +### **Chunk lsm internals** + +So how to implement chunk's lsm? + +- Write from scratch and use RocksDB to prototype/benchmark, then switch to own lsm implementation. RocksDB can provide some sanity check for performance of home-brewed implementation and it would be easier to prototype. +- Use postgres as lego constructor. We may model memtable with postgres B-tree referencing some in-memory log of incoming records. SSTable merging may reuse postgres external merging algorithm, etc. One thing that would definitely not fit (or I didn't came up with idea how to fit that) -- is multi-tenancy. If we are storing pages from different databases we can't use postgres buffer pool, since there is no db_id in the page header. We can add new field there but IMO it would be no go for committing that to vanilla. + +Other possibility is to not to try to fit few databases in one storage node. But that way it is no go for multi-tenant cloud installation: we would need to run a lot of storage node instances on one physical storage node, all with it own local page cache. So that would be much closer to ordinary managed RDS. + +Multi-tenant storage makes sense even on a laptop, when you work with different databases, running tests with temp database, etc. And when installation grows bigger it start to make more and more sense, so it seems important. + +# Storage fleet + +# **Storage fleet** + +- When database is smaller then a chunk size we naturally can store them in one chunk (since their page_key would fit in some chunk's [hi, lo) range). + +Screenshot_2021-02-22_at_16 49 17 + +Few databases are stored in one chunk, replicated three times + +- When database can't fit into one storage node it can occupy lots of chunks that were split while database was growing. Chunk placement on nodes is controlled by us with some automatization, but we alway may manually move chunks around the cluster. + +Screenshot_2021-02-22_at_16 49 10 + +Here one big database occupies two set of nodes. Also some chunks were moved around to restore replication factor after disk failure. In this case we also have "sharded" storage for a big database and issue wal writes to different chunks in parallel. + +## **Chunk placement strategies** + +There are few scenarios where we may want to move chunks around the cluster: + +- disk usage on some node is big +- some disk experienced a failure +- some node experienced a failure or need maintenance + +## **Chunk replication** + +Chunk replication may be done by cloning page ranges with respect to some lsn from peer nodes, updating global metadata, waiting for WAL to come, replaying previous WAL and becoming online -- more or less like during chunk split. + diff --git a/docs/rfcs/003-laptop-cli.md b/docs/rfcs/003-laptop-cli.md new file mode 100644 index 0000000000..4d1f0a68f0 --- /dev/null +++ b/docs/rfcs/003-laptop-cli.md @@ -0,0 +1,267 @@ +# Command line interface (end-user) + +Zenith CLI as it is described here mostly resides on the same conceptual level as pg_ctl/initdb/pg_recvxlog/etc and replaces some of them in an opinionated way. I would also suggest bundling our patched postgres inside zenith distribution at least at the start. + +This proposal is focused on managing local installations. For cluster operations, different tooling would be needed. The point of integration between the two is storage URL: no matter how complex cluster setup is it may provide an endpoint where the user may push snapshots. + +The most important concept here is a snapshot, which can be created/pushed/pulled/exported. Also, we may start temporary read-only postgres instance over any local snapshot. A more complex scenario would consist of several basic operations over snapshots. + +# Possible usage scenarios + +## Install zenith, run a postgres + +``` +> brew install pg-zenith +> zenith pg create # creates pgdata with default pattern pgdata$i +> zenith pg list +ID PGDATA USED STORAGE ENDPOINT +primary1 pgdata1 0G zenith-local localhost:5432 +``` + +## Import standalone postgres to zenith + +``` +> zenith snapshot import --from=basebackup://replication@localhost:5432/ oldpg +[====================------------] 60% | 20MB/s +> zenith snapshot list +ID SIZE PARENT +oldpg 5G - + +> zenith pg create --snapshot oldpg +Started postgres on localhost:5432 + +> zenith pg list +ID PGDATA USED STORAGE ENDPOINT +primary1 pgdata1 5G zenith-local localhost:5432 + +> zenith snapshot destroy oldpg +Ok +``` + +Also, we may start snapshot import implicitly by looking at snapshot schema + +``` +> zenith pg create --snapshot basebackup://replication@localhost:5432/ +Downloading snapshot... Done. +Started postgres on localhost:5432 +Destroying snapshot... Done. +``` + +## Pull snapshot with some publicly shared database + +Since we may export the whole snapshot as one big file (tar of basebackup, maybe with some manifest) it may be shared over conventional means: http, ssh, [git+lfs](https://docs.github.com/en/github/managing-large-files/about-git-large-file-storage). + +``` +> zenith pg create --snapshot http://learn-postgres.com/movies_db.zenith movies +``` + +## Create snapshot and push it to the cloud + +``` +> zenith snapshot create pgdata1@snap1 +> zenith snapshot push --to ssh://stas@zenith.tech pgdata1@snap1 +``` + +## Rollback database to the snapshot + +One way to rollback the database is just to init a new database from the snapshot and destroy the old one. But creating a new database from a snapshot would require a copy of that snapshot which is time consuming operation. Another option that would be cool to support is the ability to create the copy-on-write database from the snapshot without copying data, and store updated pages in a separate location, however that way would have performance implications. So to properly rollback the database to the older state we have `zenith pg checkout`. + +``` +> zenith pg list +ID PGDATA USED STORAGE ENDPOINT +primary1 pgdata1 5G zenith-local localhost:5432 + +> zenith snapshot create pgdata1@snap1 + +> zenith snapshot list +ID SIZE PARENT +oldpg 5G - +pgdata1@snap1 6G - +pgdata1@CURRENT 6G - + +> zenith pg checkout pgdata1@snap1 +Stopping postgres on pgdata1. +Rolling back pgdata1@CURRENT to pgdata1@snap1. +Starting postgres on pgdata1. + +> zenith snapshot list +ID SIZE PARENT +oldpg 5G - +pgdata1@snap1 6G - +pgdata1@HEAD{0} 6G - +pgdata1@CURRENT 6G - +``` + +Some notes: pgdata1@CURRENT -- implicit snapshot representing the current state of the database in the data directory. When we are checking out some snapshot CURRENT will be set to this snapshot and the old CURRENT state will be named HEAD{0} (0 is the number of postgres timeline, it would be incremented after each such checkout). + +## Configure PITR area (Point In Time Recovery). + +PITR area acts like a continuous snapshot where you can reset the database to any point in time within this area (by area I mean some TTL period or some size limit, both possibly infinite). + +``` +> zenith pitr create --storage s3tank --ttl 30d --name pitr_last_month +``` + +Resetting the database to some state in past would require creating a snapshot on some lsn / time in this pirt area. + +# Manual + +## storage + +Storage is either zenith pagestore or s3. Users may create a database in a pagestore and create/move *snapshots* and *pitr regions* in both pagestore and s3. Storage is a concept similar to `git remote`. After installation, I imagine one local storage is available by default. + +**zenith storage attach** -t [native|s3] -c key=value -n name + +Attaches/initializes storage. For --type=s3, user credentials and path should be provided. For --type=native we may support --path=/local/path and --url=zenith.tech/stas/mystore. Other possible term for native is 'zstore'. + + +**zenith storage list** + +Show currently attached storages. For example: + +``` +> zenith storage list +NAME USED TYPE OPTIONS PATH +local 5.1G zenith-local /opt/zenith/store/local +local.compr 20.4G zenith-local comression=on /opt/zenith/store/local.compr +zcloud 60G zenith-remote zenith.tech/stas/mystore +s3tank 80G S3 +``` + +**zenith storage detach** + +**zenith storage show** + + + +## pg + +Manages postgres data directories and can start postgreses with proper configuration. An experienced user may avoid using that (except pg create) and configure/run postgres by themself. + +Pg is a term for a single postgres running on some data. I'm trying to avoid here separation of datadir management and postgres instance management -- both that concepts bundled here together. + +**zenith pg create** [--no-start --snapshot --cow] -s storage-name -n pgdata + +Creates (initializes) new data directory in given storage and starts postgres. I imagine that storage for this operation may be only local and data movement to remote location happens through snapshots/pitr. + +--no-start: just init datadir without creating + +--snapshot snap: init from the snapshot. Snap is a name or URL (zenith.tech/stas/mystore/snap1) + +--cow: initialize Copy-on-Write data directory on top of some snapshot (makes sense if it is a snapshot of currently running a database) + +**zenith pg destroy** + +**zenith pg start** [--replica] pgdata + +Start postgres with proper extensions preloaded/installed. + +**zenith pg checkout** + +Rollback data directory to some previous snapshot. + +**zenith pg stop** pg_id + +**zenith pg list** + +``` +ROLE PGDATA USED STORAGE ENDPOINT +primary my_pg 5.1G local localhost:5432 +replica-1 localhost:5433 +replica-2 localhost:5434 +primary my_pg2 3.2G local.compr localhost:5435 +- my_pg3 9.2G local.compr - +``` + +**zenith pg show** + +``` +my_pg: + storage: local + space used on local: 5.1G + space used on all storages: 15.1G + snapshots: + on local: + snap1: 1G + snap2: 1G + on zcloud: + snap2: 1G + on s3tank: + snap5: 2G + pitr: + on s3tank: + pitr_one_month: 45G + +``` + +**zenith pg start-rest/graphql** pgdata + +Starts REST/GraphQL proxy on top of postgres master. Not sure we should do that, just an idea. + + +## snapshot + +Snapshot creation is cheap -- no actual data is copied, we just start retaining old pages. Snapshot size means the amount of retained data, not all data. Snapshot name looks like pgdata_name@tag_name. tag_name is set by the user during snapshot creation. There are some reserved tag names: CURRENT represents the current state of the data directory; HEAD{i} represents the data directory state that resided in the database before i-th checkout. + +**zenith snapshot create** pgdata_name@snap_name + +Creates a new snapshot in the same storage where pgdata_name exists. + +**zenith snapshot push** --to url pgdata_name@snap_name + +Produces binary stream of a given snapshot. Under the hood starts temp read-only postgres over this snapshot and sends basebackup stream. Receiving side should start `zenith snapshot recv` before push happens. If url has some special schema like zenith:// receiving side may require auth start `zenith snapshot recv` on the go. + +**zenith snapshot recv** + +Starts a port listening for a basebackup stream, prints connection info to stdout (so that user may use that in push command), and expects data on that socket. + +**zenith snapshot pull** --from url or path + +Connects to a remote zenith/s3/file and pulls snapshot. The remote site should be zenith service or files in our format. + +**zenith snapshot import** --from basebackup://<...> or path + +Creates a new snapshot out of running postgres via basebackup protocol or basebackup files. + +**zenith snapshot export** + +Starts read-only postgres over this snapshot and exports data in some format (pg_dump, or COPY TO on some/all tables). One of the options may be zenith own format which is handy for us (but I think just tar of basebackup would be okay). + +**zenith snapshot diff** snap1 snap2 + +Shows size of data changed between two snapshots. We also may provide options to diff schema/data in tables. To do that start temp read-only postgreses. + +**zenith snapshot destroy** + +## pitr + +Pitr represents wal stream and ttl policy for that stream + +XXX: any suggestions on a better name? + +**zenith pitr create** name + +--ttl = inf | period + +--size-limit = inf | limit + +--storage = storage_name + +**zenith pitr extract-snapshot** pitr_name --lsn xxx + +Creates a snapshot out of some lsn in PITR area. The obtained snapshot may be managed with snapshot routines (move/send/export) + +**zenith pitr gc** pitr_name + +Force garbage collection on some PITR area. + +**zenith pitr list** + +**zenith pitr destroy** + + +## console + +**zenith console** + +Opens browser targeted at web console with the more or less same functionality as described here. diff --git a/docs/rfcs/004-durability.md b/docs/rfcs/004-durability.md new file mode 100644 index 0000000000..4543be3dae --- /dev/null +++ b/docs/rfcs/004-durability.md @@ -0,0 +1,218 @@ +Durability & Consensus +====================== + +When a transaction commits, a commit record is generated in the WAL. +When do we consider the WAL record as durable, so that we can +acknowledge the commit to the client and be reasonably certain that we +will not lose the transaction? + +Zenith uses a group of WAL safekeeper nodes to hold the generated WAL. +A WAL record is considered durable, when it has been written to a +majority of WAL safekeeper nodes. In this document, I use 5 +safekeepers, because I have five fingers. A WAL record is durable, +when at least 3 safekeepers have written it to disk. + +First, assume that only one primary node can be running at a +time. This can be achieved by Kubernetes or etcd or some +cloud-provider specific facility, or we can implement it +ourselves. These options are discussed in later chapters. For now, +assume that there is a Magic STONITH Fairy that ensures that. + +In addition to the WAL safekeeper nodes, the WAL is archived in +S3. WAL that has been archived to S3 can be removed from the +safekeepers, so the safekeepers don't need a lot of disk space. + + + +----------------+ + +-----> | WAL safekeeper | + | +----------------+ + | +----------------+ + +-----> | WAL safekeeper | ++------------+ | +----------------+ +| Primary | | +----------------+ +| Processing | ---------+-----> | WAL safekeeper | +| Node | | +----------------+ ++------------+ | +----------------+ + \ +-----> | WAL safekeeper | + \ | +----------------+ + \ | +----------------+ + \ +-----> | WAL safekeeper | + \ +----------------+ + \ + \ + \ + \ + \ +--------+ + \ | | + +--> | S3 | + | | + +--------+ + + +Every WAL safekeeper holds a section of WAL, and a VCL value. +The WAL can be divided into three portions: + + + VCL LSN + | | + V V +.................ccccccccccccccccccccXXXXXXXXXXXXXXXXXXXXXXX +Archived WAL Completed WAL In-flight WAL + + +Note that all this WAL kept in a safekeeper is a contiguous section. +This is different from Aurora: In Aurora, there can be holes in the +WAL, and there is a Gossip protocol to fill the holes. That could be +implemented in the future, but let's keep it simple for now. WAL needs +to be written to a safekeeper in order. However, during crash +recovery, In-flight WAL that has already been stored in a safekeeper +can be truncated or overwritten. + +The Archived WAL has already been stored in S3, and can be removed from +the safekeeper. + +The Completed WAL has been written to at least three safekeepers. The +algorithm ensures that it is not lost, when at most two nodes fail at +the same time. + +The In-flight WAL has been persisted in the safekeeper, but if a crash +happens, it may still be overwritten or truncated. + + +The VCL point is determined in the Primary. It is not strictly +necessary to store it in the safekeepers, but it allows some +optimizations and sanity checks and is probably generally useful for +the system as whole. The VCL values stored in the safekeepers can lag +behind the VCL computed by the primary. + + +Primary node Normal operation +----------------------------- + +1. Generate some WAL. + +2. Send the WAL to all the safekeepers that you can reach. + +3. As soon as a quorum of safekeepers have acknowledged that they have + received and durably stored the WAL up to that LSN, update local VCL + value in memory, and acknowledge commits to the clients. + +4. Send the new VCL to all the safekeepers that were part of the quorum. + (Optional) + + +Primary Crash recovery +---------------------- + +When a new Primary node starts up, before it can generate any new WAL +it needs to contact a majority of the WAL safekeepers to compute the +VCL. Remember that there is a Magic STONITH fairy that ensures that +only node process can be doing this at a time. + +1. Contact all WAL safekeepers. Find the Max((Epoch, LSN)) tuple among the ones you + can reach. This is the Winner safekeeper, and its LSN becomes the new VCL. + +2. Update the other safekeepers you can reach, by copying all the WAL + from the Winner, starting from each safekeeper's old VCL point. Any old + In-Flight WAL from previous Epoch is truncated away. + +3. Increment Epoch, and send the new Epoch to the quorum of + safekeepers. (This ensures that if any of the safekeepers that we + could not reach later come back online, they will be considered as + older than this in any future recovery) + +You can now start generating new WAL, starting from the newly-computed +VCL. + +Optimizations +------------- + +As described, the Primary node sends all the WAL to all the WAL safekeepers. That +can be a lot of network traffic. Instead of sending the WAL directly from Primary, +some safekeepers can be daisy-chained off other safekeepers, or there can be a +broadcast mechanism among them. There should still be a direct connection from the +each safekeeper to the Primary for the acknowledgments though. + +Similarly, the responsibility for archiving WAL to S3 can be delegated to one of +the safekeepers, to reduce the load on the primary. + + +Magic STONITH fairy +------------------- + +Now that we have a system that works as long as only one primary node is running at a time, how +do we ensure that? + +1. Use etcd to grant a lease on a key. The primary node is only allowed to operate as primary + when it's holding a valid lease. If the primary node dies, the lease expires after a timeout + period, and a new node is allowed to become the primary. + +2. Use S3 to store the lease. S3's consistency guarantees are more lenient, so in theory you + cannot do this safely. In practice, it would probably be OK if you make the lease times and + timeouts long enough. This has the advantage that we don't need to introduce a new + component to the architecture. + +3. Use Raft or Paxos, with the WAL safekeepers acting as the Acceptors to form the quorum. The + next chapter describes this option. + + +Built-in Paxos +-------------- + +The WAL safekeepers act as PAXOS Acceptors, and the Processing nodes +as both Proposers and Learners. + +Each WAL safekeeper holds an Epoch value in addition to the VCL and +the WAL. Each request by the primary to safekeep WAL is accompanied by +an Epoch value. If a safekeeper receives a request with Epoch that +doesn't match its current Accepted Epoch, it must ignore (NACK) it. +(In different Paxos papers, Epochs are called "terms" or "round +numbers") + +When a node wants to become the primary, it generates a new Epoch +value that is higher than any previously observed Epoch value, and +globally unique. + + +Accepted Epoch: 555 VCL LSN + | | + V V +.................ccccccccccccccccccccXXXXXXXXXXXXXXXXXXXXXXX +Archived WAL Completed WAL In-flight WAL + + +Primary node startup: + +1. Contact all WAL safekeepers that you can reach (if you cannot + connect to a quorum of them, you can give up immediately). Find the + latest Epoch among them. + +2. Generate a new globally unique Epoch, greater than the latest Epoch + found in previous step. + +2. Send the new Epoch in a Prepare message to a quorum of + safekeepers. (PAXOS Prepare message) + +3. Each safekeeper responds with a Promise. If a safekeeper has + already made a promise with a higher Epoch, it doesn't respond (or + responds with a NACK). After making a promise, the safekeeper stops + responding to any write requests with earlier Epoch. + +4. Once you have received a majority of promises, you know that the + VCL cannot advance on the old Epoch anymore. This effectively kills + any old primary server. + +5. Find the highest written LSN among the quorum of safekeepers (these + can be included in the Promise messages already). This is the new + VCL. If a new node starts the election process after this point, + it will compute the same or higher VCL. + +6. Copy the WAL from the safekeeper with the highest LSN to the other + safekeepers in the quorum, using the new Epoch. (PAXOS Accept + phase) + +7. You can now start generating new WAL starting from the VCL. If + another process starts the election process after this point and + gains control of a majority of the safekeepers, we will no longer + be able to advance the VCL. + diff --git a/docs/rfcs/005-zenith_local.md b/docs/rfcs/005-zenith_local.md new file mode 100644 index 0000000000..7b078e9ec0 --- /dev/null +++ b/docs/rfcs/005-zenith_local.md @@ -0,0 +1,103 @@ +# Zenith local + +Here I list some objectives to keep in mind when discussing zenith-local design and a proposal that brings all components together. Your comments on both parts are very welcome. + +#### Why do we need it? +- For distribution - this easy to use binary will help us to build adoption among developers. +- For internal use - to test all components together. + +In my understanding, we consider it to be just a mock-up version of zenith-cloud. +> Question: How much should we care about durability and security issues for a local setup? + + +#### Why is it better than a simple local postgres? + +- Easy one-line setup. As simple as `cargo install zenith && zenith start` + +- Quick and cheap creation of compute nodes over the same storage. +> Question: How can we describe a use-case for this feature? + +- Zenith-local can work with S3 directly. + +- Push and pull images (snapshots) to remote S3 to exchange data with other users. + +- Quick and cheap snapshot checkouts to switch back and forth in the database history. +> Question: Do we want it in the very first release? This feature seems quite complicated. + +#### Distribution: + +Ideally, just one binary that incorporates all elements we need. +> Question: Let's discuss pros and cons of having a separate package with modified PostgreSQL. + +#### Components: + +- **zenith-CLI** - interface for end-users. Turns commands to REST requests and handles responces to show them in a user-friendly way. +CLI proposal is here https://github.com/libzenith/rfcs/blob/003-laptop-cli.md/003-laptop-cli.md +WIP code is here: https://github.com/libzenith/postgres/tree/main/pageserver/src/bin/cli + +- **zenith-console** - WEB UI with same functionality as CLI. +>Note: not for the first release. + +- **zenith-local** - entrypoint. Service that starts all other components and handles REST API requests. See REST API proposal below. + > Idea: spawn all other components as child processes, so that we could shutdown everything by stopping zenith-local. + +- **zenith-pageserver** - consists of a storage and WAL-replaying service (modified PG in current implementation). +> Question: Probably, for local setup we should be able to bypass page-storage and interact directly with S3 to avoid double caching in shared buffers and page-server? + +WIP code is here: https://github.com/libzenith/postgres/tree/main/pageserver/src + +- **zenith-S3** - stores base images of the database and WAL in S3 object storage. Import and export images from/to zenith. +> Question: How should it operate in a local setup? Will we manage it ourselves or ask user to provide credentials for existing S3 object storage (i.e. minio)? +> Question: Do we use it together with local page store or they are interchangeable? + +WIP code is ??? + +- **zenith-safekeeper** - receives WAL from postgres, stores it durably, answers to Postgres that "sync" is succeed. +> Question: How should it operate in a local setup? In my understanding it should push WAL directly to S3 (if we use it) or store all data locally (if we use local page storage). The latter option seems meaningless (extra overhead and no gain), but it is still good to test the system. + +WIP code is here: https://github.com/libzenith/postgres/tree/main/src/bin/safekeeper + +- **zenith-computenode** - bottomless PostgreSQL, ideally upstream, but for a start - our modified version. User can quickly create and destroy them and work with it as a regular postgres database. + + WIP code is in main branch and here: https://github.com/libzenith/postgres/commits/compute_node + +#### REST API: + +Service endpoint: `http://localhost:3000` + +Resources: +- /storages - Where data lives: zenith-pageserver or zenith-s3 +- /pgs - Postgres - zenith-computenode +- /snapshots - snapshots **TODO** + +>Question: Do we want to extend this API to manage zenith components? I.e. start page-server, manage safekeepers and so on? Or they will be hardcoded to just start once and for all? + +Methods and their mapping to CLI: + +- /storages - zenith-pageserver or zenith-s3 + +CLI | REST API +------------- | ------------- +storage attach -n name --type [native\s3] --path=[datadir\URL] | PUT -d { "name": "name", "type": "native", "path": "/tmp" } /storages +storage detach -n name | DELETE /storages/:storage_name +storage list | GET /storages +storage show -n name | GET /storages/:storage_name + + +- /pgs - zenith-computenode + +CLI | REST API +------------- | ------------- +pg create -n name --s storage_name | PUT -d { "name": "name", "storage_name": "storage_name" } /pgs +pg destroy -n name | DELETE /pgs/:pg_name +pg start -n name --replica | POST -d {"action": "start", "is_replica":"replica"} /pgs/:pg_name /actions +pg stop -n name | POST -d {"action": "stop"} /pgs/:pg_name /actions +pg promote -n name | POST -d {"action": "promote"} /pgs/:pg_name /actions +pg list | GET /pgs +pg show -n name | GET /pgs/:pg_name + +- /snapshots **TODO** + +CLI | REST API +------------- | ------------- + diff --git a/docs/rfcs/006-laptop-cli-v2-CLI.md b/docs/rfcs/006-laptop-cli-v2-CLI.md new file mode 100644 index 0000000000..a04536922a --- /dev/null +++ b/docs/rfcs/006-laptop-cli-v2-CLI.md @@ -0,0 +1,64 @@ +Zenith CLI allows you to operate database clusters (catalog clusters) and their commit history locally and in the cloud. Since ANSI calls them catalog clusters and cluster is a loaded term in the modern infrastructure we will call it "catalog". + +# CLI v2 (after chatting with Carl) + +Zenith introduces the notion of a repository. + +```bash +zenith init +zenith clone zenith://zenith.tech/piedpiper/northwind -- clones a repo to the northwind directory +``` + +Once you have a cluster catalog you can explore it + +```bash +zenith log -- returns a list of commits +zenith status -- returns if there are changes in the catalog that can be committed +zenith commit -- commits the changes and generates a new commit hash +zenith branch experimental -- creates a branch called testdb based on a given commit hash +``` + +To make changes in the catalog you need to run compute nodes + +```bash +-- here is how you a compute node +zenith start /home/pipedpiper/northwind:main -- starts a compute instance +zenith start zenith://zenith.tech/northwind:main -- starts a compute instance in the cloud +-- you can start a compute node against any hash or branch +zenith start /home/pipedpiper/northwind:experimental --port 8008 -- start anothe compute instance (on different port) +-- you can start a compute node against any hash or branch +zenith start /home/pipedpiper/northwind: --port 8009 -- start anothe compute instance (on different port) + +-- After running some DML you can run +-- zenith status and see how there are two WAL streams one on top of +-- the main branch +zenith status +-- and another on top of the experimental branch +zenith status -b experimental + +-- you can commit each branch separately +zenith commit main +-- or +zenith commit -c /home/pipedpiper/northwind:experimental +``` + +Starting compute instances against cloud environments + +```bash +-- you can start a compute instance against the cloud environment +-- in this case all of the changes will be streamed into the cloud +zenith start https://zenith:tech/pipedpiper/northwind:main +zenith start https://zenith:tech/pipedpiper/northwind:main +zenith status -c https://zenith:tech/pipedpiper/northwind:main +zenith commit -c https://zenith:tech/pipedpiper/northwind:main +zenith branch -c https://zenith:tech/pipedpiper/northwind: experimental +``` + +Pushing data into the cloud + +```bash +-- pull all the commits from the cloud +zenith pull +-- push all the commits to the cloud +zenith push +``` diff --git a/docs/rfcs/006-laptop-cli-v2-repository-structure.md b/docs/rfcs/006-laptop-cli-v2-repository-structure.md new file mode 100644 index 0000000000..ee4e432182 --- /dev/null +++ b/docs/rfcs/006-laptop-cli-v2-repository-structure.md @@ -0,0 +1,140 @@ +# Repository format + +A Zenith repository is similar to a traditional PostgreSQL backup +archive, like a WAL-G bucket or pgbarman backup catalogue. It holds +multiple versions of a PostgreSQL database cluster. + +The distinguishing feature is that you can launch a Zenith Postgres +server directly against a branch in the repository, without having to +"restore" it first. Also, Zenith manages the storage automatically, +there is no separation between full and incremental backups nor WAL +archive. Zenith relies heavily on the WAL, and uses concepts similar +to incremental backups and WAL archiving internally, but it is hidden +from the user. + +## Directory structure, version 1 + +This first version is pretty straightforward but not very +efficient. Just something to get us started. + +The repository directory looks like this: + + .zenith/timelines/4543be3daeab2ed4e58a285cbb8dd1fce6970f8c/wal/ + .zenith/timelines/4543be3daeab2ed4e58a285cbb8dd1fce6970f8c/snapshots// + .zenith/timelines/4543be3daeab2ed4e58a285cbb8dd1fce6970f8c/history + + .zenith/refs/branches/mybranch + .zenith/refs/tags/foo + .zenith/refs/tags/bar + + .zenith/datadirs/ + +### Timelines + +A timeline is similar to PostgeSQL's timeline, but is identified by a +UUID instead of a 32-bit timeline Id. For user convenience, it can be +given a name that refers to the UUID (called a branch). + +All WAL is generated on a timeline. You can launch a read-only node +against a tag or arbitrary LSN on a timeline, but in order to write, +you need to create a timeline. + +Each timeline is stored in a directory under .zenith/timelines. It +consists of a WAL archive, containing all the WAL in the standard +PostgreSQL format, under the wal/ subdirectory. + +The 'snapshots/' subdirectory, contains "base backups" of the data +directory at a different LSNs. Each snapshot is simply a copy of the +Postgres data directory. + +When a new timeline is forked from a previous timeline, the ancestor +timeline's UUID is stored in the 'history' file. + +### Refs + +There are two kinds of named objects in the repository: branches and +tags. A branch is a human-friendly name for a timeline UUID, and a +tag is a human-friendly name for a specific LSN on a timeline +(timeline UUID + LSN). Like in git, these are just for user +convenience; you can also use timeline UUIDs and LSNs directly. + +Refs do have one additional purpose though: naming a timeline or LSN +prevents it from being automatically garbage collected. + +The refs directory contains a small text file for each tag/branch. It +contains the UUID of the timeline (and LSN, for tags). + +### Datadirs + +.zenith/datadirs contains PostgreSQL data directories. You can launch +a Postgres instance on one of them with: + +``` + postgres -D .zenith/datadirs/4543be3daeab2ed4e58a285cbb8dd1fce6970f8c +``` + +All the actual data is kept in the timeline directories, under +.zenith/timelines. The data directories are only needed for active +PostgreQSL instances. After an instance is stopped, the data directory +can be safely removed. "zenith start" will recreate it quickly from +the data in .zenith/timelines, if it's missing. + +## Version 2 + +The format described above isn't very different from a traditional +daily base backup + WAL archive configuration. The main difference is +the nicer naming of branches and tags. + +That's not very efficient. For performance, we need something like +incremental backups that don't require making a full copy of all +data. So only store modified files or pages. And instead of having to +replay all WAL from the last snapshot, "slice" the WAL into +per-relation WAL files and only recover what's needed when a table is +accessed. + +In version 2, the file format in the "snapshots" subdirectory gets +more advanced. The exact format is TODO. But it should support: +- storing WAL records of individual relations/pages +- storing a delta from an older snapshot +- compression + + +## Operations + +### Garbage collection + +When you run "zenith gc", old timelines that are no longer needed are +removed. That involves collecting the list of "unreachable" objects, +starting from the named branches and tags. + +Also, if enough WAL has been generated on a timeline since last +snapshot, a new snapshot or delta is created. + +### zenith push/pull + +Compare the tags and branches on both servers, and copy missing ones. +For each branch, compare the timeline it points to in both servers. If +one is behind the other, copy the missing parts. + +FIXME: how do you prevent confusion if you have to clones of the same +repository, launch an instance on the same branch in both clones, and +later try to push/pull between them? Perhaps create a new timeline +every time you start up an instance? Then you would detect that the +timelines have diverged. That would match with the "epoch" concept +that we have in the WAL safekeepr + +### zenith checkout/commit + +In this format, there is no concept of a "working tree", and hence no +concept of checking out or committing. All modifications are done on +a branch or a timeline. As soon as you launch a server, the changes are +appended to the timeline. + +You can easily fork off a temporary timeline to emulate a "working tree". +You can later remove it and have it garbage collected, or to "commit", +re-point the branch to the new timeline. + +If we want to have a worktree and "zenith checkout/commit" concept, we can +emulate that with a temporary timeline. Create the temporary timeline at +"zenith checkout", and have "zenith commit" modify the branch to point to +the new timeline. diff --git a/docs/rfcs/007-serverless-on-laptop.md b/docs/rfcs/007-serverless-on-laptop.md new file mode 100644 index 0000000000..e6355f4a03 --- /dev/null +++ b/docs/rfcs/007-serverless-on-laptop.md @@ -0,0 +1,93 @@ +How it works now +---------------- + +1. Create repository, start page server on it + +``` +$ zenith init +... +created main branch +new zenith repository was created in .zenith + +$ zenith pageserver start +Starting pageserver at '127.0.0.1:64000' in .zenith +Page server started +``` + +2. Create a branch, and start a Postgres instance on it + +``` +$ zenith branch heikki main +branching at end of WAL: 0/15ECF68 + +$ zenith pg create heikki +Initializing Postgres on timeline 76cf9279915be7797095241638e64644... +Extracting base backup to create postgres instance: path=.zenith/pgdatadirs/pg1 port=55432 + +$ zenith pg start pg1 +Starting postgres node at 'host=127.0.0.1 port=55432 user=heikki' +waiting for server to start.... done +server started +``` + + +3. Connect to it and run queries + +``` +$ psql "dbname=postgres port=55432" +psql (14devel) +Type "help" for help. + +postgres=# +``` + + +Proposal: Serverless on your Laptop +----------------------------------- + +We've been talking about doing the "pg create" step automatically at +"pg start", to eliminate that step. What if we go further, go +serverless on your laptop, so that the workflow becomes just: + +1. Create repository, start page server on it (same as before) + +``` +$ zenith init +... +created main branch +new zenith repository was created in .zenith + +$ zenith pageserver start +Starting pageserver at '127.0.0.1:64000' in .zenith +Page server started +``` + +2. Create branch + +``` +$ zenith branch heikki main +branching at end of WAL: 0/15ECF68 +``` + +3. Connect to it: + +``` +$ psql "dbname=postgres port=5432 branch=heikki" +psql (14devel) +Type "help" for help. + +postgres=# +``` + + +The trick behind the scenes is that when you launch the page server, +it starts to listen on port 5432. When you connect to it with psql, it +looks at the 'branch' parameter that you passed in the connection +string. It automatically performs the "pg create" and "pg start" steps +for that branch, and then forwards the connection to the Postgres +instance that it launched. After you disconnect, if there are no more +active connections to the server running on the branch, it can +automatically shut it down again. + +This is how serverless would work in the cloud. We can do it on your +laptop, too. diff --git a/docs/rfcs/008-push-pull.md b/docs/rfcs/008-push-pull.md new file mode 100644 index 0000000000..272628e1ce --- /dev/null +++ b/docs/rfcs/008-push-pull.md @@ -0,0 +1,66 @@ +# Push and pull between pageservers + +Here is a proposal about implementing push/pull mechanics between pageservers. We also want to be able to push/pull to S3 but that would depend on the exact storage format so we don't touch that in this proposal. + +## Origin management + +The origin represents connection info for some remote pageserver. Let's use here same commands as git uses except using explicit list subcommand (git uses `origin -v` for that). + +``` +zenith origin add +zenith origin list +zenith origin remove +``` + +Connection URI a string of form `postgresql://user:pass@hostname:port` (https://www.postgresql.org/docs/13/libpq-connect.html#id-1.7.3.8.3.6). We can start with libpq password auth and later add support for client certs or require ssh as transport or invent some other kind of transport. + +Behind the scenes, this commands may update toml file inside .zenith directory. + +## Push + +### Pushing branch + +``` +zenith push mybranch cloudserver # push to eponymous branch in cloudserver +zenith push mybranch cloudserver:otherbranch # push to a different branch in cloudserver +``` + +Exact mechanics would be slightly different in the following situations: + +1) Destination branch does not exist. + + That is the simplest scenario. We can just create an empty branch (or timeline in internal terminology) and transfer all the pages/records that we have in our timeline. Right now each timeline is quite independent of other timelines so I suggest skipping any checks that there is a common ancestor and just fill it with data. Later when CoW timelines will land to the pageserver we may add that check and decide whether this timeline belongs to this pageserver repository or not [*]. + + The exact mechanics may be the following: + + * CLI asks local pageserver to perform push and hands over connection uri: `perform_push `. + * local pageserver connects to the remote pageserver and runs `branch_push ` + Handler for branch_create would create destination timeline and switch connection to copyboth mode. + * Sending pageserver may start iterator on that timeline and send all the records as copy messages. + +2) Destination branch exists and latest_valid_lsn is less than ours. + + In this case, we need to send missing records. To do that we need to find all pages that were changed since that remote LSN. Right now we don't have any tracking mechanism for that, so let's just iterate over all records and send ones that are newer than remote LSN. Later we probably should add a sparse bitmap that would track changed pages to avoid full scan. + +3) Destination branch exists and latest_valid_lsn is bigger than ours. + + In this case, we can't push to that branch. We can only pull. + +### Pulling branch + +Here we need to handle the same three cases, but also keep in mind that local pageserver can be behind NAT and we can't trivially re-use pushing by asking remote to 'perform_push' to our address. So we would need a new set of commands: + +* CLI calls `perform_pull ` on local pageserver. +* local pageserver calls `branch_pull ` on remote pageserver. +* remote pageserver sends records in our direction + +But despite the different set of commands code that performs iteration over records and receiving code that inserts that records can be the same for both pull and push. + + + +[*] It looks to me that there are two different possible approaches to handling unrelated timelines: + +1) Allow storing unrelated timelines in one repo. Some timelines may have parents and some may not. +2) Transparently create and manage several repositories in one pageserver. + +But that is the topic for a separate RFC/discussion. diff --git a/docs/rfcs/009-snapshot-first-storage-cli.md b/docs/rfcs/009-snapshot-first-storage-cli.md new file mode 100644 index 0000000000..3f5386c165 --- /dev/null +++ b/docs/rfcs/009-snapshot-first-storage-cli.md @@ -0,0 +1,56 @@ +While working on export/import commands, I understood that they fit really well into "snapshot-first design". + +We may think about backups as snapshots in a different format (i.e plain pgdata format, basebackup tar format, WAL-G format (if they want to support it) and so on). They use same storage API, the only difference is the code that packs/unpacks files. + +Even if zenith aims to maintains durability using it's own snapshots, backups will be useful for uploading data from postges to zenith. + +So here is an attemt to design consistent CLI for diferent usage scenarios: + +#### 1. Start empty pageserver. +That is what we have now. +Init empty pageserver using `initdb` in temporary directory. + +`--storage_dest=FILE_PREFIX | S3_PREFIX |...` option defines object storage type, all other parameters are passed via env variables. Inspired by WAL-G style naming : https://wal-g.readthedocs.io/STORAGES/. + +Save`storage_dest` and other parameters in config. +Push snapshots to `storage_dest` in background. + +``` +zenith init --storage_dest=S3_PREFIX +zenith start +``` + +#### 2. Restart pageserver (manually or crash-recovery). +Take `storage_dest` from pageserver config, start pageserver from latest snapshot in `storage_dest`. +Push snapshots to `storage_dest` in background. + +``` +zenith start +``` + +#### 3. Import. +Start pageserver from existing snapshot. +Path to snapshot provided via `--snapshot_path=FILE_PREFIX | S3_PREFIX | ...` +Do not save `snapshot_path` and `snapshot_format` in config, as it is a one-time operation. +Save`storage_dest` parameters in config. +Push snapshots to `storage_dest` in background. +``` +//I.e. we want to start zenith on top of existing $PGDATA and use s3 as a persistent storage. +zenith init --snapshot_path=FILE_PREFIX --snapshot_format=pgdata --storage_dest=S3_PREFIX +zenith start +``` +How to pass credentials needed for `snapshot_path`? + +#### 4. Export. +Manually push snapshot to `snapshot_path` which differs from `storage_dest` +Optionally set `snapshot_format`, which can be plain pgdata format or zenith format. +``` +zenith export --snapshot_path=FILE_PREFIX --snapshot_format=pgdata +``` + +#### Notes and questions +- walkeeper s3_offload should use same (similar) syntax for storage. How to set it in UI? +- Why do we need `zenith init` as a separate command? Can't we init everything at first start? +- We can think of better names for all options. +- Export to plain postgres format will be useless, if we are not 100% compatible on page level. +I can recall at least one such difference - PD_WAL_LOGGED flag in pages. \ No newline at end of file diff --git a/docs/rfcs/009-snapshot-first-storage-pitr.md b/docs/rfcs/009-snapshot-first-storage-pitr.md new file mode 100644 index 0000000000..801613e2c9 --- /dev/null +++ b/docs/rfcs/009-snapshot-first-storage-pitr.md @@ -0,0 +1,227 @@ +# Preface + +GetPage@LSN can be called with older LSNs, and the page server needs +to be able to reconstruct older page versions. That's needed for +having read-only replicas that lag behind the primary, or that are +"anchored" at an older LSN, and internally in the page server whne you +branch at an older point in time. How do you do that? + +For now, I'm not considering incremental snapshots at all. I don't +think that changes things. So whenever you create a snapshot or a +snapshot file, it contains an image of all the pages, there is no need +to look at an older snapshot file. + +Also, I'm imagining that this works on a per-relation basis, so that +each snapshot file contains data for one relation. A "relation" is a +fuzzy concept - it could actually be one 1 GB relation segment. Or it +could include all the different "forks" of a relation, or you could +treat each fork as a separate relation for storage purpose. And once +we have the "non-relational" work is finished, a "relation" could +actually mean some other versioned object kept in the PostgreSQL data +directory. Let's ignore that for now. + +# Eric's RFC: + +Every now and then, you create a "snapshot". It means that you create +a new snapshot file for each relation that was modified after the last +snapshot, and write out the contents the relation as it is/was at the +snapshot LSN. Write-ahead log is stored separately in S3 by the WAL +safekeeping service, in the original PostgreSQL WAL file format. + + SNAPSHOT @100 WAL + . | + . | + . | + . | + SNAPSHOT @200 | + . | + . | + . | + . | + SNAPSHOT @300 | + . | + . V + IN-MEMORY @400 + +If a GetPage@LSN request comes from the primary, you return the latest +page from the in-memory layer. If there is no trace of the page in +memory, it means that it hasn't been modified since the last snapshot, +so you return the page from the latest snapshot, at LSN 300 in the +above example. + +PITR is implemented using the original WAL files: + +If a GetPage@LSN request comes from a read replica with LSN 250, you +read the image of the page from the snapshot at LSN 200, and you also +scan the WAL between 200 and 250, and apply all WAL records for the +requested page, to reconstruct it at LSN 250. + +Scanning the WAL naively for every GetPage@LSN request would be +expensive, so in practice you'd construct an in-memory data structure +of all the WAL between 200 and 250 once that allows quickly looking up +records for a given page. + +## Problems/questions + +I think you'll need to store the list of snapshot LSNs on each +timeline somewhere. + +If the latest snapshot of a relation is at LSN 100, and you request a +page at LSN 1000000, how do you know if there are some modifications +to it between 100 and 1000000 that you need to replay? You can scan +all the WAL between 100 and 1000000, but that would be expensive. + +You can skip that, if you know that a snapshot was taken e.g. at LSN +999900. Then you know that the fact that there is no snapshot file at +999900 means that the relation hasn't been modified between +100-999900. Then you only need to scan the WAL between 999900 and +1000000. However, there is no trace of a snapshot happening at LSN +999900 in the snapshot file for this relation, so you need to get +that information from somewhere else. + +Where do you get that information from? Perhaps you can scan all the +other relations, and if you see a snapshot file for *any* relation at +LSN 999900, you know that if there were modifications to this +relation, there would be a newer snapshot file for it, too. In other +words, the list of snapshots that have been taken can be constructed +by scanning all relations and computing the union of all snapshot LSNs +that you see for any relation. But that's expensive so at least you +should keep that in memory, after computing it once. Also, if you rely +on that, it's not possible to have snapshots at different intervals +for different files. That seems limiting. + +Another option is to explicitly store a list of snapshot LSNs in a +separate metadata file. + + +# Current implementation in the 'layered_repo' branch: + +We store snapshot files like in the RFC, but each snapshot file also +contains all the WAL in the range of LSNs, so that you don't need to +fetch the WAL separately from S3. So you have "layers" like this: + + SNAPSHOT+WAL 100-200 + | + | + | + | + SNAPSHOT+WAL 200-300 + | + | + | + | + IN-MEMORY 300- + +Each "snapshot+WAL" is a file that contains a snapshot - i.e. full +copy of each page in the relation, at the *start* LSN. In addition to +that, it contains all the WAL applicable to the relation from the +start LSN to the end LSN. With that, you can reconstruct any page +version in the range that the file covers. + + +## Problems/questions + +I can see one potential performance issue here, compared to the RFC. +Let's focus on a single relation for now. Imagine that you start from +an empty relation, and you receive WAL from 100 to 200, containing +a bunch of inserts and updates to the relation. You now have all that +WAL in memory: + + memory: WAL from 100-200 + +We decide that it's time to materialize that to a snapshot file on +disk. We materialize full image of the relation as it was at LSN 100 +to the snapshot file, and include all of the WAL. Since the relation +was initially empty, the "image" at the beginning of th range is empty +too. + +So now you have one file on on disk: + + SNAPSHOT+WAL 100-200 + +It contains a full image of the relation at LSN 100 and all WAL +between 100-200. (It's actually stored as a serialized BTreeMap of +page versions, with the page images and WAL records all stored +together in the same BtreeMap. But for this story, that's not +important.) + +We now receive more WAL updating the relation, up to LSN 300. We +decide it's time to materialize a new snapshot file, and we now have +two files: + + SNAPSHOT+WAL 100-200 + SNAPSHOT+WAL 200-300 + +Note that the latest "full snapshot" that we store on disk always lags +behind by one snapshot cycle. The first file contains a full image of +the relation at LSN 100, the second at LSN 200. When we have received +WAL up to LSN 300, we write a materialized image at LSN 200. That +seems a bit silly. In the design per your RFC, you would write a +snapshots at LSNs 200 and 300, instead. That seems better. + + + +# Third option (not implemented yet) + +Store snapshot files like in the RFC, but also store per-relation +WAL files that contain WAL in a range of LSNs for that relation. + + SNAPSHOT @100 WAL 100-200 + . | + . | + . | + . | + SNAPSHOT @200 WAL 200-300 + . | + . | + . | + . | + SNAPSHOT @300 + . + . + IN-MEMORY 300- + + +This could be the best of both worlds. The snapshot files would be +independent of the PostgreSQL WAL format. When it's time to write +snapshot file @300, you write a full image of the relation at LSN 300, +and you write the WAL that you had accumulated between 200 and 300 to +a separate file. That way, you don't "lag behind" for one snapshot +cycle like in the current implementation. But you still have the WAL +for a particular relation readily available alongside the snapshot +files, and you don't need to track what snapshot LSNs exist +separately. + +(If we wanted to minize the number of files, you could include the +snapshot @300 and the WAL between 200 and 300 in the same file, but I +feel it's probably better to keep them separate) + + + +# Further thoughts + +There's no fundamental reason why the LSNs of the snapshot files and the +ranges of the WAL files would need to line up. So this would be possible +too: + + SNAPSHOT @100 WAL 100-150 + . | + . | + . WAL 150-250 + . | + SNAPSHOT @200 | + . | + . WAL 250-400 + . | + . | + SNAPSHOT @300 | + . | + . | + IN-MEMORY 300- + +I'm not sure what the benefit of this would be. You could materialize +additional snapshot files in the middle of a range covered by a WAL +file, maybe? Might be useful to speed up access when you create a new +branch in the middle of an LSN range or if there's some other reason +to believe that a particular LSN is "interesting" and there will be +a lot of requests using it. diff --git a/docs/rfcs/009-snapshot-first-storage.md b/docs/rfcs/009-snapshot-first-storage.md new file mode 100644 index 0000000000..aeef54898a --- /dev/null +++ b/docs/rfcs/009-snapshot-first-storage.md @@ -0,0 +1,148 @@ +# Snapshot-first storage architecture + +Goals: +- Long-term storage of database pages. +- Easy snapshots; simple snapshot and branch management. +- Allow cloud-based snapshot/branch management. +- Allow cloud-centric branching; decouple branch state from running pageserver. +- Allow customer ownership of data via s3 permissions. +- Provide same or better performance for typical workloads, vs plain postgres. + +Non-goals: +- Service database reads from s3 (reads should be serviced from the pageserver cache). +- Keep every version of every page / Implement point-in-time recovery (possibly a future paid feature, based on WAL replay from an existing snapshot). + +## Principle of operation + +The database “lives in s3”. This means that all of the long term page storage is in s3, and the “live database”-- the version that lives in the pageserver-- is a set of “dirty pages” that haven’t yet been written back to s3. + +In practice, this is mostly similar to storing frequent snapshots to s3 of a database that lives primarily elsewhere. + +The main difference is that s3 is authoritative about which branches exist; pageservers consume branches, snapshots, and related metadata by reading them from s3. This allows cloud-based management of branches and snapshots, regardless of whether a pageserver is running or not. + +It’s expected that a pageserver should keep a copy of all pages, to shield users from s3 latency. A cheap/slow pageserver that falls back to s3 for some reads would be possible, but doesn’t seem very useful right now. + +Because s3 keeps all history, and the safekeeper(s) preserve any WAL records needed to reconstruct the most recent changes, the pageserver can store dirty pages in RAM or using non-durable local storage; this should allow very good write performance, since there is no need for fsync or journaling. + +Objects in s3 are immutable snapshots, never to be modified once written (only deleted). + +Objects in s3 are files, each containing a set of pages for some branch/relation/segment as of a specific time (LSN). A snapshot could be complete (meaning it has a copy of every page), or it could be incremental (containing only the pages that were modified since the previous snapshot). It’s expected that most snapshots are incremental to keep storage costs low. + +It’s expected that the pageserver would upload new snapshot objects frequently, e.g. somewhere between 30 seconds and 15 minutes, depending on cost/performance balance. + +No-longer needed snapshots can be “squashed”-- meaning snapshot N and snapshot N+1 can be read by some cloud agent software, which writes out a new object containing the combined set of pages (keeping only the newest version of each page) and then deletes the original snapshots. + +A pageserver only needs to store the set of pages needed to satisfy operations in flight: if a snapshot is still being written, the pageserver needs to hold historical pages so that snapshot captures a consistent moment in time (similar to what is needed to satisfy a slow replica). + +WAL records can be discarded once a snapshot has been stored to s3. (Unless we want to keep them longer as part of a point-in-time recovery feature.) + +## Pageserver operation + +To start a pageserver from a stored snapshot, the pageserver downloads a set of snapshots sufficient to start handling requests. We assume this includes the latest copy of every page, though it might be possible to start handling requests early, and retrieve pages for the first time only when needed. + +To halt a pageserver, one final snapshot should be written containing all pending WAL updates; then the pageserver and safekeepers can shut down. + +It’s assumed there is some cloud management service that ensures only one pageserver is active and servicing writes to a given branch. + +The pageserver needs to be able to track whether a given page has been modified since the last snapshot, and should be able to produce the set of dirty pages efficiently to create a new snapshot. + +The pageserver need only store pages that are “reachable” from a particular LSN. For example, a page may be written four times, at LSN 100, 200, 300, and 400. If no snapshot is being created when LSN 200 is written, the page at LSN 100 can be discarded. If a snapshot is triggered when the pageserver is at LSN 299, the pageserver must preserve the page from LSN 200 until that snapshot is complete. As before, the page at LSN 300 can be discarded when the LSN 400 pages is written (regardless of whether the LSN 200 snapshot has completed.) + +If the pageserver is servicing multiple branches, those branches may contain common history. While it would be possible to serve branches with zero knowledge of their common history, a pageserver could save a lot of space using an awareness of branch history to share the common set of pages. Computing the “liveness” of a historical page may be tricky in the face of multiple branches. + +The pageserver may store dirty pages to memory or to local block storage; any local block storage format is only temporary “overflow” storage, and is not expected to be readable by future software versions. + +The pageserver may store clean pages (those that are captured in a snapshot) any way it likes: in memory, in a local filesystem (possibly keeping a local copy of the snapshot file), or using some custom storage format. Reading pages from s3 would be functional, but is expected to be prohibitively slow. + +The mechanism for recovery after a pageserver failure is WAL redo. If we find that too slow in some situations (e.g. write-heavy workload causes long startup), we can write more frequent snapshots to keep the number of outstanding WAL records low. If that’s still not good enough, we could look at other options (e.g. redundant pageserver or an EBS page journal). + +A read-only pageserver is possible; such a pageserver could be a read-only cache of a specific snapshot, or could auto-update to the latest snapshot on some branch. Either way, no safekeeper is required. Multiple read-only pageservers could exist for a single branch or snapshot. + +## Cloud snapshot manager operation + +Cloud software may wish to do the following operations (commanded by a user, or based on some pre-programmed policy or other cloud agent): +Create/delete/clone/rename a database +Create a new branch (possibly from a historical snapshot) +Start/stop the pageserver/safekeeper on a branch +List databases/branches/snapshots that are visible to this user account + +Some metadata operations (e.g. list branches/snapshots of a particular db) could be performed by scanning the contents of a bucket and inspecting the file headers of each snapshot object. This might not be fast enough; it might be necessary to build a metadata service that can respond more quickly to some queries. + +This is especially true if there are public databases: there may be many thousands of buckets that are public, and scanning all of them is not a practical strategy for answering metadata queries. + +## Snapshot names, deletion and concurrency + +There may be race conditions between operations-- in particular, a “squash” operation may replace two snapshot objects (A, B) with some combined object (C). Since C is logically equivalent to B, anything that attempts to access B should be able to seamlessly switch over to C. It’s assumed that concurrent delete won’t disrupt a read in flight, but it may be possible for some process to read B’s header, and then discover on the next operation that B is gone. + +For this reason, any attempted read should attempt a fallback procedure (list objects; search list for an equivalent object) if an attempted read fails. This requires a predictable naming scheme, e.g. `XXXX_YYYY_ZZZZ_DDDD`, where `XXXX` is the branch unique id, and `YYYY` and `ZZZZ` are the starting/ending LSN values. `DDDD` is a timestamp indicating when the object was created; this is used to disambiguate a series of empty snapshots, or to help a snapshot policy engine understand which snapshots should be kept or discarded. + +## Branching + +A user may request a new branch from the cloud user interface. There is a sequence of things that needs to happen: +- If the branch is supposed to be based on the latest contents, the pageserver should perform an immediate snapshot. This is the parent snapshot for the new branch. +- Cloud software should create the new branch, by generating a new (random) unique branch identifier, and creating a placeholder snapshot object. + - The placeholder object is an empty snapshot containing only metadata (which anchors it to the right parent history) and no pages. + - The placeholder can be discarded when the first snapshot (containing data) is completed. Discarding is equivalent to squashing, when the snapshot contains no data. +- If the branch needs to be started immediately, a pageserver should be notified that it needs to start servicing the branch. This may not be the same pageserver that services the parent branch, though the common history may make it the best choice. + +Some of these steps could be combined into the pageserver, but that process would not be possible under all cases (e.g. if no pageserver is currently running, or if the branch is based on an older snapshot, or if a different pageserver will be serving the new branch). Regardless of which software drives the process, the result should look the same. + +## Long-term file format + +Snapshot files (and any other object stored in s3) must be readable by future software versions. + +It should be possible to build multiple tools (in addition to the pageserver) that can read and write this file format-- for example, to allow cloud snapshot management. + +Files should contain the following metadata, in addition to the set of pages: +- The version of the file format. +- A unique identifier for this branch (should be worldwide-unique and unchanging). +- Optionally, any human-readable names assigned to this branch (for management UI/debugging/logging). +- For incremental snapshots, the identifier of the predecessor snapshot. For new branches, this will be the parent snapshot (the point at which history diverges). +- The location of the predecessor branch snapshot, if different from this branch’s location. +- The LSN range `(parent, latest]` for this snapshot. For complete snapshots, the parent LSN can be 0. +- The UTC timestamp of the snapshot creation (which may be different from the time of its highest LSN, if the database is idle). +- A SHA2 checksum over the entire file (excluding the checksum itself), to preserve file integrity. + +A file may contain no pages, and an empty LSN range (probably `(latest, latest]`?), which serves as a placeholder for either a newly-created branch, or a snapshot of an idle database. + +Any human-readable names stored in the file may fall out of date if database/branch renames are allowed; there may need to be a cloud metadata service to query (current name -> unique identifier). We may choose instead to not store human-readable names in the database, or treat them as debugging information only. + +## S3 semantics, and other kinds of storage + +For development and testing, it may be easier to use other kinds of storage in place of s3. For example, a directory full of files can substitute for an s3 bucket with multiple objects. This mode is expected to match the s3 semantics (e.g. don’t edit existing files or use symlinks). Unit tests may omit files entirely and use an in-memory mock bucket. + +Some users may want to use a local or network filesystem in place of s3. This isn’t prohibited but it’s not a priority, either. + +Alternate implementations of s3 should be supported, including Google Cloud Storage. + +Azure Blob Storage should be supported. We assume (without evidence) that it’s semantically equivalent to s3 for this purpose. + +The properties of s3 that we depend on are: +list objects +streaming read of entire object +read byte range from object +streaming write new object (may use multipart upload for better relialibity) +delete object (that should not disrupt an already-started read). + +Uploaded files, restored backups, or s3 buckets controlled by users could contain malicious content. We should always validate that objects contain the content they’re supposed to. Incorrect, Corrupt or malicious-looking contents should cause software (cloud tools, pageserver) to fail gracefully. + +## Notes + +Possible simplifications, for a first draft implementation: +- Assume that dirty pages fit in pageserver RAM. Can use kernel virtual memory to page out to disk if needed. Can improve this later. +- Don’t worry about the details of the squashing process yet. +- Don’t implement cloud metadata service; try to make everything work using basic s3 list-objects and reads. +- Don’t implement rename, delete at first. +- Don’t implement public/private, just use s3 permissions. +- Don’t worry about sharing history yet-- each user has their own bucket and a full copy of all data. +- Don’t worry about history that spans multiple buckets. +- Don’t worry about s3 regions. +- Don’t support user-writeable s3 buckets; users get only read-only access at most. + +Open questions: +- How important is point-in-time recovery? When should we add this? How should it work? +- Should snapshot files use compression? +- Should we use snapshots for async replication? A spare pageserver could stay mostly warmed up by consuming snapshots as they’re created. +- Should manual snapshots, or snapshots triggered by branch creation, be named differently from snapshots that are triggered by a snapshot policy? +- When a new branch is created, should it always be served by the same pageserver that owns its parent branch? When should we start a new pageserver? +- How can pageserver software upgrade be done with minimal downtime? diff --git a/docs/rfcs/010-storage_details.md b/docs/rfcs/010-storage_details.md new file mode 100644 index 0000000000..8429a2d9e3 --- /dev/null +++ b/docs/rfcs/010-storage_details.md @@ -0,0 +1,144 @@ +# Storage details + +Here I tried to describe the current state of thinking about our storage subsystem as I understand it. Feel free to correct me. Also, I tried to address items from Heikki's TODO and be specific on some of the details. + +## Overview + +![storage](images/storage.jpeg) + +### MemStore + +MemStore holds the data between `latest_snapshot_lsn` and `latest_lsn`. It consists of PageIndex that holds references to WAL records or pages, PageStore that stores recently materialized pages, and WalStore that stores recently received WAL. + +### PageIndex + +PageIndex is an ordered collection that maps `(BufferTag, LSN)` to one of the following references (by reference I mean some information that is needed to access that data, e.g. file_id and offset): + +* PageStoreRef -- page offset in the PageStore +* LocalStoreRef -- snapshot_id and page offset inside of that snapshot +* WalStoreRef -- offset (and size optionally) of WalRecord in WalStore + +PageIndex holds information about all the pages in all incremental snapshots and in the latest full snapshot. If we aren't using page compression inside snapshots we actually can avoid storing references to the full snapshot and calculate page offsets based on relation sizes metadata in the full snapshot (assuming that full snapshot stores pages sorted by page number). However, I would suggest embracing page compression from the beginning and treat all pages as variable-sized. + +We assume that PageIndex is few orders of magnitude smaller than addressed data hence it should fit memory. We also don't care about crash tolerance as we can rebuild it from snapshots metadata and WAL records from WalStore or/and Safekeeper. + +### WalStore + +WalStore is a queue of recent WalRecords. I imagine that we can store recent WAL the same way as Postgres does -- as 16MB files on disk. On top of that, we can add some fixed-size cache that would keep some amount of segments in memory. + +For now, we may rely on the Safekeeper to safely store that recent WAL. But generally, I think we can pack all S3 operations into the page server so that it would be also responsible for the recent WAL pushdown to S3 (and Safekeeper may just delete WAL that was confirmed as S3-durable by the page server). + +### PageStore + +PageStore is storage for recently materialized pages (or in other words cache of getPage results). It is also can be implemented as a file-based queue with some memory cache on top of it. + +There are few possible options for PageStore: + +a) we just add all recently materialized pages there (so several versions of the same page can be stored there) -- that is more or less how it happens now with the current RocksDB implementation. + +b) overwrite older pages with the newer pages -- if there is no replica we probably don't need older pages. During page overwrite, we would also need to change PageStoreRef back to WalStoreRef in PageIndex. + +I imagine that newly created pages would just be added to the back of PageStore (again in queue-like fashion) and this way there wouldn't be any meaningful ordering inside of that queue. When we are forming a new incremental snapshot we may prohibit any updates to the current set of pages in PageStore (giving up on single page version rule) and cut off that whole set when snapshot creation is complete. + +With option b) we can also treat PageStor as an uncompleted increamental snapshot. + +### LocalStore + +LocalStore keeps the latest full snapshot and set of incremental snapshots on top of it. We add new snapshots when the number of changed pages grows bigger than a certain threshold. + +## Granularity + +By granularity, I mean a set of pages that goes into a certain full snapshot. Following things should be taken into account: + +* can we shard big databases between page servers? +* how much time will we spend applying WAL to access certain pages with older LSN's? +* how many files do we create for a single database? + +I can think of the following options here: + +1. whole database goes to one full snapshot. + * +: we never create a lot of files for one database + * +: the approach is quite straightforward, moving data around is simple + * -: can not be sharded + * -: long recovery -- we always need to recover the whole database +2. table segment is the unit of snapshotting + * +: straightforward for sharding + * +: individual segment can be quickly recovered with sliced WAL + * -: full snapshot can be really small (e.g. when the corresponding segment consists of a single page) and we can blow amount of files. Then we would spend eternity in directory scans and the amount of metadata for sharding can be also quite big. +3. range-partitioned snapshots -- snapshot includes all pages between [BuffTagLo, BuffTagHi] mixing different relations, databases, and potentially clusters (albeit from one tenant only). When full snapshot outgrows a certain limit (could be also a few gigabytes) we split the snapshot in two during the next full snapshot write. That approach would also require pages sorted by BuffTag inside our snapshots. + * +: addresses all mentioned issues + * -: harder to implement + +I think it is okay to start with table segments granularity and just check how we will perform in cases of lots of small tables and check is there any way besides c) to deal with it. + +Both PageStore and WalStore should be "sharded" by this granularity level. + +## Security + +We can generate different IAM keys for each tenant and potentially share them with users (in read-only mode?) or even allow users to provide their S3 buckets credentials. + +Also, S3 backups are usually encrypted by per-tenant privates keys. I'm not sure in what threat model such encryption would improve something (taking into account per-tenant IAM keys), but it seems that everybody is doing that (both AMZN and YNDX). Most likely that comes as a requirement about "cold backups" by some certification procedure. + +## Dynamics + +### WAL stream handling + +When a new WAL record is received we need to parse BufferTags in that record and insert them in PageIndex with WalStoreRef as a value. + +### getPage queries + +Look up the page in PageIndex. If the value is a page reference then just respond with that page. If the referenced value is WAL record then find the most recent page with the same BuffTag (that is why we need ordering in PageIndex); recover it by applying WAL records; save it in PageStore; respond with that page. + +### Starting page server without local data + +* build set of latest full snapshots and incremental snapshots on top of them +* load all their metadata into PageIndex +* Safekeeper should connect soon and we can ask for a WAL stream starting from the latest incremental snapshot +* for databases that are connected to us through the Safekeeper we can start loading the set of the latest snapshots or we can do that lazily based on getPage request (I'd better avoid doing that lazily for now without some access stats from the previous run and just transfer all data for active database from S3 to LocalStore). + +### Starting page server with local data (aka restart or reboot) + +* check that local snapshot files are consistent with S3 + +### Snapshot creation + +Track size of future snapshots based on info in MemStore and when it exceeds some threshold (taking into account our granularity level) create a new incremental snapshot. Always emit incremental snapshots from MemStore. + +To create a new snapshot we need to walk through WalStore to get the list of all changed pages, sort it, and get the latest versions of that pages from PageStore or by WAL replay. It makes sense to maintain that set in memory while we are receiving the WAL stream to avoid parsing WAL during snapshot creation. + +Full snapshot creation can be done by GC (or we can call that entity differently -- e.g. merger?) by merging the previous full snapshot with several incremental snapshots. + +### S3 pushdown + +When we have several full snapshots GC can push the old one with its increments to S3. + +### Branch creation + +Create a new timeline and replay sliced WAL up to a requested point. When the page is not in PageIndex ask the parent timeline about a page. Relation sizes are tricky. + +## File formats + +As far as I understand Bookfile/Aversion addresses versioning and serialization parts. + +As for exact data that should go to snapshots I think it is the following for each snapshot: + +* format version number +* set of key/values to interpret content (e.g. is page compression enabled, is that a full or incremental snapshot, previous snapshot id, is there WAL at the end on file, etc) -- it is up to a reader to decide what to do if some keys are missing or some unknow key are present. If we add something backward compatible to the file we can keep the version number. +* array of [BuffTag, corresponding offset in file] for pages -- IIUC that is analogous to ToC in Bookfile +* array of [(BuffTag, LSN), corresponding offset in file] for the WAL records +* pages, one by one +* WAL records, one by one + +It is also important to be able to load metadata quickly since it would be one of the main factors impacting the time of page server start. E.g. if would store/cache about 10TB of data per page server, the size of uncompressed page references would be about 30GB (10TB / ( 8192 bytes page size / ( ~18 bytes per ObjectTag + 8 bytes offset in the file))). + +1) Since our ToC/array of entries can be sorted by ObjectTag we can store the whole BufferTag only when realtion_id is changed and store only delta-encoded offsets for a given relation. That would reduce the average per-page metadata size to something less than 4 bytes instead of 26 (assuming that pages would follow the same order and offset delatas would be small). +2) It makes sense to keep ToC at the beginning of the file to avoid extra seeks to locate it. Doesn't matter too much with the local files but matters on S3 -- if we are accessing a lot of ~1Gb files with the size of metadata ~ 1Mb then the time to transfer this metadata would be comparable with access latency itself (which is about a half of a second). So by slurping metadata with one read of file header instead of N reads we can improve the speed of page server start by this N factor. + +I think both of that optimizations can be done later, but that is something to keep in mind when we are designing our storage serialization routines. + +Also, there were some discussions about how to embed WAL in incremental snapshots. So far following ideas were mentioned: +1. snapshot lsn=200, includes WAL in range 200-300 +2. snapshot lsn=200, includes WAL in range 100-200 +3. data snapshots are separated from WAL snapshots + +Both options 2 and 3 look good. I'm inclined towards option 3 as it would allow us to apply different S3 pushdown strategies for data and WAL files (e.g. we may keep data snapshot until the next full snapshot, but we may push WAL snapshot to S3 just when they appeared if there are no replicas). diff --git a/docs/rfcs/011-retention-policy.md b/docs/rfcs/011-retention-policy.md new file mode 100644 index 0000000000..fde36c8108 --- /dev/null +++ b/docs/rfcs/011-retention-policy.md @@ -0,0 +1,91 @@ +# User-visible timeline history + +The user can specify a retention policy. The retention policy is +presented to the user as a PITR period and snapshots. The PITR period +is the amount of recent history that needs to be retained, as minutes, +hours, or days. Within that period, you can create a branch or +snapshot at any point in time, open a compute node, and start running +queries. Internally, a PITR period is represented as a range of LSNs + +The user can also create snapshots. A snapshot is a point in time, +internally represented by an LSN. The user gives the snapshot a name. + +The user can also specify an interval, at which the system creates +snapshots automatically. For example, create a snapshot every night at +2 AM. After some user-specified time, old automatically created +snapshots are removed. + + Snapshot Snapshot + PITR "Monday" "Tuesday" PITR + ----######----------+-------------+-------------######> + +If there are multiple branches, you can specify different policies or +different branches. + +The PITR period and user-visible snapshots together define the +retention policy. + +NOTE: As presented here, this is probably overly flexible. In reality, +we want to keep the user interface simple. Only allow a PITR period at +the tip of a branch, for example. But that doesn't make much +difference to the internals. + + +# Retention policy behind the scenes + +The retention policy consists of points (for snapshots) and ranges +(for PITR periods). + +The system must be able to reconstruct any page within the retention +policy. Other page versions can be garbage collected away. We have a +lot of flexibility on when to perform the garbage collection and how +aggressive it is. + + +# Base images and WAL slices + +The page versions are stored in two kinds of files: base images and +WAL slices. A base image contains a dump of all the pages of one +relation at a specific LSN. A WAL slice contains all the WAL in an LSN +range. + + + | + | + | + | --Base img @100 + + | | + | | WAL slice + | | 100-200 + | | + | --Base img @200 + + | | + | | WAL slice + | | 200-300 + | | + | + + | + V + + +To recover a page e.g. at LSN 150, you need the base image at LSN 100, +and the WAL slice 100-200. + +All of this works at a per-relation or per-relation-segment basis. If +a relation is updated very frequently, we create base images and WAL +slices for it more quickly. For a relation that's updated +infrequently, we hold the recent WAL for that relation longer, and +only write it out when we need to release the disk space occupied by +the original WAL. (We need a backstop like that, because until all the +WAL/base images have been been durably copied to S3, we must keep the +original WAL for that period somewhere, in the WAL service or in S3.) + + +# Branching + +Internally, branch points are also "retention points", in addition to +the user-visible snapshots. If a branch has been forked off at LSN +100, we need to be able to reconstruct any page on the parent branch +at that LSN, because it is needed by the child branch. If a page is +modified in the child, we don't need to keep that in the parent +anymore, though. diff --git a/docs/rfcs/012-background-tasks.md b/docs/rfcs/012-background-tasks.md new file mode 100644 index 0000000000..8692b187e6 --- /dev/null +++ b/docs/rfcs/012-background-tasks.md @@ -0,0 +1,38 @@ +# Eviction + + Write out in-memory layer to disk, into a delta layer. + +- To release memory +- To make it possible to advance disk_consistent_lsn and allow the WAL + service to release some WAL. + +- Triggered if we are short on memory +- Or if the oldest in-memory layer is so old that it's holding back + the WAL service from removing old WAL + +# Materialization + +Create a new image layer of a segment, by performing WAL redo + +- To reduce the amount of WAL that needs to be replayed on a GetPage request. +- To allow garbage collection of old layers + +- Triggered by distance to last full image of a page + +# Coalescing + +Replace N consecutive layers of a segment with one larger layer. + +- To reduce the number of small files that needs to be uploaded to S3 + + +# Bundling + +Zip together multiple small files belonging to different segments. + +- To reduce the number of small files that needs to be uploaded to S3 + + +# Garbage collection + +Remove a layer that's older than the GC horizon, and isn't needed anymore. diff --git a/docs/rfcs/013-term-history.md b/docs/rfcs/013-term-history.md new file mode 100644 index 0000000000..0c359028ed --- /dev/null +++ b/docs/rfcs/013-term-history.md @@ -0,0 +1,147 @@ +# What + +Currently, apart from WAL safekeeper persistently stores only two logical clock +counter (aka term) values, sourced from the same sequence. The first is bumped +whenever safekeeper gives vote to proposer (or acknowledges already elected one) +and e.g. prevents electing two proposers with the same term -- it is actually +called `term` in the code. The second, called `epoch`, reflects progress of log +receival and this might lag behind `term`; safekeeper switches to epoch `n` when +it has received all committed log records from all `< n` terms. This roughly +correspones to proposed in + +https://github.com/zenithdb/rfcs/pull/3/files + + +This makes our biggest our difference from Raft. In Raft, every log record is +stamped with term in which it was generated; while we essentialy store in +`epoch` only the term of the highest record on this safekeeper -- when we know +it -- because during recovery generally we don't, and `epoch` is bumped directly +to the term of the proposer who performs the recovery when it is finished. It is +not immediately obvious that this simplification is safe. I thought and I still +think it is; model checking confirmed that. However, some details now make me +believe it is better to keep full term switching history (which is equivalent to +knowing term of each record). + +# Why + +Without knowing full history (list of pairs) of terms it is hard to +determine the exact divergence point, and if we don't perform truncation at that +point safety becomes questionable. Consider the following history, with +safekeepers A, B, C, D, E. n_m means record created by proposer in term n with +LSN m; (t=x, e=y) means safekeeper currently has term x and epoch y. + +1) P1 in term 1 writes 1.1 everywhere, which is committed, and some more only +on A. + +
+A(t=1, e=1) 1.1 1.2 1.3 1.4
+B(t=1, e=1) 1.1
+C(t=1, e=1) 1.1
+D(t=1, e=1) 1.1
+E(t=1, e=1) 1.1
+
+ +2) P2 is elected by CDE in term 2, epochStartLsn is 2, and writes 2.2, 2.3 on CD: + +
+A(t=1, e=1) 1.1 1.2 1.3 1.4
+B(t=1, e=1) 1.1
+C(t=2, e=2) 1.1 2.2 2.3
+D(t=2, e=2) 1.1 2.2 2.3
+E(t=2, e=1) 1.1
+
+ + +3) P3 is elected by CDE in term 3, epochStartLsn is 4, and writes 3.4 on D: + +
+A(t=1, e=1) 1.1 1.2 1.3 1.4
+B(t=1, e=1) 1.1
+C(t=3, e=2) 1.1 2.2 2.3
+D(t=3, e=3) 1.1 2.2 2.3 3.4
+E(t=3, e=1) 1.1
+
+ + +Now, A gets back and P3 starts recovering it. How it should proceed? There are +two options. + +## Don't try to find divergence point at all + +...start sending WAL conservatively since the horizon (1.1), and truncate +obsolete part of WAL only when recovery is finished, i.e. epochStartLsn (4) is +reached, i.e. 2.3 transferred -- that's what https://github.com/zenithdb/zenith/pull/505 proposes. + +Then the following is possible: + +4) P3 moves one record 2.2 to A. + +
+A(t=1, e=1) 1.1 2.2 1.3 1.4
+B(t=1, e=1) 1.1 1.2
+C(t=3, e=2) 1.1 2.2 2.3
+D(t=3, e=3) 1.1 2.2 2.3 3.4
+E(t=3, e=1) 1.1
+
+ +Now log of A is basically corrupted. Moreover, since ABE are all in epoch 1 and +A's log is the longest one, they can elect P4 who will commit such log. + +Note that this particular history couldn't happen if we forbid to *create* new +records in term n until majority of safekeepers switch to it. It would force CDE +to switch to 2 before 2.2 is created, and A could never become donor while his +log is corrupted. Generally with this additional barrier I believe the algorithm +becomes safe, but + - I don't like this kind of artificial barrier; + - I also feel somewhat discomfortable about even temporary having intentionally + corrupted WAL; + - I'd still model check the idea. + +## Find divergence point and truncate at it + +Then step 4 would delete 1.3 1.4 on A, and we are ok. The question is, how do we +do that? Without term switching history we have to resort to sending again since +the horizon and memcmp'ing records, which is inefficient and ugly. Or we can +maintain full history and determine truncation point by comparing 'wrong' and +'right' histories -- much like pg_rewind does -- and perform truncation + start +streaming right there. + +# Proposal + +- Add term history as array of pairs to safekeeper controlfile. +- Return it to proposer with VoteResponse so 1) proposer can tell it to other + nodes and 2) determine personal streaming starting point. However, since we + don't append WAL and update controlfile atomically, let's first always update + controlfile but send only the history of what we really have (up to highest + term in history where begin_lsn >= end of wal; this highest term replaces + current `epoch`). We also send end of wal as we do now to determine the donor. +- Create ProposerAnnouncement message which proposer sends before starting + streaming. It announces proposer as elected and + 1) Truncates wrong part of WAL on safekeeper + (divergence point is already calculated at proposer, but can be + cross-verified here). + 2) Communicates the 'right' history of its term (taken from donor). Seems + better to immediately put the history in the controlfile, + though safekeeper might not have full WAL for previous terms in it -- + this way is simpler, and we can't update WAL and controlfile atomically anyway. + + This also constitutes analogue of current epoch bump for those safekeepers + which don't need recovery, which is important for sync-safekeepers (bump + epoch without waiting records from new term). +- After ProposerAnnouncement proposer streams WAL since calculated starting + point -- only what is missing. + + +pros/cons: ++ (more) clear safety of WAL truncation -- we get very close to Raft ++ no unnecessary data sending (faster recovery for not-oldest-safekeepers, matters + only for 5+ nodes) ++ adds some observability at safekeepers + +- complexity, but not that much + + +# Misc + +- During model checking I did truncation on first locally non existent or + different record -- analogue of 'memcmp' variant described above. diff --git a/docs/rfcs/README.md b/docs/rfcs/README.md new file mode 100644 index 0000000000..fdf6885929 --- /dev/null +++ b/docs/rfcs/README.md @@ -0,0 +1,95 @@ +This directory contains Request for Comments documents, or RFCs, for +features or concepts that have been proposed. Alternative names: +technical design doc, ERD, one-pager + +To make a new proposal, create a new text file in this directory and +open a Pull Request with it. That gives others a chance and a forum +to comment and discuss the design. + +When a feature is implemented and the code changes are committed, also +include the corresponding RFC in this directory. + +Some of the RFCs in this directory have been implemented in some form +or another, while others are on the roadmap, while still others are +just obsolete and forgotten about. So read them with a grain of salt, +but hopefully even the ones that don't reflect reality give useful +context information. + +## What + +We use Tech Design RFC’s to summarize what we are planning to +implement in our system. These RFCs should be created for large or not +obvious technical tasks, e.g. changes of the architecture or bigger +tasks that could take over a week, changes that touch multiple +components or their interaction. RFCs should fit into a couple of +pages, but could be longer on occasion. + +## Why + +We’re using RFCs to enable early review and collaboration, reduce +uncertainties, risk and save time during the implementation phase that +follows the Tech Design RFC. + +Tech Design RFCs also aim to avoid bus factor and are an additional +measure to keep more peers up to date & familiar with our design and +architecture. + +This is a crucial part for ensuring collaboration across timezones and +setting up for success a distributed team that works on complex +topics. + +## Prior art + +- Rust: [https://github.com/rust-lang/rfcs/blob/master/0000-template.md](https://github.com/rust-lang/rfcs/blob/master/0000-template.md) +- React.js: [https://github.com/reactjs/rfcs/blob/main/0000-template.md](https://github.com/reactjs/rfcs/blob/main/0000-template.md) +- Google fuchsia: [https://fuchsia.dev/fuchsia-src/contribute/governance/rfcs/TEMPLATE](https://fuchsia.dev/fuchsia-src/contribute/governance/rfcs/TEMPLATE) +- Apache: [https://cwiki.apache.org/confluence/display/GEODE/RFC+Template](https://cwiki.apache.org/confluence/display/GEODE/RFC+Template) / [https://cwiki.apache.org/confluence/display/GEODE/Lightweight+RFC+Process](https://cwiki.apache.org/confluence/display/GEODE/Lightweight+RFC+Process) + +## How + +RFC lifecycle: + +- Should be submitted in a pull request with and full RFC text in a commited markdown file and copy of the Summary and Motivation sections also included in the PR body. +- RFC should be published for review before most of the actual code is written. This isn’t a strict rule, don’t hesitate to experiment and build a POC in parallel with writing an RFC. +- Add labels to the PR in the same manner as you do Issues. Example TBD +- Request the review from your peers. Reviewing the RFCs from your peers is a priority, same as reviewing the actual code. +- The Tech Design RFC should evolve based on the feedback received and further during the development phase if problems are discovered with the taken approach +- RFCs stop evolving once the consensus is found or the proposal is implemented and merged. +- RFCs are not intended as a documentation that’s kept up to date **after** the implementation is finished. Do not update the Tech Design RFC when merged functionality evolves later on. In such situation a new RFC may be appropriate. + +### RFC template + +Note, a lot of the sections are marked as ‘if relevant’. They are included into the template as a reminder and to help inspiration. + +``` +# Name +Created on .. +Implemented on .. + +## Summary + +## Motivation + +## Non Goals (if relevant) + +## Impacted components (e.g. pageserver, safekeeper, console, etc) + +## Proposed implementation + +### Reliability, failure modes and corner cases (if relevant) + +### Interaction/Sequence diagram (if relevant) + +### Scalability (if relevant) + +### Security implications (if relevant) + +### Unresolved questions (if relevant) + +## Alternative implementation (if relevant) + +## Pros/cons of proposed approaches (if relevant) + +## Definition of Done (if relevant) + +``` diff --git a/docs/rfcs/images/storage.jpeg b/docs/rfcs/images/storage.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..1d72a018dc462a74ad01bb17561c98efd0745bca GIT binary patch literal 431075 zcmeFZcT`i~zb(2$C{jc35SoA>B3+6JHj0Q~p$jOz2-v8G1VMTS6#=D)h|)U>n9vac zL8^d|f}$cwA_`jw;cdRZ^Ugi@-7(&H=lyZ-x#PMWf;oKc6}0XYJh&yCi`3 zw28S10D}Pl4Eh7?&H*O?IMd#-_Z1Eu%&g3N$39k87FPCs931TX*x5O_5L_IbJe=(8 z+2blR- z_+>SYu?n2K!X|fJQ1eb^@jm(Eb)7=z-%}N|+-^j(a|nxwiiz)6R8l^mqOGH=cSPUd z#K}`8re@}+FI=>-wX=6{bocP|^7irdyLl@(Bs45MBIa&vTzo=eQdai8`wwy+=H@*q zd0P6c?D>oG`i91)=9ZVQTD!V?di(m{47?p1pO~DQ{xCC3rYwH?y!7Skw`JPT^p`ueYd{XfP9O#+OG8JbeIy>Y>qLZQEKK4unK4OaeR=h&`X7m(Au zvrq7NW^r97yS&zUs*u}__Z-3s+T{JTy`lXxvj1-bi~j#LvVR}ge;d~t3uekBSnj?0Z7 zs_}T)jIT{|S8jTlWP7|VfjPQZ^7{K;AR?~ph5gs9fQ6bDA4^|9PT{}w@yr)aNwd=2 z^jaVfpm#tYA3XrpIu9L0?ekR&SklY#n*7W!y8iPAlP0WBD<)->Z5~-4-|w^w%o>uK zb7&XNQ-9+Vuj7J41E;Q?9hdu~rO6}~Wpt833&FHmY?uo8qA+)!&IeIvc$C=93VEk!d%;uyoOjp%rsZ%sUs zT1VF+zxFq2w3yM}jK7d}*wJTFY*x`k&Wv$Q@A})?j$XhTtl7TIc()5Mx4R>vu=~Sy zfo)aiyamj&uai6dlVWJw{_)G7(4 zMEla2KNw}yiAcN!?)1&wE?MiJLCYH>Pvbhxu&TJO+MNeRr~G|ye)e$)nUFm{|Zd`-n3uKg$IvJ!}F8T&cE&1$!*za3rvZ#@p zM1QxU&sWVeCR+db1c34OhnC2#oo7!7+cy1i4CF|l)o{R0?yYjx@Y${}*HE7di;EDi zb^-VFxsonq(@8s&c>DYO?gE{0HSA#VUY9NY!`sE;RVFU>WRHtKuV z-=r9ZA6yX)S&Bl7?gAdGhNQkK+FI9aX#dt|PhsJ+(Ugy0XaKgAXYjT3G`&pj`fldBK?a}MTB6MGh zYdoee{sFo^M;&;HVe6vs=%9gh1eV!=qg|Vuxc1ju$y9) ziW3edBp2+D+t(!JwR(u9{ZxuAt*<=u^OsE=RrFgGMVngyp_69tkE>4 zo$Sr2^Q*%X@7`rn!tyaDB?_*kl7V=K=7sK|Gr^~A$NUR*^Ddq5>4*3s+n){}FTD|Y z|4B>P{YN?BG_Zl2>1S{Z22e)$9m3EXQPk#~BJShom{Brgjwc)9Os>kjjI#|%)n7U% z3;euz$iW!(73^FzgpwIyU%Nkz)3l}e)obc>ypELDlxkF;m}g6K)Ib4jwUjL4x z+Tmt(H*Ib|Ms#6W$FIVdR2>dh2O@(?-grCr&)Z<{!G0lEs-P8YRtLUV z!*1J7Rm(Sy2Ro}hvL7&{)>%kaj_`SP*z%Ab0I}OW@U_{$)!P4}YE7%%bQh4@c(Dt( zT-XJ^vPPlVcY!NMb^9H6DllwMmP5WDeNd|N`~HySn3-qoOVbN0HVJee`0b6*$=`^X z3mc1LzbpgEi!mlNo&MNhy`cLZO?f8G>-jr(TJ__%i$>XM_izyfC7Bbnpl%%d)xc0k z(9}|IIspn3^_H;9l}Bww$(M288n80bq1!?9gJ>3pIAdfNKzO(O45~6Z5E}kMc3HjJ z+o6nkbM}YzZJA$12m+3jL-^^XGUAy(c8kY2|95Mf0{6d|5+JE(+BI zExy5B;LRYlfzciuv?CT^EJ7e<;~Lhw3q?SSUYw|rKiOH!l@MbmE}WNlzx-*`QLjiQqR6MENpN*CL48% zH-opR{e~7VETBU&Rulr5jZa`OxhB@N37tk4BdzL6p-rNfrhbf#@iiY|8Gfo&FPA0+ ze@ni24FIkw!#DIcT+EEDwhv&m8>XwC;9Fi(ZS%cva($IjieE_-vtx_nDq%Z%!w0^e zvlmFwZkW?l*18vA-6Y$5^<<;{2h*?RS-w(#@$LByOtIL~XOXB<_KXzIvz)vLPLO;1 z5p;9J8j!S=(J(9ifd18VgT!|5SZ?cjNu4K0Jh^0aX}iW`-)=Xs4Wk z@Wmc&7+no)Ts@W&+G<-O7>aanPaspS9Xo%Ke7{z+GWE_QYc_Idualu+P#WX#`0DDF zq$za~Ki+;O8T&DfFp|u<4(lWRXRO(N!LLw9$bI+2c}hy@XzKjq`L;ya0-|}ezWkN< z_wU2lK3cPVGNlVS^-bgDcV1Lu)c60YcMP*Kt*=k|ST`e>ZF0yt`WIK1&(DaAazhSa z+inBC^ALsjhHkKnGqRx|;~CP^zt==&bxWVSEJxh3JL{e~HyCR)I>+a}kE0r{yCkm^ zg&45{m6bXRIWfFDr8+nsP{}IYYP88;HNfv(-(gIf>k-WXaSv+2)f+>R0~G)`U_~cWYTqLJh=*K=H9~BoM%fkjIz!PDQqf?vV9imK*wI72ZH@% zbnG>kd<}}!v4EvlHXlbj$~|%?La&JH1GEgSUI06dG1`IHmgg=Ir(UuE29Z5umh+@~ z^d7WUt<-;X;e7be;EZ#A92fkG@ugqDWox#jX}Xa6zh)-TNNxYGEC&#pnTDi~Ed&3| zOw?DgyYD~!mbDwTX0zNrfL*{b;k4Fe-JUJUa+i3ntDQ{P~ z4;L*$5OFasiiLfJ`;533C7vt=-0qI>DwDnWK<#Md4avewO+bkDe@D~)r`7HM$Dc@S zL4Fs=ybY0pnghGQFSs!>4q3}aFA(Xr+TM2w_f6p!Y`KtX14W%0i78V>|>cGOB8@ur?(Cjvo`1>Xp72dz{2D?aEN7Ln{IQ0 zg7UF3b6no;ZjF7UZM=%t_od07ck%p2Xb0VA}mhQMod%VfWoEc>sgcwTtAN|(|Mo)$M z&cW9ZCnAvY3V(VsV7owdUJK*VkLU1pZw3dXUMM|Y9NGH(bQkzmKO2QMF;b!xGQ`#x z?4L=5ItizH|88(sCr*eO2a+J*Ld?df!cBK@9YfET%j6+Fd;b z3*=}#HKgy;T}{;$rNheHME*JU3qi$hc#<_+H=24Lmx=>Wy7c4Jg$Alk$%GE4s$oLY zs-J{Xa~hb9oEhIts*yJlzVzzSl?kTfBp@;p8d}n5QyA*2cLyF4Q#1zi0-s8SbUHTjCkrgr6u(;wXco5 z&!u1F+G)4;CZ1ugOifj$<3c7E;yymT6Riv<2hF)JHm7czU`HeR(=bYsUyjFdMk<;6 zx*a%3JgYiRd1->dm%%2A65Mz$6q92(wQrkYGm-eX< zw?dTgwQV_k$^Damvd`44eZ$UYGqU@t+$h^KY%fC&dU+&BAfxih#8ekSDk^a?QuOHS z-|q<9cPB1{YkXArah(5Rtf#_Zm9S3MsG}CW1q#qxE`+%-x=ov?IWb12&zJ2qW;^|h z?2rC%_$Z^MEVm!1T_!CaVU0q49sT93)a`hvkUY%VB_XACx+&@H+bN+rN#2upOEW*1Xr@+GmbHNqY51OJG=>N=xzW~T~+EDj3^5*1>8e&i}xyDK%Yj6|k~VA}tVFo~bA$iw91Kbc0LH+jzVDV_ACrt%)DJX+?-)oW~ldWEY3_!io3N4sB9U zWfWJwV*xsq7E*Z*Gfz&pIca3eH^4^M&M}68AnZ(Go#;+=usWw7lG=bH3!=oMe@GR6 zAOvok7QC1IGLq~?R43}MrWqSr&4_IHBI{eaIY;8*eTCHQof?NiJ3Yb-(QR(&1Q*sd zXKCP3xc+?G;BR*U{}E7$K-aPBnL{QJsV!)-$hL4YYx)Ly1<}0rbE&{pYiRq7 z{YYVIo7Mcg@D5-TJyS+1qiYd5BjIEOU4cN&z+GvY!};u_uM~t921D}Ea?3zVP=3qW zqty|y4&YD2Mh*l9UWi%wMbv9rC_}`R4yE=vUmpS#LDbZpnt`8cPhxh0W=rM^RYT`R z-|6@Hz=?r4FEl3@vvIk0C#T)Bnn?87nAM5PzW%~qW%knP0G9*&9=7X3JfQWeVc!w` zB1DRjIg}WDGZrk>@1l&JH+c;m_H*R%IQ7fdJ)bACOMNWL038RN&omw4J3Zn#y~tz{ zhJ1Z8*Opgq8P2;h1s(fE^Gz?}T6yZ%t|-(FwLi`XO(GzC7X+x)*+I{I#VuPMXJ+QA zKadkmnC{RAK0V6L@_~Os%vLbiOUcMtwthWVfp+=Da3fg?_Zd~cYXg@LRjDi3aMZK# zqzYuPQaD-rLZYD%5$RM16+ZWOrzg6Ig2@zIDjyNaxjYroqpK^fkv@{Q;BjB5H2kdK zJ%Fqg(_`( z$yET?DPGM}cCdg4{3ibG-Jc{WtCMQl=3mE5l^qZ-Pi|SW)nMP@s1`M`y8xS!mHT3i zbl8){e#eDem5=rv8~(mmf;1mh#;Ls+q;j~A4MTrpTncUNh+iPt-?sTbT*pSNs+DF zFY^|B<_2FjJYo{|@mNlS$-k7z=!iZBfbu=i{WN)zR&ua_3;ddVdFYJwa^3;o5~&z- z$>K}fcU? zq@(Q!{B^?rI^lnv@c$QWRm;e#+|>swKG{^IeyY&^_4tZ+Yx8(fK)s8Frane51XdoI_EEtn`?Ngu9rQ_d9Ltwy(3XVt*i2VN$mawl<7iZ zf)NSu1lblhdxL}+6DVy^z8V9Y`h{i#gRJx%aVcf%*x~19;xfe!C47f7PJcdbHe{6Z z2Cdt{#u#-)VpZt-!DK42OPXn?u!$)B%|&AZJ5b>_#cHK)Zl$R+cr|s>%r5+pEsi0LPO|P$z4bL4GLs9Bpi`!fvjA=b+-K6-T-=Bi7*V*B^Oa*v(2cNdk5f*c;sO-c=2 zr-rVK2EA898N9d^LNl%_JQvlwv{N|z{qvu&b@cU?AtGU^Zx#qzb*F7pYC4=%yIP2x zSeU=7z|yqN3GdrH`AgZJqCoV*#q+$EvVOxuHV=NDJ6ZZ9(*&9|M^$(+dP`TA-yBDo`f})@)BhzZS5izZ0J&#=i9Enzu z>Q?5tq&s6K{&Dv1p(-2-7{qIptgudCt&A4<8qxK~=oWq-A=}XsSnlrjm)XAH%!$)m zk*!PD!u@DSe?8X(T-yce34K-xI7x1GH+E^WnD>m zo!1%7->C^lkrldnHRQz}I$-KU?@r&y*&b^JgzxjJ5Fn3qII4%~% zM+3~Zh0yA|gq;>40;M2ZRX^#r^MN3D+WYlWPqXrG+rJO`acD-J=`O8@E*_bN>&?!6k4Kjh=V2W!nE4yt8>SN*vjQ>l;Z#!tjzaVcle0r==?; z@WsuF&a&ZWzE93?Pz}t64G(I(`4o66<;pd2Yjct+5W4jxS!YL!98$bkfn;`F_#TqS z+}eneFh%+md$ z9<&I?4n&h{5>SR<w~1@JXlhN~zx1ODAu(sX5#1|^NY zbUym=_Y8UL$n08oz7_lO^_g#W5s4Yq6%yLTNo%QaA-)pflvX~?^^QZp2tOSQR+V5U z84@7k867tD8_Dur+}*_)oJH%cG^c-l`$56P4d$qwQeh^Vd-2ZZi72O?D!e`fg)Ag) z^zR};mW^9U}JSF*y3fb1qKFY!R2Nul=VZd$wSfGVZ;P`!x%yf({IyS$ZF~7SOtvI z1Vie(o`_b**3n(yp@Pr*+J^Es3Crp4;D^E#`1dKm3&5jL0xax7r@Qi_gu`m~;@64` zMA=!&_iVbIW3A-BB7SdP)MWi)?!A&`-Yc)P0gMN@O}juQ_6NlgIooCTFxNZY$ld7r zcj^62>)4T}s+OM{^zCOv2G0!5&D?0`2`}PF-gpEyEfS(}>^cZ(x;}W}1^vWSRkcj? zQRxn|G3LzLam}8zl>~j+i@(KWn2gz+$VlPu3>olB3HZgHYEoSC-7zkn^~Fu{%q58m zr<^FWDUVv$$M*LISw7tm`~)EMce1d1bGuFAzR-&k#P7{5SCfRATL5aI`<%o5>Y|G# z{N=-vfrsC{&u;PcnPbzp0jkI&=>#F2^x5tn9IS=``yL|A@%~ri>qc68flpIwM!&#i z0nvmwU+KdYvX`&8nJ(;ufY)yEOD{T_Aq(DqiqQvQ)pXNlTHuTQ4=RG6E&|!^23spC2|bT+Om~ zmu`K6%@X^GNd~T~SBfz#jUc>w0aU*@wy?3-&zFlD2#`rK?L@iW18$imHQw1g{S?t! z4MRhP(l{lw!-~a+p#VBlRUN=W-1B9c{U!Tvcwc{e2m3!MvPpJ-9v-~BIxE(9_F5(E zE8d6}lwhPrCAiS~?vV0#Zaa?sg3QtJ?oquh%g;U6a)l>P#~IE4kZndiE;k4<%3 zgKke7=_AB!+4Ue22%Jp_z(2HpM6mVf@|$XYJ(UKPw+YK>tj3;O{D;lh^r)_T*qg8m zNTT-BHEHP`YB#&2k<{S9-jcAi6^o5XXJL2o`kd5r?LADMKWCyYMBj`ymZG`t-9i|4 z6pcU$gt?Mf+bD&cVB>+wb~_)BZQU$KTN6aH?x~-Xwzdo3UtMDPWX(FgYC*$M*dUYm z?lgobzSR74U}9zO?I1txd_Gl!Q?iC-o{3U3b^ve1WTyh)18;=zJn8q2D;b-=m=8g( zy?Kf|E zg@&m>?XS1yPbr$byu+{SaL-aQDC$4NZv`2VUMNUEv3(w_AqCbzrQ(>8RIzVGxcj2R znL-bIHXm=%!=97+S;oa4UaZ1zwmSidZBu$A_=0+~uaFyKNZp2Fg$6%MOMAZIh%dgf zbZ^WZ!4bW1DNykK{ZI3aj zVYpRY(`1NARiCe<@vchNNssFjz$H*=u}}bG0bU>%N`ZqNEqLy!rhK7SuSeHH!>7)B zq*jXA#2ItJl`o4jY2^XyYct+Z9&t^>QS)$J@Ak+?9GaTum*`!F^Akf;+y%n(o7Yk` z-9NPD>dCI#`I1KA>R2d?H1AX&M8Qr_*_`q_HXWT@EEbEGcCrqzv<(jYq*sO~)dpy? zEs%b7$nayQb^&R|0XmHOET)>S3L+}$R_&9)Hg&Cy<0pp#GwaNqixb|N za~L00r?@7!pCZ-7xyaC3;^+t*CBWZ&>>0k1x>JFB>hdnZAz-2iQ)h%JPGs=*15q~% z8@zkfB`{v#u@Wdl$5Ok`QCVA_HxBA1N4A?KWobSwGD{$PSc|e+Ukm$qxCfpMA|UK- z=~Yk031DE0=}`v6=te7+WrY7z|CR!uc)L`uu!(ALQnFQLi0jO<(1gclhbTlMwAk^g zVM-gBR7B?pH$?B~=47+~yN40UgKf~h30MbqfGCbvz=VPB*B7mlkgQEs zakGsEO}%#`yowrBFB$)fqV|z34Ex_M-a2#a3jjClz(I%=R@||3~jQ(+i)TGsxk{Vg)bh6Q7Xzm*E5-btv8${Pg2s02v9x0MnS;&%y@J z>qg)!E_?j^m9W{@_V8(7c}RPfmkAWRH#c(u1LCY6#XALyz=+dCdaMA<5dv+h?dem+ z>67(#zKTRmlOHGj-r4GeWJbR@a$?iO!B`j13l8=`)to5k-^Wm+TCUyIN%gl%`muhZ z{h(KEX1oUevn?AQir;I-`hu_}Vm%H5(?aQ6BSH)Gb5QM0*T^|BdFpv1yR-`wn*cI|`+|2x25b~qm?aCMq9ROt z38G*;RQ7cv`9WeAp5rymtM19S`UQutAs&A=$I3sQY_oZo{k6PjG3qP}aE0*>DS+{% z#gnGv8FEm_21SxALv0*}-z7rD!Et`?RE3_Tw2FVqe90qyOjKt{khCrh@H$dGlkbk3-(RLI*zrz*yQkrSLA=7sred z8N~wrtq_)AEIkhCAVueLQG?IkxY(%ly8BF+$eRTGR7L8Cq~&*L1>#4RcH%>bzw>v2;K#3ThZON}Zkz|-(*za?UEhoXXojSX3M$b){o>17 zXH{$Sb4zhL$6NO8KY8tGo0970ldIR0fMM|F-WXEQzE+ZSF&d(SJG2`wK<&+aIc{|2 znul&{exjU>jiQFwy0EF|zLkr|JL7IP!U)p%J$h) z5u4EkCZsYM05bB6XtJwLra#Q^^HwmSic!A{PBUSuJVU_Xw}0k z=3gVCU{W*RjZ;cbPi{NNlh2y279Vgh&OMhhMR%sUl$G{y!* zc#ooa3v%q9iBN$@+hk@<+j@mtT+W1lWl=~M&;7@3j@7|J?Kq+8Y!JD3o2x#mqI=IBv z(swnig}S-V5-sLBzR7E^)b`6~z7p1m?eIOO6jcc$`lHlnJddgM+x;O}Rzll!00w=i zMkFrFIPh7Ule=;q7fkxff8eJWHxRN3$tO_;U@kTNJQ!Trw9{L*VMD#|VxJv27wd3N z%BDrAo9DTi0-v!cj$@B)(WY0$Xl!J*Lt?nu1 z6i<}9-2LOQ<+rdg>^K@~1f)8VHtL{red*>l?VtUel3~i4>Q=8+KOk4}c2K-Ame1Tx z$b9i}$w9!NFy7@LjV+lW2IBe_W75ed1D%QD3+&-0XtpOoCC+AQEHyf(%X zrF0Bf7P)P|OoRoZ0RQF%_S&(r4TO|NhQsiII|?!a$a}{^eU4YgZkqI@cqqehe&}?f zAcJiojGss@gGx%ud9cvt_EGa+?+%M(8ILiM-JMw+baYdIfmJR{P(18-iS3;9_93Jiq zQS5n(&e^_)(=B*CE4`HmLG`wsa$39`@1OAgR29y>hWeVpG_vYX8>F~)pyOw~lTjiV zfk}pyaDP6Zfe`A`%zomBnqGVj*(x2058NlBo1+e7R*BYL zGN?PS&LXq(584t}9@9oB-u(plGD8tUTP+9h%5&eXp4Q0ManBUFgE`6zbxyuNxeKUj znVg+3%^-bIDH6gQg*1t0RonJpGU-q;M6zqCF4?7}tA(k=;cc~vIORfE3ice=n9&v6 znuW2C==9UY_QPaN6m zKzxJNc7e8&(^C@_LF^7|S1s@NEFFCQ=jEiFyRuuH*{H@babb^-PtQF3EyG>|slXMb z*eNU$WcQ?UI=!Zzt$1c09=w=9ANLQCi8Jk#Ln{Ncr>7i^d*1#-P5n3Oz;93vSBh@L z@na6{iB?vGiQurxLkDjWwVNsrR$}^idsXQ}!iPgvxfcIXRjec-Js0wgovOpWpP(LP zysGxiliVt;iN755E5Get)SKocYhCAuSO1*g$9{E>GS;`6N#2VQU5v~)N(N)H7bEDY z8pVPG+PB}!L?&$8dP1;!u3tuTJNg)N)J+`A9_oVW(3-OAMvCqg>9XVzwoB&1=@bis z-kZruZ;cAsut|v_E28>?A)zq+HwsX*moQW=*iaFxFzPzgFN%Zwi(VH|THwfPkkG6c zYWV3|Zb`srcMsPC7f;X4hpGbYqpZaLEH!PQt|c3tsKgu^H@f*eOm6$^wE8X4V25_S zgN5I8H-iHA`zyM!y{kSGuA(n_2ilXfws9IsV$ zS-LWE4ZKOK@+{|1_stuOvJj$O^l9iGLaekHjhI5`yiJ-9a;O&(X_X=e84PUlHMOYJ zzJWz~GP43HF~-uk?L*MkjxyQ+-4CjHA@CKeiPy1d7(e^mV;4xgXSw82lPm7gUl*$R z8Y<4an?t;DnEJdZc6>!3*#BL?x45OKBHS29Ws){l^L2v zA5jd?>qF$l%3=;rU|?0tYMR4Oh(Y{vPg7nf^UED^5M+hjla$uu(f~f=S)oN(LbIT( z^^C+bG$DZqR6Rj;-iCIltnLDNa+B{p!@3p0Nr?#MhoX-sRBMz}hJZrBJu8N4x{8Ff z57Js37WN&xpIVkwHRWvA+sp`>X>9iOFtJq?OT{$(LrfpoaE@4W*UT0!BGVbH7?tdP zS8hC?AC9jnc0z3xU4M(FYIF@xS+9j^{E9|(&dG`FC?9ysIL5BV6{Fci>tV1Hny0F@ z1t$x;gJeX`IqST<_u+vI;;{bDdSi+B(*OzWiA=@u;f*kQZM5{Bwdh$w_fcD--OF*x z*zf)KUBdX}Js+QhBG+?P>wc%t!5yk3fn#fO;;He*AHnb>Lm3j$1mx(2x5M5gZ2}@Xa_76H54o! z%t)v_t~biPpQ+pRac9YxXJjJ|%v?m?#j&-YfD{!p_ZmI4`kO}ARI3mq7wRrz1|;x& z&TsqD#(2xk_eF5k0+Zm;MJqncNys^^kqm3;6j?w_=~V3dQKPL&S%|!wI;HoqykoOl zrIs0zfr}7&JNK}A+=EsYR{x!n-f=XDoHEN(ez4MQqJ1>ziEN_kl1Z-eMOM$ZkGrL~ z0fD`!6Ly}%zv=<&ZJ{4*qs#~4P*YjDpwkR0K+

z7l(=RCuMO8-2aITdx{aMA&ZLpREMSeS2w15purz3EVkPKPU(k@+Hy z1617Xjd+*(%<{H(v167J{=SHu(1&VsR8r;H!$prcfUccPm#Yb47JW&*_xY{e4z+oRL9`TxmD>X;5@cRDLCPMh5gM{D@UqnO z#LAwnoRol}LeKF_?d6>bsU=K0sbVp2Di?0ViuLUor-Vh*o}fEOP71B13)&&wS_m1Z z&bAAkzuH*t7wl7T5UUn*Oa1uV<0KjTtK7T;GEUKCH5waTWeq$F_3f|`7;}sc2Bb*c z^&dj|ZM)9UiM2oG7!8np+ER05)mRb<5wsG>^~c5j1m`I6o%2?Z>n~h!{c@i3*4r8* zzRIgD9QA!^f(@-V2oZ`hBog37H-Kb$yFO%NOB#wREM^1RQE-%uK@JM7uQSq4fC(~ZtD_s|7oA+ z$^A(tZ)4BRp7-=PUspTA^#(SL9YYJy{i!!eORs4N@>;hA6c7G+m*ZEq_Ge@aD2Jbu zJ}z$>Vn3N&Yoo+*snX3SXgpDw82Jyu}~0{-tL-xSNH5xWO&iM-qxvmm9Gvr zQ}60Eu*e^|ybt21BT&=B8l|wm3f>b@(+jOAV%PV_$cwtR54Li1^y)}iZ}%tZ?&lGD z$n>-T!3?k-gLJe9|NO@@G@EsPd-KJSUI5t? zx+mAbeyy?3L&X9l*PP9FnY6tB%P}a=GR61+3uHdA4DwcfIM?{)c@e@4giLZij3Ic1 z;wl+t0j;{jQ`%1Khbn)m@-IZvlgG3nzkjo(AFTn(c$PC z6MW3GWH}=0o??|)md)F>RGYL9V`qBa<$JkczaN1u)<8wBk^1&IP>+O7glfa7b}5S5 zk@;n5&3LFdyqlY^Vh@>5%;$a}cm;@k1ZB~ zp28{}yq<@Qij3v2e>0Y}T4|sqQ`q{^u^M6DDRy0IR^)fJo14Nh<^=xuEWx?@UxUBtaH+}t+B7=vhvD^wZAwJVX$#4S~OTTnqWzfBz6PuZx6mxL#+ z$*qLU)*Uwb1%&n^AZmsZ#asrxsq38tpou8<6|0T6aCUdCU&lO_61Oq&o)M6{XX2aw z@GD0#kmg`)kUncg$w|d?(PgHvGW4TR=?(4zPUCq(@u%t*Jc7f8T+Yjz-l%(V5HkCr z7M07eF^uk>Z{bZH>0Z#Gi9`p{b*lWf&WLPAu#CG=-jWx&rdBQ;64jVnXF)G;+yL0} zumfmmyfQtK`k<`_x|af#^uzeEChywjhKUEBdgoZUU!Qz$GIRX224`dKk~RWhIMJvd`Sq+uMiCF>nC|t2h%-2l z+z^E&2#EAVR5&b~T4>^Wn)m;tIt`{$Jr?F1nu~tsUxyTxB1`~nB#|KwktV&pZDvkH zG5-ji%$N0}D}&+FH7~g#{zoMJav3OSBLO^H!LTBDm-gHiZZoKwz_TNh5YnE3-2M|$4m(xYBlI&M8+F~17TUEI9nsP=e0tp>=IqPy#?QHV zvux&~LxWs5^jR|9*#bw$4*kn^+D~&KJfR<+G`jvQOlAAR)XJ>`O-?wT9%vtio}PK2 zhtvMch^-JFbt)uHS3=3= zK3~`2BY*DQs5CL*2ChKF6oT4*Vk2ZU-({%m*%ch1dlzJb*JJELAP=dk-ZM2s81kJa z-BqCAEkAM7mCA zwD(yLcJ#3WQcA0aH1B>KGlSO;#|!b=gA*tjf7_eK1S5`SWZ?TFtF#W9{dg0TB@{UT zKvb3h4}*A(66jP6e!m3qn)qtJ2ubnM>3DC5*I4ZNrYV;WN!hRbiO#@M_Z*Zjjw2Oqj)nB0FRcxO^L>V>%l_)$NLVB1zuc~M0*wWYGece_G9 zM^A#0O}V*uxm8*C%W-+zghOkG^f)e=ebTybEPM3}#7m*(hW{Kf#c)7f8%fYA+)tfO zhg7I7h?$Bn)%lG>ePX@s-*x#lx##ZNv)MQRo?H+$jWL!KS?S%gzWe6gDXfJwB;$!r~#9%2H{gj=I+2jD}nb9O<=_ z#qGHfiL(3+CkN*9c+P$FBB2tqw~cp_&%8tiUt*euj8lmGcUZx{Nh?CeDMbDYpZkgT zI7aryA1lLpitS(>L2G+X}@CKOEP-a_$(D!b;9%(L7?mgWibSXh6*dfJl z01rtk0-J+%4_Qb*kIC?WO?yIA6jbFKg6GIqkS3T!*P>8j>}Zz7*(0>n~xCOOyh~-Zw)HD=?=n zsG(5VpU`BN$YA!v0ZsE-owJ7?wlurw<_rtx)CVN2GWW>I=9!-QcV>zmbKUFL5u@pK z4@;20Kr$3%e_?X4Y5io4_M+mv;(nj|7FSa{4l0Ds%bsKs{*q-4Z-#OV4be;FKq0^! z3gQk5=>je~zpq++Fmk+}xHSJDmoH&oWsjVp+3^?q-DM(1Sqh_3kc`+v+~cKRQt0fr zNyGofOMk2xytN!^vp1;Vih{gDbFtT%VgW^N!0;j@rJe2Hs2jsF%|SFa5F>yqeZ9gkxFFjAvC=oNJyGRr z?R{VPk4!Vsj>~M=DWWu1na)Iwi>;v_gkS+<^?Fk0R_&{X@w-D)bwX;d?`oLs0{Rky ztFxjhpE4?8-*9{w2MS^zMvKOl+OAKkSmXO{a5nS$bBC|O!^z1rV{+fGmY$Frf{1D5 z&CL^%Ot3-h5D^lLX=S91bQ&9Eb#R+@7<~Sf-H%Ge)I%bXyvpecxyn0I#+Sv8eM#Lb z^9j-Dy*qIR?4_wgd#IpXi02$_mx$QdLMN!7DKM!MA9`OQ>oY0Zs(df}>atLWQpsUB z_Q0M5MGBfVgq*R?U4R<{pQ2lk(Lx>det5NJsAHwH!9q^6Zm(nIqPfKF>d*I$zdVOL z|8r19gXkc}3WR-!9Mhu~ltc9`8ykf)wsN*%Pa@}_8s9X;W>$p5;H5$oHxSZll^nct zf*hQaR&B)h-0^#;0dquSfkbB4{Ha@0lnZV{#!SXv-YTR7LWY0T&OMYa#tdAiE^YQ9 z1yLN3*TvOj5#SPulF~ZQAODZvKk1dG2fx|qts4EFAF=-;Tn}O?d)9B^##5-*VaFdz zAthmk-Lqk>wX{bIK{G*5ccw-~`{XV&+*hIC{pl)0p92uHdVAf#F$?;TX__8Sgxnb9 zw*`%f*%8}`N(@o`v@uQjpnK}1xr4Ojn4oiQ3pzXl-n3JUR|xATBRki4p)G|_-|8+P z^}Y4XwWjJ=cf#;X$TnTOYj>mWAH)B+bj(3O2gMUM-2)+t2nbnF>QuGOnBd1=y@&SL z>6r~peyl0PPD!i~J>`M{&}O}1f+tkV3E_?G!R9ZN9Ef+2Q}TG8H-~5}y95T6$hHKm z%b$7pHfreAiFWtmBBn)X5IzL|eAg%qI&6yAvF;lXu9ViFzi~3dsqr<2pe}uxH-JdE z4Iht&sjptyV-Fq9@b0{tE+WJpdLZ|A0DgqfBZJ;e}BN(-!4a-p!C`%ay|%+rqKvPFs&AELD-u1-!T5)C5)jPi}SZafZsQs(P~y+`I1sY zM$S`R@>)}7+sWABYv}8*j;H6swoePXcSp@1dBfH|`d>Ux!Mf=x;5@Zt5&QW0GLge5 zs0@5CZr<+CIdgT>O|0YH_)D&6o)cUzq}YpiD1V=!_0Kc3;8|mkMW8v2lJ!S(H3o@& zZNX}eu9m!-)D8iPONcl$0?p6S!}I^6r}{xPUwGoV8{|G#?O&KAQMKsR_pd+ogb zFZSL%9_qf|AD@Y`lqJa$naZ9-_K=|xl2F-Wie$@L7>pT_rR)l&LZJ}Zx5(I+B-t~z znL)D8SjLB0{65|1ocnv7bD#4)=en=^T#tU=$Nld-jQPxa-mm@ndOnNx0jb(#c-t8V zBJYRgDv3OfFIfjJxH~v9-QYO!pqAU}8#kM#RHGQv#6Jmd{)O;HQ>Xz3Mav*9C|Te# zZ@2%{o1#LbY75Q>H&@S{oI^eHpeB-Y|BIGd<}16U9<_h5H=daI|3UaisxHP8GjLfy zA=%>XOJ;a?kZ$o~eTA6%dFue9(vfgMHG^B9B0J2*_=NE!{a-)kK}=Xe(E}Nksv&Sk zHce$%oTe&~!xDI5r(8w3=B1hCae}@JWs2J6RrNs_h z3<>=}gMO`ew%)@IoNG-NuM9N`Gl`Qbyzwlk+)ZZd0j53*IX4k<#)fj2UXNyD^r6KO z%t-5=mO#CzbU{hO!{;DKomwpFJF>+-}wW%mVJ>^*zO`A-K z%|y7JkJXykp1!cD=6JPO1gj{uod#L_@sA8(B+p1J58Nf#jq-~86YwrXlj>hD3Rl1I zq73kF@h%9p8QDC)D^-Cvp*Wz^?7Z}hl-M*MWHaqI2lsoJEVV4?uQ2#mSaysP?4c@b zAIb*5!sluoTanH0uJ!U@*;#Y5u}nb;^Nz#FOirP*KaZNxjeEIj?aJUIPsATc&>B<@ zFE5eIyD7W~Y)fZy71yy%O|sC8NQIHHwB5wLMsDW@zJ4s6DdUNcINlHnH8bK?!Y}yZW zv@*RbN?De_w#vpVlJ)9{$ms8jj!-TY-!`VI+&kCfP#kb`743)W7 z;{&nJj&pk--yeeP2FW^`Fb}P{+T9kgV56o;*a^t2rCD=eY0KPWslE$>D{?A_1LE%#_$sC&K-TvRJiDk zLy&E()nLKB!y;7MftmeQb3=XOWBQm-sev+B14ulr!swo?8g52_+g8kzT$%^)@qFLx zY+TA*A77t8WpiSGvrt>4#OY_H*4LQg)}iYg5MJmScBzwU0KB8v07ls~Z4x>>dw%#s zWK?Ev`eA*2d&}&~nkU7pBbOH&vfj%ZL0lMZSQt&0s*gGlP4tg0kCnUlG5F$g=d0p1 z=A*}CBIr}6dNkLxMb44+80;0)Gf1u$RkSdvEt$wA*m>*wX}R)&M>2c~Jc3uOEEVzK zN|nj6)<`;3IaQL>RfV=KA$L#QkOekr(`c`vY%h&LWMQQ@vN zYS!(%#2v+VN?r?vl&G%aWypp}5s6YY~ta@(`-cOV*v zAoFM;^e0&9$|LyA!;(bF_&U)ybL#H^^C^>R^0Dih{QIxVJuAK9?|wHSQ*@Wm{jf9$ zDFYC@T~vTW_gR05zqO{4k&85+Td_CSvRqx3{p?dbmwfv;m#1>%U2WJE#<$6GFe$uC z8YV7fG{xV?u-zDx-QduYJ==4os&CoXFgjT*JIvl*%}-x|yRrUC%*BXS2xR3cy1lRAJEO&mPuTEBb*C!yOC-z zitEX9vA{XzP#)j1)Ri5m(m_R8t`*NP?ndAac5uXSb48%!s1BIM3tDQHaYugp84q;yy`Kh#H3iF<}p=omu5(krhUFe@o@G+OmT&azuul# zSrzx6ol1EmJ*5_>&FMA&StBo#D;uoEqY_cU;IA0~@^n}<)w~xg8B(IkP@}HhcFuf6 zk*jE;1`1T_!-VZs=Q$uKJ&@vc6HAYKer320PQB=&Zn%*4r1*VwP|>b?gOM&=B1Avk zqYf`uZ`^0=E+>{QZQqgVUhPBGY}04V z;r!mS`u~j)#NDJT`P6)ZY-{gg`EF{Q((HMLpwjs0{Fku`9 z6GUk^yheUZ@+%2?RNhl$|wl2Du*nGTgCALCS`wO_VMYrI}Jr6l&gxF#B@3ui=2UNc-;#dVdYh zEW~!&K{(^`Bq(uEd=g}McpVD~mZM_OpD?8$5k~w7)`Ywgp?UNT(zsRSu9aSP#mnY4 z&+H`UA-VDs;#g6tC`xD$2ao&tIz0~F9&gEI5Jo6^zrst1 z1yn#Y21UfK!s12Pff5(eFeQVOTxw$wz!C9!>5WQ8o=dER%4agE;Hj^DE%dm*G|1t# z(Y{nhy*`TmB(l3-nM31zm?athP%>pwzq~r3fc2P}LeI%D#bcf^?pN4Oo)%7n05~cI z3)+5?B~31p5?)Js6U`DLdHl)(_2lTMZ+!u--{n-7j;Z+Yo))^Srx~h!#*HFO&&Rs; z;4FyyRMSPj*2&2=e^fA7+8Cwk^RVHcj%sK>Dm<02{J5)=^@6I86w`BH0lY%TfviH* z^;G=i;wyZY>}{l+WB*l?q~4Xc1Mj&LmfU#c%^v#IOpG>5jj!fTGch&8NTw?|1U{qX zIgFVxgyp;$(L``?OFeYg3emW?a`)#TgGcuSQ2Du?SD1FOW^(y~Um?qtPrYgpg?s95{M z>i5D7ss}2>o>d7+R8CY$*D`tOJ<=9p@`8jYK7!EZhcNiJ85I^SD^Wgk8M`!hJD)gT zBpwm%_S!Iivd1Iqu~w78nVyt<{v)@hynix#7lBJ&@$qn;LQSiwB9^L%I@7n#Bxl@U ze5#lsCD-vus%7>Taknk>f>@}~V^A!n2XzP7A{dWU;ayRe$k*lkj#O4u6!#Mouh>v) zxbK9s;k^XE;R*Y{aLQ|I?d}DmRS_TsZ|yR=+MdCV<{vR4#2$0eoaslw#Ro2CdaS1D z`K*3;b!Ebi>&O!1zBbH&(VS6*%BWy+)q;>KAK!e1oH8m?P3XL3v-lwfvg9Hf+bQ+K zO7-aN4vqT|?o1IuA{_kxOL8PG$|I0_#DS{qf)=j+s#IunbKp&XdEU{aS>wmcb~YKO zN>$_&PI6v+qISB0)D)Lkl{Co?5LUIq&BAPX*`iHt4m(Nl`w~ z+EjQRcZ`Z5=e?jC@rzTrG{+$Zk26TYM*46^L}qYc($LYtd1%3bDa1Z}pzMGY;@S4i z7e%D$kU-x*Iu-HB;6fRZs#^Kf!-ty+gPOK;UVfw%yK-e-AMFcYcj{4O=o!Chacr{<{h~|6N{xLly$n^e!hWDY#7dTpfFQqNmSS}zcKCMV%r zQrljAjDW?1`llF@9xQS~FS7m!?OJuD#Oms(cDc`KQ}%%VkYKj7Nd}O~R)2~ygsIw? zYy&^=B%n*OS361?U&FC2Dj8JpkcE}5o@vRbF%`QLw`Z{74(O(4wDw~fFQGM&GQ`%m zU=@IAGVgR3pH?{(UHKG;&UvHsbX2jusr2UHqebStv1}6SUj)n$JCJD{N*R=$?9O03 zU-`lGf*z{`Dnztw@wCNC>1_ z&(R0!FcE+7@td#diOvsUODGp0wX#%2dT!mn!cd^8M!5t& z7CcruRCih{Y53GW52|MM)dbe9s05DYBcPP+s|jIxX1UzM$+Fuy>+Kqj)e*;*opxFuF@%u(_2d9x$W zeo35Ijj6=!XjJ9##H3jP7Wi&L+d3_XX)8RH~Pyx+1p)w-4tEUwjQlo zT^6fGhjgf9Th1e=z*Z6qNNyS$%^1oMsq@AVCPzF8_fE}Ga&JF6`Zl&Df172*X? zi8EuUS4c_c+nNcSn-qKS9kI0YgZ1(@G*WU#TGRvci z>Zy3s!R!a;-|?&-3Dta=xICR?U=LM;YD2d8iK#SRIOQ=&vGHLFKfG9xZRtYTIK5tY z98r4iZVu{C=Gdz9iSV?nE{nZ2oc?#eXtR6;)f5mT2qGo~Wm|yq0AdMUE1bCNKUT8mR8+}N zJv>k&4vF8LpMO8U>&PDbUe68z!6UF^C<_=57)CsLc;ALy+K(Kp7qgX!@YpBBH|H5u z2A_j{ICHFx9M*m~m+1#48V&)e3Gya|X(vIcgFL>M?|+bO+fgY?-mkRH@0puTqEi3^50Ku>o7+{p2U<)s6hC4u&3Zn+)~K`x;Q&n`H+cYmjS3?yrd z9HfG~FpW~0;zZM2WECZ75K9ml6wm~972+mJg4iY>m$Dgmdp`cEvfc@~V;3TF4lKaR z!5u;tuRqPuFzPlw>4o?*J?P|5jJq=PMj#kA3 zQsy6w>C7~Z9Z0syH7NB=L08b8_+YIU_m`(Clkrz4w6k+*ph3qVNQP1OzzlE2;JNi` zc5S>q7=}N$@TuCFvt8ftl=>X({a71jh-tj#eoqk(kK})R0zSI;-9+o1%bLx1Zl04r zOj+S}8jH7vd?p3Cg7pKl?_e7J+2h6G(18-MEm7+>t>5?f~ zIiq9q+v+Zr>i1SJ2o_$!lqSn>{~mfyASS)*+;tzq!Mg)V*vfO!uRw0T>}GCc@9t_S8sSJG>#tSuOWuO z&HlAJ`1|Xbe=+nKC1*6BT)c8k9h^U$CH4?&L^;LHot}Ghs*7J79#r;boObW`=Kz0& z@S#s&8bKY<>7@;0>aIELw$xXa%sDLSA{Wl0%{shdxCIey$Ny+}W)FVS4itfE^L`mc zK9;T#5cU=fN1KyPE8&=ZI}rPfkid${7>~DoG!4f9B8MwO_o^>O%$q;y>>hp%QPwCm z=(SrflAt6^IM&^G6iX=sq5}x+(PV6=h+1~ocTh)O1rWsA)5*l$xx&~>>s;}H!8yD?SU*2N#oe=k_W zF5^O+WkEI-9qVduz@7^GF{8T66ixB|athQc@}pPQsS*)jv#?RMPKdxIc(!;UO&EQ+ z-*bwm;FUxKOg1$zNl3^$LH~?1)78JJVzd$Ef|uiEdS*}*{pN{8lnJkTwg(R z^xlVbiS38SE{K3yZ{H9kOE*I&;2?gK5^mI@C98kstd^r*S@j9%x`dJ>y%$_;ZQ9z9 z0v%mK`bl*^YZK3}G3((K0xej8a95ifd_2L$b=7(xFIT+<*JA3z-D1fxcc<+~{RD6_ zaT#Ptlw#_H;|qoYm5Y{)nQ`{#t5nf^#m`NpAhKTK)!H@#Thgqv#k|mEILSiL_Y^6f zAiQ6aW|+QASQa`X!B@f>Nnhfb1|suOin6Ai5?4gEeeNKoQxzRh>OBsEGIu$Lg0OGSPp8L-{bvL%D&uCS-a%U^NKIK!j zNOqK`HmGS>4qf9bB-gvGe4}au9O82>cQm~N3Rnm0XP#E*94cN{ynbDN@!~d^y}U!W zr`=AbYNDIfxgOE9hy{{^j*TGAhW&F#jPEgLVmkW@ok)+s@@+8FoTzdvJa`G$7 z&&KEbbRo>hah%)3B&qf&Wr)=`DM9CkX=)neGyM$~{}j|Ckeff08p{xH%^-y%^S36x zBQ?t;QUf%6tg>KA``X{>CC{Tu4V0kd;l-=utHk!jUaHwQ8r!hk>>=|!q9P&IUsrLf zvdeEZK-!z%q@eVU#_&pJyxj52Xj8wc#{w4V#G3-9^N>ugIynBF`3K?2(kK1% z(OkGCn$FwGw4T@|pX>4nj9B*zKNgU<3&UWWz&%DcWe71ZwsqbfYONN-!zt%4XrEz` z1|DmR$bg5DPEw7a*;4iA)r+FaYlA4DI=bY*z4dhYg@Ys2q2f=O+GY&I+J*U;Z-CTG z>w6jO;wpJ8i=Je`^<-J;suF`!1KPtBA{e(&d_9$*46!Ojx|m+JTu8H#TTINgvd#?> zACAcwK|2T=Q7{0_+#S16zr4rbi?KG>g<*{S3LwSOA{ULrv`$7%9O~cTKx-@rMaSy+ zo;kVa*m!~{@BKsrGhM;GkIO-=$>HB=F8}Rx6jY1^q^m&k|1!#hYT&M96W8t?i0Avz zwBi9c(b2_i`dRGTWN_t{r@dC|T86*c=obYA=z9EO4&IZOrc8fX$A&jPG=~|%eUHtF z4q8BHovzDMEg7v>x#ht+WBCZO0k(qTfmc*ku#nlqfZL%A=`YJu2i^IvRS&G6xoTWn zVX*iB3_W*GsP3yd{=P%s{DNI|rwLT9HRD?tWdm3sv;c)<8`DToc_fY~v=^+fmSPli zZkS&PIG5G#!UyZ3<9ND1K8uoxfNaXe9-6pMJ%edfFNcL@2=!m-Lzx!+@O(Y$^5IUR zv081Mu~Nys$6fNTP6a$T-4UE6p35{HKho0(Cz;A|076udExlLM<_!{I{g|q&5^Z_2 z_O`C6rprB@8yycLO(URQU{numU)+IEy%~e`5PFu9acD-lV{2Ou=eu_}=TwUNo0q#k zE@z1MA8%DK;E{IxTv9Xs(MoGsfddlwb_x8N%&TNfD@>$|+>93=e_?mDZuQon(nAle zPyJ)q_Mo+m%rm*+M5keR9bzy;8kwa%MM>^t0f_jt10kMF^mfa&o0ym{gX6^GtgaJho|G9x zURS(v3-M*-DH2__)-E{EXBy_&78_rwvddkqv~W&( zI71#}YbgPBb)MtDKje?%H!H2G8oak&8?pDPVi#L1ty-YuM)Z*h8g2Zt5d=B81TthW z&jXYAp6FqiAly+Vqg>Y;jVr8(E!?(3%&DGi`Owm*kezs|QtL*e%}sNryAbb5QVXUk ze(B7Vk*W#(m9I_g@wVQ5FkITuD|avVn_nz-T<*+%mppuSPeT`|U9`Ls4}*9_E;_wO z_h5I^;FKDCZ{fXhiHIJGAh{G6k=X>7DVaUOTYjaC%A&Ot2=~;1cn!-GnP^zC^H7|5a5uY&vW3V5kDx%H~c-HTAzI@W-m^++m zV^r7ww)|t&i2}B`rFzq3md(U>yExTOAK6O|W3Y8ooq!ZdGm)=AmKIE$YYhjV7?DANHcYhup% z%^r-45$2b8M(|9rz$9o`)4&p`i#Uq1rYFi}B~32%A?7G5RlSyWKbE(~d6Yhvba4tk z;b?f`9#f!Txod9`6X;Fs&?hd`3y>u~zH<(|H?5~k7`XRQ)>LnEqU%6u`U9yuo1jK- z1^uHICruydp{lBeXbJs*kES1C^cPpF>_F~BO#0~7e)d%G*PS}^#W)Az@UxJpX8}2R zSV%SqQbudWKFm$m;d-eI)6mJK%1xUkT;Gs9H8XJl)vMNcdMofoObBX*s?2BunKZ#X zaMZ~ds3A^9!fukD9Ga-|m&l@*+&p(AgKP=ipeLe!&_h;l@%i%vL4*^bJMzTr6_qe&GiN)D28&L{t~4}z<%5Q zsURk-srb{hUYZY^l=6FBS7`KEsz}kd?e`OqKU&Iq7##0Joxt2&8SSBfg}QGrN!+Q< zV$#L4$a(u^PMyEsXPsTnC&hNFOCKjZk>9c^J`^GffQ>NbS$Y{wFVAH{eV`I`mJE+t z!mS_@<|!pw8D2uJ+PA8oF=u^Z3lsQOdl6c3t(fJ<%90RO`3Ly@8+QW^MdKWbS1JsH z3KdQFKQxM6bgAD zj8L6>0ZI|L@!@Uw#u?XHlm3$y*8Q1#1jo9H^-lbl=dLk+rZ?6jngrcWBn1)N<3qG| z12zPaEH~jqmvoV^ivywpGD;ELC%J@gzRZS`jzee(7(fEPWUqM(r;bu*(P8wfG+Qu6 z`@Jb}RZkoWTON>ccXqVO|Be%=ZyHX{XpTM524mkSdQf29WiYrdxV{@?r>zRr-*@~I zm!OHx&r*GcKNNEd14u1rlv@Tb`ox=4cij}Z?>~1fF~4=R@kYBJFH>6ub_D$1aH`&pnzV$6`!EVT%<71vEhg>pK*(a)x>K*$4p$H=g`TGJ%1`J@lC=HcH)kYwlG)<)oXg=!B(6u30;Oq@L5K8!)(h^+E#_} zZXf zoMcAd3EE=aicrXo@zN1J4S}fvA{ao%zZuKs0_NZAT#yDzpz>>%?cmmqzS}`T$N@3u zGQqk^_Dl%EyFpMQM7?OFmcj(|=hM8(=-F5ht#`oY`hs6-@BG}GoKIup#=(4(N8cu$ znr;`?<<38EiC(gpF%5-oOa3vx`Crb5k*dw{k=XJtu~cIuh|IIo1Tx;bX0ZFZF-JCQ z!5)rvJ8J#V_d6$GNy-0`9qG=nf85Xh$FcKJ693W`l^bOVRFNew0ShFRAh&Lhp}vWU z1;eB{S*HUwHO?MKlFo`sGVdzrihZQx-U8t~i{JrMl$&+B*)Wb~g2274OZI}k%0?m+ zgvhg_6Sg6VkIQh;JkaD(7%I&JRh5enN40Lj^4FE1%j^GJ5A{EP{9ocPMc6})!MYs^XN?)KW@YhS#eEBAiw z)8p{h@6ApO%@4JD*uXUKl_eS7gDGEWLg0eV4QZYuJvaMUSN?V7%S%6x#fkB%rLp&~ zK4i%q+1P-9^S`41C2#QG-sWH8I{cr9fPPQsre~o!ffX>4G4>#BTIYhMj9D5fu{Oqh zWk((|9Xo9$c;2%l=b??8FhA2UL?rcBKRFg@(n@2-UKIaZS3U%Log(2SU)y(v)ynIy z?l(fE<5r)tmSb zdY^?k4A|e1$+e%JjJegr#OSm;G)z;(WA?jk*NGe%yqu?$R^D6N=Er;uA@T#A;IiHgD-ZE*6jg!}5T)k#spd#bQ~j|j`a2rfnhLW@wN5fRW~;9DMDLav zQtO)c`#$QUyxmW{F3Uq$YX8Yv@-OZS32)}57p7940O4T5u4T)?;8M=dU=4m^6CM5> z6{IstU8(uLmF_82c?TTIul(W>k;CtJ%%|`U=5T>diXgZ$DMQhxzwwR7fx=Zs$s!e` zXe$ykj;}%@Nmf}p?W)(ogH@ru7tV2vTFIYH+>Ap)Ho?*M-{Q*t8~618;v2YFftyrw z7t8^S^A02mQ&xx#QSMe)AkaUI8NM4!qy!b)9)Nm38M`(1OfvAD$f-xa{LwLsNu6kc z>@hf#%M=Fal1B-W?Jp%whqd-2C%Rfv)Xk<`Hj_rhqK>`Tx76||JM{EEsJU>cF2uk5 zR}S-kVpbxIiL5_KlP5dHRjfoz93s~z<^peT@M@%g?`h} zQDdg2d;dfmi@9Y~_|*`MxjU&=$g3B=Zd5DhiS`{ZLh7e<+0$Bd#iGl#@YxB|oO)-q z&rJN<#nqpdy)ZR04kFhmnZt1Bq9}02nM!KE*vk-b*+1L#VT^fev-|rIqwep;pp5G6 zCW)q30o{*R)la!K%p1FT1Ut=& z@DXENS)Veo>dv_;6qU3Rk$AmDpVNQO>FEfT1f5LI_y2Sff9WYPC4ZHx=Qo;?MfWsq zrt@L=&L2}>H|Mc=sSAbGs$R_aC>%VgNXq2Q|FzExTB;}9TYjbOeO!)}bDhi(z(sUqSKK#+Qi4@pi_k&T|LK$tEtOMbsVypWUUrhu><2MJSz zZ0z$@_B>6o{k%88ZS|YagA?aJ?Hc@79|Sfkj7VLScv8<26B*VH+s`;T z0&dH5ZtOs|4$|NU^l$ZJknGFmu~*LI3To-OZ;gFB7V@j-WYG6krjJg3ha@P*WL$)h zibjNpJ?$&e>72tG@wmPCS(;3@b+b)cmvQ;(?dT3&-*(e1&rn}b;7K|jSKdK1|lnw1*b z%|6b*{MiYKJ>2JhBIe+J!&7HM=AD^3u}*MY7{Ox6fG~~#w>{ZCHUtaSZO_kD9bi-`5rO(Gmh2NZ6){Q=o7jO$Oh+_cB>E z%8Iip7iZm;YN40AcAaC4)IPB46OhUI;n%hvXz7L0w%$`W4qr?vxQ5@}-=PH#L2M}2 zpxhk{-D!7l2$bifS`BY#1pAx0UkUJyj+#jnvVYxHZ+#4ZJkGm1$v~|EA~O7|S69F9 z2mZTn5|jXE#oWT~1(*5O$dXeNsYJTJ=|H_`$E}*;55CkhZ|2Sy@C(-q+*I7;UJ@yX zE)V>{SpjslXbGfU+|3V&Qni4%sfMP}D`d}y1z$$HWTo1Qt>CcQG=DX&uuYTGED<u%cyk^S-WzBV9Ac8cLi2;)%6m1_#%V6+=`!M835cn_nN7yWI`5g_rjt~P? zY-Nahg%mDq6)Nj{WFxZgB!=xkg4rQde}W|Fm?J@SJL;gz)D9#9w;uTB@jYt=uPZhI z7Av|ZoSu%|@(K)P?EocJzx;y?Q&2wnwKvo2i`O1FUHK#=%mWn^TuR*dOG_B`0PBii zMJn2s((~&Aa*$cqGptkGPr7lu?JPTMo&AMab!)mB@I zby@2Em{;Lp@9bPp+1eNcFSkFbqVW#5EiVpXOD1D*vh5CrxlhO9TJkE@v__P_m43F_ zfu#C?yotscsfTWv_#r>Rfm84fIN`gU)(#{~jqJ(b04Y?C_0m(LcOV%)RFL+~wp4C= zU(OCBjDL%TrrFg#x$Yz)3nC1`MXG6QcqtjPwv6z?ydUAG@3{iL98C*jh&WEIY%9j| zflDOcb|9;=un{D7Yg4WoK71#d5hjhnw=#5gAY2OMO}b!vG=_8sv&9#`ScjldQ^#Nn zmAX3+ohMilNF8zn{GFd6sf?JZRxF?br$y^g?Bp21UCUe>qsMk2nV%5ADNF*U`Ewn} zvUkCoI7t(NF?6%2a{N0Gt(i@h?WIJ_S4TOT0QUd#9mD8;9oQcVsf>MyX&7lu4uFI!Jda3iDk7Ghn{5PV&3l?w<+jmQ4@q6wa7n8OaFqa5Q1GOEpv z)q{BZe;(LB5A5Gt(LZn4KTYzVKJ1@P|DRFvKaG;ClccN0BDp8P#i3}@8RaX+GnAi+ zg&V`RYx&UB^@p{=K7R?${`I@S-SZUGp5YA{3>QtWug;j%WA*~V-x3&LXzmlp)~^Oz z(l06c=wtTkN_2&OIc+fBpXJ%jyR1Q%Wc3H)kp>26mcg{!Z{I9f2iF&7&GQXYR1*t^ zHdXoxN{oc|z01lw`Xo+=WlyQbRU=zIXMta*6h=xmcFEz0lT))TqZu)Ql@Gm1A6y=i zuJPr0NQP_nJQn9QWV~D1%BF#|1)vDZ@*m53IqwqTQOn@c=~}CZY?*Grx)I-|=|js~ z{NPY(m&WSr5V#na+C7Is>$s@8G%GUZk(RtwRTSZ?tiwkYe;-BhnVEs2XHKU-Km9iA zqIRO;#S}BsI+h_orC5VS2^8;1J8+J{3&AwQKnnJCZA}jFXLZIjg>V55D7zr$f{@Z6crRhZ7hwjpnBMrSG zYyirHkH>5}MASUXyOb+xDih*U&Lc@aZ&_H5dxx@uvxR0jNiT!nO$8%>@+56}2+vtZ zMsgON&@FvU#LcrNhAdtjR}*tk6a2CV0=IPj$8@uxrHW#A7KHT83|OnXU^+>A(Ag#| z;ph%*xqmd)#hAs7AwiC8e2E|oeu*zM%xQJWH4w{ii8ZxBIz1AtG333$8}nXgIzJ@n z^#9?mkj>$xq}wbe51@>vqN`!J*)%-g}M`@3O!5d_4z~ zDok468h^R_o`~9?Vwi&4E0OEQ2j-G~Fpl;WR_v9or>X9byiO54!@8&^TH$0Jt4{LD ze_!|I^PZcJJ}nen=FEC9)U5#rWIhBcrUNi?WDBB1GX6w4htt)%K`nOV>viopjc2gt z)GNXdO)|Ee28v+;=Y;tY8ThrB9K3jsu_e5bvI7y_j&5dX#`BsY@0evh46QYEDf(`h zg>XR)#dU2Q5`10#P%`vdk~FWny8Q7^7j&3?nJpfm)3pAW$^Qe9`%H~m`_O}MQWZlJ zWegkzt+v?)v_gW-m;k);WcK);R}U`*>@5zkDXBkIv-!p*?#UlsLh{0jz^5Zo=RuOj zs}?(uzSAHzMGu3u7d%Y8R@gSF54-LNvY%!BC-(+@7SnhXJ3XrjvgO$8ZZck+kYd{S zgZ}fchk0l@G0Q-7w;6NUfe@CwOim{JaQox~OgUdJ_it>^wH{MODo zyu7BO@;R#YV)4<2)vdY+R9!~!RuYI)Cg`Je46$FL{&NlpMPzvGM76jWUFx7ad zC;Yvrp6XCsLW;0h!)-~Lury63RVdQ3PV-0oJMtWmKFN8zzhli>0MPI;Kzz|KKuz5X z1}SNow}9wikPX_HWS;fTRw(wIUW7Y4p|kVY(1keR^w|#X>2!n6qAtW;YTzLmKdqQ- zlmFyG{M~KAKhoqAHhxgAj5u%{Gas_4^w00J^k7x6Ajxb=@AVOc;R(%j5hW zS5&KMny|S4u}&<@Uh)(}-@nC}dQcfd7flI)gPfT7SlFr(%5(=(#32JVQ?8q^%q#!9 zxbT;Yz%ngK;}K$9zS|0?h#@d@O-&(iyMNGF5z4xcy_k7_C|4kaSbvbl2Uw8K>D7PL zIsk}XCVk1sv#WpQyF01a%IOBwp*m>~FC=TmL>`jKiTjP~V`_$b9MfA&Cc~NIfkcCT z`uf|$pXX6<<*lqKu5gH4(a(^kn>rGx7Bu2p|rpJs01mAoyLixpYd|X=!hEAj5a5?W`+%v2t{A*sUR_U*^7w z_~Q{HoA}LUM3l%0AO|1MJnDj$BG9bw@Nf^##QrdJG{CQ2Sh-6VHwIfFaaezsS|t;=rQmJU6zvq=1C=C^xd%c1JzmeA8#=S(i-Z|8@kp3!KAkqi+-56RepoV(~f z_T_E!8X`?g9smQBSZ(7Z1=7C*S$`?BA&jN>r2-CHqCi$Od>Ru6TJ9ibz2>l)Z-&;N z=ft3MAVb1M%$AD#IYt{DY>(DcCG1>3m`Cmw$ zp`d_OWULk+NKX5b>|_f}76NY#$V>g#Es^MDNy~c)jtaZYT`Xa;`VUp|QDHC0AzT5! z{fYi7$jM)${@?hGT@fE&e*+$$8$fB~9}SYd(&r|VeArsrBr7iX=~ouwa5&0>+|FRj zATL<=S8(mq%^g9s4aKE;*O{k{KkED-^E@DO{N}>ios;uj=^-#3O!MpxWPTL0O5TAi z9|B9FwH?TYJ9Y~+kWCJ-*4f&DY*$Xw>M+y@u%trmfHdl!3_Ccz87!8xFpNU5w=D-_ z5Ppt?;5!iWQE(;zHvPT6g5?$$c_7%9cOWz1pk+)6EZP2V zA8ikSQglMZSchhdZKtq$R-ChX#Q+yaeb!wf-$$F>f6Mw63qyO=U-$Fdut(9+Bn5wC1eD}L|&#HXd4&60b;4o1W4&yD;I zKJfFYlEk~~8ZSo1MqDF=h=zR)(F?pcf42OfzQ)^pK1CJ;G%EeTei7>g4_VLKFVlyO z2c5SC(>cg159*Itj=9=#4;?VAtavecae+5+DFNagX`TE7h$Ie+a<4_U73E8s$gMkSkYVm&MAM<$bAaNF@elqX;i_*KP$Afksa?Yj<}?E zz#V6}H1fCxZ9CHVMY{dKMYHt7@Bo>L_Y<{(gPkSiKEUndqppgNV;p_oS=6U~@9VL# z`HU;FBzt#hu`he-BnlB#*L zl@|AMWcgMArzrfT^Udkh(znOs)3$SQb5>`3`yIcVldD?eA*j`z4g4YbkraOZlh%qxlo5uwlkm}()4cS`n~`3T z!D96uvN>xS>`PkC|LSd;OUdc?bCG4HB%{L5vF}{W$=AnP>~G{JL9Ru#ahdiHFIegw zyzHA3X*}`Bh}L6XZy3`&Gf;Lhkv@1~ir+V=Jy_M&=+oHh@AJd|mX@f1y%z^GI0Sr+ zFY3(qr2E6`yfDpF?D~`7$5tb|IRy zcCk(x-+g&}pgV{w?NfMEV%2igiP+>Taial0AZLhwG%zXD84%!mkzJS7^`dFYIc_N@ z{Do{Ue$jYpurYE(bUl+3#t;mnfTo{?1e|st50WEyAcsKT`#*h=i<6#N2WC9Xi}dHm zRM2MtV^Z5nivpD5$uTFw-M_qH@QvO(a{?KqdOJDM*xr~Iig%-`(Ol@I0vJd17dQvz zc29)A`XQYGCLBms2nQrnUQWr%GzGRdyerxT7`BRV zRUkiGOtAHf^;c{OI_%r*NO168UFAxduYK++4T7G;deJ9P^1$KzQKTGYksd~4!9NpE zuo(sVwMmZ2lyKzu)5V8r3fi4!P96cVyCK2Bc+3M>Noq?EmktpdImrgHQAF$2>_996 z-T-%v(vJn@$q3?_HS-z|v+_c*VB@cm3JQw!)}a(ZT<^u^9Z2k7+JS%j>h2{W3L4l; z==!l?%!S%d;!F+Nzegilj>Agl`2LQgee!y<+K1_XTgm*Uvi?0`q%)hlWlaIp$Za{6 z;&A^fv{P#UN`#dnmJkGE+*D)Qa12eFVC1P2d46zGKaIom>fPd4v3&&(-urbZ)O<>> zx0Yj#bwPP#olmtxoVE?Fe^2KdR%_qepe(tZ_R#r{Z`6DwaoDt^fkb*-4_!n~4aq)3g(MW&L-s8R5gA5Gc8U;T2$3~gc4Oa{?0d}E z_ZiC=%uL_+{rQ~l`P}Ee&wb8)-{v=qbd-QxeNf+WGV%>Z@C~;Zu^|EeAq{$U@ zTy|MjqnULzia%*<_=t&fRd%4*q*`5?z-A^qq9A}1!<4@;LEA^&9eV1l?jq%^)j1WM zT5|lHI$u3q1r^A8kC%|ksd8Ip-3Ib`?NW6|=10>HJM!B}+!jp=itmNV*erd{(v^-i z!AQJo#N>l=&5izDxh)O=Xtqp5HQy%~VGnmS2Om4)-wV*)uaf-ThDL zqPH54R7~_oQ;t@AL=fDBAE4Gc@E~AXCa(K8hh->IxNWAoQ@Igb`8~%66};x zTzY(eeZrKhY^SIQyM+lX4<<7-;b@ILLN$bBk@sn?t;d1jeeqvCH!1rAodeo(2p2;> z_sbvV_sIuWn0+#7*6H|?v+5R_HG)g7^jLEmNAbI!CEDUzW3e%+IMsYR2M>%3>#Dg; zzcV{blvPJ{k{<5X37Sp{cvh(8T3@1|qQE4iDWH>HYM|+f+8a6y!P@GDVgw1N6*4(R z&v6dh_;gg{c8-ouPdbSjR9fi*C=T+PJuW1hFxOlK4&GHXJ5+?O8<}`BjCBZ1KW#Yp zW>IW{{G_6HM_-KMk_Np@FN3G2K2UBhQM_?Jftu-V^WDzu`)OwHIOlpq?R(Uo?ha_J zjtp*!5C$=j=J$SLh{h7EWdZr0+!x-N3VNEY#Y>_L5Rz`? z3aP)xRRsesN&vtobd)$k@Y1X1hob*OH&Fvu^AC`^+PGB0&>u03KR0Us+n=Kjh4YZ| z>bQvZgeDxzY}V_iQD-OC5Oj}I2JfXD_Va$Nv`$EKrxhxye|ttcXH$pGP0}Ci^m}IF zx1IL?<`-~~32J;Nf%52@(#Q|c*%}+piJk0gWUhgCrmHbA`)biy`xez#Gnw#wKz|5Ec#C`ts zz16a^q_$jrHW{qvHN|YPt^$Jl&LIf`&}6jB*(vyKQly`N6NE|u2)HW5=|`8fhQQU% z^M8G`q2psQ$MrGwbTSP8ov?5uXV!p5PG-#}fB%r>9p(aVhMc-SnyAvy%ls+A%~GI% z;*8IA0KGlKE@9%EkYRP4VJ)cBH4Yhs81;C{4R+toc3w-#VwI!*ip@yMGZ=rlKidfX zSER{L^DnC%WX=F(BWe$bmZ1Xc#J|<3eVR1ua&mXgFjZ5yH+@<%or#ZH{-yiSmB-4* z{r{%ZA2-{PE21M44s7)=j6Xoj4qEaFLj*sk9W|gpEMh+1OP*ZMjv&Wm`lSQNqzce3 zo<{H?m|X3MQMj4C!$NXAZ%#qUQseLO51>XkwLq5z(#J{kx+`Seg?@DeDwN>k9B{ej zn{V>|%gu9jYV<)dJ$hqSwD z?|TJj0eup5qoawUWR+>ib)3Kpl&$p_Y=$I%<+;$uLpLj9ZR7Wgow+o!Jt;XCROR*G zEe|)^D3tm^O%A44$#&NyApw0}D=?`cb2>nNn_5=)`aadpodss5T==35H8ouv_q zJ=nDg&1GkUJBpgfxxPy4*;Ki>B`tEU_R!w5P7I(&gs@iRrsrUC@bfJoSEa%N)&lN8 zioHX0+)DvS@H}mDJYzN?(8A`kUq1{iBE=yin(7PF5n<{Nx)Gcd{<6(@t@XNO_p<8k z)d6A@Wl-*L(-Sj0o$w^AZlTv=FU*-95xuiaGRt2n=yzl}J-PY9va}Mi zS^(m_)T-V_y1Lm|(tPn9X5@n4&{9UV01!Sjv$NHeb^Wfadnxar=)F^K%=5~ojq4Dz;ejhXA|CzuQblV|>TD7g6~s@M@DzN;ENdCObk%wih_wKKx8yu|OD|1OFrBpmy^O5?0Xp|sLE5{}OKaFxQ|H^wHHpdF>DDb&tS4lx+UG^R6!e`= zT5@qZy^NLeDfsH^Ja5;)8m?+^x+%!Q?5eBV!qWwh&Lx&S?=rLPewUGnNGu#x9G8iy zzr&2b1MUJbBm+eaGvGf8TS5>z8o%|C)5eq)~7c;9?g%`Pb(8Y&) zlEwvBWsPGOG`?%AMn_Cm*{P$&4rIlE-V9?9;5zuGUj@|3-ImHHv!-wZWGA1hDS*rz z$&a?TxE!h$U(T7Ig`e)Xh?9o|CzhpTISD=ynf2sfC)2UcB91}s3U#zJX0q>et!(J4 z7xIzCx;CNv7n;$o2WA{$;+k^r+1<1F$U`5aGYJ}qMY}Y=Z{I0RkhETAKR}w;$<+z- zSA`#)Y6a;g7rxNstiJG;Ihj6Yy-QUqymYEZH7V_k{h;I8QC2I#8|fqLCmQjb+*!a+ zr z;WeEEW_s&k{BfRY!@l?V{;rD;XI+)JH2+l=!7FQn^2m~SjFgzq2IckY6&M+(H1YDe zb8PK?g}~+u8szlL@r+AZN8_pBl(t})3u^L`GqxZZFimsV&MmT~LCydnnoLQ&m(}@g z{$aScOdrFoOO%r+yR9*oS@1eeV(pMR2xv|jHXU9Ly^Ny_>rqvY5?=1U)~GJQs=*(N|!X74(S8 zls6t@w_ZLMVKo~SNoC&sEjcpwse5va(Y#=q49DHz`VR3jkQ*Rh`O2~(btKxy9dzUj zXOB0)u0KAn9b88f)>mSYL|y3*;^jNK`2=d1=npL==;CtG+HGL4SSXaqYwI?S9~rpA z-s|#xGN#`)q=-+1+?9iL1Z`yhdg1;qT=*w1QpDU~Zq`1I<9mB6lZ#7PU+0Q4b@l~+ zg~3^|^%-mw-VsaALL*;6x#DjZ4R1~AR5MI>xous(o9gm5geE6dkfh$h_Xe!>b6ELr z57$XqMX)9C>WR8FpJwn~E>@AftWz^QM5 zUUxnLmg7FX4mdog=gjUMYJB`&xq?=t*eWX}x*l8ys-B*u0Kw(c^7+I_%$Af*Yk zEnC`;1QQU?$k9>7PLkO_KZ}36dJ67y@f%K9HT=`z-5xL|E7;}gTCLf5!G;cw#W*4}N~r{NYF95EGR6dcvtADI+Ao z$n5GXCkFl_XG*POYr2(v)04-LZ@Bx;--LY!Obs{`;Sx0|v~Nb!-?zR5Selo)tAYbw z|7B^^wo7GMEuw@987fl2jwIk%Te$d}muw6WVs2rH-}!OxB5f;h2Ej>Yt6=ek6p5xs ze9_)2ra`JA94=-e!?U zk*B~>aE`~GK(L?@dsBHrrVTeef!2E-UUat5W(RzgVo$oEaDs7mbb|tjWw{XfaO8x- z#!z<5_sc+o@1@d_&ErMxZQZHd&ENi2smg!b8KM?DPMaL;k{kcSAnouLGGRbbj>IsGLw%nvYU)u z^lke2yVECz$aecdlADn6=XNB@e8g5KARXr($=ZR{$ReMRy9Qy)Tuwhgeg?!#_%c`m z3Eh1Zgf~n00a98sKiWp2)yh5-`?k*5O~HzH`wpoA=4B=p5Wgz^GzKqsD4G=i3Gq#C zluRrn+}m(ONFFsM?1R^9CCM*FQ2Psr#waJ<1LSwk|BUtjg?UB$$*oMl5mazzKF@Qj zU1?@#8!sM*T&~R>YG2h{e9GuD7q$4*f{$E8_dIs0$H(SYBS zQB?Zl{ht2nmnk^S?QcB8GUAo_Lt=>{$OraQFUwQJ;{=N5 z3c3~_r@RVViz^eCTFm&~n9bC*#d&qneTjD8;W?w+Bnhyx}Z zp`ZhkIWZz*$tor_Z{ka+l2SJWUio;M1H*JhoZk*%GbBcBs0-9ag%_25rvGVw^ z0Yn*;5r~dy4=>yVyJtyyOXKV6oU1V^Y;&G83^KqMXa1RW{rN3S-#d`Opg@1h`u@eU zU#EK_seJ54{1v3}Dum=C6}S#ot2~!H06XOjKap?bBr!f_JlP_A0n>l);D7^BqfA-&5-M*QHPgvhucx}c0oYZ0)%81 zH5muK?`v_y32-dAL_ls`WDF=ARA@-+N4#Upf5O`ZQvvxVxpU^tkO&yF!U-JhZ1Ps1 zg4pba(geZXERJnJ(UM*$r>e%EuX^tOJ66J7v`G@fpKVaYt%%#fml0@>q*fMz)Gyhx zuR|#?iOHKKK-g%gzgB+UP<;Hsv@JL~i&2qJSPVD-6xwb&6skd{^7qgzI4pSjx9#7c!!r zb)4$;v*Ex#>7BNxT>hsDUT}h+EYNHcAWR+(Bhb>{M8y0!y|&rtTFOHeE((l2KIb=} z&ThdcJO73DJeiOR7KSjwT(rkQBqWf`|1xH7zbV%YhUUl4o9S2EfyhE(Phj4G$CF-j zO*{uW+|CMYxMIQh=wc%%3t@9)PWS$d>qmg)OLz1n98Km=E%(48 zPr>I01wdHitrQR^_yM{d(g2{us-N)ba}*Qw&aZyLZ?66SeJABB0-*KfGXZpMc#2wp zRMj*qPyv#g6kvq8gJc=lelUO{z`sBdcH`WC3PpI^H9;n<&@8}Z7+fb!Fq(Dt^g*2^ zLaUq<9oa;<`Haq~g%HavLXLHCoae0*2g;O!b^p)~=iqzW#GxS^8XRhr5P9syP|Vm} zN|DF;h0bZU#TB8L?-;4L%a%`8E?ns>{xTbIk|(m1LImVZ-PSkJHmkNi`iTP->L*)P zih3(OX+MHoVUHdC)a4=XuP6;%Ir>0LL{=LTAoUHz&^9YWm?R ziTYTqRXa~W8ns8#H|j4$N>Tv%81d8)by#n#$DBE5m%xN5aN&4EC!Y_d+<8lNa9*~T zO)cSNz=je;?e~Vb;TyOu{8<78ZgsCl*C(|P4P)m$z>_ayy8~KD0-Payg4gc2M^C5onnV5n|G;X_`Jn_e5X~2=f1e)i>a3_AfIHwU=2TYtz^A&Jp5&f zaswn8B=@I|BxO7S@rfugcr~}8j(N<>a`2ANO5z3_#$5A+V=C%uI}ZhXYR=Q|aYE{u zk3V|=1)JVbdbW)Pz)4>4hg~M~c!|1<&CNw-VUG+0^U3U9Bb%DKjfo%Tm95v+5^UGK zOkd?}Klgs@cSOHidB%bBYBEi4e_S76Q~3R)F+nHvdL4xOD7YPmhLiMnf>f(JVLs-3 zBnl%#DjfjgMPlk^$qo)#wj#%V`;`9EWA;Z3FIWhtf(nM&YX{3A*1NC;x>CJaf*lxN zefl~s0c}+(oq5;lRGzod`mh=Tc#w7Xe_iGW=@U<6bs<%r$vBsSzxUY&WO zbT013eB*C@&HufE{^q;?>qQyuD9E--jmrQC~)zQxLCU+d^v8F_04k-KIp_+JY&* zI-csm{PUrUn~%=vS6WjfMB-gOM4NudWl)m)r|{rsQ`duo1n+`R+8dBB;j~KZGFI3h z(7Aj2cuTa<)!dfo-JAklqQt$#?*X!z_W%dL^&QA(g*C`mPcxzy7X$D_TaF&@=VzO? zR33RZ!zEwt%tC}gej#Jz2}DU-BY3Yc84@eOc_gGxU3ve2@!tD5j1TBB3TlIaBb+U4eD7F?O+5gT((4Cv z8}hAn=b}ieFEHN6rN4Ge$Si?WXU`mcq>OsAS7=V1Z284ORieg6(Ql~|```w8L<8BW zmoxYzhdd>6ZpIT?MVv0^y}s0dDnEm~TwYCzRaoNO`ikR93RWAowm2NksrZy-?6ma* zvWpJ};AF6|@xilN9LL^DGZ;+r)J3;2tVApdXzV3<=D zKvgkwf*24I1rt<%fSeJBS0_$mM-XygBLH;l%iITQwt@dMrT>}I|LoHL9Q^;>rT@80|6`v1|7P)nn$WOG zw^fz$m-xF0eC~k~gxCFNA#S}O;;l5?U*Qy;7w_x33INFfxGP#t8|>Exzz?++pXPEx zNZ~Ss1}@j;+-+=HUpLIBebas{wNJBAuMHyP2Bs2&wOFzUn+tBw0HB0-RRTsUq44Q; zzMU%suWT0dxFekL^RW8UtxDj z8xochk+pQWoFuAzy8DTt^=lou+3?Wi`uiR36DZv{j&Us$na;z>ezAslRl_^hwJ||L zK3Dq?-)h{`9KTZyDj>YQT{2e^Bau|817=!98MGIuCN^)~RehqJUHG=}MXm<30-x@? zj}!f7c+Y-r>*Y=~s;es=6kZ&X5^5I_asJkEb*G&{X2`?Vs-AnXW?47Z z=#sK&ZS+)wfPnRrkfZBx`e=hdze`?T5*{L6X^3N9u)ybqK|41K*DvmK!E~AtymEbX zgu*5>@xjPKw5eN{UNOT}e$0Ty1!@pOC09M~P9cmju<#|aq*=MuWIUJkewLheoMZP= z24Mypz-4|!qa9+#W%7+|6t#hyg}Ro0^?cpO+Urzgoe-9_p?Avd5Jqq}&oOXaocG+7 zf2a(|))WGSx@22^#bx|^8HU$zdQvptcuQiInrFhkbK_J|+2%wol=S}BXC*?)jy+dn zuG|5cEF}cQ@#Wxp)p@v=ezhNeYok*`a8{TZfE)?nrW%4?Al|_-*F}7E=-TNz-_>=V z`jI%#LB`uJo*x6W6<_dBVo~@}QXTZX-^la1Wl#R4Ce)ccyO}I|4IZ#tr`BXxV-{Fc z{8h&-pgeH^{W~GUE~>ogG^H-y`F_e1(wTYHM^WxkwdnK=!1D1w`{JI z4O_*$Dc4nJJA?0c7GR*#e@O2cHY9(eHJ_I8&LOSGD0_qvuk-?qBqGUXu3 zE^0ypgIsT$%dYXjwwYX!r{E$dL(Tf`y@ai1h0|AfG@X`6xgQE5O_Bh2?hz^wSyuik zG7GKoBGW6b@v2-~kmx5mciT3$r!w46`LGx7%2UzIh=6<+{-}db-&>-li`6Wbg;b(LDIjAY5uK+14ak4pSe0cKn@k(TC39 zhZUapPi+UtE;;)9l>O0%|MFr4_7Zg~gjFCJF`JE|a2|0uu01>bq?k%xeOpYHH!T%h zvixgO?sqNVZ*zJypnsJ%Ja~qGJ1V73NB)L9?NQ6p6fd{8k(obvBg>?&L-f?tN!^#> z$F#ME_u;bFQv?KB26ClNmUDEhM_Eyx9BLd2PmW8B@xpB3LkOgg<^DW``zgo2?Xg51 zsofje8gV#%8~rusg#Ph2XO1P!alB*HeeqiJjo#s0vyVe7u_&~-HXo=Db|8Mw@BwFS zAxzSQFG>ry5!5h4LFD^_MK&LWxubodv>L4GRif4vUdP@y<{{)|x3{nF9h5Xkp&ts` z8m_k*hST_sXx55kT{~Kit3^mghZP!ECqqJ}NlQHd^QWMVm25Vr6{Gftf>$R@hjUGO zEDB5r4v#j{yR*+6w4Tz}e<*Rru+YzG!%acdDX6V&?R_lYW)kN?N}tUEVN_0Bo&;e7=~W@>be z$(&2{TIY0bJF;vcwuz$d51Fh&lk^WKw95gAgWU%>_$vEOrP-vs%P0I~)>|*;(8vpWF3v8#kx}@d z;XOzoP$P#qG9hz(a(N=M1TP}GYgqp!FcvN5>hBLSyt%5Kc4)u?zW3TLrPG5W$*Sk# zGhfEiXB$tuzuYqnw{ja6szp$~W=E~hL7G~z*Wom{ms7s4>ZMe%jlQLQ<)mjwl$Z0& zHOca?;w{fpy5zk2@)aJ4_-r3(drs#&Iq$4h_IHKi>jo!8@QZ_+icxr#A)qfvUIdRALvMV_km|&5@zD*`Gig{&%|hLO}#_1fsFg521}0D zeYz7ui`ojm*!BO`n3@-?zLo=DJV?F**R3VYVImmsV(CUQOzQDp-(D|Keq`fTc`0=E zBs|kXNL=(3HJkA7Z>i3$a-`u0+)1=38ze=?oO$p>f@CkZqR_3cHqwbnv*7590)cEN zlF;|*W1>_2{Vu_62vJ8yhz2EvrwW?R*rOJ?&3Gr?mJIueGYPJ)Sq`qz6dc31<7GG& z+)gRLbC~Z_<`cXDF?Rx~2>!(KgAw*?$n2!ilbcnW`P>_Ib$aE=JkahO_hq0c#3v{O z(Qx+Zt#AVfv!9%~=`onM9eP8AXOg_pynaBmV4~O^1#I9?!Pj-0S%{!5!Hj&dCJeFy zGe6J~elA1&+Y+r_H%=dYK!BHBY2K2Xn{Po(V~!JqsAryp-;O zKx+OVE)WyYej^Go8w_QtVCca%&3($ZRK24=fsg0Yd+f9zVE zbi_Hhd(%9*A(oW$t?KQYc6EK78^`ECer0}5Fzh|l3BN&o&pHt9pcFgW+-^yWH=Pub zN&+WpTyG1Sjq^{gcgw7IPKDV2-T>uaV2YYT#A${*UurVXeu)sTZ0Dl7#Ri?fOC9u6 z30Z+il>N-HHK+(~slnF--7@kaU-Og6xFg+pr}uB^YZ?|?7;TIauN`=9-gub=)R4{p z-!OMANLIwcWdy*;uHPAj?W!z*e6#+{3jY=i`^zSvZh(D#MoejI&^}_>Ly(vi;iCZe z+MZZW`iq$K2WZ-6o7i7#9tz?A)pt?90%TS*!DWyM1GHyt*0NkgZ$eg+r|6bN*|UJD zSh#mIMVaK=@VAMFHHD0-De7UFvYD4<#;+CN#~=4;fOW(h^}?Q4b8nen^3Ljml$pO$ zF!wF+qQtNS)d)$43d1jED;&O#Ic;mir|Mhjr*ERxIRBaT>rgiw^v7Fmh%rbodeU=PU&f7 z-kj0!d$bIdbuy5nUT^v6B`Tt$Dczhn@;1u>e`ptbzPe`OtuTI#J)nMew;sXS1@S-L6z+Mt>F*r%P_ zavsH{RY~;J6(P)i4zvo~59cn1U3)&iCp8n=Al&*m9{(|3l3>=?rEW6N^=%^aqG<)m za({&+&z&&WBd>X9o$&3~as~!5*5SSG!O$Xm+Fs zhL-YbJ-!k<>i)4nlIIt|{0!q<8}Y3R2M|Z40zv^QQqL<{h~d~Dpr2Hvvp+!J&#y-1 z!K{!6QiwnVUdRx!oMv8&wDtimuRk1XrI?*LhH&Ikn^S86q9o=tn`HX66qYD--N#3k z!!9;r=du;XdCxGl)|sKX_b4(>hjZ{&P}lrVPT8+uEQPkw$kjfu{r&{G+=n?O`ICu; z_cQm)jW(tOk@{IwwG6&LRV#o)LkK2V{9~mw{X7?|mVrEb26( z;f8`5rjMXS)WF8K7>@?)Uwy5pt&5e)Txb^m+?k;*Qo{OSdIO{8~wzfk==qCL*NaFW+F5z8vz|5&ZZhw?}srX8`op(F^8h zC%U+AQT-(0AqZ)cD9QXlq+JXQFjC)ukRB}~nFfG5N@;V4;MhYQ!73f?AFhWvhsxvy+1Lcl&a$FnSjJ7QremWY{?IhH!XR9>EF_fDt^w7r8`q)Vj zcjZ4{TuS~#A+Hqgg3PFdDqUj6V7SVuEwyo;w|SCcqCp zrUFKMJR?2sj*Uz3G7EM#_&4-QzEJ}08vjgm_%ofwFEIE&_Futa&?`ScojPP-rpFQr z5X)hmpd+n6reyfXPx#Bo{&(biU>3q0cC@Vu6<))OGePG)_a&F0B_*!KBLw2&@7=oM zDSKY;i=ms^UH9!-(lxaNi%aNS8^UDv{qmZ!64Ozawa?2ILIV-9{kb3G@7p<&#Yi~{ z2A`(v*$o;@d7Bu`R7bWx&kzD=vu97A6y}XuKT?jrHH52m-=MxC2mXR zbu5mJWAd$cH99!Q^3+BgtAgCk?sfki>%l3|7IkppJMfe*bX(+5$2~W(12*&zP#dIs zrQrvt=Eg9@(60@)4Wy6L05DGf3&7}JNBKb8@p6A+OHjVT3besqp%~!?7z8B&+$QH& zQb3ScT2EYf$I&$2$@S*5a4M9GP5pZPS!LPOGykY?JD8sI_3`GTA0TVMu1Okq_g8y8 z_X-nx>PFv&$y1wCf(QXp0glpqSvWloVsg&ax6V|*bGYDI1o?L3I-{a%<=EVaKTIY2 zms{!#CH_TG=@6hYRxPS4S5N)Wm{?pT(+*pX_DkJXJPFh(-Xicyb*~i=^-36!{2H1U z^tD=IH~nET(o0H^?7uU|zcQgeGvW|Mz;&~ZhK=oDbXC_g#!M~?x;E7K(-Q)KpTNxm zIBH%4rOS0E34;$JdoQA@aQ8|RXD@xnwm49#zS?Un9e4wqgqBk5_xutXP|PFCdLllh zABn;B%Yo7|>Dnh(V{$6m8PpzctIp^>)#skQOWlUTs)q0ZdgpX{&r9WpCMWtp?>8k} zw_Y3obRyR;P|M#r_dhG~Jr89i@&Fw+tyHS8TaBpWD}HQ)E-u_;%tWI|$i*gqT4ck0 zifgK=4dKn>{OY%2jZn%E`EjX$#~f4KpE`9SC?QVD7v z52Znjp>hEw$>+7j3KaR~Flu9lBq%^-M_v<)UyN8K-aYsS^1a{s$^TX;SWlpyg*Czg zzl-1uB5NpTeMBH;xU$hnkI6AJRi}<}$+)H>!v;HB!57Mw>THYt@~i*gj4Mdu0gw=L zn}7rqg93oP9`J|S0j669(Q%akATWzbbCSSSibIxPnW-m0)&v0VLg30kjK~0xFAk^! z4IaAwyTOBWi`ZWqJbe5IF!X0<=&$Dsz603W6TqP439&%2muo0sM}xe|iS#>U&7Bq$ zURL`DDC62YihbW-ot6benH}*1q$TQzVGsFJ4SncU;I~{ww3p$u$*58!iN#$BlD_-* zp8B^ZUQOY;7oUIo%Hny=_DLf*@i#t@=OFCNI164MfI8MQ$JhMm28oC~LcT|>t*OPP z6chcyo#l|SyOMNmwnjTUjNgFn%|9=J|IfDPZ&AcR!2Fq^{k!a{EL`x?pi_#nLczvB@H5EQNMn&B*W?qm`pt}w!FN~ z<3ewGz+xb%cws8=)Sj$4KEn4=aZ)_BqgS`C`&xTJd@mrIMZR58xW*7)t)Dp)*Jgpe zF{e!waq_j&7=bH_zQCB%P0^YfXgw394|&LC_@d#`9G5cJEgq!2dV;1 z-&+kA_cG5d#cZ*Z$da6ZR_!9snyWs%Jt9rC|@GE%(icOJQp)3}ht$WFyEw0Amh z{5WGcrfPXXR*&XvepK_?ML$EK(wgZ~zt8X1&;U#3Qb4v^%N050JJ3iXkKoDsW#+<)2hSI4-C z=ne$RFXE`lEv6(a3H3j11;T!HC;jaq{>5JuJAFhZ7V@1fi~-3D z5J|2)VxqvzVAc8#r2EbRV0HmUelu~Gdiqz_19dEU7`gojaSpbwF@o5QGzJj{|LdB& zFxmn;Bo82VO?yCoQUCm!qu?YX-U^HduziIH%btH`6+V|_>#rb>SNVW1MAlbz1Gc^( z!UZFluSrVaPI2xPzQ#X2W5=&py}!}1BfftCWDxwn&gi+r7&k?Q$?Sx$n*-iFDReaC zTbU7|RWjvnY`5mmswy*2$Ej#@sY)1Vxf1)oatRop+BZ+wiK+)^f)p_V8asFgr`-$- zGZPEwJ|*(BT2qvGU7;ewMxwhr&&Hj{BBq^2b{*O$-*e@FU(O58%}uvFKVO(@P(0B& z0=TooEttz_AsL+jlu2Rx8kC9<)cXA-p&A48hf3it&ICK><+XnPxM~npWpzsMwj_fg zPT;zi_*a<=HN+ioU@K7D8!pVm0WS`nI9!{|>RtQ%T9j$^TO@yC*{=PH-z6gdAn_i? zs784X7EyQM+f&&KH_Hc{rDWE5B4U+1MaRL_Y(Me)b0Ak6?^0|jGv~E)l1L+&N|=Zo zAhqQchF%;D5|MML!&f%JPmLDBg>aJ(2%4Eh^$Hv^&~^Y;x_k5?Y_B%@yPkc6wz&Ge zCK;ZGkax50NCU+*PJLbNE1#8ha(g-l9(>OCD4%vo=noJ0{5rz>Mw~&(Ky$RAlR1|k zl+f4koevjRnc?N$P^8TKH8yklgMO6*ldTb}TN6)8GTRk3`ON#7Zx@)vuU>A|6a2m% zU7O$U2}HdQIQtvWX*%>;I;(rKJsjwoLxBxNNYiB~FJa{9hAE%ZwW0_|CG4$&laV?y z)(0lx$8Hq(EPefI#eTpt8cX8{pDLU-J}Bth8c*!4N&!#0smM=SxnrJ1JSOuJpVSc4 z-Eqn_r_kDT#(amd-EyxypRorv<&?)oi*r)&u+G!v-GYhAx^*+Tt>?&w+xs|{aeYj5 zyLbDrv=+u}TPE2waR16`RnXOx3A0v_^59egGtRu>&?Ug{O5YsplvlyBL;0caVj=Tx zoQ`CUaM~L*l?`9U!}q>GgHrKRe#87~JC=WBbk>$Z{Je)F?1nCns6i<`*K-mR~A@ ztpwHu8;mw=oVXK?l!<1e3iQ}?%@H^k_q0!oXWVET44V-AjPzyI z8Gi$2T4yG`-UmUgNGyb=VyCciE1Kc}{%i%WOsuQZ>72=@N2G=~|XSc&UMOr#vMMtZz z+4+jC6{fLPF?|^xiGMi)__rrv8&&`TTit{toR0#8mLow3x*R{!~8!!jdMA=5`V7dAqFq@@UnKg)da=3*sv;*UZdp%tiCk_t>@0)oyP2 z<{u{QEnWA-in@yR^8GwKuiveoB@8HMb}+*IajIvmie&KeMK$B9hjNJ|oVE zRq(>44qE|#5F6iUd#gm!JIjLcMZU>ms4GGD>I^GTKR(}mPCtP%D{z;CG0owbaHij+ z7*6gbk>_gAn1b(4R7sCc$F`%l^~{#-!8|G!p?{RHiZK{|Qq+VBsYdax$Pv;z{(|)W zy^5cg@MAd0hD9FyJ|W8jh;te9N6qHa>2_gNrr#AMh;H00_SYKcde3_EjIm1iYs+JM z_d)xK|H#9+fuAf-S;K1)(zt3gdBuVj-vn|o)099OlwpA&csbylKOPcB)+6Y5Gb_WV zTXM%AD9G2jax|{yM_s@7+BchW_t*5+Y#q#W(vWzH5aA7^Bb|ma4ku3oS(6STAV1)y zZeUAI0J0TA%Z1p))$B4FL+_=-ndH_ z;$RQI%1|oMwx5EdG&NV$T$_$e^k zzVFFhM+R>p_v0lBxqOeNq(!^@-D6kU3-d8GQEcW{;sofZWjZIFY@_vkzoNQ#OE@nY zje1aR^l` zA!y`lcQoFWP|#&gfJ9lkiXVPGbJYeX?f!~j;EH|OOqRygl^xw$^>o4Hni?DOd>s`v zmN{RpquM?k3w+AG+oR6P28lXl)*>HTb(jU`${ zAf>cCe5%WyPG~RK?MSw$W;{{XdltLLhU?vGI7Dp7#CRcF*vp$Lr~;y)wdvlCYr3)T z1$kMUK>n8>tS@yoaXL?I%UC*Ed!}g`0Iisg=i3()nmCR6m76}shGu;6e06_sd_78) zKGN|Umz>4WO+$4ojEjp zeYweosnXy>2_gz18P8gzMLTgLPUz#w z+vVpsVMSGhc-1f1jlx$29v`YJ;i(Avzl4;dhEimq#uXch~Ch|zn9wWSsMCbG)*4-m@%hxu1+;s-qH z2gr0CvHMAGVRq|)Q+GF>9NoQ*+Rr8ecr=leUL%#iYJOaS1+#IiV@p)wyeKkP^H_j} z(ii`Y7k!Q{-XD@t{{Q zvgA-0iAPn47Le-wC(Q=I5Vvzt=+fx<@!9 zGWS$Jq%(7D7Ulyr37cmi!SjU>Yw`&zA2RFAY&r4B2}QUTR|y;C?5jawne0fPgSU?& z3mXu$ew;+=L65Do%LC@d<7I^t9&C>{Gb&EeRaM7<=EJGnTssP39En!$UC_ zHH?f9ZzCS4Jg^G4L1`!KBmt^1IQ5YdVSW|CRO8@)-U2PN$PU)PB)@#neRqbxV&Ww2 z%Rz->_YsVcl1SaK_A52-X|%jT3ZFaK7xbg{hIiGnzYIUQzo9DqVH&>+{eU%ZnF%x# zM`;5R^Oh>-z3wY48X9-Iika$Xok0|D#LdAGosy?H73%~)sMpSTdNV&6{w(_Zt^GkC z2GI%>N`^b&@@D7?6g;S3ks&L9ju0QUu6> zM-D!BZq_jmaJ%%8**xrWgBrDIwI?s_Xs%atFU^XDAt-pF)Unk5aMSdhY!eMy+J|!j z->lt$5}irW>H_?A29S4)1MaPsD_zx(+mcP_LZ0R7i}j?WnbEiY#aaKi)@M0OU^y}( z(9Uf&7lt5C!!M^20)T99bYv#!L-vE!bM0?TWfJW*Zt8W3^||n$Q_3@-W}9N7oLv3~ z7Cf!c{<%TOuA*9#@y5GniI4Lgq#_n-ke>DzuScK93pPXYL4J4ON~9MkCztUq%*D~r zA0S&_HN)Z3*LFwe(mt-LJ7=p3`FNm!JqWN996(Gx2MC9rfUE)|CJ32(kLyPzuGe{c}G!_YH`hPzN%W5_YW0F1pW-L#4}doMzVL;gZ_@#%E09p+nf zr9vqbG+IDRTpO=Iioc2+H%Gs~eq0q_t>YP9VmVw)XLnY&n_#vKrmyY{w-()0w|gYI zkxDuJj~3zYAO&%v63m{D+8K}-x%PrPw7|zGL6%bQVZlo9S@?>_^JA0L;an$;BnqL>7-`J>wtduwD3>DZp-}pc5eR({TZQK8dlE|d2 zSw@y*mk5O+*^*Ep>r@B{MYfC?k$nrHY@?*fo;^#(z9iWRF=iC9&RE7UOTVky>XrZ$ijfBqQq)uA+FUy~EUUU}E;;RD#k-^q{QSkYRyX6n zYB`^?jKz&q^oKJ!3iLDE{A;-1ul-XX)xbb>?ttuEfHgrWN0j?$906Jo*FP|3)4yXZ z{sLb7?c4m91g!*0+mSosjX+D>GNc)RzdM&u3u*KS@+_dLC<<30K3~L~>cKdJ8ORbH z$cnX4F{09vmoBF{T&;!DT+voq`=ciY_lAq{^}UYQWrPN{rNF z-eup(6K~UTFj|di$YuWn!7NKD;+#_#`l9&eq2t9#Nia zVV2-@Jn`j8F4vN4hIxywll~5^77rdBtwKjscF7mJ>&Eh&oX{(8zr%iBYR1W-?<_+{ zpvB>K%S=Hv=-{&dq9>xT4ix)QwIy6E|Wb!>q~-R2_x z7|$cbO7*3Qoz$vlKu4xdJm#SsV!uDTn~@3oVj_02Q7q;nZ>S|Zi#*3LPp;vb7;;w z18{wG_e*kwe))*ZEh)z;Ai0e9dU@!As_;y(zM!s@@*-}oz$`g74QSQyfYbSerc1rz zu!6Sd9vkfg`fM(u1_N5=rV5-4O`vVE0Rq-2m z6wAGEaSm~5?UcE*-_c23@v(t0LA&75*Gm`f`np{m{T8Rf<5SxW(8ivUHNJz+=Qd-> zB0t0qr2FaPA216dNU{Lt19r(AP`{up5A*{qYM_L7+ay#toE5hsLU^9A!hOb(dwp5; z{i-fAGh-~N$Zbyb!_@%kktb=+$H(wpSr4kX(8tFzP8brrah!un-q zU9cFOzrF9(E7$m9=1Y05Z{H0iyHZOeZ%P|`y>~CQ@KJ8M`TxNIVz+DoS{TFdf`M&y z`VZkoxGa6Efi)&g(dxmHoYc9|rM>5-WE{&etK$fSH^02OFv#cXZ<@KEf5^}T_aJ{G z43Po|g4i~bXiedSi$k&)KQ;&Oy+AdlqC@(b-J;it{UEF8*J2>J_YC$C4B$x2Jf|NY zYLo|5cX_74WZ4^v)BYwn>J=sL+e=g)zaoW6s|N!6q&j={zibE=Dx;Xu+qdnZqnmy4 zu!_zoaIoe{|I1S%L-H#hdyFqki%+DcNrpW4jeA&~Sh?)u)X~q1EH7Qj^(6XTt0&76 zQLSho57mJL&j}i}ZX@dxOr;cu##1F8r0kHKBCpgMq5HHTX8)?W|M_zON`;=$`yrJk zEo0xqA$6fjm1b{oE zz@d$&T>4+%QMHjj)D&4D>tWug)y@HArDOLL#&h*EzMwsU-dQ;^JG_lyqYJxu0oo>j z=2~-v9if(}nd2Gx*haJuzrBgq;H9qotc|k@rmu5OsPani0b2h0 z|2Oj6}9EI;6KBU@LMQR%SUW z^?Cu+#CdwU$l>)NV8tE0Z^&~evPffp*YnfW48y-%Q@`{;{$%b~cjF;C-~{fcGm~dK zA%_|mUfNB%K@WKvlS;+5-4&S?4JEH#B_8ETnH1P!R0V;+R{#8UGPWUa%TM&-`j%^P zE}ji~PNQ(MLX<=+o#)Nj(kjF5iRV}MT-IpY_c(On`O`3AA%21m$(Wo#?CVTL0fEp; zm&mYkC2c#b0c3f`Gvc|JBHs(K!k(6_xXeRrVJKE!=E)h5!U+Z!#wN_|Qa}@_2~#9UFniCJaIMoF<6R*g5CiO`XA|hgXn=( zzsnR7B8Lins)TUfO@~F{K7N{VY|>4sV+^ynlh4t0T7h}uj~K{*_)3MCDdzM~LQJ%$ zn>I!)?#GGH1fTU|gtbSNc>8s_vwP%+0#SDog|;*g@%b_k@NOG45ftNRfdJFCBg zsw7s*0>F{*faAye)g^;>VPW`>3&Z+J#ZttFT^2puNhVoZf1!_PdeZ<7lmqMleIS*D zP$aAt%ZF-8lD+Q$1-8D~)@t}f_Ak+1Pn}~l_Md&6WEt|lNdUS$NpU2<8tY`GC?Gx^ zy=Z@kT;W^|lqO%#e?@+ADp84p#) z)2>$E{*QNAdK8%J{%?5ka4TX2{c;)sOmFK2P^RkLl64K6ctEDf1v#PL#yZ;3g&X0c5t` z!R5yT+U@u&ff=kw%fUFq!XcHN!6oVp#Iw3Q3Ig2@NcO`U#wgw+HeK2iQ;O{75p4T` zE-P@dY$Sb5w1n-;Rl(IM6kU)q%CmBWzX&3$z%QU==pn)L-R7N0P4=y zDCi#z2KqqEM__Q8spt2=GVJzhs7)u;l%B+l}a z9^`2kzdPtXv>6OTmuW9U)J;7$nzuPSZl*ZwX5{!54)taoPkM!Es)f&(3$sRCjU{y;FRI6FvrO3J<_x!~{A^2Azw!K9R~d>k(U|s+rw(D1eh8|BrYo9G`7t+T9>0aT?OA<;$%yH*mnW64js=R zy(GI+Ed6!NiKVU5c3#t+1|yDx&c@aQ!ig`h4QefwO8+j#*c+amX477F|UUJi3AhH8hdHus-%36q8GV061t4Q0zW)7(m%~i;-WD-r!&U z9OTnZ0*$c`sIp1hVIFYfes`KVYAKx?~*|3abLQm?Q`(5IY~zg>I+SoI$lN;cgVN ztQn3XH59vN6CN%X4)ZIt;O(`##!B=lB%Ga^ofp^_dIWF z>uOURqW3@McJ{lu=QSwFun8zk+e_^0D2;^>Od(v0(7jNnz@@$eRMDaN(+D|oJh9Rf zf$4O+sd|&oL`z3OG$4*$475v`F=>(yhgGBn7bAq=a{ZTES_BNey^jkm(&0DIZ<9gw zsHPJwugw8aO!sF_03#IEuLR|CNt@;;$|buNxLQ8}y9YS=9q3^rDzu8`Ix2{k<$xc; zjR{M$%+w>EZE4uUjr*(??N!MPuWk7;9Aw!%S5I{f6voG;H_1%vE2a*Vrh4QA^8|{y zQIUFvyfI@0iNvs}>+UPf#U6ZF+9O+;bFjac4{)%&6w$*Rj{)KBKO~Li^`nU=4p_4{ zO@LBT6mTdmCsZ2&=U2;F60#zwFyWM6;`$EC@7Jvdeup#hyZhrab9M~6k;R%bl@d|uA+60;iH_|hlK(6&$MZ`DhRZy)}e^tHm?%V4&1 zI7LPs&FqYxi|r%d!aRg-Ufe`d0Ll`%fSOBl>_yH~6Eb({5<8!et;$pu;AnTw1^Up* z|3$^R{1#>d@H40;8*cz4-i>?&fSdaCzk?cK0D<(UZ-*|CQS*Rr#5M+`?4z<1zJuTZ zFAEUX?oJ1EsCSshh|rI%biUFV6WS3C2z>|nL!diZ8K{_A2bbcJSrODzuqM>*@dr@a zI^a_PO5eu;4+Z)K0t2e_&w_t3X6Wd}F*>qK6B9-Eg%Em?dyosy7QTbFfe;0d$S!(T z3UI$32O*D8m!Y_HIulSzd}`o3Xf5rpYU5U?U^_qjw0}?cher9o-|nBhm{q@nVxb_! z!3F4n2Ag6)9u-_!Z?hyQv=|sLYcVAd!zb06ofExT9{U}nSelS~A@3IcOR~MCN~^=Y zBgOmQU+T?W#e8a+IBPO)g1ckJeXK#*RkC|^=`Fp;jdW@e?D8|a{ReHD8jRWh3eGQL z3w8K5f`PCWu0*r6cz2i_9V9Y3Ege%>_I7UN1w*#>yS~1Ub_qm*;NQuz@=l*5riCGd z$PQ*iRNNLtmgoH74HcfWv7|_R7<0acp(=x(p%{oo*p#_hWumXcge*CVIURvB^ht<7 zR#>3!57;l}!Ly9om*YYOv^HZzdDV0mDWCWEGp;6tV-Kpc5+a)G{DL(F{5^(r`YYd( zY$hh<_93ID5&}&=9(M9ql#dMg@BcUgueZ@r>!o zSLA4rA2jXIF$-9OFPmf5z!QWRh=d(?CRLLIC}!lFfyFFN{Zy-5#C)43H|DHoWq0$$ z_+CR2r+i8xNjcW!lan0_ zO`7Wjm4|Q$VW=hD?;&+BZb=q=Zt%t->uVGG)v9B)Tel>{pGDmgOEMJe-}h^dd~Cs@ z*|6f9W+q3DZ(5K8)El%^O$o$@Mzuy-9#Can?_>sDYp^3sGB2NL1Eo4wMh127yB0ad zbQoUpN&>XYPja9YAovi&jR!qB{EvTIu8M=+v7V1ul28lOqomnoo@M$XZ#6Dc<^484RzDNuw zTw5sPl{Ga{Php#<;_-!MrhpGgr_Zp1_C75LX=0p3GE>EVL;!ma5%sc&fuy9eMgp?= zEKB=*m?q?9ufXnGh}Y^?IKLZX^Gu4E9*50`$Bb*mqWXhyXr8r;HFt&^+I0dh>F(<4 zcDS95L-=n}gQk1m|Mq-#^htTNnAL_r{57bH{P<)WV-xIXrMj_azD=TFNhK%pb%W`F zGADk&$J($Yt)94G9;6d&^4u7nRO&;3&B{mg0m=bnq{3phVEsPTwP?AEB&Vq9JVekl zG0D$)L-RRce{voXe4cDXz_wwJ!1c;CGjKt2^H<--i>P;7oaos6(ZP}70| z{6BWOqJL>;<6(jvnzWP|qcVxVRG93LY

pq&Jg!(lj9ahf zY%BI)(Qyhaw2`kUKE#~Pnn8~phpmD7k{=2R z<9UZcucP1okb@Keo!Il>AG{+lDU%lKvtrXBN=l(U*=i%`cW5Z^-MSXt>3~;SBupLK zQSW5r|KyHf%6h5Ao=dt+tGF+paK#?3Oe96p_mVNV-u|UV&c=gB>T`BpRbf9QCT;-4 z6(K*labCtxnu=6&0v2e!OBW%R;zzPJOAk))=g$T+ioGm<~-jbt>0uqs9}IZdBP zkP{4XZwUtp-OUmD_8;ctSIGQtEr;LE0soICxBe^h`cta`_R+G`l_?er_E2(O07ct> zOn;Or|JqH&glTg8CUu08?8%qz zo3LOz0azQ{!C~nB;>F|M8_ZLwI}z^BCWR22j|3F-IT>~YDFTR5gfQ(1Lg6#qkh1q9 z2EAC#bK3f2x(B?vaayHqPVZpVsE?kU_SuV}WS}5%iFBWqp>9br^-s{Jy+OLsV8%qT znnc;P5GucuVqdNcDON{cv>?^cQI+|IY}dYXy>gMbrNBH(tfLef$%KmQ#0)xcF`>f`6|D@kF7Y9MMi?G45qm5>ZSK7w(O zkL9+$LClUSY(54)=5%ba-^D{Of9iT0yJ?0Y|`3I zX+92R_iQlH{nHuc4{PhL4{~Rix+!Is?xM@PtKPB!?HwzBdxihqBp_VKW@G4`Dq1F; z57|;rIGz8MszWTU=)6tT6dSDEc#=w(P4WFI@@>QOF48ZAVaJWaN&jTMz5I~KY)v$8 zj644B<#JFcp50vlzEYPd4wYEkDg_YQCgb-`e9+l zl-792rDox0v8?{Sqvj|*0<92E)&msDjzMSrqEh(>_G=9qpdH=Et-pzTPIa*yxaNsrb3m~&0j=6&+ zp96ZW+%_pC`jj_?W>ayrIvf3Z5`;!ONqTaI)CMjI31wq~@{m!6uePkZn7%M3# zt?TnEo`1RL3tJcyF?ldp-H4<|{zg=p-40gvE46j7H=(mb6AsITCv6JIE)kXG3obP;-9zJBP(T zmq%F#^?XFVf%H#?bZrk;Kl8Gk1||hqm-bA!U8kbW96|!c_#_G%cO( zE7yqsbtLxBgHi%=e2kPrgmgk7@GFE{H3V-f!W{_v*7H1T81~}F^cQK*R6q_|@k~-* z&0jLyGRb&;{Wn=UvzjO2)&$QcZ1}`lySxyLmZ9J89=pD`^O<@?E z?mNm!gLQ>H4_Voc%suuE1~+3MWX4X+LAbFG&NE7rYh?4F$39u*60`^Lf|toh6>ye` zm}-}Xwi3g~5N##_j{bh;RV4L@DCq-$>qV1qBl!I{%nws_2zKRS>lb#CEr!AmpKT0q zGhxp8UPd$H_2{foQ6bRY-nh^AflvvAttbPz3VXM*5-b*3i9!l~#+Z zpM;`SPPXI=GwysO_Yng-V1o4HZWu20fIw25ja}_2#e7Y>2A!|b`yYN2aN*+gY3deB zGGOLsI$~!mSXC)6Gx)|}R#)a@+YF#+kUS9#wSdlKK=wdqa)kDz_n6gIW1 zNsfQXcu8KJg_?t>bFEXA!^dk4R$?cuwAw+4SHBYy0!QNtkuXUbC5PeoWRSN!A>Vwh zr6(#MWJlb_+!$H&-hSlw@#}>|Dr1*wm+W!ivA1CO{G~gf;07HZ98AdH`D(v%hF%-V zI;F5#(fPU5%we$U1CFla!6$L)ZUIU7Y0KzqPtTk=X<@-E9HPb~w8=mLm`6;rw6Yl* zn$8IZlj}`&h)0?Uo$f`2hahG(lrq@swR(QXPD2J&(}fd^OaCCG06^;f^V$3^VzqA* z9iqkLRy!H4vL%`10B;4w2zBv%gsO}P_iTU>0^3%Kzdi7wd`nZ*ctR>lE)bn{?26Sk z2Wy%1mMqcE8du(xW6kix_7>%9#4KDCN5#?I9dY7shFCW~aXOIi9g%IQed zT%2ToGaf5`d{!<^Gn_!LN0T$OhZ+p?4Emf3taSrDJ}6xD687m@zN@DT_Z4|W7|A<_ zZs#7_y43kh^^Diu)P#bS^}hj#B2!l9`$1;gwm#AwiU7L4RmTQC6vFo-j zrN{4B#eaBo04xn0y0h;Ql(!1Yai*Mydr{=x&9XwsmYUodVA9mBeU$seY{K{#wOJ;< zi?_AAWVIj#u!@9b4$2usucT+bzh#SCv48dv@c}QZK~XDup+&p}lm{++^5!MghA1 zH1qispIxyRr#(|Qr{#k_pa^kp6*7LT^xk*Vd!RX1=dQCmIsnig(AR)GP7#E=4>lOz z%emdiNkoQwoCh*$!zW7zjUDb>ZTF{jTzAgXhJmbQ*@~(`svbS z5%~Dx8$5m_Q>mKAt!;F0HaGWP=8Y$X;FO2LLcgPl*8M7cn&<7?6p?6gxz%MZbFFT% zY!D+LS$q%}ye{SrD|R9mkM}of8uRX(fgG@XmmS3>oLFF&Ty8X&n>vj!c=$y^m{Bwu zG(pS2%shcrf;(X4giyU!_S=&e{F z&D_hRV$Vpp<2QXww=G*_^z_tinR_8-U92FD{ZXbIV;U}uSeJje0ROU_*qLXUn>-&u zg{YbU2H@HB02W!ysSsimN&0M^Jn}qif0F2G>?thJ+59BN6+#v@iK&EC`KOUwiBPJT zuhBE0JMXBIf#+kSFFz_(@?B_T7vmbxAR|VT&wptavg|;dG$uM`Lq^m!@G$TtfM%~h zJd^Zk+WKT$^?D)O(=TSNpu4_x>jf^1yIf>tqIWY!5OLH6#z|!kB4vr%RUZ+lR__U5 z?h!luaN9AiRh0diJqxcGW7R*}aYq-DMD+oTfk`p(vHtTMc1s+lr7s-Ca^D68rpglR z<8WDui|aPyE0y(5;c9aH2UeROWJ;C3G0g=8SkL_9bLx*1JO8(cIfK)Xt`MNly(sAs z*#TdCX^=EGW>QrjAW{+2`|ROojMNv$uMvo6rS-TEGQ@FBdGNmEogr zJ|*Ct+c8}5AVP6xt9%&jK!a(hjmvEPh;od!o2Tc2t|Ei$9~xE!4OBlesG?JyRN)ba z%ErS*gubru)>p51$}jnvLgGOxIzuutD{@s# z4cvj$$F)7xTI^Rt&}wX2^Gt9OM4{b8DOH7)-L<a6_Id_TVFa&%j-HMOQ@+8b7hX~KjE z9Ux0eM8PHVCS7+^Dke%Y?g`G;!nfa>lYU z1J73B-!SnkzV6cVxr}8GY=uW}J%u9^$&I+Z-B{L!v6;K-u1ET^E!#7(-H0WO_Ak$O zQjFFq?&Z+*192zF*|XBT@RNw8?$)#Wjy-+EpZ9o0t1wLhD4O^AE3U{4pbWG6E!Yt2!~V&5bR1x> z!Ph>_sUB=SWpuZd@7i;%!3%oY$2eyMvL#int3&8QffONRKcxUU6TSiHP7K@ylLOY4 z0XnA6YCfPa!QV$$(QSg_<}qdSG;kf7zTXo{p5g%kdVtyVPSmF6BB06E0C0To+cN(8 zx75=B=of=vrdpGO}pC??~ZeCltag~4WiD1UFeko7k zRq2qKdq9^2pzl0cJYjh-bvNzN)JlGhji%Mh4`3I4^1P9b#e^AyftVE(1oB8C_x99PRGJqW~%x&32=3uZ(jvY#Df5< zh5yBj6p$Q*I#l~ukRRsfNbJqfde^kAnxMq6328+|L%;wV$U~los!a6A!XG?tF;0q6 zQ*^k?ASoiC?!my68{EXvsU>O#1cT03Fy;ujNF|XY0nR4%gQ|<^sYALm&W|!vcVo z9@J5uW$Dy{AB^jWY4)d-Bs=*_wGB0FVBK`p51w0kqSNSf3PU0=33XaL;Qb+o0ov(b zr%}KBV{U)^V`d(rB?$mDLhi#o2}}N#_=X<5vFdz}QQyI7KH-}R_Iws6#boJ&il_&2 z6owg8#e4@KGiXE7C!>O1!@0@C;@e0z>eVJUd+sqMqoI4~k2yP*zTQPPj?!7N@dX#* zX^Dw@d4hrgSz)07iXg!G{SE?F&QHt9?-Q^r{-1r!zKxS#K~t3BcgBvi&>6{TZgN#8 zMGk82PPj>yoQxMaA{E7ts(Vz;pxTuC>`qH(&B$LIKS(apz>oNeu&JCV7awb1> z+qis{6>smhb<}0L@iLi@PwFLG?5t8|k}qQ`;2h7R%CI!>E}(AvWBE%rHbDp*eV^9o z5i}DZ`cy$tJ!1ZQpZQ}}^eNud1LSWrHwZC66fWQahdd_-S8ARZ1@13|N#9t8w7a_P zkKJ?YtKHPF)E-dK7fHWQY<=UC_%-U`A=0f7{!#|}MV@To^OF*hRYCh!m(09l1>m@^9--yj+e?2RDqDi4o z_=>Sc@PcdbZy;zo0RhfvtG?H_c}yoJCOU>p)CZa3PYgR{JtY)?#FCr#JUpFLuX&pY zp>jr$poHmC%{cjX?qTujn@Ps6zWG_6raTl5OK}(%o;~}x{IMPj`i8f!o-fX5rli3$v%L~WQKR~dLU95>m7CD$yp?d}1NSe7$89fy z#CD7nOEZF5jXg8o<3a>Cb${t)u*(nqkZmTjQ?-~mM-mwuiXdyVe0rGQ;EN9Yf55j#`i zJru`DTb%;f03&oirZds72m{Hoxb+>o0|+ZTiy{5m;s7W-%d6+%DYuV>T6Wcqp0&v(LPT()x3 zz|l}m+;UFlbq+(U$9#@vy#5mh1-?fot@4aCu3ky`oMG%2G~RfG)gsR43Wv1s=t{L8 z@~DsRAjV;7KR!sMx1qnX@P^>G1esz18)_c%ov>|U!c&L4UkcOhknArkkYD`wujhpq0u|G2^?}ly|woW)BJnhlU#R-QLoUb(2VAr?v|Hg zzfjLoOyO*y=fkmho-p=MypchhbvQ~cG)clNVlr&zNOE|yAh`GeZ#-|WlX(yGmF$f1 zjm>Xk>$R2ZRGpM>XFGFiS7B^a#*g30bMuZ;JS!X6X* z)xL1RWS`wAm?0`Yx4wPZV&rUbsoM_1ImTk5q@stu&g&dKLm(Hj0l-F5J(J8X_f<-SbLw=5}f`=(DYPX83kQw_;m z1^yAFH?*26fHAyul-mEz_h>IE3cVtpG-OhyxaX$c!eGF~d0MTZI-6xm#vEk)nk0V3 zx7cZM8FLE71zQ}>va#JdPkTgX71`^ycBGbZZ~Cqb3L&xwa~kgbtfo?Pp6(nB{=M8_!4)pTC1XK&yyCFhA-c0FCD&O9S**gg8*+Lk3v7 zBao$!&Jz#-D^sA4wFLxW2|9FM%b8LtjSP@l8D|0S^(wMd4doKlM{T9RIKFNpo5AE8 zun^3~HXcR=0~CumGkU{Z(DQBR26zRopEd=UXz2f*k#Hm@{VF z(PR5$t|SQFa-He`e*vBz_r1S7UkNS>hkr&JwxVyPin*oU&cfy$#150zI`PD4z4$$+ zZ}roXyMho0=Ojjh9O}rfFD&*NRaV{FV$R59>BH^;uAnN_;Z?*3%YZk(VcOxR0Fsb1 zxdJcGNagBp04R3jMV;|JEIGqYk+_%THM-8B8d`oyGP4DRlDNo|5&M5MFlvx z&uH@cZ{Q>%=)!P)A~^EaaD)D>^jO~s_cl{)Po~TcjMANtU+)@r@u@P#qFUY-w#)R$ z^-9fn?cOzg_1RNw(%rMdP`+ON!KH(#o-&f zQ0h?~Am9MVsgdv6AU%MG$bh6iM*jPkfai2-4moK;I!mZo9>^gF;w@VthnGSoX3dQ? zua%Np0-mSxIh>aQh3$R`4FTWSm5=xw5QERuwPn}jxEvdh9PO%nKi##aq|oy% zuJd|scyuP;ZU99DZZguSi0Ex(BuAC@%`c|MXZFL4YxcRtxW24TI;UZUyQMlSbFV{s z>)P~0wu45<>pj6xPGvz+ds;2heI-zzXc;M2q|}TII?UN>=#*zwFJpQhEhP|FtJ*Q| zC%g%jZinm8i=B>qxFO`~@*;aI5Y{b^dB1(;?yzQ=`4x}O5ktTxG|@6f_EhK1>I+{l z8^(2SoShR z5=^k1{^%K%mHd1v2xT3Z#c%m;M2UZ4{rut2)?%f52VY$h*xAec;)C|L&P`3r_;}Zl z+=P(weBSp{C8eIp$Y;2#5fvGs3w>jCo3SmwPWdrzEYS~a6CK(#d4a%bE$jhz4yig@ zzP6!$go>70Z+82#bGQ0TN8f4u*{7D(UPEQF-pWCkLWm^%G%cxttB5RbIM!fZusDca zOii>`wEmFTez@!T2JV-C9@^zu+pWSg9N^?Gd16Kq17uUf}FfCm+Y=<27B-1TR)FqxR%FnCK; z8%3Y-c3tuS4ZYWYqcYx(#zVBMh5*gn*V^JaN2k+eW%}zAM*1505w)R(SMNrjs&n zC2ic>=u!_|;8-a7(gky=j-HD8v!3H?xJERg5NDr)llAcbGHjMtTgOrTCEie=x?O`5 z!XO%=Em|A^*M8cn+}X!r7+5F1kP65)qWJA@N~?7`^naO2j zZzz4-YwCq{;;$m|KQ~*=Px1r+O*dJi4h-ipK%`vd#kWRC^{BZzm+YfAsp&oUCo8_z zpEM5n9G>R3>dfS83J&;dj*gMYjDt~8$Vh;Now-GMG{cEkUm+2hRUiGkAn2;5FWY zez7B4eh0zoYUnZ5MOY$FE{(Z=mxp8vG{U<_b_MdojadLKj2jd0-%7v0=1ha5=ui3z zTE2t!2t{rk*lErAB}HV`lPr}$%>cxJmc0Onb&!98=G@%3BLqB}Qbpizpxdw;m1}AD z6hI%C$$+jqA=xpj^4~#dxA*iXn(*CQz!s*u(;}c>$PI?{)%x$C+@F-B&LqmXj-=!w5KP{c@ z8$p$gY@$_v_1(Q71Tg7lCqUBLg=`0O06bbsj|G5zBP2WYOSV6Z-c$|UJtn<%k%v+Y zC6snUTB(3gj+Ia z`OeA83=z8@EiU4}kbs6_&~kYoJ4Crgg^s`j9cBb!Es5VA3{ODEw^*36@5QTru6t_^ zSs!wf)3ms(=)(AU;a@h3zy8Ai;2syh9_%k)&EJ;opV~zehpIrYxXxfF^xKgfK%a6H z>(Z|7SZk2g{ra(C+*O;TK@(ob3BxOO6Bl!@2U`vYGpyeKYq}31Oak@Uwvg!k_1DK5xf8cWeTmiv_^OS6A>{}O0xMG& zthm^Kv)r34cJo?n5{Y3uHC5*y~vPvNWDOShUrYdSc)5?wF3;;l>pdaL?7y6KZ6z| z4)bQ7isFfaPK_ZqgAjIjG&Nb8dY5LRFZ&(jzjJC!2=lb>#jobXxV|w1R3zsC{FzT{ z#mI%ibb1#Kydyz?ZLLC*G_P<`19mG&XBwZ(|LUut9B>bcEQ6uOL0EyNfB}&@bLGz!Ry0ZdTv-8T5btOS7ehn82<} z_y^Kk0HgdO4&KR$+=D7zK#a|xsIf%o)(Y~Q0Zo<`2K_S7$P3&9007(!VLY;Z3w2O% zFa6GTbh7E+=pIK`6LSqk_nI?R90_L5&tcccMF7t&9ClRcEb1^-nppA-@2>cfxVf`> zM{GLrv#^Du8fe}vUoi5zo^}rVZv+kh%5M70%KP8qLjC94%`IL^hlW=cX8FN}i#KVS zy()d)_k;~*ZD$-&m$evcg%6B|9eA1^ox)pdxSVMEsK8K9YMB_AFnMZb(^D+f)IZs@ zEHY7?R{aF)Y~p2G2&T+6rmG7+Q+kr=ATiJb1J2853o<1b8;ObeV)EGz1Rs(9&=Zf` zzc_IFsL#i97lQNIRHfcGDZW|VzppWnFQhcMXyK-4k2nqf@n|Jl&odxP z0s1L_=~=Fl=K(X_^7a~v2D7M@({!`XGl}t+5Z_9rmAcEU)N=)lOQk11QUu7cvKOT< zpszsjuoY&)EbbPXJ+x%n-hSzbaN_+bMeGU>gNbqLv8&lRhgk$Ri_51W9dnO;^2Kr* zhqDr|DwyA6O+O>IJVt%Hht* zH`kwuN5v=Hsxq0oB+6>ZHdbj9OU@z|;(%HW)nX)0eniZsaG93gF)FIumUHi|@IvjZ zSg;A_`iC^W43#bag&NQEjW;4%M%u86E_F{D`2gu-_C~4OM;6lee*KzVdMDd4-6PMF z6d3oei|<(3XGt;V^fAS*leM;7*4?VmDjpx&gA~|aOZM8ln)?rQ{n5gJXYeN@-6;g; zHgLGT(r|+|+Mls~T5!~J^DMS}>5Z%3<6hakG3jcYgIFPbC2ekvAB_8(9QMDDgnu6i zf9yyQ&ED^EJ!t~3oFr+Qo68n{SwUOqkts6+!DxoL1vWm`s*-((tbtSNzh6(;QI>w% zIHJ*Gc($PAV)EI>JuiUl9>b7W62rnzuE?)~kYD=YzdY*y{tN$eOJ@WiohJNlK(hzV zCAtz}*XH!|SDmcq5FR#*oIvA|5$h>?w*Szch_N-i69PeE*Rb;#9O0YG`qw~PqI&NXICEcMM{Y{mWSb+(I5%dXwwpf|w>~Yz>TCDd=)foAvnzCeiB3ek8%0 z?@x;Yaf(OydHkJ#wSn{(RiGCk33eeG=^pQ zO6Ul~#T}+?F343GIA#ZqCo^Troy)h-847Mqofjm>R!DWb4u#+7J5yoXtNStuIYg=B zHvvbq=t;f%oZgtlM3yE>&vwQ&iY_%CCZ=>gS~*6Dd*57WqwS`UeHX74$Lb+nbAtU6 z`~KPVQ$;nB&DZns3h&)VEMwfQtjFrAcEjbukEWdXm_=K`%%nAF6TR{1c)G|ElwFhQ zTiszDT#;^(TYCN3b7{u+Wick&~SIW?MA#|-dm~hd(S|oZ`DNC?rU6wC&*dZ$@K7CHoHXawgNgE zo2^vWV7dl*&#;08bS(x)=?}i$kV8np1Ib3S(1WfR-j#-v1q(tF@9Z=@?>i-0k2+u9 zXB0N$@$o?BYgIK~v_Pq3@0$_l)Y^_b<-iLQdEbzIh~PKmJH~2fcrM3RI|mSlX2r)^ zC>o;rgwe>lu^4q@Z<2;cbH;6+89oT_j*cRmqqBwA5|IsVlxy&lgEpOrlY?-cQWxKPr{N-{Rz=%M9qPrRCU#YzY)MVE zN$kN}I1hNvM>9V>#lV;ptNoD1bcQ)?Uj9Z}vb-un02@-@9jW93=Y zfe1V@9=nGw;JTg8wHBrH8dDxRp_$;ozHI?}(juZAHXkd%@cE&gzio{22j`<@q40?f z74j6oo=U&cjCZRN%YJ9oL+_*?B7(ykj+ET48;z2S@x%+>veP#ia=IM1x7Pf?g=|aJ zCg$?c{^(Ua)k)xzRMwfx^LL|T()?(MTjN>RhV!|20;Nv}q%BWT&QVX0{cs6~tjT?C zmnUvpcSd<^$;-+(r`@?!7ABs!f3dj3d&Pd9gFwIW)DkVDB+uJ&a!S^mQc)ooQfz3IjXB|ShBK(OqlM|QSit*bKjyHHV5HwzfORzI1?!CN6(4p6T^x-((((0 z&AhJOuqqoVCjiOOi>a}7o`S8P(z%pL#uqdJL@XQ`c;Mhj^VLU+*Yulpdt_te_|gU%4FMkKt5Q7&l@Q$WX`$@wfm)-FQ}nHE;gTsCE>-;% zyEn9aU0P7?2qA#sj}6*XzD+ zb{OL$MIdVYmv*KLVI@z*11@37%6$9Uvng@VYuSMdjo(Hky%X2e9(&od=w4YovSYQ^ z8p3a12Z8owe`Z{Oo`)#dXI;bt@jnpZpYts+{%oCnyLsHDx)1c~oyF>5=gpG>=X|In zP};7gC5zgr3_zS1k;wCG%oO9C-v-(w<@s~8&}@D{Uo%crsh_xHbLX~8!TH`m z>TxdU4aZ%6S1pH^i9k$UiF5!K)AWs{lDrnvG0t>%$1#HRQ*#qwk<`INBV!pmLm&eh^PM4W5+F z>f5HlIw2mz02wO=)>4n?jNccO+yvvup%rclX|rJTH2l<$lB)@jd-QvAiA&!>tkBWoi7GhyBB7`K0OkvCbHV``nQXi*kVqLINc{GJ zF-bww0F2~*WUj_bcN?7qR{KS9f2w}6B_(OJmcvYX)K8mts@Dwyd-O4=oxp-&+N3&h zxrx!!9xkXJl3>39JLf=W#W{5)uI7EEr^xnut3nDcng+*xp+>TON*;Nh?cP+x#EU%% zet%Y-f-m^gvg=>}mR0*D0|fyP3n5iFehf`)Gld-o1dY$FwV#Vswy$P>V_xO$L;h}e zVTeO|cZIDSrS>*8QTHftVZ|4@(E~gbHCAtKFsy;5Z9B5TyRu*n!djsT5iF9^adYva zgbO{m7s?g(@nVQ{xBOfXRmUvTk}qz!R8Az>`fP0Pz-0d6s!drNXk@iO0caWksEqv}ZQ0Z>AB3y#feg_Fb(Gs3;d(;V{4vP8)t)#(|$SVOWkNC0e z^K=;S4)9;Ziui(vEdr1`iby8(L-*)0OoX#w93V?iwcFu+G&m}HaG|58B*3B+|r` zwcj=M7qD4tNp}QQQcUah+rBpd`nBhSa*EcG?k!I(4H_@jG(nt4SS0p>xu!rOWQnC|ge zUqH2y&;Uoeoo^ocErLKAdZJA;gKq(_!2I~<@D=`fQ5-!K3#hMi*-yWNQo&eOnoW7A z&mjQ9EjR#f_^RM30(}Q?R8;;4uHmLX@N`%4FE&f8v!>v5v+7amii|1Mxsc$-C*Vj1 zPaMxYe~+^I86S6GPW6{8TSXCs8=ycpNKo%5ZJh1eMbP_dX59CX{nFH+lOmhwUKvm-qP6K{ljJA2*&sWJb%ci^c2C{`R)Yc_ z>kTjV<=EbW_!BNMo&Q-|fzd!*MFw9!1aiE zBYx1M1kHmah*zyd^6n^ zKy;d?k80-?R|vyoGRGY8Y;#~^<-W0?%JF*PszmhPMD5gtI*z4rN4qWV+DBFoOsqPw zJafq9st0;0P9hlOo63^CRKhCYGom8 z`pvZ~JJ}gBgJgWnGRvntOupfuW^10Vz3ysK{4=xoROEPTdjm87$>kBv(&EAs0VOB0 zPI%OKO%PY}#C&by6v=3rEuE%E`J(Q|Lipl2@&%A#rv5jfN-};$#l~j+;*{YP!~4^1 z!%k*_l)w><#%Q!5XbD_c@j0rl2vM9u!8(<2ukRB}$?6YD0;^1k)06sL?-N0sSCL6I z575jpj7)SHg#w9jnW;WelZ~q-+q(jY@|OZ$SEY0v3tV1oz)2vqju22IhAA3nUpl=; z5X3n3nmdS1b?VNZYpm&Dyv}{9E0K&ynN%#uMRs7-YC(f0JTz9oVMyBt@KXyWEVL*s zc_owYX_{XBf(C~oXe42D0UFpBCUeW;@@MvawM`XcQmmFp8CxSnkd(nRvYs#D-6YmlH18A!xk@qh|Rq z)H!dq&8$arO~hj2^28nyFIzo^Nj;s{b_y*F$%_c%QaJKltHx2ESvV?`)d^8MxC&wI zV8NFM@%QVySP1ke?vV=KS6P7yto3IAnHY5R$uMrw9zX!R*R^!n2cJ9uW-*X`v?UZ! zK=G&4&Dn7eRq+04OQC|;0s^BxcG-(50$~pi%i^2hrnTD;5lS_hm)^SI9-{4R{dDnw zWyA4yoG5L=!e!lYKd79`@l_NP3Htj zk<5oScXQTZa!ERcuq!!d&35@#ZM_K^deIb2vS^*H_^Vhjsx0z6p`U|pB;dllvAkSy z_lQN_z)bqr%hieVwgz85#=kIOX7iNXvh~e8Kj^4UN9}7@{IXg{!)r1>uE|1OgN0AW zBB(|=9(NH|?Sf~Dr2r-@ufVZUUF$d~&2s%~e54F|vQ-YsA z2fQN#h1++^iaa1@?g-3x)6H|{Xw&al*UL~obekZMmB#G5=_aEkz8TFUJ^|ESA0+_5 zw#*weGP0QuC*IuAoKxT%XK_QCIe6HtXQ$}xnh|+~LRjRwv74@MI`eH$$(UCuxC!kJ ze1B$#N9#-LN<$Wm@{%Z15P+efk{8ZTaN`Rv+49o7X?04tjj9*?nXm?-!R=|AJ~FqY zi@+F1uDp|ESYACu)Q7YC%QL7Usvkt)uB3~S2aO@mH;ESBpVg+DHgjlrjmr} zaTpc@_;hBS0G}Dr2cUMYX;}(HSzPi^?84DYY3SioznpcNFUtf?7z&1u8V4U=HrlqG zZ>Tp-_ zbWL7Hc`gEb=#PHoh#%hCG2p7k>D&e2`7}QomTzzX?pNsgAVEec7=ZRef%ZOfzcK<% z85H}C5d=h%cL_iz-6g<9f;0jCda^O`AKXmdr!Bj}XTxkBIu+dkuLRtt5@)&WGgxn= zW!?CN4k>FMaS6G5#vTbzGTEmat>}$u?*Ef_{->5n1O(%!+wgH?WAVX~QlJq1^@-<@ zuJbn0Kl+j5pCD<1&S4!y$pa+Nso5wzFR$1m#)P@-D*|2X zfq+bRU9Q|B`hRqlEY4T-KpP+3WObr}yMfQ-#0r;~e#H`Nj_k8|*@)CesIE>wO@h#`b0qXs!Rhy-E!X|B5_e=Z@M$AL>5i zKb_-vSa)y5?V|KCtI)~gJ?-EifCB;L5?+D>TCZCVSV?0YG*s+u89PMW(yYe5&RIWC zm0nEeVqgzmc^CY*Q1h4U*l*^KJO9D_@mHF>#e?aR00N75V#9x)O4rGRz|>RFbQ7wX2HSRQ%p)$#>>66U$HKIdbrZh!heXP*=3&;41_%!N)OaB^fnwH(s{0povc z5z6-K|ItZ6%kfV-iHQP&0$y#)kCx+GKxFpw`>r!Nl#e)gi%*2!ADr|-k!*Ji=*`U{ z<|+II8y%G^YA}+c)VBxUUpu%DB4ckHa;@C*TZEx+!@exp4cp-;-c`tk>q@SFQUuB- z!k2-Y0b%DKe8n%yi1!zr**aJUZ+riPMS)flh67u|F})&;fo=bxBP0K&aDRLy7)X$O z898vpvJ?ZE0tKv#BWcyEV!znU_KDhap)EVdmGAIhe5OVky+^I&Upci(|LfxP zxtS0B-j!;$XEj{Q%cGW1Ig$HhB|+*c4qDjPT1ZInlw%kH}w|H}{VrDIL|_ZJh>gve&-`l!Wx%a9Y2B_kR5yf1m*WasB*%ZpEBK<-{21M{D+N zB-8F?z5CNfjW}OoJ{>e!se5QgA-|>Xt#OlXs025g(PPLrtK#8e_pU9hv6vG=3>kBu@jFsK$9z9W8X2`OZv5}!Ke}1gapr06BonabqfJjYa z@Lr#rF&(NZE7153GR^A1h#q*OY1#*z!3-7}@9eTJhLNzA#=8q|v5`y>Cl*tav3z{! zmgxqQO2tr%3g@CR>Ym8S#D*g$f(i+xlPT?@q1#3`KurCKz>a2$;=*LkiU9p>)xP(9 zFJ%~yo?bsM-R4B3qO+!Hc&}Zue2;@gFYMyXR8;lmF8jlW>o+HNa+?x{R^VrG3a^q-iRQ`E_UnNz%{A*^ zdVM3clI=b`L!2it0BwGG`3ctZ7Y3APB%IWRb&0JxFWKKY$8eZq9c1Jn>ce^CFIUwrCZ9T;Q1{LF zlu8{pTZocv+LClkX+>FhO7`4@P2_7!-$%P6epwdQCl`IseKm=v^X=;l;u)zrBA&KS z&wtA?=6Z$5;JSZygHdQ)%GO|Xxp&(8ytD49J2753+@ZRWJ|M&h#=8g?v~A~qhSu}z9i=g%GS z0)^b8%$Fizw@|(JIotq}>0#l^=Jl|VmQ9l;N)Bc*hTt-lw7%hxE2WYEZ@XkidDMfQwf5dX#VpvA|Du2)m-v+h?~GaFRhW6xoD&Lo|fr47Ot2D1D{*{g@{byEi7$6UO8w%ICa8E|`=_+w$pZQcB9QZ*f7o zL@`Qufya}e7|}7>W}%{gtPq0?M^@H8Ca^i~_}J5h(X|k zUEq6qMnwtm?m-%YG{IJj20zd7_PFf;*tsO5A6+E(t0M?e3@4KZD&+t;`!Cx$4yqH0 z03-Zs2sbu+-sT0Sac;+#?>2h23P`cn8{5RJr9R20T;Awan`Bnl^reT<1eFvKtg1Fe+KPCk~1=8@d+UKXp^ms~(H zOC}-3_pG)l=veetwLd#!Yp=o7!9n@0z7+!y*WN3Ns-MHmzw-;o`e0|8&*N?ZY|Uca7ZhG7`!%utAp(N&J;k2_{eqT&*WQRa45MQQti4iRU%;7m`d z?5d4fDMm*&$Cr37CMPR5jwW=G0j!K;f5V;sdYi~a%Px)-mr7D35;Uu{B{I%6I^9ug zjY)Ksr3RRm!5fGZU7S3=xQVV1yN&vkQTpY}s%cgFEsP`MP#b@ORowOUZ#|qT`~u%W zjEdbCT~*2cOA_^a5&R!Y%3|enJciz@> zCk2CgWQvOnUiu{#>CEanA&iQ6bv=@jYpr|xpS?PLkv8-?hfYaQFA}uBsjg{wg4%EL z=OPSP#7x#BrJnp}fF>CG{dKe=xQD`xnTk1L1 zdjId_{`aE#FGgT@H^BtT3{R{J67y#A^?l>C7I|lN zl0tWONR_WEc>gyvn*OPV`Sn5iB~tv07pe77&9Bjwx}Qt=ur~uy8YL_)1sW0PuQh`X zqGFUS1NDbyHhkW8w|A-7M!o`;D7jF)T=}pXL>d_s@~L9okR$m+7r?U06cSyQYFYEf3sUYMj&xKClA?oq&m$jyMsPWrok zpUH;we}9ADtBPNgypngVsH2!kZ^BoJhnzkZy=C#q315lhE)y@qzJM1dV&QhM;O%*2 zq_`m*hXK_%YZo#H2szf$%jf-=q?)8-#OPN0&QS@gN)>|v z5rfqoQt=7ybkKgoKkaY)`N99+!lV12>ac%pSs;63{e(9_2LGZO!Li6T>nukdF5*uNQ(9s=0Nybx@j)un4 zH%GN@I7#nX=fcLi7`?~0h#T%PeQlJs@(ZU=(F7yT76XK%ENGurCy4P9jB@pgiKe_` zwX+ClM*gEWQcxA3gmMU$Tl?J?{h^xr>7qQ-!8`m*+|3ikuQ*{%KG;arI+z%{tc@`& zV3?|RtP`GIisO8R>aASxCRJU$L-qkk{Qh!OertbyRa(K56TmS%QNdYZ7vB2D^&Sk) zTiL^{Dx&yI9(EMpj^VaybMEHiNG96a`=`$%JrN2~nY}Ry-M|9~>Zs$6wCLdzMQr4} z4~8c^CRLed_(9U6=Vzo6cz3!%;MvXHFFZip{F6|aJuiYni#!~NR+0E(FmRXNHvvCK*_bG zYl*6pkdc=O-T9oVNq5_LnUjKvcGEgUFJMK19BfEh00~ApZ61GqR~bgK9y1l&{F*-X zbwKo|+wVHAVn368eirlvQMpxyfnCS)&kb@+`XV~AE~Y++L$>V*Z!kvEYYSDFn_t>_ zCeVqzeE;a<^Z>};;E#3JZ(sOd`i-mwTH}Mu>T1zGVcLh|859C007z$ddM&;YXgt0( z*Z*_l@z4iY6qxrf-T5bh{MT|cfDay^VVi0w1&?`2E7=Zn=%A@;)D=e}~gliHcTy<18)BWGep zmU%^C<A7l@9T6# zXlwd;d$c%By_dfzr9!*mi0H6(9tdEP`BNSE*9G_Ig$KF9TNInY_ONsChcv)s$qDDJ zh4*6tSN-WjhG7EwS2A8G8ZEQ%C@#E$R4=%ZUdy*g({CSE>oE&7tkBq(ajUkz5mvxH zh7gYifIK2tO#qq<2f%!Dqi_tvQQ7hGb2xxZ24ndi;|pOG0Q!x&5haBu!2!j@P#qvm ztV5_@0s~k&G6Ff=!DyoyfF}R$F7)uyM-e~3dO*Q01U!{;A_@SUqnqjApJ5#ctRxtx z1YXa&_Y)uxJ{NJ27t{x!lkabbA@`UtE(9u-?;u*0UzTy{->dldUi@1w{yWadLMpc4 zgcm1{E^C9+`KD=($=^OSwMp^tIXhTL7So_?OlpxC6Q|_!Ex;(Q!^-|n<~@5gKD+gz z{cCMYlJ5*+SPh*#p|=qC`}Ay2F5`qFbCHd0sl&_6A6;SK@Qomb~e=RHF%2@(3STL zU>4@vo^Q}c4$0tHOITBvz;S7?ws#kDx!ZMmyDMgC(D*PnafGzp0N z$P3sJz~u+Z*A5r7&tnm$V|xQ+T^jkTjA>J0wdd?HT;}0%Gwe~V1r#aQ$$Aw?K!F~A z=63&q$>H!ixryT}$70YLozOF)`RCxRD*0+IE;8KY&S~e(y4`vVUU1!)3s&(1?ZB{n z@J=V~`Rt8DYg9aQWygX`c&)ClT)TH=HnK~ytcR{{P2(1W21kX8C_sW;u<}O^NBGv( zRO`6ode86$f8QyS2Odvu>7?Ed1fBj&mKi{d&w}_dpHJ5ib^r#?-pX z#Hm{^7+_nfL;^jWS0GsFi@`*zRexg1|FN|FIwT<*MAi&jKN#u?or{N3Ay72<6mCr3K=X7Nn`5od%HqA)V&>}D zpvS0}5hr^@hf4qUhyOvKBfpD+Zjzw=7h<;~J%NgL(6yS0E%{4+68L+ZA$H9JS4Ege zZ{1nD?G9S4*XnEmGDbIllkX>%YPY-q)GDF@z6wfg+0FOk60yo$Br~!54cablENf*K zLW&ihzM`n9sTnu?JZWOq;o{=L+Ci^#@m4JTJp+ALpiEDu4odwC$69{k29{_JMibx* zs0;?HYj0--Fj@(u(SC_v1x@psEDe#u*PN9h>b6T4ciaX*et#Yik-flO!e=+pp^e(% z=j63fax<3xh)9l$8SUu}$U(M9_R*M)K5~8C@+IYu_qz%th*tlSTT2R#Xrh@zoA$a=J>+S_4o_=g?5w;@^IN5Q~7bBqn^;E<=$a{Po zd~uAx06>yT?g1-!5_Axlu;KY%(2`_K=3sfiGY^4xI2A64Ny20`6niC ze-xD7QFgPmpje|=4U&6D!WpAp(=1@5G25Xk?}Q|Q0l;D7WR*<$UcsV$L*&jG+K-u~B;^9N7(mrn`s55)gB zvpoNEUGOhNC(xW+3F^fJ4ftiZvnxX*)3>8(;w4kYUPwxUOwK)!28+9R?`j=EEL!Nk zF)#i$Z6u$A65Z%SSI!N_$-7{L@Uv&02T{rw}$yjuz|q z+J%=^cV_ULtqcIX{W{p;FNPbwKAUpDh;#pgxX{+)DodUZ=!lpmrXt0{6-&O6`$mnG z6r5G%pmtKDv*d$k*J4AYxIq2^4n3?ns>2L-C~CyIM(2pKfn8$%I}ckipf=XpwZSui zXEL&#N-R|=iOl{kIkznfXzvvb13uv)+#wcMEdvaU5`VfMfZ8M4o!Gp2a(5M0dF=zrh1wh^GZA1CIA2$qR~@15 zwae7aU*~i&l^f!_kzY!`Y^Z+D$mPwI_W8GW3uUn{-YPmT4r`2kpasCPyV2Rlk~EQs z!-eml>Iv-wZ*f8(fV(4tFZv?aoq+?aI0{EUfc673f|#It4q!U?b{D_{m~$e>@*M=p z`@!VCL$hCd5orEhgJQ+NO8_GI*S8$PzXU)Z*}6bK3hapNc={Rqvlkf0LIVsEc+@e7 zXN_n_o&e1mQ!9i|o$$dz_?##%6z|jxU6G9^P(~a@;N<6*39&clUGPAuV$&0DYXw~f z#se`C6bUtp;5~LA&?3EwYa?*Dd3*;czs5)raPWf$Xk~f!?ivhB@f`%L%*>tUm^uf@ zU0osoCXq(E9^h2DgU$F33WA({A{}0rv4;%z=XHbbR#CJT0&<>*E9UQcyr4p0-5=lnZ<{e-*-p}oH z&uDIkn(y0lG{{EQ#QlPfU%f~nm`Ed)Hb2p`rib1b|NyCXqY-U=rvCYz*i_*WceYhkXZeIv@|W2`O};;sEJyTHmhn ze$eHB=YQu7ap+)t7J6|)7?DVnh6}O7EcdA7mj|g+>JvIS7-3g(H!{F++_T@;=5pStd zsx%bh>PuH$Q%v7`v=}`x^NgOrH6J+aOsYw#J6OgXKD7y7V%`$|rglH&W=;+%zyuD= zU?ks$@m;Q~WpR#|DYEMt{(DmOUqZWoDftvBKFt~MZv2ao9Pv}}8lQ_dlZ>;w&j8R2Y$}-otv4N`eHf|=BU}rb#{<#f`T=*qXxsAHxo)#C{ zO|3g4fx)ePdLw0YeRWk$k9otbKY2Gu3$#6Mp(hf_F8E@9jd98_#mdFn+WM^7w3%6Z zc?YHHYaLp8(7RV45HkpL`BcZBr8dJ(T>+S!DMd-JG%ZkS2_^I(keN7R&%ri)-smN8 zCZso=##Ll&Xgg1ZXq%q}&Hkl81BmHPl@@w=Smj&HPTG$+FfpRlgbbfGaHmnfQ|~NZ zyl8U~WE^~E@$mo%Xu$n&VvVIccqW_nL}V{|kR(7Ey_cv2Fd!Q_)uwh_aFn-Bx&4-! znEmy$OtE17_hg?vf4gTyB>NLz9+rv*4Jvd^&OZuxOBaynbb3>1tBaYq{wGf!ZeIv> z$d2&Kj(>o4e1{cUZJ6nd6WU(hjhQu7Cfsjs{|?H^F!9C}7mK-F=UDi=Na5En2}}g| zn>P@P3qSnI2`V7Db3N*Vc@mzGU4(mq;oye%h(_cc8xMthBW;u9^gKw+=X|(nIx9;|vpA^pj2<++tPiq?8gQd+y*sz8YgA#@D2Qk= zue=v%`_6cJ%`dQ3+tKH|_UX6o70MqO?rkWNzN3p(E$o;PKf5Pz&!P8;wY^f+$c)6s zM9&L9pI97U5jOm=HQl+<(XD8;*K}=&dHd-pBEA(9#pCfi9Xf>uLt#mYGeAF3(v_CE z)g^{i>7hb&&&DA<(K?&fH|+*GHI27=UP0>u1@jA?rR0~d%0IICrN0|nMWHqY8o4MN zg*uk7PoA{sd1N$42TEE`YZUpf8@o91oL6*X?W2o}i4miLyc$nPOxHCNPc7m9Tqt0` zkjFS;kv{aolqObQc;0_gEV1c%3QI(b_ItZ7E<=XrD| z%)lg95?N?jd>Pv{R{4>n3Jvd$;c88&*@#nhAn&Tk!)-2;S1%yzuNP_4`A1`o3tMKc z-bYuzms}2SS4nmAw%>Dn&4`$%y3l%j`*{1QBzf^eVeg}no|lF7A*F6>{bUR3)egSC zvMCmcYxZg1u$d#YY`oOeUOfIhjTKIWGnI)E3xh4|k&N8f(A;pVV9Oh!7$Fctdu0HH z_rcp^el*khk$CZz!NoExpGR%whssf>`^_U;sQQ|hH#;Om*x9vj6iaihT83G(W6ZN} zno7L%OUPqi96sp>N6g9(5xBAUX&CJcC!U#_t8qJ|*%e*9+@W=&k~(L+{=ILV)!TQY zBWj0)7wc}+HzQHS0TmnOZ|`YgjlPMJ%`Y8=zZNl*^eR7j zewq7}E47%Ci{YCbEd!nkO?=nOkXCJQWI2M0Mca73?#H*J(YvCY3x$8{urZ5Wwi`a>`kgWB$y6ss9UDRz@@G| zo7(pw?pVe~ls2p|E>E*Te3cV#D2)Un99!DUGWg^Y$#%bUDY zdE@KRQDbyCTqLH*xYYF6+Vu|D)=1q}&P240Ssgb5sR$ZiY7i7BzrHT%yvdP$- z*s{&ZK$m(vYtiq&vJ+;RHdJ)b-~{VkIw&bx^`m|XDJgL{XJTR|vTFOXm6JL%X}iNy zzwqo;kGC6XdflyeG0ew9ZK%7;G7l3Q;ao4!$WCZDL}ZLK8nw-0JcATxP=(k{-5}K( zm~)w<+gW~Hs?L&1M+dc4f8$b%gDf|`aCP=izH>Y-MRj>(ZO`6svN2%8t5)jTxZ%#s ztTx-Z8{uj^2_bJTvk0jH{ZJ7LW*lsN7z>bf=+L3e4oGqwXftvFUXDl|R zG`xqMa+`*}PP__u1z2%VD>!&Y(e2wTHhdgPH5Ib}pXR3RM_{RKp=ef6 zG~rARK*Y?bz2P(`Q;^;N0sWQE*LlTO4$^Z;6=*;)?xOEdM_VFvz)cB`RZxf98nS$ol5;K zs_`>yDtmsSb<+c7m^W8)p+ ze;pquP|Pl^yjd?|A#?)t2j6v+aMGn>*cLw6nS7jr#VaWAMuf#B5{B2g(K!cbsLwnxK&9d^yRv#^tTu zZ0!p^e4I^1*v0qHDl*?E1$B3tai6;qP@Ke@7u7YE6BYE0hj()#WvUB8N5Pts#|luw z9I-KVYE$C^v6|?u<$0>?u1yua^p77UDeOI0bj6oc%R`ySm~AbucK395(v&34i^WVc zzjtqWTi_3|4W#qi9AcL?L6u7y@TpHs8~T{0wOg3Y9e$+28gpF2C=5ndUs!ovKeE_V zt>M1y{`7uiD&$mIe^=%+@=tD-20{&4ZW(y3SV?t()yZ0Pt&h*xhKF*Pn{l=^NpAx# z&KgI1V$g%ShJ7sB85Uu|n2+=I@_ZiDPb!7a>>2Tfm?z6)8nH$7R_-KEkJDRcUv3We zZ%-X;PQcS{x;TX_A6O2~OdU>)?n57E(;-mK0oO(XtR>%>S*lvj9x(5gJ99}p)pKvk zUpbvfY?W!##I`F$=;<7!d^x^)1OcSBE4mGDT2%0Pd%auc#_!5-|JoW`7(nP*UGN0q zZ~@6-f^}?_-Fg~url>SU&O!B+`IZ={ulC8^`+-B5DJ0^DZ;5B#f?kokhzjkktCj6j z)Hyjh4dtcV=bmA+Bh?V+01+nwU2aUlaL2OAllkn9Xk;kBCw_6tH@;mqFl_2c+bX81 zMm?d)uDkZ)BVEKcQTxL-qWT4TqK#XmB*v%6TgX0uY_KkE92d~e(T*qtITo%pB$2_x z?{u%gH19*x#Y9!r&m`uF+Qfyt!4fO~d70$5YG5Z2b`XtjPC8LrfmL1{lGOMBH$(3= zb4k_Ai#u97#|k`#7+(>(f6rVK2UP^*FZm< zgWJZlnGT>ik%pto7Xri?2CAoRx41zK_^Y()yA=<0AGv2DTRaJ-1Vcz;`|Y?~=^A4{ z3OuS+XV>EphLjkj!`kweqqJGDQVYB#LLoHlYsXLWwtR1$S7lJ!4&@RLE4d#Y_|E!3kpQ}nK%ixL9=mnxxsmuE`LFo$iUT>m(8ka92rr|Us(6uuqh#J>SfjiWj9fEFurWs zWeHbN0~t&0lOhvnJ@m!ycAM}8JuLr%Hj$%g({5l|*RhA`nzF8Tnh$r@1GCD{s5wjc z>g)Zo^pCn(MwYQ^n+7JwwO&VM?fCdeOqKMsAmG70s7LGRK51wbOf}pYcG2D{?W)K5 z0`E^@9&thj-DTKgsXd8>-Q0--0?EWQGJ@oB5{@4DQ4J+}a(WtI(m7Qm_#Nb)N1(Uv zMDFNV!PgfN;uHtjz@M_fWmAkEtut#Aqm z^E#NiibWEBYK7W%sN_Kw{Cx*o*mM2L7xbiLg~FL*_jYhR=0Y;7UB)F38-2tE>WUOv zmW`i;cZ?bqNu)XPfa#zd_t2D&}SJnLnT9y#`bl{j^`8v zfUyr}xyy%nuN}&!bnKYjITNd&9T}BZ#eE z7&4|fb9eclO)*%|RQ#l-kXhpIXF6qi*kGa3iip03qz%u@L)z7{f}cvL-7<~KkaND8 zs7hJ#xZaoRRm;J}tR}=&<5@d(`O3J}EG7omWUR-eQJ#Te+qhBKc-oGm88id&&fddx zunT&AZI*vDfCn9q+Gys>Q1xKg->B6fsd5hud7S~;A@8}D$##m=F|pLiFwb76MR(b4 zX0{hd2fWyvt79S#`>5OR8_)cHt@M3Jbn?{frm&G#+=P++iq|Y9rXWfLC-=?~ejdl2 zMK7x-ljQZ~1qb)_&C3_>US4$7a&@ewh$<1#EsT!f6Meb26|m%$ICpu$4&^SdG9H*$ zvb9sqyOC37v{K^fS_5W-T=wnd_Zo}9Y@mCe%MHz`yCb|AG`J3vP7`9}W>M8sq|INa$0fN|~JlvfT zk-P})y5W5b5m@D#sZ-Z+pZ%i+>C-Vql+UR3Lr4Wb_wABf%}tqC#Kl5ljay}>9HeRT^=tP~LUp7(G#KfZd!Jc!Ez0ahBdwM}9IBJ}O zqkbG`kG5RU1~{S{$?9tmYRr}=F zw%PMtxa56!8x5X^eYw5th|dyeX@AdiNOh5`qhHz1x0-n~u+jN!BgNObIW zMW0wo4-?o3{eIBsYTvw63h4q0dSj$etPkhA7_^w`^821LLD7yjlH}H^10Zx5esO4S zlrS$HAGiMPi%oL=LeV6TmN23|U^Z+hKo@($lWe*Za3)Ig+5P%Js%SkfyyM zJah?e4^wpXake=sF<7v5CkM+|gjvO$7+KzX-bBJ|knCeGIjw;(;{cy^P4+DSKm9mV z;$nM7U1UmqSj}rl4p|as`h%FdV>hp79?v%yHh`*|cT7oL*S*z{_|=_zC!!xADn0d; zZ86RWCy_XHzbZgg|+?xU?M{u3pk9XBs1OTMmz&^;BqtwOydDYl`Z zb!V-{z9|2L-qAw~45U^xLPGF^1N((LV_9PcPhN&uTDMQ$y%PO3W_g9m-m)l1b5zH@ zR;j)(^={7ddDN2>;}P8xJ?|)>{Yh)K;1d-Fcs64!I!g%1guoW~!yms1j6M-lkglF- z8ZLLac5AS^IPN-M8N&`xX+K}|{KUly>V2)cT5})Hbx_@!>>9ucd{Xw=a;i+K%A<;h zxyZd>tWenB!5k}z5!l)8Y8Ju>;N&m{Q`j=okTK`E3Q?+L9b8l;#Ptbr$VbmpWkOx* z#_838$=ULWAF9;s%kuVRu!G@+L%mmwnPPL8!t?mu>!Lm6a>sIJQmgw0BX)yE z(su)tlWq_O-lK6>l+)`X--wq-c8(>5Oj{g#ggC7AQSRE~9n(^j0t~D70<{9D*yh>J zgqv^{ZrYk^HU#wSXSYRdWuVCFYDrj^RcmW@Ilsz~1jnMB-4Q+O))Udjmd?g;`le9}H zChUrwaYx}o#Ru)nkT-semr1Efa$LTQNbo-h!&;+^AuKc4h&Df}@Pv0*kKU#VK(89J zpOW1&f5qm+9j^=;fAQ&5RI6;&o1Dug`6&plq_$6d{d`$%B>x|K?;X`tyKawypeRyA zL^=^^QUs9>fuMYVfPnN)RHO?C(g_4ndJ_d;{gZy(c}odA3}BR7VuUf=z!z~h&R zMX#9O8ec}Hq#`QBERS=BI`qR@D4K}&C2N6Z;oca9=VJV6ZT@8w_6g?0l_@tkvjjA@ zQW}L6Wai>~aieY8**8*jG~L{rjt*7KBK=YFdAXyVSV`v0?9oiO>O~=kdS6rhnM%`j zJGFVn=v|l%E~WjXwm>*}n>R=+C+np`IQmXUZvN*Im8Hpgm7C_-*MyV_r`7T#)!ub` zF7@d4Vbj+xTD~jVx)S^tou3CH2Z`~t6o4QG-g{3Pa(bm4dEi}xYtknlgK!lCh_P8f zgz_`2;w&Vcob>Qaup%2`^Vb79L>xX}hqwyMVP2?cx@n5l$RazZr z_)XE`bWVbJleHuM#IFPShSg}XCwHtPP13RaQ1<9{qKEh}pszYf|A+hn&^l29Q3=4f zzR)+E5xMgWpbpjbp)$Zg!Jc5*Zwi3$S~rlFf*e`EkkTGz*vOvnzo&R8_a(L|7iL&H z5!H!pLyZ2h`U$6c&9#kM+e`dnp%)%ATvS$tga>!sizXmCq$0pY*UZKuI}V!3oYLkgl{*( zh}rUU^%5FfO8AVgi#o`0{b5Tn1~!3&A0V&0Tvhw*GA!F;7j4Uru5?@iHS^~}BKhg3 zj)+$0HOj;^#0z8$jVBgaYYu~aF!~59?s}hF+ESkv`?1x~fE8qQ1+e>t!5Ko>2b9wq zi(NuQkwN|k>k={<;tp>wu@t1{(KtiJUjBIy~P^QEIQ6HRy5p8O(OvkEA6?YUmQVWXf1JV}H!sKc*}r0|x2iO>Z~4;3JA{q~SKnW3{Q%`I@qqOfKM z4G;PjVP>Goyz8UCD+vSPKmq~2~k=fXM zXa9Ko4Sp^*{Hx*DP`Ru{$@kNmDhig)0#_kX8dJv?rQdzEJmZ(L%~HUuT{B*I^pOPLJ*Oc zYZ2?$iDXzqJ>vBp8~Bw=e6{X+eLZph9pyTQUTKkQgGpL_-mIqIlVVJ_pQ|T(uMS-d zJGcHv!;2@wGzsL9Kdq6%ATF0kGOl>WP8HGGVQ+T9wB0{OYcy-CC;Hy^UM^s#=8fC7 zM=u6F38(Nw(5W!O#EB9M2RLLK^4vNP$Jjf1{4Y#+K-AjkPp>_|9j$-%fb!SyKa%SH z2j2tTBC!xn@U**xHR2Q=*#DWEN2OZ})@NC9*>&Q{0?Th{#|D`0zbM#F!i79pz2fD)W_CKJoi2@+W!rZ@}GvHh^5%{E)NVV z!H}4a1#$ZCw~69k&Zm|7AAPg-4;*+r0haLS;&Y0C{T!j}QxNHJzuTduO>vUG0P6I( zK=L34k?miljBQQ|F)> zV-W~D8DxMsm%@%nVu|ZtN-yb0A-{bOD<5|rLPHc?dXQSSnqOh|L=Wz|EnMdLu$9F& z(=^oE2Tk0=;sUUQWxBG%3J<9t#ml*aN@H4Q7&0qFc79MGRZq&lcaA~OFxJ*6WaC&% zJ&+68ZWBaiW$ccJbcz3wMrOx#e&Yuvc?iaNK#QFVLFQ+rDO6UcI+F1rZ^KT}sC~HS zuBSNFncjj(hRdjV%--SBo9dAG`x4|?@sN?B3YZ$cC{l&NU)`;+UOm|1(48&n!tftY z@84DZ0gHRtEV5r`#A%y*A>lU#bIYt5xgihH9W5C=}#1fd0Kw*?#F=DFFq+ac+EUYP^QEL&&jW=JBw;p4}i1ofFY19 z_!)(nuNiq+hji;WR~Eo=VmX0^O5pfCv>Hene>-&bHw6YkPE)f!J|e|#z>o*|fwQcD zcdFT&-xS{wCoYc3#L{{3;<@wNFoT6%4*#r2^)}*YvW@r(uE%}&eQN%LVG505ue@AV z?gtf|Oz-qJXlq20-Fv2*Z@U?4u4U+ho}ieADC62j-`Hri8ppW_h&y~*!UQ$XW;SjG z9<>KswFj=Yf@(<0W4901ep6JH^Id8{32REgy~2D@lN!h-wuk@11__iE{1q(C%`&1X|5vjy4Q9u@;01#U{SyntQK!lN2r>s zlUhFZLeFYu!xU}-0&q}=E}J^ZmO}Aq2WM-Wc6S> zX7Q%EQr2je!o8;4__MjkIZelpkrnR|)rGAH z_Og6mc0WYb#4Hn-SI|Ry5cwYE3Lqfu7c6A-dR>UU-5}DQ@%ZM6<5{o`fQe34NJJv1 z`2bQ=Y8wXl3Mc1Q5CN5VeqIwX6#!Q@JD~*X7C;5gXq%vPHd#c3E)69uR2*|Z24eZP zA!oOFJaDnCnUD^~DTF32?K}VV^x^4qcv)2Z(FPmU5Y5M*f=x|zm+2f#Xk}yp zx^y&yK!O4dC_JHbMPBD+Yqw^VUbc~bPa(>uVY~~8SWN%HEe6#so+2)`p1d$g8N48h zF@O1LNQmlHc2A-@Rh-~HT`t$a=XdEn1!sY|X*>v8&oj6^n+{D}&bCKjb^&ts(6?G* z<{QL0-b@c4+0IDqDjT!XXFmC*NZm7HZ#hgYO!>tNry&xu2)+s_dy z{dXo*6pcR%xmB=tIxm0R&5UzC%ka70TJ_ThlOS1*hib~~$Sj4ZN2U;PsfP`up_^yP zmmoB5uaqTrLVhsncTv$bX=e6ha|wouM*K2w3c1-NdH1AW;c>~H zib>xylq^~lX~SqaLFRy|;4hR4)|dNBX(EvsCj7Z8Bm9oEcchYfC@6EkQ}|Xm6w=TG z*0TlDbH0g#IGAqsGSMfz?P(FM%l56F{EE^VL~yVVSO=^5#v2q4XC?EvA{e%fv>Q+| zyibDcofmOF>qGj2XXN}MMS1I&bmiSQ{&picA&QCu>_xznzq^^t2)#w~|2)!O&i&kosxqp^o}L<^oIHVL248)=gS{H;DE z`k^1dP{^x8zRA+4<>w$F3+bfvH^$agN6~@za};`7qMeY?A4B-EN}?knh6pX0Nlx2$ zPH`6HJuSS)Ra|;FqZsnEs#Eiq!~1|fS9#LbZ;Ao#00FZMoUOH=5XQ4^+LQ;kRR*@}zsv_;V7*T$M1Pmv?n z>IHcn0uW4(*OYp{xOg4!?RQ!M8Jy671BrwhonX#ZOG25?4f8T@Oolna2}3oV(!B`P zjV&&Dl)XFx`AKrU**3(P`2=?`FV?!q^l2t18NBrTxYMxH+N zFC4v`sMyzby*8JtR$=X_zN|f=vtvK8E%YYvHY*ZvSPb_$eMZ#2w7U1D$?7~RVkf~9 zp`whdXmRJQ1I1H>ocz*sm2=Cc~0SoCH96*dmj8Frfa8h(B9D_VD)}tFbn}kXX za83|E?mhWGAdHs{$oAcfuCt<}V2@H~o{*XkrR4~p)!vo|FVtW$t(*!Fe~zuZ_fMUk ztcDkf-VZ9h?Y8QA*7dDS{@z&Z$d0#3*Rz+oio}5VxoAuIVVK)TwUVhXVa8?-n(;rn zmb`i2%_p_8bMc7QQN4fah0aA?Rxvw1Daw<{vn{yv(#Y2l^eLntMCE;%bMrFjoN8Z* z>Llu}z7M$aJ*L>qaRxL&iwHb$#jChRWjiV&FE>CgqCaTOLbK?#(-Zv+e`i9j?Rs~ElRVN2eZ?uKMCJ~Hp8&<=~{u9;iO!h z4cZDyKDXiX*eKh@B&JypQIfHk!${y6A#n%uU-b_RNE?})@GjHViLDYJt8$;ZG?VT1 zn?mB~`+(c4icXntlKL5yDl=@_QF}rT!0k z)3d`%O&8vR<7^gdn!iWf)c?d6Zm{)%R@l?QLz^q6=FHgY6y13I=B1diNj;J}5E=o# zy#P#TIr+|V;#$(L=M+vwT)}kal-9>e_V6cb@h5v}RWP%}l*I$F?CH|ETT3N|CeK#o zd`qmDUd6Gq=v3!yWFta>gq-+IR&i--m(& zqHi+tPJfN;+t0s6@#FSf0R_MX(<0{)&`QlV;hW9FAy=F;%Zsjao@#cSuTh9iNVx7b z(lv|!1UO|fb#qC6G0-TE`epde8EhkAs6?T-J`ZTsJ?g#YT#h4psz>DDB4 z>s5tKqM2&tAIIIiaX>3i1sG`RpK6lFa%s=rOWQDZkRdq0j=e(K#_Ex*oZN{jP-qHC%90T{2{)Gm!>R0KD&@D`>2ht_EXMWa zu2+as#k&_|zMJ7nr+TjXnbV)O1nZygqCQXLrUy8C6k7~E@ncKA{2tBz=-}CtFbJRR zmrq5k?9YcH^DO-Xlgf0_2F^t*HJUlFYeZ2;oY#;^*zM;VRW2zS+mbbV+8UwZ4?UjG zS8?G{1pxuJW?YqbwXFwjO))D&)r;=3C0!;?gT~F-&V^+I1HRjSm9>4kzSGSMjNz}_ z5m#Wkcvd46q(+7w_T@!l)W0`XYQA3-F<)^qG>a>SyiOHD^2mtg{aV;qcy~6nT{xG} ztOWXG_x;#z|M~ToM)arXOTs|_J6#8joS8tb%+AocNz5~L(zEKBZ zad}2W4=gk)ougnS@2=JvkTf^I!9Vhoxo=+q)X1k4IXm>+%;sT3!E>&!H7@&dgtC=f z)ZUK2yt{boGD%HG>BoGk{5_ei@dLLm1Eicdf`!Zz8L+T@!%2l_Oqv;jzK~;2kI1U> z$XNcMi}CWzc4U`X8!6gy*2vj?mOc$jvx_|;-z^uPZrXcS7KF*Db6pRD2636SSG)n# zt6-A~mq1Wd^srCee!7P2dn=2VZ+iqukI%Jp(s15`NPyYI>NaGG6upiF2yRt}$n|58 zK?m#!xpJB4!qx!UzGl)~A=l9;zqeb3l$zX`#lI?sQA~IzUV#!{FX(r;?p}gB*Jo~Y zSY3{Fc$=O4+59|L_BjRs2g24Dr3~h1nY2#4XxTt%Rz%{d7~@*UTJa3D@(71A_I>#Q z@qv8I{sh#|uVfOY2$)Eag#Cg`{OhnPdDz&}@dpzQu{~iF`H18@9p_&5unnH~ef%}` z>-J!cnJif=kZ<*SqDtSsi}#xSFSdon3$uxthR6U5CGl9Jbd<_D7vB~UBpwbh5m9Ej zR~shGdxD?Wwqa+TFZDbw2<1u+pbyiN%r+CS{^VIELewtLO4~p^Jo*X_zF!X2zdiKT zv!gBdZMsxkG8&~-hta4tYfflvwWXe{4yG|01^X>tPGoKQGO$`Ru43)VEZBav5PL2u zdy!k81ry$8T%`A?Lm&OpZyVh`xEip-${t$~-TR`Cwgf$0zOz1~)hIaUHh6Vxo-w}Z z_Uby6q2=y|CC_Vc+@_sO@^nqhVltmsJ1Ep>Iys<1ML~m)(f^6-_tt}&?X{sVW!=ZN zu6;uZpR~A9?Tq0KLTzUv*fG8O@5q*Mfm4l-#Wq|6J`!Mr z>fIL#8^CeEb@<0);rSbjtiFeCJ5zfwgHHMKTOAq&PQ^xH)x4kx(Y_&dzrx-g{v_7+ z#0V&^;t0>{9CshKxf#=$y{Q%x0BJC1G{(-`t^3R^|*lGeu|MT#jaNTrs^I;?X? zwl4YW6P&kly3oXWF*K7jsRgxnt3{0u1EX!5_lSsvXqZ$*G}Gni54mCiZy!=pqQKt# zOw(6It$k^8RZ3WLG-DrD^CV`8VwIuz7u%HU=}h$kxG-meopQlh;J zI#+kbr?cN#ZL)dgBU~+KGNwtlh{3>C+}`RtdxeZ5kDW?a_7M`Rj+N@_pSw-}>hhdy zrbE&k($YrCYzMo!k@uJVtD@kOaian0RRmbCXc`G4>S2G(IV%A03k$rsozf4%c|cEv z_BEP1Zj?I?d1f!?_H$4w`yg-l8;JT}sn6?Pa&{unAH_Z+jq*l1@RRfm~ z@^J7BbZpa^;agBnZ`YOUI+~P|rL(-kLj#6k9?bkaLD7R7jB&1)XyauF zX*nKqhbnn2o#WrK)r`Ou;c<(~+zKl8xLdouHKXeojh7seKX#*Mg{#S}`0j4bzBBq? z0fb4k^$RUUs<(8ZC!qm!^6SETxX_}tvk43}i37(CdsoiR(ONG)@- z%l=!ZK{%U=$%V|(Q&B5#e=g-N887R0miMTlZAV+F*rR%%EJLp;0!H6xSp4Rxdec{T z%nF~NXZEwF&BCIEiHSuJM;Vntz~LfsVTdF$M575fTOqa~;#dpv2m#Kg^iv*F-(Xkn z8Y|ukcN{hjs3@HTz>Cb!!%xmq)rcI znCct#dn3b=ogp-bjNxzni?iY%_MJ(GnWRee4G5g4$@G$ZP;85yc0uES2u+0%0JFW=pSnSv_sXf2g-NR!={cMq%GTAfN zdBLl~-fB}mCLwD1r=`e9x}}xb0JVRkzA8G|`NVlqSW!mSv27_}Rw5FQ>%ZeM zml$lM?@dC?O}GRYBdUsGYh0D!bi~p{m9i4z)2Y0!+}qS1H+eY?@@KWEsMu-7DP!=n zI=1;HA`_Ux2CKvR6Ka@7-`7<8gucD+MxEL~-Ij8eYcyE4I;^rOA95L!#&C1$){IK` z`P5ZahToo#4&-;DFafEa4;o+KFnAPJY00-oG8HVBZNkee4=B`Pb})_b0GHw)|4pdm zmMeNTXQ7PI0MAn)AJMK|ok`eGBVFkTpFy5CbfP+bjbDwe0fC5%9V$u_J#w>r6Yhtu z+1X?Q)*-e2r2fMNf1ysx#8+U)55zf5c_fZd5t$r)x zR1v|HXaVZL>|Rm#hKUwcn*x|hQ-;F@-uUzjlJ5F99``c%Z+ZS1{`3Y0AX{Ox>juZQ z$crj8Gt-VLCP&dSERi0Co31a8NkRZ-1uMl1k!e#;pOOC9e_TflcV6HWuF8Vauj4Ux zXH!f2;;cNQ!!-SX+6@^G4l3bx>{gMOB&mnI%qp{bl(u7+iv)ie0N8rB(>wlo3rDYh zQ#gUn`D3uzW8Djhj4pVWy28tLFP1Q`#kBcA{?a9EDjk^=1#`ie3dPpVV8_-#X~DKW(K4n?y!Fon;=3Y(mdW8PNmhnzhum*0i4 z^(Wsk=)3Y+k47+5#SoB7eGTbA92guzNkSe-;siyo{`}l^r(9)RMC2lZZo0WE5qnz} zV>-6tkCshFV+GO#MSk2(PAH;2r^rFkMr_7sEmZvQxX^>A!Xl#~Uk@(2->eBjew((& zT3ou=ctbPf0eJhzt}2gkKG0q~L_WR!(xhrc8X2D(yz|kMyQ~_Di{IF6v0xJsw;ZQT zR_(bF2n0f1qg(;8{iblPcm=OI987n_M`BruA_^z-(yXX+(E+f-xMDAGF@{aN*>f`V z4w*d!4XOfWl-MkZGz&$YS@bWV&05duDkBCrWPRc<*U#{4H;JV5Y^(C1fIkE;M;JbP z-$GAp+x!8!N*4R&6nK`bF3|aI($Da&SQPgqC#?OlIS*qX#WeBu=|d93*~e9s8=2mJ z4|KIIqO)YYU)Jo-7m(9F7?J5Ttg&;^9*sD!HMqW~YD;<<0|NSQvsAZP=0 zwdO*@m^**<_dSwwh_QB@$qzx^>^pqQ@{4?#6i51DNX`~+@~|yUELNgz+cH?^e&&kn zAGxktw}169QFirExL?|gp+FNO7p){BdCsF?K_$+Xg+;C(%yo-(9SeR4HhzCF{_TOC z*vIKbitFskYg9|S#DLw1iea4vZR`Y|j;#s9FABKXxpyfaB_ zp-ia5(AliP$|dcNZB6`vt<6A5vJ70rn9R=VJINh&GgMw5%?RvSTrk%8EE^pHoTzV) zr|GELFtE}ooY@#&$G1A!rRL7}h~E50msB|OgS-!fMNoBuFE4r!g^Qv{t{)`gRp7Jp5)GAn@b_E(CI=sA=c-n>lyThzZ+tVV!-E0Hv7F_{ z)9>Vnr^>8G`=mAr&T8FE8W-t!E$|^WSO4cNsL@- zZ!7HIyn=O~x{~~e(YTh?1^xEK*MZ?>74c*NVi!icE-34d`UGX=JL37zo(!`;?qTEA zxuS84YQ6U#%u|9j3C(ADD`)8{%<6a%RGTo?HRCW#`6j*0ZhTSZe%fmPMdA%!i1#o} z-G8j8`HyF#{yHheWR0=1#ppN&cyJN+^VG!*KRiofdy@JNcaJ)Zp28pT8g9opb+jPf-+lwQAI34A80zW2Xx{NLO2*A?`?i{gKung6X6e<`N_`zZW> z=26ID+)BbTw$3V^H63u9wKa5O%Al=vwKw&v`gj~olkWreDn|-~k{AVSi@tK_NY_&n zZHCQM@zTc^bADmP_W79;PGGNyI8Um30zOC z!IgG`=-isr#ovI9inI#!;n&VN#-KftUwpQ>=MeStLrSPQ7uVxZm=8XSd};kughYFm zsv@wN%SQB<(Pwg2VH$UQZ9Q3@-`L2UPiJak_=n_oz8l>f*icTKCi&9tsVL0%>|F@Q zX09;Nx4|Q-272k&fJ%Lyjh3^oqyH;si+hp}mi9CFTqsUi=Zo6265oJm!IHQJIFn9! zMR@hW==Sb%=>0v+AKE|I)vmBVW%_tyKJ_P+JOy{nKR#6cPsc9)<`{a7a>zdg%NW+g z3omlE6~Nz4Km?U)^H$cS>CftQhQ)tY=z74uo-)~ZXDURL^^);Nnp@qbCw!LPE+%@9 z%pR<{#{Jx%yBabKxd0O=QAw8l6?FcGubXT?W9p}>+~&@O4rO9Gu=KK;S(L!I!(k0xohX-k8n z_G(^%t@vFVo(Se55wGRUDg^{9>;+M0UI#S>Gg)lDJGdtnkk-2`J+gY*!{z|(F!N>R z@0T1|Z69^>n+o3P8vmNQDcW{B588^nLSn+x4(jZlCuVk=#$2WTH2bjWkAvAry86~G z)j`oL_H=CJ@vWyHuyY|TiFS(BB9DBFzx+k^;XB7fVhUIkCW9>f6DB>bpu6ldTjf~e zRjHR2o#gH12LsBd?0*$MNxS>VC2K^=@&2cxZOfye905-@VCdjK6Ak9L_1eS3@AMW- zqNXuS;bVGRnlQiDEqfnHx-dp+vt7q2cKB8J6VMbNwjxU+CgcLq>vFx9<^F^4>iRx- zaPD|_6W}P@I44cHSJ8nW-t=h8u^Olc1os{tAzZ1IeQu@b&JtC|Ba@UKAD*Rpp0it2 z%g|^D?g1P<*Q(;y4Y(aCEXMxPCp;=66A|1}#xfj>AQ&Jp(v-_{=MRuEbKmB+`Q|~< zpzA7}#G^>tFP#@>m!!BmVJ9pGx)nk@2#tXhL@W=o0pr+d{er?%%6I$ zef#>%n~(Qh!o?d)OsWjY@+O9#E^{jqfA{)>Ab7(RvGA?eRUsr}LXue(ARh7Z@U)HhuM zz2&q&ynS#^U4HrA$*Xl$_7)x@1ee~X_$6>RT!qzNQ+@oSsrjz-_Ej|6o9^Oc{&*^i zdpG9y=qY^0DSr|p=K+lyl+~ruwMCR@v%u|Ym|^m)Y1~`KHDme~<-O!T#8fFLQ+`~d zEL#}5KvZkRYtg)E63tsn(k@OG%&}vsZ1})S$yEIFk)AT(>Bf1DG8xRbpw0%RbLPv& z+k*g%Se3(+Kp?y=sQJRwPG@(C%Z0@_kV5_%rNIALC1_rU^ntD-ogq<)iywjT6`@L; z;X21u4Gv$c#k^gMP%+<4I@(6RPxUR`l{csjRbKgTODe!dozN<4o;!daKliEDriys7 zIhg6gjB}u4Wc*vnZa;+=s9fLQ6is%i@5+bfyBJ!l}5gJxfVel_87z&jGwR?#L3~k~kmi+Qph_EGlLx7<(y= zaFD^`+Yx&80i8L{-$)e%CR&kceNJz?B0aP|$fEfY@!b969u3#alWJnDVwcfUYTANx~* z+RpV3P1Qf_@o83fN!PTydiM!&L}u=pe5w3Nbnk;o4K~H_xX^nWl>B6w^`#NRZX-i8 zOY%b^^2)||Ub}q5yZrIwgpw(e=Y|&?7wvCG4$lz+%r5c(tpAL8;{cu%{FAIjM)gBh zNxvy@!hq9io4^9}!ca91odLvhP9L7Lx9#X{rK&yKfIL#M(+dv&T(tE+y~{dH;g4;by^<-N31V2=)4Um> zX(b=^I1m^VD*Sq2*${J@bag=LmR6TZVJ7(i*&oKu`ooXa>zCEKB6Zy(y1sMwE}NhzQ4usB zX}Drs8P!XRE9#lV#_qn!wzqVy*wrEZyzLF8oYV0P>N1BzB0qdWi z3B!YH+VK~~oz(@_V)EXvypR6u{dxQyLS0zyNYEex5{|knCW|LpTCgn`oNv4u9QtLn zO>4@FOk=!^Fie!5dDe71pilgxN%%L#{lOGK%WD8gL0u06AwOG|UI0sb1+-6w6R#lw zOZcm}inGg#`eaH?^0i>w#65|68bZqx018?CP4P+xaAjwb_{&ZKpvZr26`B(OcXjjh zE&TtV*GO1wS7EFP(x=jd$3Jl8@xuB(sp9#eM{Y@Gx+D{fLwc1mu%62+O6gz z)QUavAxR3~70Vh0lgM9}Yk>8yrNAAc_Ih8myMA!Lc<|-xZwlj1BNL}i`4f1iGToG8 zJ&*AGw?iS?n1S-3o${KvOp?w?TKIeppI*a64Ty354tnbT5pxgWnp@JRG>0FCdK5ww z4%Ing%JTKSB7J5byCb?C-Zu=lkCe$N)Lg5dEo&4gtzCxFTags;5s|J~2!j(^cQ(hL ztyPZT)!ih@0ulZ2WM7yoVdC0#R%Tt3f@@e3a-N$#=i#PMr~di)Qu*=l@_J0osH=%! zt*e7Gu*3eFIwA@!9U-e9@kO3O?*I`1`#=Oh!Jh{F7t_PxYZjw$94!Qw_b=55QnSx^ z46GXxEffe4_}hx5QlWXwI{o_p@ryjWy2e8jvB4=^Z9}UR8~X53O*QY*j~;aXvw5FR zoxDh0^-h~(KJyj!FYAEtd)zZ&H`CM%y>mhxNE0T|kk)mZSuNoC!G(FhE#NQts0gmd&E#NjG{^e0fO8>Xd zOK)^O36gG*q8a-Tw?Eg>xKv~QSn4Le=b<}Q(x6gIxIW$T&vzaeot&$kt4yE1-7PUO z3KX`rE+go>$EEGJlP`{y&F()MkydizTVBpYUsFFxz9Ukg$#{O?R(>;64p+NOhpy3| zwUD4~=&|##q`;CSg{u2a@dQJ(GuS(ZR|+&?LK8;)}k&;$z-ciD5b=VEUXpyu^jOwE0o*G=5g%1{iX|VX64*ZR$L$cn==}R>(X+eo2W>0!0ipP%x#r;2PU;I*GsRmm2a}0hDUb z@+P~UuXoN(+|MK}_LfyVLz2wS2Rf^U87Nr>DH~O zdqAT16ZvxIGsTLQ@mC5T^n;-SnC8{{(s)k;(<8H$p>+t&H@0}ytS74f${l2orwLf z$5Sn~Nca;xB0vl^2=04@f*%FnCEckFc=%LzpC9f|73VReH?eNdc0ObA$s4cbm6pb> ztTTo<8PFFK<6YNWE63A&BwKuLTCNZgisM-{8D^n$?HkZm^80QNYP!<#qMyqBQM<42 z0nMwOp~$^0ZcQ~%tU<|?dZTY6#?7rDp*!rOwsq=fOIt+78vKHaq|+QbSWXu3*cvKy zQH1@bcs723d7W|Ffl7Xm(jGkjmXZ3(#hXpARiV-pl=0GWo`8SPbJX$Ks<`N7F|lK- zh9Id-yig}&!T?rTUlS+vL(AiNbq|`U*hTS+X^AflzN4&8q@9g7(e}kXmYImYyA^h{ z@0)B21cEefV+F z#8^_lN+l;;?56gkW**3SHaDf0y*`wiqwe45KBM9kXUn=V>FF0&>nOq6jfl!sL>-nX z%_G}6gI{V7i(3vZnvDAh)FuRslX|Wu4HYY_-Q)F=n%(2#xe%|g;xjEEoVag6h5}98Y+|SI+*{#h=C%FP#p73HLG6H(Bg{N)W9U72Bsad*6 zkk9iPT>E0}M{FYdpMh=B^CeAu*iCbl3GUKqG2BdhsE+!L`J|p7(|5`u=3nJHCaGPH zNKm3NLyx$lYBEDy$(&W(2q{jVIb=++JRYyv0lIVmi9iLC>z+>b6m&X! zU9n*ew@)f?u)ZI~S)s7|Dp~RCw2WTux!fnF4R5%Q%WJEgt!%80VFSYR*Vek4ik|Jl zH%qu9x9RaT?JCm4Bw>8sf+}tR87*|ZmX**DW}_h>#$rHl=eD?~^oo5@@~4k{5KVNu zafki;M5p!AYQV9?tZsRqv(>Of&!~MU6H{uw+ad{jGNQt`W{Ns$2p%jc*+@A{(HY%_ zqihD&6yJVGr4H4zP^Eq=+S9WaSl?`Rv4*YF)-XumEHE(pm$le8LuVy{#;t(C0^o^x zhEuN&5g=ya7=q>6C2n3S-|~5|BUr5=Plt~?>Yn{|S9Q?8!U4g!{C!H1jSDJc8{L_I z20FIhNT3Zj>(MA%UMYJy-lUJWYPaRUsi${P*N)f3i$2=3oI4f4Y+r5YEl90>=J{%< zz+e9P=N2IrF-{k$_}%`k>V5h3g;m46f={mXzyw#jwLd5M4KxZw1m=OZHUWa%_#1Se zfFRQVjl?>6poBDXI;4uD>9A44A|iKvJQ&wc29uT^5?-GgcBtJ*dHU7oez#ZrB9foU zRs$lQYi23EDc`$6-fXBLAb&2G-CJ176GY^x*b*ynE~`H{ie7=AV0T$;h*@16t8vdq zPYqIIy1rfL?YtJuqCqo8am%1Hc+w=h8>G(^(LQFp*7mN9r*3ENaMd@6?YUtY``DX+ zr=PYGtIBULG}fg}u1NA5@oC=ViOYZdnrOC?l=9jzVGU0qXl?yCKktvl>foI;vU1pz;3xtk&bWcSw z7Ur-=#JWPFnio+6V-zhmV*dW>wVtWF}BeFypmelRIn*Sa=+EcNXn&=JB#l z(%slufn=(zJ+zFD>0eAbSPjJ-`4VyuNPS(32O2!pm#VGI6>|Q zGjIulHkpsAVnVbKUF~A{sLWSdIIPibz0pEP3gNOT6{@rq9q#_!Az=`TwUGl-RM(O zn*gK+|HVN4Dm9SQz+$2a1cFj{f4JoIpPwwD>P9*yqVpCax8(q@Oh=Na|KU5}XrN@o z0SpxrQa`n>D9qG8sBLuD`1Nb1=KHcHV6!Uy@8TQ({EG&YH3O{efzKO;qH1EfCGKMDP4(QeJw3<1^ol|b8*6(=$lw?(q98F;9j9sdDd#Zl~J*+fB;FThQ!I1m>u?wk393uEJDDoAepdMg-O?y>#o}N~&)B zV3}t6mP8U74OK$_8~F!g4q&mQA2-MPo5C^iwcVmzTd;)ML>`Xt@!aN#ZDT z5n}s}^3bf^+)(+oj%<272sYBv0^rm)Yb8ZOAmJ83!^wk+vH;_eAx@3}y`fm;*@YD5^p#t-on%Ig3NAPc;Q;nW?o|*k=bJ)Zvx_ym2nskX@k;#vA3_sYt(Sajv!_}(!Sho zF!mraPBw?!NGatoH{Vr&Lo;wZEJhIwwQ}Ik&C|+)WcS^@Sg6!^lOy)F(!x;aA-Df0 zftL8Eu&C%C4isVrWHA#s$06P%FRZ<{=s_9e4;X`Px+xYso5paIYOz7eYC`nbAQWIs zibw4bkiW}m2g*ws*~zV_wH$CqGGL{uL%&1>PnJSxAIFuVAA3qKTdZ3^SA#0Xb|Ef{ ziJv`H3}OpHV+{KWlXZ?2jP3Vyb(S9<{TUXk*#wlw$E!dj$!l4_SE+O@#NY>X&k(@E zyO4(PzcCUqv4`&%BirJ>W60WK9%r!hl_w!%fJc}b7#L7=ho4x`zg(T$lPCcWdZS63 z{(#~|z?PYeg|C;qS7ATPBs2O){%E+pjf$|3Ae`#dW?B_{6fv%VwJm;#8!7NFx+|CF zMrgh&i+qzU@b$km#89>2suuyg3k?P!{4aWH6Mm%%kxWYUfDRJSGRx&4(p@*>^k z8X7My(uBS2^ukKT7`rrF%M@BFE#eINztyZS6!$Io2)KJl*VegK&#qe; z|A3)3mG_-ThQ`1MK^}Q?XU?L6_59TCQ9v%=0+$}20=cVC zM5E~o0t~Hp6`VO7fkKt9H*lGtws{L|*7kMFU=X}}biT_SuqnD`=GCe2>>FMOZc2U% zGe4;Ie45{-dkyaj*qZYv0WrR<r9v9v*}!rx-uuFM8!xA$>`zkj`Btu^z>@ZN^{b&6dpSL4XSus)1YNhNR( z0hlFT9Rh62{-hPigB^7JU5;qH6Y(a%6m*+TI~#qXaIW zv1#238X?b&=KRu0Z9yS_)M1JV@w8PUkXkcOufy#*;3{qpj`l!-J9xy&g;=bTXWyZV zbacJwyb%B?kwD0wnZ&^Lpq0(k@5^$7i5ic+NVQXoxA!g)T))cMNv!ID5KH-+lvM(Vg7D|Z;L$rdUOLJx60Sc^tZ&{k~H&( zDi50Q{7niS&uxqMp@@tIrPUykDokJLikI@xHkHX1YFP`(%+b2;Dy<(?0gY#5YR}Y;CoSc~fT8uq& zRevrnA>^QO6Q+a=%I?KOvC=smI@Fq_)ljD~S5~w26sJqDjFi)2(9)~1#joF%qK+hK zo!zA@wU!Pvn;?=HWzM3`$v5u3d!z9!dX_xrn_AI9;!3_x@Zzt@Z0P1Yr}uSOqIR=4 zEykLq`SXh;ulVXneeb#;cdRzv*Kd`!NmYXo_ zyH9S)ngM67!0)HLxPrYD^n};3Za}I!=V5kguuECGQO)6ct8zbLk)Fd}d|{yMfRG}8 zP-(dlOSA1Ly4X8;%o{<)b3R{NRpt_E& z{IS<4Cl5Al7c*@iQvD6mM}_*MZ)0hPPQgm)Whz%P%{)dXpfY2G@yMw}-&5rgyL%FQ|L^3DW#LbLU%_2~UD0 zj(+!=QN0lVEo*TpFDc>JYM!UIe1G}%OF{DB{N6h)dUB^Q@{CIqF3td;j?_rP`}PM% zuYnTjdI>;X1A5na`VjS!ReJQlWs1pm%KVYfe6-&siFb zU$98ggR|jO9G7XKznF!5->H4Lq08?v?+*~a>i4_&(y(`rR#Rt#A(SGPyv^WanC zTiM_;xN{+p5s|}`R!Ot@XdXMMKC1V;2}MTFcnog@GT9`4=VHsJ<#q}0SgfdNiJ5VE z(NLahkk~kPawLA0+9ch*%*Z0P#3g9DXahpg%1#{ZnA(jd^T8iQjSN}glYe+-$xP@9 znLfy<&6PVgZFP2k+#Z^kKoy$3V&eQrNRE?fR$l#dNc+Xfblvfm$D;6P=5y7QVx^$$ zTR5Eu#HLdJeQZn!1u;6`-#*U$o}PBU5$TpL2^v zX9lnc>p6-t=e%FDUVR`{6l5Y(R#3YY7#@KGV>9+H}I{)vas5r<%l$N>jtMlwJ#&L@~0qu&p4u-h?*CZ~Nibr;^=%AMVWa1{p3D zHAk=)!OftfWcEeCE8j4_Jbyqg52EU(8s#DCFH%j&1jjh|wZ0&Eou+(a+s(9@XGLcA zG0r_#ncN)$4{*xC8h2gYdb)CvkWw4&(FD34X5gL<%bz zJ6c*HzL!b)9Iye{{PM`eU0AUY6v$hZ>_foZ4(DkhTb@;9pIuz`FVx8)Pn9T8lmP9F zpFVq3AN;{x$S7rgJnAhUkp-s;l{8IaMAuXc^bRj5%140r&=ow%iu5@e_2rhm(fPZrpGPWcG20^*SjnB z7fo|4!~Ntybl)hZngKaPS%$S0f9&)ifm;g9|C2&A+4peyi-Qa89nPy@93e) zBSJWk5DGa@d`;eMw-T>S5AY6ZP~5GMGo?>sBVSD#41CAR`R?M0(_wBWo05gE&?UpX z5j``g3wZw6WH&QKmPTltrG>j?&PhIANB1R9i`kD*92 zdupn>`g{;`MgW=)TvgwmXB*PeF)(Q>P7O2&_Y&sqrh5bC?NxKe$Ju>uB1_Po$wS>| zz?jw%2IU6736U^e#9_X&->Gn`li2lIy@=1sTvYC^iBs3IjlXmlX`W3&ANiU=>nE-T z>rd{N?FOv{rxwiCekwGZZz-)E@S=Y5Jk1@?{M7gBma2PjR|rgxd(idL+#q_#U(cfK z^T7QZ_c#T5;*W360UxHXmv_lgPKTc2$85l6s|WmJy$w)isf>e7LV6&3PJny|=*1fW zmG0(NHuWe$jnz)Owk`%|=2MRJ` zTOB`>5`c#}jdH$tsXPW4d@ploRSzGOJl=LLeHJ%0b+P-J;QI%k1-C#T`h1Y--^^@_Y_$>mNjfq6YkWZeu-0-b`PWZCmCY|*CLjt3m5oqxoC(!)+ z(Tt&;c7hBvy&)RpjCxiCXB|T)tR|=g|?=A#K(B;ps1s;N#(Yl(pLGye|XG{-P1Q@S+KmbD8*gbc>rG zN609d<|MC>M-(D|gLYHTJ-uN3x|IbkO>~|L1(2l(%sdSs#*x>AOSwpJ@HKd!6MlTt zkXChNX}|X`3QA?+RPrUS1qC+l)pl@oYLCkWd+Ygn{KFs$?h*M_;NL+ieJ-b(d7g6u! zKIU06*sa;l5=WkgKO-9LBmzDwHRun@eWdSB2MJQv5N+@ri{h_+K71lV79lJS0CPEq zm;L-eT&I3H340KJJK%q0ZF2BPByR37}}(BgoMC|?5PY2Qz6@<64^ z{JE!8`lq&5aJ3fV%GC)_gMbi+7b>L6}V^=!@ydjfl^~%wtg8YTfnlYwD)|-Y{ zU+zgi1o0wL|IW8YQ(GHwR94Oqa@7kfr}EY&Zc$?#wyDFsdLhRiC*G#EqKliW19$CC zvD`yjvsL2e!=IyUIX}7r?{Aa8Uka(0}7VKbg)-%QfA%V%Oaj z9PZiN?Vg)<>KCb<5&9-oGI4j_XW#w{aktBNPVOv2T8C+Bo94?}E37#4P3rZ>IXD(t z2=yV}KPBo;m)B7=eo9~GDl@MpL=i7ve!{%nCYRLHP?pPeHpP%GSxjQAnAVCZ1v zDr3e66ZYGKsPyf-GW7ucJf^+1_S_<=!drK5X{qlrmo{WiZ`uZ~b|~mjt>v88&c}6A znZsh0gT@^>rFH?~_nT7rdb=ri5QqOF)Alo)2Pu>}7pomn3e(6OmHlk~k~Ew8++X%X zb1{otOrJmbxEhkm6psAR`n|yF+8-Zl1f|z=0|9+jzrf(}rM+H{V8v(;-iE1Z!q+tr z5w>z1Izu(CqBUgz}+Wz}7vUuA&q4qoS^!j}l*^V<~%Xe>mMAb|`%ni4cM%9rR>R*_AYYifZICG$CHE*_P%mC+y~Gc;qq!8)#YevkjcOuk(@7%DuXf5!qWzrm(0!s` zkO`0XbHd-N<%f*sT&<4fzr&p+oy6iLA)v-z1>*?G&H?yn zSVu+ZA9ntkDlLz0J(5KUc|kpO;}`Cq-gRgUB0VDJV%e^%wZvOxkKDSrBKKpj3WS)t z;YD8i((CvLTOn{-wpmynoy_0I#qQInP1k1(xjjQe}iP~Fqzq4iVW=9x)EdW+@^Xhdhm{t zSC3h6T>e56}S zx~6?m5&iyEyNItY`(+?=lssp@#;KlTfGJllOM9R=Q!Z<6DX*(z6*ss!IQ>oyFMvI2 zgJ}kdM`bfaTj0*h`#s9n#Zw-h7Tc=CWQVR?N>JlHOpfbb_R#HCNvF|vAfra4QZKu& zvKu5@fji~p-db68e;#g`7bbao)!BxJjI)^eMFB332ZZ(oEt}%almrzaeB-U_Nuk?q z`JLBd;i9)5t=|%FD1e3;l-LE#$s{{W~4DXA#BsQ$aH4HcW#K6C#!v& z%~{d%CjkbaU&bYn`*_b=9#T`A7emxO6X(0wBi>X;9*WAg2Np)!1%RcI<93GnWN1~Y5R4rl|?t%5z)#w z;ZNMcLvEnLPfRcYH}4tiXqrUYCO2Jaq6xHawWs+o7Uo1m+41LK`fBaBj#n>YD%gBY z--`qb%JGl5nZ@NLXBxlQNe#*p-|7N%4E-JH_7P%m3S>ZF%K(7f*z7!?Teb7-c=P)s zi<9(??=UP~P&ZXr>F za}gZ!i=ch&yRA$wbQt{l`&}B-3>HK-EWosB50od3R_i3{MAPS8o4D+|9FfX0qz5%b z;oN8iOZo0WC40TG7j@SxR7v{Nd7_9^H^*jMin!=+R@0sn!W;@rYzNu=oQaMl)BJ8%Ur?K|o<+RgfL!xN1W!siohj{vSmH|ffP!TpWOF4*j_!IcHjdqtSzfuj zQQO_!EkO(Bs0Ouk8!yN{bBlcuE*9d9nPyuZ$_fHB{f0T&)(+|*y3wB z39BK;9!nq%rQpahn|K+z+q2pL!94@ag*Z_R@;){^9}5F9u|V3Ami|Fz5j6;pm{uce2>|Eq z>}?~GoY(?VJ7qxh=E83P^ko$-V9zLgk9+bzVV3`9Yx6(1EK)2XYT`*5tI?>Pmjq@7U1V`ELctiy_5ja1CKVs4aS$P8`V{ZF2Bg|m>X$bIgMiRm8V|resSm@i@|wC#Wa-p)HRQodFvXP zH>HhtX>#?dmne`!tklxsf<)m(aFmQOiru+Y7)rd=?C2$Z(C8LHTg3WYxJcqo>WX#S zhu3l<0c_t0`xcv*ll$)4e%(M;s(WdpCtV66w>q&!n5m9>FVA&9K#J{cb#a``3%_&r z*&7);HPl&x;&5c2OBG$L*$efs-biRnarx14_LQ^s;yaIbmpv987OE7}1CoWPlQ6v1 zQ)c2-JWnZ*#Y9F9H*aI1CEsIJB^aP!|Azj!_WX8n*7mN|uJ%Bg@*d1u zaF8=;{mb#rfDSE*mz#^&hW!f|2KO#FmwXN8AllkS(8{sj8<8GLYu+77&2(WMf*Y}B zPH(cgaH10C2I_)#eNW7_PAi411J+byTe&~s6EOlL(Wc86ddS!4w~$*})?DU(lZwAV z3jIdhR#ABA2;0HDS=hM5@;mSCd{GRz?lx^QJqEL{ENtULo1upDth6xD!w| zzcr5M=$as$(09&HQ5vJ+ch5zRIN}`FosaGZ{GqOr5U{#J|7+>WMFOVtg9wk$S2?-% z+>P1owNexf7YwYCxRR@W%dcf9U_>fQ;r>YJf!57oQD^7tnlS&T@QUszCWBxQ)1FW9 zV2irpmtFpy`3%KL^+9dD*_2I*2QL**vGeWE@#-Y8RVeufplNe6jWF(+gzr`dAohlK z%Yi=y<>UvkX~WJ^ip#a#99g)wVTY6RKQ}t8Ly!acJEcBkzEN>$6x~LnOjdDk{_Omu z4($AahUvDge_v@a1)I?0XA<&0T(E%-!&vs8C35?`qp(IDx9o1lrBBlQ<*6#GZkNd4 z0X_K-CLq8?<8>A7n16B5L`vkIs+lYh=C-xj3N)t?4McWqh@*~@BNq=qz`g^@P zdIfeGRn6;Az<>lgK!ndzlllD>I_-_|Zat!!UA6qP81hj4=$nh2@88M>lzneo7n{3|r zi?Un7o55w9(W_8aCl&n`_Ogwc0PUvQ9_BHN>vJ=;tBQ7U0H;axKbvL$n~bu*Zql(C zb2H*vRwK6+pZj21#9RW~scLG*d8=c~7haJ`>UTKCS@#S$xv?*$|E|cBvYUh89_}_Z z0C6(UVQH?skDIjQ3Q4@7$3)1cqoY&6zErd!s1S|u^B9(nV6~2%YL+e}(S-eMu+Ae0 zh9IQvC`QXZZdX}(QoOoi3U-w#KP!IB_}}2c;p3+v{w#}#uAx_wBrYNc=7TiNE~{0C zOUc~Kd>4!^r|@CBv9)EWq(Yv@_9L5`yv0}dz}HAJ09TL4zi+>sa#A1j6df-p2d+Umvu_~b&T;@OC~oFSDaken@~G0ow~OP zc5Xdopy2xpIpGMp5cXfcK`%&(IB-O>q|-g4=*OeatBt_h7sp(cS6<4Sfr#{EY9^ox z1mjr9^)STAIm3;moZW*nAb=+l&qG#+L-P;^^&fmr?vs3h4;)pL|LFqLZj7>&1L_K< z>8lq1^w!9R%wE;of|Lt)NeZS2fNAw$$!41+u;sOo+;<*lB01rA z*raj=%q%vp87*r#TJ5YCt7Pd59qJTP1ASnca(DSvFdAYs0$1TLth-I3oFS6 zvbWL`?U5!VQ#=G1<1x*d=-pP;xh0m*cVSItdu#9YJLag)ir{xX{UQcw;N4!O3~9e|Sui$DdoB2_ckk|(+^~T) zeS=SOrjCGyDYHj$fpe5apChuvnsa3)tHB7_Kg9)p+k76c(gVIwgE;SBW%Ba{p8J!O zs&FLjTc?}q*moC{t+p^6pR*sA7ZaaYiBk$O<#YSF7;f8+ZZH4BOEd!wqwmD@aL3fq zFJFIZU=U(Hu?PC)p?$N#7(Vfrx9{j*UovyeO1N~^3E*1O{lxkzUA&%yXMVY&E~RNHsk z(qqcJ3a6_pIWp|!I5d)`43K@=`&m&Qy+~QH{*478?MsE~8z}%2e)H$aju^I-iV<`B zSHvYXj`RI@O+D59v^0)%40lSrq=T=9!f+ATx(-ou-PDDlx863NHuP&o9p&4VrvvCo z`+{`$K(x#LAg<0A4T~v@w4auMck4DY1aKCG!6y>V0SqJS5(9j*HJg#%IKI=G3R2d0Fa3!Q&I+J!86Blrfq2`QZzzZ0X+J}$UOsj*v(GFurM;X07nnOpjRoR9w;G|+ib$s_xGPjtZtHNppx>Y+FFZ;gseEbkRd;aYZ z+W`oqu(yTU(9kaubL=-7O2d+cEglJ(z(vLF!ap}Sb{vNIYPK}hPCFB|aEsyle1|&_MVhGh8@p)D0z_5cYUb~uH`#wAw!Ifj(3ci@%${x?F)jK zT;4Ukr>PPe+k#~xbLl=+d*W~?5emcWbkO8+tb834DP#!@#=d?c2~v~0zI}Z^bS?0e z*$b2NA9R;m@@jOd?)pQbgxt%OumN}12RsLrfXyaNd5-iLHXjGsX2G_IHiH)-Y(yut zYBr>5-fI2lcXZv;MOu`^8;*!`FI#lZx%*o&NOr3HGA?3b$#pt{?bH^IsVOcr7dQT> z{2^*;q?3L%LnG5}cx@)DP7`Wy2xJ{vQI3;2;L26Ff(9>prP=1u??Vn-!|!%Xo>dmD z1q;^+x9d=b8#qZ#wa$+y!(#WhWUsAC{PF2b40dqGwVi&Q{Mg*4dAx#c56>hNoY;G zc<;EkvJr#vl_|+ph#QSWOy7W#j;Bw}+%=?6-FAtsc8 zv!6nvMWbTkPDWHzhzsmDC_j(`b&Y&^gPR8@{)WVYQRshcgBw50Yue58nLl?&!d=%~ z%#;3}{GMjMtm$b;z%FSDOh!LhxFcGHIQbOcgpSKZ zztt2)YKmtAlqch+i767Y?){0jLiFB@8xv+f;?kK})aw`*?x6dYjstGdr_-?3hDtORynUbnQ?#+0_?)i--fb4px5VbvBM;$Pm z~4UIYuzbIf2-I-j2kXY8+$9r&N%zT6Nn74I%;hxTX{hQWYTKUtN1Z6 zV+f0XRR0rn*w?D2T9hgbR{x6g&eO*W6fs^kK+dJM$&c2iP7$kX>e>^ITj;|xo~8&Q zQnWCs|Hjalz5;MYx?#M{aulHgu{3?zj~K8 zQuJev9T#~yxS`DZ6dH}7QR}qi(UCX$;vMVtpq?dAYeW3X_d9!;Ue()HnXiijy6Hn~ zUY>06d_M-X@2x%kxOm3jppp;iIVOeM4n`I=*H6kf2P}6RT^t)RHPA2!M+jd{b_CwI zYl=~st0v2?AwtcV!76#ZzVapHo#OU)rq!{2JaJRMTU#+Y#C(OF@})~8IvK0gc`=jk0UnMKy0X3Ac} zS?RtFD2Ki&-rQJZlRK5`Z>(~it@d3@tk|~V{OP?*;~lU2cJb6 zLr>(*h*ko>LGXCM06Db2d;T|Q>Ao=;rTH85xF96aco1A3_OF{QZt+tGVDp?u0z4fR zvcLp+#P8!pz*XmVEamnFLJ)LBo$xtOP7D5-M@;$BSruxDpiGVUg(?ALVMUuO^(gYg zan$zwnUKH*d-A4u>PqYe>FM!pK+832`y{e|B z32}q`!ICdC8PN>}neR*%1bJz)q(I?xSF>_8oWM^QK-2tIduglu>8 zkv!1-8#I)LSP&NM zi>Z58MkOW`&53R0d27#x#bS4NkOMg*b97HjBECCyms&t;Qa0wTxrRnIju#JK{;O=3 z5)c!*fRCkeq5-=d{|4$r`+x$x1&}{hE$71pU@SCiw?`UofkuJ(|KkAQzv+Ab%|rfAsH+iMUcg}^ zaY%9(jQ)&K_^4_t-sbO!ga6xlRS`@uIQyu~2M?&h15&Z8AyiHP-ZrBnaN4m}^9!3~ zF!v?iOFSwKx0|Atweg{CAL;%7k(Z$c9jA8YR3YM8Z;+>_SbCt5EP<9oFO|V-=M`@h zc1S9glxTQ}?05Gunb<`NJw6+6SSSJlX2<{jrJ%salNhYY`(0n*fy87FKdD;sO=4B) z*?VqP!FL6ds^9!6&#uQonzrwO?f))Z^8aYA0$e?c^M~678Fn6D<_F)In|1CF2xkhZ zeoV310`b38oF@dI<&v30MnkT&;8E?tdOQ1Cqcv<&B{?-yO4V=WZ$f5Wi}fBpQ!AId z{*q$-@6Jppjwb=_xF{|=3_BE$I(HBpGF2(0Yh_DOK6%2J7$z0(-%o!ZlUkv>UKOeC}UEg;S=0j{S|VI=u3)a5KR z6)03t-Ov?;ETU5ni#nkOwmpD1Y7dat(l+5B#7OjjV?-Jl zM%(!SdVWE2TmNs+>iptgP-9UR#y}ASJ?4Xu??EuBAyh!wl>Qq;lxPGh_awx#<=>$F zU3@A)Cwn9&{2TNUNlM58DljmvA4Li5t+Ws%HcP;T0{Q9*9MuuAAD}JXOl1O|#F(tE z^BeSzo@h7!3J_uNqsX-7DB?2s*IfkZlw|i80^e0lU_ibGJS%S@@D-*aN$h_64LV@} zwtp9lp(M}vElw0Um@M%d1UQN#^SclS!1E>mqVXllzd5IB1Ule9DgOqs0SwW9p3#4v z(SLqM|M{)|=Y915j3cI@5q#)i}%>T~Qt1hOWaxl-0c?*{pffXgHmF z*m{`u(sr)J(OfI~X=G7!7!36~g3?xPbwpN1eJJfKR2)}h+)9bM?OUy9UK>(i92FAv zZ#CWjd%(&+ZBhPpH%;y@h@3~wZU7>OWB!bBd}9f^C)oik>5#P1(BB~ch+^fhXK{$7 zaWb#~{wf1j&d*^Xgy8pE-v5B~gDM@)2$V{fLx!18meK9>tOYGB5#s0LJ{wIqC^pb( zT;Y2(7sC`)xx@)DnGPEN0c#(Htl+b-aO@>Y#dm;O`_4U@KUI5nz`2RSxWxt8Fg~1G z`}U=SEOyv>4fCs=l?EHd`nhiPw9Hk9jh=jvpVU9_j{kFkDLu@q8uy5{^r9Wl^>~$g zoTv`OdH5Yb0s_EH8T@TJ(wU?AW=4BmK2CLxj;waI&dgpu3OeF(x zIK`S<_!4zxAb=Ri6^^M4OIZDt$s{)=WE@4_YFW!)%W74NhTkF%VQ2joEuu@E{3QIL zovAE@bVbjrOSunKZde^p{gI)6)wb{=a#J27+ay%|acv;H(27H4q#XLa%5aMYxi63gd?1t_Rh z*nipxzm)D5?el& zHc#eWebKT(xDcR1Se^(iw65^8(~_&i3AQfBpKaRtvMeTQsM)s|X&okQRc`JRVQ*WKE0*4y znB#7~i_PBb+chd3HAb5&mHUu+fI%7LqD5Ki+h+3uN3O9RUm@fuQ{^g))5I=b}cxRjR06nW4Yz+!1ss{WD# zqE|M}^-HLS^$T{IatFE=@T}BB^iiR2CN|UyTA%#Di^n%JElfNn>0)DLpCmk*dvBp# z!h?MB=Cm^iTZxSM6h2Zta%dl7`tC(qpBqI)*28KS{*mB}X}pNPS63@5J~(&YA8J)O ze*JCjW5OuauU~_#bOwtP#o;$o_!Y-&^O|EMMRLK3V+IumcG^0IKhhibcK0L>$OyJ(v^+L>tjzP1bv>80;u+AX37ojr{d|$-u9m5S zQXt__N!~&!9bKj!rl>fxp_VP2yvagHJ%STN*T^EcIn-zHHAnXGvhEbhL|^b#P}+

}UUJ zwdU3!(-gUzjd~-pDLt2lhI)>?EQW5kPs9|Be#xlXTnaKE8Y}c~=yhVhR>evS&ek`H z8lAQJ-i@DBAwOn(>M^x!9~KX=k2?ht@a|8=}aRbM!9Bjth3KFNwC{$BigNMEhO|!@0a0)E!F2{v76eZ z&o7KN>uG2qEp-Jt+*mf0d5Jmr^E)-Wl8;Brx2tDm#`>C=A3fj~Qb_27?B?oFzD*4uti<#NW)oM0a#2>l7K2uZ`O7`{$+_GLDS(b~!t zaB@;Qm^d5NS#p;L@@wN?g-GD)$3m_H6yuKU8?Bb|SguE2Qgoe{H#!r$u^s0qUte5< z4Hk(Q>09MU8_Q`;&U*)kTuV{!33ws?Yhf|G^Ji$Lg)$p&cC159P3@-lk=oC^u=5{Y zQt@Vw9~3S+LLb5#7Tsd9Zv(Q=VCvnr#$X>C1lA1) zfD>`!&WZGIlP&f#H|`5}-|=m3(zJI|JK73Rivz@cp_rU-*JeaM+-Y1oI!i>fYc}hE za%*4h9jg%Yvu5tB(<>fUGU7C`-$wjD7<+n*7gXD?-DURafNRBt>qPcpG^E8@`{?JQ zYg2u-PR`a#vd0(VuhFymtEl4dy}ak6DOhfH-IM3VQ`7r96yFDeegWz!(@)eBbrMfV z9SZ%3ihUs>&HQAaF3LMQekHbXX7B%J{HFi2e)+$b<5ZXMi{!_3u>cu_G6>k9q_~I2F8@iRJaiGs<*$j! zd2CvY-zLkt(v#ua} z_q1uLYajh*>@%631sr~)8T$v`(0@JyhGPdrX&0GYjfMa{j$gcW??1ik##m}3l-|`3 zZQ~5R$9?})kle3Kf0hsec7o6=vSCHtOsmIg-qyUeALo6rvCHZ4#xT|pbcxz_QN&Y+ z!|x;v~YTBy!nQoTgek0TIHXZCB5Lh+sI)j zyouVI{x1_zM@2Q_C`5OUvRaP*P<_*#{!PszlP`Th!@S<{OAWxM-G>-?Tk)Z4h*_r( z{pocoZU##}80m@RJdt{-&ek(#4kj}B7^nBFc-wHW<~@Eq*LTyIKc4u$Pqufa3$PC5>c$v%`sZuXeG#yi zC5rwAspPbaB$&*M6xa#wtG;nf4d|6-LMG2ARw<&f@m+|p?LvP6U83Wc8JV($j1PJf zw?u6t)twkGs9N9lw0W{x1t|+KA$45pRW(uD)pwebE<8X>onGYKpH3a`nBv zMUSRjnxqoR`^Wjeju6pPEh#7**a|H*sn~C+qRMi+u2{|p{0aH_N3m+HrTn|e?$xCk z@SBoUyf4QkOEdS9zB9v7X+V|L%Zp1rdkwfW4k*l){{{9NM+cZY5~4YP%8l>%0~FA- zlHLtn0~pKzBUHuW_=ps}0Y@F=14%Lz8)(kp7-(}}jNOcG8cR%LxM+%u1C|r6I(-NgysXlim<*PP5vXIR-R5dkb z^FhPVV)d53?lOYGQ$x?lcW=FD4PZ()bAdbPBhry?%YEv7TGWi=EPeL~S5J%LQ!F32 zU$U@W1ts*n`o>*g8E!%=RovjEVVoGuo%)klE;1^HF+Gn_dk_JJ=yf&y+lj{2E(Ca8CVj53cq5_f znpHP{)WxJ*?YVY=wZre*zfiJC?Kz7odS**9Fx^sbIaCk4iskl+!9;xs;qKkgh!yh2 zPDd{%Wwe%+D5nN)Orum##8l>ZTm(iSs>Y2Hcw(t|*5`!Bug_;2J%0F-j)Ein&bJnAGUpqM4-WpDEV z`}RiHGDPXGE`*;?uya-4fCSaI|Ha;SMm5!L>jpuw(Tj9Yno^Y_orp*m5RhJ?0wN$? zdLUTnO}ccI5~-2i3BC8;A@oiH0s%t2>)U6aefR#(J??V%IOC2xzMmm0i?!ah-uIpJ zna_NlmqgzP{7#&PWmH5XjN?%8w3I2-DbaRiK8I4+_Hf}utGEqR5mlB(>m%cF1HDK9LT_T31_C7O95 zPuukx70&W>S1L~M#PGHQ{2VGD8MLLqvu~d=HJsF^B6jfJ%g(KaX}l2-7+{T%<-$E2 zmDVRdib&7EzT4Uc+_#KoW>iFyvpi1kT@AS;$8>gex%=)@j6wB5eqjc;<7)h~Yu{qj2RPAh6 zv)?sU=N8C+%H=vIj-^MBEj2JiBxHHWKmJ}-QWUj~;NKY)V2>k@HNSl_|53Oy<+!j% zPxpg|^+R%%tPC|{giL=*2X;#uo}%6I;+}AdrAQX-bg24dA;kaHkK*AkN=HwCi7%K9 z8y}s=huif3XU2wTjXb%t3B=2+r>au@!H|88T3rAgRogo!KU`lz&f38I@sCef7=D7H z#>nvXfP~c0;3A$GjR=yqK4UZVfHjk(KJ>7B6YAS9qkS@&(f{}(sc@q>F=(}xhtBI$ zew21r4YGgEixs{qQAoy#xs@%KA781voyF@ZhIm?L6j#f*80mMT{C?U6tUr2K z6P;alUB!XVD=n#L5BjXKvcYvm(Om+Pv!xTzVp$y0{KERSnw|a@a?UPg1^WD9{sAqIC^H4DT33|P_r9XZw?2Lr?lFX0n6#5cD`O49j?SbZKw@<2*wJsEU`O>DfFDk5*GH`v~uw$!C{A$R$m%ICrj`bs;<~Qzp-@sL& zoqFi~Fa-HSRzG0l<$ilxp$hF*qlCcIWVM&uyVcXWBPZ{hjKJHIu1oXu-HTU5uu!R2 zV}q|gik>$G-wo_@eRh`22e6O#oFLqR@HjOp-O-9$6#|I0k@F?7Hm6p;3<4&Gm=?9f zQRv5E-L9mS;$oj0>gveJ<|d#IP&AKUXwT4?UfNtBi+?2!F>OWJP;`HNgzZ^DsiN^J zF?*pcG=?l;G|W=U+qgV#cC>tJ#ibj=EwY$TV>E)E>Av)ZuFE8Wxjm zdG6SukJu8tWe|08sP?M%+QLRlLYhTaXfEWs-+S)Wa7U$VF_@?FcAmCGx!|POS*`OX99Y`8ZQgw=c$JSWIVWB4g9irC@w9WtO??8)tfErUF?n%bT0J89KPM zSfDK@a(%)(`d2?1HoNOBpKJL;xwy<&b+SUl0B^F>*g?8Q*bMJ61%Zv+pG4Z7|W01*Wgt*hR~1fnKotl2cIoRE7HfZ z2&Xs!Fzrb>k4|)hmRoFv4?w#lKAmN;y09GkeZ z`(AIJ72P(}vS0hdvHBag5agp$wW1%oWus-w553ib;6cm8;mvn&3O>>nnFv~6LG9S} z?&wS-pQIvGIvrFKhEX#Fgehszhb{elL31}p5- zCk2J6cFg%~DXBqU6g6GiMRy0Y7^s&t7&#p(am}pZd5JG_&3Tc&zftPl1;+x}o?K*| zoW^Tk`2rMw{9S*7GO`fI#03C!OZy0-drWUu{W2j>ub>yGP*?mrPyz==HzJH#yO!EVT2ug-- zHm)zLMWX8it+QrkjCS`eqi;9Gt2f*qQlbY95P$x=DV47yu&Bt@N-ZKN@e{09MF1_q zdI-tPtj7yLR-NuMRb9=ODo^Ey-Km{rlE+7DZHrfbI8M5%mTjCV=b4F>ov)0WR!BIN zgiBJ!F7=cSuR6idUwx%vgCm>s*qfSoFa!v2w9`=agP9QKGK#K9F z0@Bb3xE^o?S7De_aD{XFDJT+u2_UDCpRO|Ep$H@%1ZxV$0$_}iP9AK*AB;U`ovzY6 z2Pypo5yEgmJ}$sXW%cKQmU}Sj5zYe`@0?L2`e8cGH*#kNj`w2+iWWdSfoxR(K+NIY z1IWaze|vpON4o$?VQ+Vuj6OCVCxsaSRj;Mmf!m1aI#+yka-Az0tWX|`#G~l0nkKE z4KRCt08#;e8Nr)>4B#IF_{SUgS4+(Q{S;=ZXa&EA(3ipj@LZ#=kyG+IquX{Od5ML5 ztcnI>o1$UyVHGKE0ZdlJ1@4xfeTK%2s;SbHVF1vdbmTsdTDJ>qeZHc1FPtNFtO8hK ze1q_#yt*>znK(P>g1f|wT4&>n8*f8E$ADhxf6HV23o`e+LkLsRV$H}4P@y5K z0a+HTnksrX*~wEB7w=_P+<2ujc2#NOGf0D2&JRQgB6z?|^LKDGV9CbOIGCkBJ`b;z z`#fbagM}Lmz44&Xpr4t|JX0Dmj{4CH@-al2UL^zc!IaRwh0>|Fn&z6M-%L~6y#HYY z8u?d7>A&3Eg~IIW(2WY~Ocf&c9in%mYea1&c2d{gcA8G~Nj)JB!*?XH?#O+HiXuzS z^QCpOZR`1bY!=ldwzbJA8#29VlIWHmLK}q%DsCFe$I^^JY3OUu52sm(ZrafvZJTD7|7ZJEZ%>^nuZkQ2Ukfjs?tsua+e z6=%0$^}Tafuop^zzhS{c(FLK+OBJZgt}emJ-TyL4zee9}*dp-y3kih>G+4Tti<`M}c5-s|a*y>+&UTg_j?AF}F;`I}qwhe{d^l9)^W5|+Jbf9y zXRviAuPuSpomY(oO^7&MikciwiyKt*f2?OEtzb|_$E&yn6P-uQMY7-R`Ci3+3AJBK z@~W%+me~-3FUCo=S5ePh6E%3KJ#Yx{7u4i2EP>x%0p8dbNFGk9S<}CG(#5RYArsUm zcSy6X*F22Sc)DC_)84RhQZFdbI^3q6y!s(F9+6R(9<%8I_Al*chHoD(2?m_b2e`s^ z_{sW9zW2t$cfOabK%@1KKi`Gg^>^uv==Y&E7HJLjuW6GdiK!{&?(dD;$AgV}oeUpW z_7kR}Y?~+bR$|ZB&!ScP(;@C^4tAcQvV_>iCGcerF-3G;dxHbhj7rvIyUkE``fYul zdmpbub)O6KU~S01qKBcm)8PGk{gqODeZrDh>`SaiZnSer%9|R1h+Bn`(!;Q}>`flG z2u*tsn+?0($*u_D=kZoy*l03+&^mUvCa6u5@N3Le^}R!$rTxh=4ONXPk-h45PZPlp zjgIYfs0?XK$=K)^SL%dYbrN|JW}S20zTfuVz^)nD0?~&^|7DNpHaU{9Ks#$!q7dCB z2Tr+#{la|V&C5p!#dsFc?@nKIUoFO*A%&OkB^EqJ2Jx&K(C4+~uh5oh&N-|morv@i zE-0jjL1jp5&<^bk0$H2^pDay`6_+tt~6CHPVPJ{`nchA zzWLm?rJhACM?j=kO*yd7GCXqDm5@+InS)%VZ9%T^HUDFc$7w8C+~Q^%?U33d;aV>B zMMiiM$|)9&4W!O}n6aX-_Ga$9R9)0|a+U>--dm)tTk6g`fT6bx(7UPj!AG8lcg>?$ zI;H%=zBPbIon4X+(FR?gDg1%9iRH0s(~@7epuszJRRH^0}IDrv?&Wr|y-6k?2Iy^f*hTT>Wz z$(|jKC_e;0=taeOL_Q37e%(@%uCn!^)3+0yvf1H7`?30nzFwAh31aiivdq}fM*sqz z>tV(_dVef^>Xn`n8?j2rGjbw7rIc=6iW1|cWAVlVs&dl?s@`hy7;`fP9 zFPLODyv5#BXo%I+IQ=L)5=UIbX=2|3^7`fVK({~>n94@~RQg3<-$94pD|={luZtS= zHhp*+Ymv@d`o_%*%$?(Y^bUO^2xs_p9e)#TVJlFFGL>^>R5oU<1HJ1Kj!eG&;rc;o zM5bP18)f0G6rKU>!fphF%mqiQ6M}fxGy5I;NnGWqZpfjl7d~}7DNL+|t-%ntGhFQd zP%DFWZ>e^x`8HjaInMCm5~GyRR$xr;GqGos57?;MiNCC+nFqJ(ys>@~ovc@*r+RMh zHQ`I%xi4+u!t~NgHS4Ogw$}UA94V=+uC#>BVYmI6b?{3&MuP!gO6ybmnk+NxacdXn zUI%wV`CISru*E&Q+)TaWxq3o}stt$-=edIDx;F2f3mx%CL0r$cd(@u<`98ObP{ zQEP>#@9oy3we{_OC?6mL+|Bs0urt|p=jH_;C9To?+D3ust^t#6+ng&>hAJI8mDgR` z3p_pO^%_^c+axOBm;#Z`I_@-HF>%W%V$0 z`yjeaLGAs)}_LX)f_~Ctp_v8&G2-yDKAo3 z3!9x}REn?>-7_c-x*qLlyKa?lcp`c!Ly!L$lDGmZJ_^nURQI1UxYcEv zZB)ECMfKcp+q?c&g-*(V@V%L$ylNH`L%4G3?hl*xd2B{R(<+l3&xC4J4ykfpYYu-+ z_Ta`;VM6BS7}9F&O^xZp%hSCSCJ#yk7RYWIusAB?Y{zmmXlSxy%xu=0o&y92P7SE2 z(iItj3mRlX0hOa1~mcYqpzE-YI+uG6`Eq*{>b)P<^<6?K7!u=6fo)E2_!@e~&aA7Q#I zs>pp1USU^`ohSJBcX7NuhWunZ#OlMo{Rw@5=7oSymL^3BXVdrAb{f_~Z)4F{F3yJM zp?2Q34xf-LWtyy8vOMom=Wsu$uIoa$aEF*(reV)cj_-~MTkIXvTxAoNt|AFG(1F*d zq_TJhk>w3uNF3N(ZO^WnszLm5I&ng`curUbldFb3#QJuY=9~?#LSx6)N5qJur58cRIy>9)A%irn>oZU>VWv?WY+Yd95sTNp`CHw`<45VpMp*$e<`V?RN{tIl&0{gp7X^K`$$IPnSxH!`eSKqtE?|Bi(u74|GGnzCn86K5LL76Rv{%f`Nd@iQez&7L zas2PZXsiSrThKH`eV4a-bWM;XOtKK-f1hh3dT}WMQJ+Q`J6k|RnXmK1uy@ZHCTFr&$wae z=ncB$z|GxZS0;j^!x|%dZxQ1;#?Y$C)yKszw#Eh z&hFbp%Ya0J?NK^5coFgeSu)>37R*9g8nI-AKJs_*w%}0TV9KqeEdB|i%vOsU_;AZl z)(Fda#)ZvTf;9?6a+S-3G~2@ZLpW5J8hNfWK4zpoP=!FRIsPy-BY^5VI9do`IqW6S;16u`Cq3kTp}not=%^NvC&lm#YiGj1;)foCOr*cYPdQ+fkR`tTKS3-85Npfk~KT zdi-4=8_UiwpUF&C+1U+#?DS^zdk%fPlRHHkx$1z+w zUH6dp&K}Fu6)t0sOBSDUqbZ2p#2b!92$sOb+&k>Kz9V1~2Rxh2{54n23u=}HCf2;q zeWcXa@~s&?J9elo!aAOWS6&Yja%v#1{3V!9+yFyAL7a&HrdM&;3_z3vw-EXq0HqWT zlt#IrO2MrAYK!Lp9-Ysp(3l_-& z64IE8sxUx$;lErfX2v>_Q4!Y7r#NS_?WTLv*-&mOmmJzaD0UQ&O@(2Sk8ICl>hEkO zz4r1s4zB0JO5!)?_)eU924?pr)7)r@3RR?(mk&Ra1|A{M_=>YTIQt=BxeAoDYi$oi z)P@r;W=GnHSABPT8me_|pqqXo6VufqPIiY4C@rJa>s*vEqVS!Ii<)FMcQ+Z#3NY{% zu`w~n`|G>>E9AA#H+S>%FUtbA+VRVgCwm8by>}uL`7ZrP5O{K@JYFY-f&kzz>Tu=` zND2ox0yN?3?v;}aFC?|>G!ISF#ZYIBdpH?_&QV&tr&~B7dIlkxH`%Q258%K^ z**K}DDdh8V41S0r>y3WC5O#jJR0r}p{AX|Pzu6TOTwNV3?L=3WH?ET_4t2Aak3Q0h zZb|b4X}lr~#mZx9P>^;6$qcpz#TsS@Rmz;4>x64M5ox!t%&_TM@bcvOj6Wp3D}Uh% zHNnilf93oAZ^k3>8W=|nG}{FR)k{NZhPI(f-Wi?VmU#*^qZt(ZHG=UyU2;s{Z)$2> zbhj+cg3w%biV5}6+mc`PXlK^s@4(RIj+pD+Q$;5<3wrXu@kO$?B2jW4)A$IgHYAoD zz-OK7yU*Bx%9k8q4sCVNe+t+@0n1i0*|8x05%_n9Ms9~GofGw+o&^4J*q_Gkj{*7P z4f$g({Kq=-$6WYhF8oV#VGCwHPqbx5ab7De1Gh1g7=_B+!YUr^?79-ajLL8jC6eIt zeKG+n^w@(4rlR2fEw?r7?K@{r>&T|1#e3JS40RtXvXWev)MZ$CS8S(9mi(y9y`qyf zH*P8Y^KVJUfo3agLcRe>Bg?-z&tQ&i$2c_l-WpdgAJk_Bne#=iL zebIe;7QM6>=#L{_m|m_Q_yG%KxjBSW`ZD#ZfSRIRAT5%)_Bw(wl8xg7%@a82dBAPL z0h|JAhpM&uQWnaAa^s_oqz1y4Km0JYX*Xi=3Ux+Xb0v>5*a)=0HyL_lgmpjT_K?y; zgPZL%A$7xLK1S*X?%W+lIk_WA*$bR=LS{H$b1EqT}S`A4}9_Ax)^ zzaP~TchwVNbum|A*xP_IRJboEx^5KVZ&yq69Mw;1BNTgYHsSt(hB+wyqJ>KAGAX?1 zGgHv->AB$ft;ADxh`4X{ zpZ|d~lE(L@b@~;+xjlW+t<2KZ^Et{+U`ZFSRhV8=W4dh`Cq`Zq#ANaI0SQ(Ub3+YV zgC1@MQ~gk5%#$Lcw*F`nKXz;kUC)Rkyy%lzxIiY(@r2m$@4X2Byvx?#4iUi`nA%sy zgBW$`%kylHx4l5S+*Udav_Y| z*YY00)S4ez)xT+i{)5w;XVcBuM$&FBoo4&U!G-8!K?^bELqlcKz>@3B-XS1N`Xtqn z{V3RSF94t~ksUgO7{Mb}>c$F1Es^zazwFaK3Wvrj#y`AdE}*eZ>n2XWOgQs@UNHWA zJ%kHj>}`Eq3akNgiUL@S>lFHV(JuD0XUEf=l$?4u!cNF_e}WK0z(2l|o;i9Dxb@{f<{WV- zehR#A4F=}>w)!aSM=HR{oJhv2Gor}OLl_+J!A78?u3xZz{8rFNA8EM}>$csr@93ImXiYbs zzea3#Claa$WjsXg(48^v+xAY4Z5@>P$DL`~SKZ&5*ebQga|4Z11W<82-5|d!_(tp3 zjlpD7RddR{du@dl9PbPXI0F_`bow00QjNqJCd+iqG(=3Y@^w#ZjK{MBsGeE|CrFjr zA4$)*&B>hS%95|=VF=rbJXj3}c2nptzpy-(q=b z?8M*M23licz#vQRyrfTRSfKt|4jr>II`(=a;fr9Hh%{H+8pnv6110R~3y^E+!A|RD z*FrUD+i)H}(Jo$gMU_?+i6tox#OEovvdwdyKpNP$suAb9svj%$^RBphw~_TD0~)Zo zyAL8O@;O!u!wOhAFQ|*PO_`9`SLAk!*4$qr>QJq=B4ndzZ0KNHu%bCgEHstQyCKc5 zE%eGMZpd=~s@Soig4HreNialPTDw`vpxOg? zIyJ!1PPW3H+qmU@_wrswSO7)qoa*rzP^>@2)qJp7mAJ$% z8*cDz&~3#V=&JjHjZ1j)51ozoZ%-z)OOf|!nJ(;9^hJikblfhUacg){&s1IyTVtMy zGa9O=R{3h$>6B}f99OHyNVO8>HysugG|is|mZlM=8*XdAw7AFVeGX9XrMWNbwW4?I zZ|c6t&@id3Gb=Lw;k#eFb>Jh*q+g|C+h1%ugvb89^eDuQUT>sm9V)_{tcKD_8IDdpd)}?oQ1t#nBu? zB|>xFc49n2^j1T|5xLvmbPUlM{yE~i`Cft_z?P7A^`xWE$N8m&xxV<=H!db?S9$wI zkjlRXb1f*!@wVGwil91)OLwZZ)A=CEdA?G3Zhn(?3O(t9Nl$uYsa4A!J=RzsZPnaQ z0GYBjf`>aK4Qy6u1sXphVIawE$a!JO>=Ir{$ai@<(7ES6Pb%~!zl35ml4q;ttQ4Pb zMogxWX{GnII%K+P6;hhsY--yGyDZ9Y-U?%e8Wzr(sM*?)rn^YWTY27zqYwNjocgVB z!Rnpg5yBkTHUUSZr*?TQ%)W8@eRH&K9t&HZYKo4Ii?-&8{WL7F z-@f8rld2ly?N-zK!#}FHau&c-N=pv!x21V8Hw@7B!Kf{agVVR}Y`)@olUs37`b3}J z&QM3^UqVGiH9jpm;Zh5ZXTM0_mua8TYe_RazPGV@d#vd>kHnCAr=1V7C+4UhyrS+DtBm6t$IIu_F2#_keJF|2G)W~LJqb@2C=YUfW|s8H1aW84{lx{ zQyz8kqb1eUMWP}Sj^uac4?dCu?!)wKESO&|t$!Q3vY4K!`VPpFo+s8*+OksPp0mZh z9`6`_P1Dr zVc}{CkBv9JSZhYKBSRg%}JU}0Q(W%F9sWej-h2% z(R8{jCM?O5hDG>MklWLL)7Gc2Gr~6XX1Y#qh128p=jcU+0Acnx1-^&hRw9T&&gvjf zvhA!hZ4dBK8P*eqxzXs$I`+m}UCJO`DSrOxBT9Pq2nj-7#hy&+L?MC>3Ff=X#HGKB zasE2P2TUIS?S*-B`D>~rrOFl$9#&~p$!!wsD8%g=rNo+04gRSKqWzjKE3 z`w_yaP~I8p<*y5Ak)P?F3+1nJ2<67<^N4}G{CsW~*R}R@vZ!rL_z?a7y~%`OSep=} z09HaAD9gTiBPq-)J%I9QX$U>}gT)7O2h?B8N0zSh4mH0l`(7zod7HCad%SKBZfA!T z{z@0Dlf|QE&7A59muSKnV8s^p!i>}llNV#F1MFmVMB@%)zeNX2hj=GUwIsC|Czh5|ZN|TkDH8@=GP&_bpV6pY$h<8J9)@|9WUx~fL`{Als zw9@Q@D}9$pc^`9Azns{9WbJ*sZ%?=D=&(KwVV|Ylb5Aroqbofw0-zJvSC09zK|#ap z*pHEUBoC#M^~D-$OsB5UKkd-uZ)jWU1TjUsm}ye9#Y2*N7wYDWWhA8_^2tmYN|fjE z@#VAh(EuEOX;ON?!)lQ%QiS=wyOob*yZ$v$zvaa%?Psl#b#Gt=*u>1OLAZoq=@Me? z!~nS?8EM$YfBfnYLEynCk99&BgNd>3J6vrY9a@E!2b?e6t=wKNagvp!*kF=2$DS zp$l6iZjfs>PFoD}COKSVJC|>^U7jJmq*Yw-_M_ung2`X5WPiJ^{d+zUT11PRTU$nH zQK$*uYj`3V?R^!rD$9iJMx{qUWl#H?U{oG1CWCqa2Y*ejI>7t6^JQ!5QjWr(pilR9 z-3ezX2}TBfbtQiyF${P#;-zskn#7DmBlWF$KK^?yfVE-Gf!I!GTaS4}c1`iT-QhRy zYc6lP96`sC?lk9MFr+g9)dk0xFL{{&G%EMKqHf~iiH zraatnpVDzLxMqOpT$X?XBzOt;(*f_%uao^AzBhNIrSOk6@hbS|N3f}GJgk=BMDr)8 zW$?FOlOLrq1KccbfQ|#m)}MiQEPsN)b08cXU`)R%cbQ`i^8U@4D257%L${21ime1WVBhUW^kl0jhuOVN8L z_rNFICxL%^+X^LVXq@~X_x#6m{^Q;LV?O_W4HM2kj~rio4XX$YOjx|OKD0^PBvv!6 zK6A@hb8hDhD}|e@Z_)2wv!yJ$wxzdPJZ)fvS0>a8cfc~%FaSw>k`Ty)&5h_morV30^?YkfJ^(}6E&I?f&;V9$O^w;UB0 zIKpt4i_vp1k}l34+Sk(DQxVQKjNno2gGFr~9iffnm<#eLny=qxj}RZ>`TExHGbxBL z?CQyhN^h0QY)NqW>?G&S-v=_l0R9hxt}DK*Gf;u*;t@-%A=p2w^bDHu#DT1=M(JoU z^6ukjOT8O4?!8+Nn05aito4^oomJG-ceHvh1ZF!cb!3l~KzWl&Zyoubr_|Hpd+JGR(PB;EJe&QGkt3F)_OhdI#wk0czH$r9 z>|9~E&0LhG!r|xT`pYqe5(7p>SAlRM@;6&V6H(;(1)f7^&Er#F#J*LF2T%~>s23ap zpPD?DJZHm8T?MVl-t=gM5~4`tTZgP&Y$!ImB00z_^?6L6e$-^L?>xFxRrD!6XCUq+ z!alI9c~YwXfXC4J+viPFuF=sU`*|tjkvbHFe9EJu*$5JxPZLd|JtUQ>$QC>IIZk|+ zRPLB2o2Ar=IzvxN#bl+|c{5Q%viwqcF`S$BY;tO=b2oy!3BoKbv$z**glx8zy0Ta~ zggZ2bM?H2Yx#sVxHD1L&sqXmvz{ViOuFHA zINeV^4?o00cw_8LIqIaRAj03iS4HnO&Fy=;m3w|sG2@oF7yUVodm4^1b&QF9npCtf z4rkEK(x6;bDNXm5!CR$6?_yU_{UNe!>FsZPQTl4^RECr$gzE7Bu`pZogAj$1t| zuaCvc=5I6_Ct!euuyAbxqeN)#VN4v@_QE15uUe&0#ZXp%cJElY5Q7L|L+ksCf@w^q zWO|G&F1h*B@rn~UUfCF{anc>54a*bt=1rH1wsh$`xM~RH?Anwv-44WpTO6^-rc+1= z|4!PRjP{sRR`3MvgJy9KS9m~Sk%+k=uM2Nfie+&p=I}dJf-sz$v9SCOR0&g7(VTwC zMpme{D1M(A+%-$|nkKy`j-8G>;@W$xp2v)KA=5EhYw;}5x^16k z^w5XgT2a2C_qnN6p80!a1#WvV_F?G7rJo=Ys4^Po%Z{#_R^jFs*(rq z-;rrwW_zdLHSySz&lPN|uz^MBpj>uY+a2fbEfir&3*L<7s#W{wFZr;*l}x$+)8*q& zn+c7pS`Ocv?wW}>wdqEVT*favnPF#6I@%iK&ZZRgw9yDN>a5Zvh<8yNjuNwqz;L4u z(?brsZN&w4xzg^dJ+cfVa?T(~jFj6lkuVeR*cK|#;+Q=;JW1DVJ0WEZC`nwws;D&a z2<%(eX**aXEWDZHA1xFiyJ~B+lj&4@YM<`nPT6rq(PQZ)!FRc~2Yr_d^VO9U!#0)P zB%ej0Ea{ZZ53bUwhJPDLd9hQM#gSSuu0w?)3v>v@%grulNVB&(lG=(Q&eUtHvR z!zeFI(G-z+kkrf%lp+27J+s_>>WpMG1UtVu;$0#9&G%8H#uA2g=Jd7}@q0xR3_#`Bjw zz$n1N3#yf~QBb^XZ2akq|JjA@E61}H)+HPLnBPlIboFhlJvb3B0mD^*K9&8Rf*B5_zZNf!2^)srhyjG!35 z_Uy|#@4R%HUJfFRHxHkoWcv|c(mWf=oO_UG#)Lru+@z@~Uuw;!3heUKyt79KtrlTz z99Wn9_0x+LA6&uYkga>&9QTh{MLtE+H-Xd~RhEjAYJDva?=&U7kBR$XrpP4N*dKg% z6cf|Z#$m_ysoF86l#yTUsE`-eheY1waixVlx#Xq&~xUyd%@G469n?H z<)JSyDq|-OOw!EL=tfJG2wZMhxEps*j zo3({d=knB4Lrqp?Y}k(0Fq|WecpP+OUNrKZ`aBB~y*YVME7oi`!hrOCF+Kj)SPAO6)JohQ+Tnve1`Q zr}27wax;WJf06L~iCfM|(Sf6e&zE^1Nt-0q+nfgBH%OfLgbG(W$0Q>{0S=>@Sq%?1Gd4AcZhS} z1bo!vb1p;**!0h!j5OqE5ZEYHut=a1W}o*Hl)3kS0^c&f|7!j+Z2QSD?1$wtKf81WpsJvBFS!#B;DZITeQL7tR?-+cJe=bNSo4) zzu(x{e1~n}9Z#-yD!ublb=iJJ35C`v)z-9>q<5-bdKF){zSp${=S|vBDB_JfZ5^X? z>j|b*qX`3%-3X>C2p^R-l^d|w(DS{B90tL~sMB540+i^QyJb+>16^|C$1 z@`p7#S7P4&0$WKM>sAdXYnfyt@WT{N8n;q`u}859#)HOB?MPg`*>i4rhQAb`>c0}U zdvDE6f@!JFcAMu6Y}<9>J*|m+QN`KaYj>?I>LxH8|DymL^LY}|(jkDgIQwm(hYRXM z^+ z%`}A=sXJWyV)Jd-uwz~KWtg~yz=r^WxAcT;A)tN4r1_BrI$^sCy^fWCgiJ6RJCuqa zk(OU%{B|^JbDXatBx@L?+j%uPLP&)(G0{&Z{&!`pzeW`Q5@7;=Bz!lFzYLWe!V7Q0 zSo2F0huf0c!jecy6Pk@WRh9NJi8%^yWtJaka z?*T?3NBasFJ07%Mp;tBpNi%1@kF)_{kdurk*0#v^tRxP#{FweFlKvJuwhn67^$bNFTOP38wE;(dYaI{{=P*w$}!vd;=;g(~=; zAk9(0Up$f$yqwT_ssxl!g99O%tnJSq(ED-^EMUX}4mC+<&AUdnS4+*;nXj&)`)qlB z#Cd6F9$6!5_qs5{-Sd#z6DCnyQ`2CTC*UdVoqmIs3e)O)t`zzdAMan@hizJzL-=$ZE4z}+?RuM7HwZ=*JTrn6WH}O7-VF{CYkMp3^MLD6H4e)9ftiJQTbFYX;i@ z4o5bh6&#qr6KS`e`I-*XMtwT<>%Y@~E+l6r0*Ek895jz_pu5))C+8jK*GK@aL5tj% zWJ?D_v7J}PrTo+hwpl<+r-MpbJ7(rR+Pl?)={+2LK(qUDepl3otk;7KuWG`I9pBVM z>)=(mKOncM07}W{tNDoYt^}-BBqOX-Aspb^MdNw|r(k$xt6aHWBp)-bHrlC?+#xCFhqTT+^|zOIQtXM9EZvDAn!5urXdJ%Q`hCh%G8CL|o5u)ss;k_^Jk$oR5E zwaXZ6sG{9#XX6m_YPiD@I4WEoU~9Cyz!rOn+mYx14;v|X#n8gLR;#3in}CzVO%93? z!UJ=*@LG$thaNN2NoA60eW}}}HNLMmdhg=I8hrZ&TT!wV!A{!WW@gM8e}u=JSbH1J z14=Wb9%pSgWce^Rn$m+3?jKjL1wQPj`Ei-t)VO?zKK9uKqNp?w>bobsjNwb6+PtTY zi>Yg*b>lw<7oFne;)lA?wer!eP5$2SOMBg%K2+{*$Cfp-))~dCZ(!s=wd|n4R;>Oi zCdHQm9g*;6?EW}~VKBVLwQ+bmJ7LQ;*S*ADwPWv1XRR(ey}=7#hhc0}UyR&dIKIDs z&GK#S_s@J*tSx;`(HcS$TV~;6#sbZK**tKWG9%-a<8=1m07z?9J>!+==krjf36!gdW=}b9CLKsdzMH3j zY87r8?KRH_X*Eczqk?HH(=JXWIHpDWfj~c`L6=<@oOuzvZDV8kJ zqGbxcMMd+z8n{gq9pph$>9@V8G}ZWtW5CDGs=y` zsXi&=n9@$?iG_2!f34P29k*2)F2MW*g+i^+4yd`f&q@ffU>u}iUNk&zr?t|wb>ic+ zkm6IJ+eA^(S2{WhXI^sE8~TTf3V21ksUrqX=2*Lg`h@!O%sG`dv66S!QgrIOZ71!A z?IvI4myPs|LvI2nZ!)8)$2?A_d5$4EI*Qi(_Ud3G_j@Ou{3@TbVDCpEMH=S^r%bu1 z5G^lpNIg_?;3l8!p|i;1$a0aD!(AqwioEheSlRNA_XO1=^l=7a0ojy_iBu z$8J||&AaYFC*f!E*rhgBl3Ef}ufD>l%E+ZWyNgTtsu!n%EUdIErR0T3Ke3nDrYvqP zIR04EFgoDC{eUGlqvn+2F#@(B&5Jqsy~)m2)3eB=J5uxd)Tg*E$@F)ySIGsWFj^Ii zn-sqxn{DQxUFk$?snx~W^t&<9l9q8*yIy0icdJrzs{87)Z1*z@Hegs{vl~(i)dt^I zy7^tcPKD<2M5g{&T`_hqjvoFxu`7E8dvvg<)LbCea+;#m4qa6i_+Wn3B0OZ5upAZ8 z{(7JIxh&CCiGp)4Z_nM_zLGS|-w)m&ch*lyHStV2;> zjiIg>vbI^6cIk*T1JFGj@Mc1!J26u&vMAq$RrJv_ft{0?E8?u)1<<km{QTw9216*l~q>a)f^xCRLMNULzZ5m zDoM1q`XzA``HM!O?#?BGs?KRM&Z}EuJvWWgqTqAmFoE_B!+1kk0+i~B+r6r>R#9}0 zvjVSAcJykomxH&prHjtuQC^OR%hLRN98iC}9AB3|VeO^fcrDvdWP7NCEArL4kWl8q z$IFKWE(|eWJ`ELg-nrenjmWE(#*;S<^i(P_C!D`b<%#1_!bWQyn|JaakM;74RSjcp zkHXMyWsAJ9tF~UUHK=gyojCrlWB-f2?~H14-PR3)ASgtn7YPbT2kD*I=mLs#gosKP zkRBiqLFr9EdQ<7rq<2E^MS7DGK#(R0H9&~>Tlbzj&N<_pz1CiP?Xkuw>n9=k>U`%r z=QE%AOaXjcQX*RSvKF~KW%D^WNjiUVpxd1yJ>@J5a)sKssEcBlHL;c_*@@=1qwPA8 zWvhQi@(+r%l83@lw@q&fwy(a z8D-u>cxY&>*u`{CaB~|G@R2}g>HabpWTl2Zu2vJ-H8X9KC6xE?fT^FO_yvl@5lGL>4 z0VS?HetUt0)&KSo$z({T7HlhhJ*AmBa?R#%Vm0;s8s#p3k54ttc`S&SuR!=oE2U zo{v+*#^gICMPJ!^)78?=|zD%36+2MoqJvA=irwIpE^ z8KfdF-gqn}afc4;_CZbCg1f+U`^+;%<=iptMvwC8W9>j>MP*P2ghncG9~MXusFoaM z6#nr;aQX676XSMbU~kNWVv*Ft+@zOZck9?@ir{huSt$)wvzi{4l!?&jA>I;U=v|d1 z?rlnR2{A4WaWM<$ZHl!DX(*936;rC*ODAE!{9K{zf;3Su3C$~Ir@k7mCfT5g4K49? z`vGpd6Qo9WG9&LqP(|#+D0gF=33jb@f#$#jhyZRR6u^<~UQdq`j1@~9SHAKgYG2Cv z)jCr#K4*RM+GsV24pEkBSF7y$t>WX4MraXv`;)bx(6rI8jqoTs-!ZoHn4oaCp>wv( z%C#tK7j#_YJfbW>%uOe*^Gl&N{6RfxxZIY6leVqV6yu8BXEjf6>Gian4*|VY z))GtAPfTYl(54j7XD;OHgfj5g!!T|63B7>YjkAu1H$VbuM}-imaoXr=EHqp=oyCWs zd$~PjwdS4wJ*N^TX|kAGSG!9%1p7poDv{=a20pNR_2tQdZ1AMrM=FV3rzYRM5QpWs zQFKtGym01%YH3S>Ewi}^GUbB9X??ntO=o;1|#0@7X?j0EMw4P!lEB(pW#3^!izqeinLuhBbmS}m~mw?rh zoLR@(4y^$7@)iHFB_#UOPY@89o$wr@aAYn&L7kgpKS6rs$m3lC4p7V+K1&NFXm@Wn zfU)3~j4Q6w2T>%O(HrsZqbW8Ps#&+n0`&QV9(ajnbT;(6!$SHj^OrvK&Y(1#R1|FK zcodC8`vJ|T^c&+%yzoe*9~*gn3_qpfbfqxE#?5{IInskl%2Dsjq6+GfejPrFcLQ_d zhK;It&2$hIODEO2QXdI*Y=Km%FEJ+rn)A`Hk^_Q@$|(!r+nR7lLj3}AT{o_y6S^y1 zunsxnVzSKG(K+_L3$+Hb+SO$fGB6Ny`wBdxLtW`$#96#ij$o|5;xp3sR&H6kjkd4* zrg;7xqbdpCjr6|b>|tT;O^vAY>LS`S2I$E77j0*uK%C)74j)yij9v-s>PXNZu3vWH zQ@Y~s)cKOXAcC)MDwK|^J||;-I>(M%_HFDY-AUR)7b>(GM%5tWm<@lruswehD}b6l zp^JG^u0BQj6pYDP^js``0PkVbzZ<6_Qn+E@o~dnw5?X-vu1SKa3I_K^`1OZ4&*B=c zc`j;4z^LT~^Eb5fV8(_vJD$#4hJ7yA$t`5$qgUmgq^q{bPkzB_2}y62Q<_nK3v({p ziVwr6I6Tt>G>Vh;jng-=ua^Q7FbOSEzC2C0o!nKcM_>3I7yp>L8XfKuDXRMY{ed*y zZmqJ>BgqcF%fpV`({9LbtDaLY!_fNa^bdo!gc)Jni@D_rTp_~ z;ken}!#ZUBC7$+kFE3tECDP5fbxYymy>Xhy!!IOiwn_v>A5?~}+wVNN`SlLxtro8E z-AJ*(N8JFDSpOSX^1m=DXo)DleJjqeQaIb%q-?CP7atL6{m9cNatNZi#Rvm8U59fo zmAu(%aKr4hyC?s6owsu;c#wZXhC)V#nEmoyeqNI2mz6-SuD=cf68)VQ19||!g{!-L z>WvMfTbkQr^t&48KflcJq*F6&k*91Y%J2eV7p5Ug8C{St6s0gi1AB_*#d$BW_}H#? zE@<7`H@L~MWICflg4iSH-GpaR z+7iJhf(S`-rr{2~mNtS}P36(T9yAB1Uv#O9tJ#V4+NNKCVP8`;K;rAsL5h7xhBck5 zoKU=In|^sz5I5>N=1uaj+wKVyt@qm$I5u7;JChSAq>1lU@| z0CQ*Wk6mMWvCJ~R=?(QdI~zeV`u1t~po?w``d)`u>(+;oC67!H_BAfWev0u6oL>u% zO#0Nm!n&UoXQVcJZ*IKb8A&^L$xZ6fdF50Vb)7f%T=WMv3m+%i-F+n;S$%tE8przs zO7;gN_WQnQ53DLxob#Ec5IMT>&IxARlQq9?ghwqhm~a|Ff?at>VfFbEItoSnbHZw- z!kiu`Qv5@73M{~kuBr}8pLD5o{Uv`F*H_!#JK_H1-4frw#jUH5Y0B_VyQy?z;|(GU z+8KLBGbPj1jPk0|n*trTx$+EcVMGlYF1cp2b9YlD_SoZ0orT%EyQK##(gwx)K1tUX zLHDyf;C4YpflFJQ{4E`)C*q?7*3-n~+-Bqjc;8}9axyq@zYly_Epx`q#O$_1bPrc^ zfQelV)3x2(LoVqbYd0)tMGP-|dZ~`c&vmHU-0(^?Q|K=<Ia#?&`=0IEf6)>;8g^X`L|T*NGq)kz_Mn(UEg#8Or3R8Xr!0I{)lV zoD9N78U={)LI>9pI;Jm4Yba)id@Kn}jZ;+^FHVU{t3$fwj_pGHAe4$Uxca+w!wu$6 z-WE$~e7n}d0<#b8+}Wir<)=IKrn>SEJV_J#PJK-rw6E`MXJ zd0IBr2{(=o3$LY(FRM;5_mJ^iGRduj!XoVWBsOVcG zXY~~T<;(V-5+DUr>g%+jye7OB0I2?wFRcCXqjs52(m45%w7k7Nl_NxcFYnfNag2vX zZI^Fv9U%#QMrMSwD>1!oUSh*svCUO6NS$?uPD9A|kV#c~yW)tht?l50aw?|sYA%xk zI2&s~1iRVsW$0$KOWSxZi`tD5uX@EvO$p!XEe{CGldw)8!)-9if?Eli_(7qQacrEgco#@HCEwVv#2 z?v#B-Tn5xwBv1jyB=IMWsrK16pN-5^IC~>V-gZ(~pYwg-zCtrbt*jT8Z9qnv)i@Tr zTlU$hQz@z$OL{{r_r@nJoiMdZt{W>wa|P4Knc+C78#tVxh`)^s`B<8ubSf|VTDk_vJAIDif~I{(_}^yrlZsA&}p$<67U9nO*L4exV)|}qJ*AC?=OJudXEBVKkHeAw6J^MY4(0AH!_5BlVwc6v>Fq@`{J0^sjrGw?`~)7 zG}34K2#v%5-N@tkM(|ziT2uo^HeP;Q{_c%>bBWXwM9x&!_ttOeVx=@>s{*R52DxFK z4|4vA|O$2h$ZDP3P+m$Sx#6-F0_VZ!d!;bJi%6xAyH@s`y+*Ttf0UuOElp zxZ}b(cnTWQR)l<5X(%M2V4_)<%zi=7MsURv#5OH&RjKc&H`($apUd(a!%>b!voxQl z74>SO`jrNo(E?h0rgZk6V!#-31p>t9kAbBAw#0rp&ImUT!F@S^B>_a6GAKTJ7I}IB z5FxqKpamo(PG*H?+#L(YycUEDzABlxg|@o2kc!}aK%W8Ew9}Tr`?9QZu7aOK_WYA) z*Mb?j{Dw}iH{RD{jzjmUJ5ZehV+m5DL|QEp6HJ%_Bj6R20x4>8MGkcZ5_^hIF^)kE z&k01>wGMCX(7Ln`27`cp_inA+749E%i5UqlGLNa^cHy=Xgd!6r<^kIig#PRf)P2x} zwbb(11FYMJhPt!#5O&f|d-(01$40Ds!$h^Ayi6gjRU!ox(5ZeZMY!(5kxD&w)Cfm-cP+UkxzZOxsR~uC7838UZNhciC+cY+m>_Q)-wM)9*>`A5W`Iy8WA&zic+daOiU*p#Y zs;U(ZwQ1oFg2tMp_Fk=m>YPT#BN_olccheOv{V&8HbpG)@}5d@4a}Tmyj9`v9HJ_9 z%H-Cps*SNXbttJEqC6|8R3L6=Rt#9&reB)>3F#BU-R0$W8`UsnQ70r+n>r$_ z65$+6M{Ok`b(wxSw=72-9i05AU0*Vl&R*$Wzz`c-0+oyz##+2=2MUJsIATp@AWMHa zZxwAmxdRhaRYCD`w=I@}9bUln+w2wgNP(I8X%($@N}T1~haH-D|GO?;qKx)*tA?M5Su zJ02{meJmFO-2ydpET*RkglFJ(sSAq1S}AL0EkyjizO1zw3L5nDzN1 zYFL?!FmlgCUANx5)9oAWrgQT(s~eBmlt`7qa_8Nbg&%>H8V91l2uDC8%}g#0h6cA? z<%5D%dj$dFLIP*#iorunGf4rOCk(_*$DmHX}cJTpR934JZ6 zq+b;_T^#MDN^_vG8~e4w_Vzygus5*FkV7OE5<#HZAY8(dg;+)z#p(ubwoPmzuSGt; z6($;Zp2e1TEv}VWK``_+#HO z2*T#J5ar8+D_^$1*n0BZSg#^p3OViJaQ{n%3&_YYtxNRekj=m0({$vcySxPnXe${R z@kvo*x#IaC@lEuHz&Hx~`puGV?Vx16w1w?#EQfAQfSDa7-_$Kbt7r0(??3O$-G(SD zR#?(mP%*WRYOAR&tgO(jOu)4G3y>*-gsI^yFUPjI+q4%`kPI`0I?JoK^g=+V?;hbO zhF;IQiQM#-bv(hV^&aJ+>be!BRH#Z%wGr39R3^43KYqHjEAH~n&aXBh^39J>;lUD4 zhe)M}0*$3%1S+TN4WdT2Eb@e{9viDQasYcn5H6q4k*pweG{p5qUe3C|0l-xjmlT>y z(t0*@vlAc2DI90w>P^;4I|5=O;L`|Q^~HG{Ut7Rrz?TW( zz1nz+N?@*{c+-b~pP&yJXkmh?S?4b;MR>?@e1j>r#0=+&F)+kNbhTPy%nFZ|HAf4t zE=RK&ah!KBd79Tuglp*uzqE*DydNKLp43Lks!|zHn{_Y^4bsOqVRc%So31^2w5nf^ z{II%ku}KC`hI7LVJsf_HCC&C(viBwbQ5m*2`c2Tqn@>`CS5!~qGWq77l)l8aWfGfx z;&P_mZN3LKN~SinryfX*>{WO;+l@vGeq(+MUtpSg`Pw$aViX8ZmT=45&7%b8K6v!N zTjZjE1M|x9v;K8|%P56r?$_bEPGnR|rN?DOA9JPXyf-nMz#wrJzkCy#fkV@TMO_>n zPy${&dL3(k?P>>KUND2Nb%<4*q_x`Y%Fecfe{3&Hqi=yyomwOBP)y2qiwQgnWFRBS zd!!$_YTQ(G{1eo6@T_C8V7yXR<7?y$fooyfYB}MRI2l$u$n9cZsZTL3@KnDnLNMv| z?D6egZ6)c58pZJh+UM~8JDJAh&aKp8lSSJnv5Dqi8g&wgg5^>Ac4H{IXU2mwwpRQ|=>8CT@37s%OeeoIEhG42- zGnSJ}&b9`qDG~UK^$6jq1!Tsm+%v1k=0W0_ZHWib4Pd^*NW^SWVO_`*JD*2WvnF3p zIkZq!7%;bSI%PSXYHUk4~g4R^;OhQ3|UU0XmT2~fGx{|t~J@1VRDyANf5 z6`=V~D@=B!x|X^^w>Wt^wLcHh5}{#bDqXNJ2%9|b(p8tGq2{wk=v=h*E~F+U1OpjLMFqu8bM7M7k%+P`DvwIl6I*ua4cWsV)fbcOS}1?br{ z#rgY-Sx1SLAwe~sV^P^4%~M7=4S^1|m5Zrv)h3G;S7fPZsEuOeb#g`7Bx%>gSPeCT zyp?{n@8kc&eSZf|M+9gi6z=;-?ih@5?H0Xw60_>jBKJ&5po=*66(z{CYis9J6(M?- zCr{(Hyc3Y%v1e0d{H{)~6=vd-ZvBIrx|=_?MeU&Y^C>Za7F07xRdNQb&RPZr4lqwL6INjXbe~sG$?10Ke7w5USZP($2j5yGw z=APKkqMk8jdQK{FKA7e1`4`y*ovD zx*qRdRC>ADZd!OtxO`;4d>iz-g|tqaNzOLXyCnnM0KptitoF5>h^~K`*IeuWXv6sx z*Xg{~e#xEH+dz_m>@I1lwQ|qqJB|k) zk1+Lb!l>yigbb|k=F5G1pK!6QU-}Tuq_u%Y?9te>Wfo9ZmC23I?cWUVKd#+qGdn5i z-gkD88Y*Lk8Z<94=i-Zaxz_vBn)Jox!5CCCis}j)0ePPtW(wkd+s9eLewE1T=}S>K ztA5RWQpkCvISXv^KzeVGXP_p!V^yzq%w1oN|5)jILWJT)ggEwW=P)A#FLLRq{!U6_ zb__WqRZ92^DKEryhgE1VCb7q5B4EMCV8wN%!7lyfnXvh*;$`qR8#Tnc*y`TS5`SL^ zP6IukA{2|J7ScS<4j;>0_&Fs{_MLS;Vv50qgzi--!`WxYwLK& zc*n+gPkvC2ke8>ERquiDCt6B3Oi~NzOJ=6dZ~p`t3-wsn(II3OGcLi^H}x7Zhe?NO zmGwD|O^&iEdpVpjQ|{M1pS3Q!jFf)}$P9FjfRY;AVqJ z?u%079dqJa!>!iH@~)O0ux(F~%{*1uwumII;WQ*yb#{!S_VE#k|LyhAz4hl)%wmUc zb4!3|Rk8Z&68Du=X^()C5JZWJb*JD-O%`FT3mUKW8JsyeNiSb|F}~_3 zb2#vnZC6bwwVtc5tg4GY$G0D76RfCTL4~z!ignHo3puPN-}D(yEdX1R*7B~vDV$dy zoX#Ti_ZdrQ;F)x4!|^wIzJGnjh-&07iB+*#0;>HcomK8$$dHN;o{r|u3Gu@rS-~oN z=jc%GeW|4KYzhVS)Ne9FLnHKD(4t~gH@!t0=W!Ka@_EG$;OpaLH#Dn zuhChh6*`v+{0kLs7Y7y>4hf0AwAFBZhEr*2V!_;qtRo9>UHCl38Nm}a81q=nO)ksQ zapH5EdA|9zt?vcG85>?kO)il`rNT^!+Uc2UW_incQipWcG&J`7+z0r7tl zc@P@zt`cTkfu-XzE@;S_b*%R)A8JrJmB(MlYUL`Hf5K@hZI49=?6$awD?TgbO8oTs zaf})_lT3+Zlcs;W=2)tPPYNvmP{uJBp%V8Xk^vlibUj9`Q?$&OGvyZx8bF=L^WuCk z2lD{RZaLHIdE1~yV7FiQ0492;pN#mGQ5+=U|?DUR3F9C?#+>UXtTQW!$nc?SlPF332=>6x9Sd{w9+X4pn5!JZnfZn3v}yDTD$z{P#S;t9W;)MZttNwFfcStvHtz{`5Pm- z!xCq*7dKhrW(8=3^lG~m+u3fjz8!wp3 z0~OC9xG{mn$v=Y7JeiFFM{S&wSB6$H^sb12BD7*4nDqNTXR zx_k+f*r4&!c7Tgl-)d(>Hci`yOjm^Q>Gi8@*%k4Dp z?V(2V17owYq4j|gVg|RW&o&^?o^)@_W(%xhD3HKv?S*X9W`^tU z+!?Fr^X?NXy312P+JK8!R^llv{4_W)Y4U>$~3EvRU?VHTSB9}(MIaL@^LIH zG7cJYtd|zy(@j1CZ8TchjJtQ@Y8mX@@bq(64rD~6yUemI&RblKZozS*xwoO15{-=J z^z+!L*W8$+G=X@J6z?vU`YaHeI|t)zRFz*thA8)Yknbv(-ZWMs;3G&=~8 z3;yWi>7v9vBZ2bfkzyX;S?l8n7w#pnc8|(&+DY|Kr^KIwuoLpD6 zbzj01acQU6fb(inr92^>g)AtyhjT__Yr5iN?1CNao>g#r7YnoN2Nv<|Z!16xM(YkK z!W(OU6s#zOzlk+VS{z{LVi>XIweY1U*j$M2T%9AoAjQCbPMZD6D@w2995LQj<#f}i zoQ=WIhjmB#eWx?~wd27BfjOCXDjuimRg50UMG@mw0398m0mQ1MBrM@XvQA=K(PWlV zMLVe1m8VRS_58!|yX+i}I$2T=&CKM4`kXwU?+MhZvqo^!?=)5|0BqyC>6H|8FMl+W zjHl=S1esZ&=n<5-AoN0TaulE|rXwktAUsyXQ|Lj@RYzP?Iv04o zywxgEUYSaobtWYysEr4&(Wl?2AZx8DZ^}EO>m*_?nEds76T}8u;hN4Hric5mguJNq zv5hv=b%2@dhRuc6a`}+jJL&Le=}R@`6t#Ts*B5z*(q`vlr@go*zw}moPEDqr&Zb?qg&fD;5^gye)&6Uz<_4C9cspe4+N8*LWGlP4rpIpyS zkeu~0(LbCx^FIIho>gD~U0n^r7iX{p4o$WSe9@rbnQPJ%0W^iirc7TYz8Bj3W-227 zIFP9MD3g7F=bIj1Z<%8<72m|%tas)6Cs^vXS{XNeZX6n8;VWP>W4anwBnga1q?+@o zRvC-)xRBKwQz*O5b$P{^;&peoSd0~P`cbN_O8UF;Y_`o4#~peax&zCxxTPZ%qGG(* zS$-3j4M&R^6pMNz-gwSAKkuszwfdxo6O}p|P-tA1=H2IDKiL$?So|4R_btboRWAs$!pUs8zaMyB9 z_C76@5Q+O1{Aq_*u6!V0N6&IiaJuO63MG6*Y4$olSec1-)XqF!gzoSnvR^hV_ez0m@u>W?;YpT4mDuMhhYR9 z-HMAwZJ)bmLUUAr>}2aHgk~1!?<2aTY1PgkTU1(Z_3~}^CpyDAA(n44#2oXD{YMhn zNrUh5a^A6j+?j)wdtHV1-*ci8k{agpa-0@KuFVLaE*b<42(58VuL7)e?LwU6vPz`y z^@V{PwH$an|Gl!n8!qz7oO<>f5BOFhB}?)AUprSTN@Xk|=Shz9=XWye*Nc9FNZiKT zm2=o=<91lm)9vlmIA`yv{RBPv|3_wkkXMEvtxX(5H-+NG4H1g2Rn-Cu)?$v9Ul}Y1 zYRm_>P7P{VCt`Z$VOE1{1HSQPdyNDYy1#r!Zif`16_{iHg^m z#ba*jJ{`w9B6GdkOlO}_L~4Sc9-ZU=wc>-B4i?Q zz&d#YRFoqi|8l@YWINp;YVAz835kD&Z2dk75WyD+{75t~5M^2AWH^AJ<_lScV9&dh z07uf;$@mFkhhXX?07@xSWvZeSt56i<1bSsaHPOQux9{mdenJkSktaxrxkr#0R zd(?)!)c9#*o(pqunDBg3hxU!8!2PhsSDZqsJ)kolqKxvWwlfd3L9-(fLJ})C(^P~m zIya=*XlHckZPLeZ{ufV)_@i6cACP5!`7L_$@00*>hS1|l0CiEq)lD{%C|y&TO5G5p z;h+KPtvJrplmwaS$jl-y87&2i#O|kGJ!>7ax9jYDyi?ExK4zct?WU$Mf&K_Vv;s{V zrK^xubgqdZlmCl7L|&YfeNU(R8k6*p@oLB9$;9<+ZLGGyTY%oXQiBFJL(=|Q=(V;F!r@he3msXn6Ic->qj-j;Bmz4wp(D8@tii~MyqYjFc0n&tZ2W1k zm8A+GKy#~^TV5v8N8aJ;lkbmo^SOQX#*dp&9{F*IX9Zx33wd2-3OWtM!T)K|@ZWy; z=jy$4V>Z!IZNdR{7C&M!^)1PeAOT0m=MD8i>XoWGCb4bhZ&*J!KXwNZy%M=~)#A5H zc@SRTX|v0K!n!Ojr4^{h!`cQEciJ+j8Xl1(q)TB7b$I!$XkMMT+ERV
~jwu<*@ zPkyBsf^)&_0JT_hJar~s4nrSC75DzTexIF{{pHh7G6G!%av}o#o|aFw)WnHKepA}! zUyF)l5vIN)^bZN+Rmifrs5fg3bqTEzODS`fa-TWMv}KkrH^@1wwOhIG9NRWLAS?UL z74xr0{;jp8J6_ReZ>QckymLRp3RN950~733?9wT!3}RnQr5<@DpK)k>mVlr z;XD?vI+~3OxKSG0Keee03toArj_qO_r~Y_7PxTSbSxZg28T5}0x&HgF_z!%O?0?K8 zMA9DSK9+kOuiW_5OCP=duEKsA3c~D6iyc0|3~j}tuX?vCGGcwJmoB(%n;&>ujP(+; zFizb3beSkEPYS9p{T&#q`NQ?!Cwu?#FQz(=5QJR8Msj*cw%v@+YUW2ZR^zzr#$Gjv z*lvH-iRU`TqTP2v&CgjwK_h>6&Hn!-fX5z-2`y0}^pL8AF5xH?VBlwIxpw--W&voX zrtwP)Hlxe8xSy2Pi{EOj5UgyidW^jVdH6e4)Nk8;?4wxq#wexq3vOi9(e|8a8p^SB za!k!Dc`vSz{a;xfhBjzLSAZ|$yv;(Tc6i0k6?mDZ<&vF7nBW_wCprqz<(y9WicgY) z`daJ4v6Z(Jn*M1S@W1u5e@l?)xAW+KJ<1=PYxZv$7ys#L|HO7O%Gmzh*-6xdADXiMF$@5D)Zi1!ld#5dVTHen#W&r>P4cj+EEoSjO6#UO|^z~}Z#)!zaUY`mY5(Wqt8|tv6pXNDm5lhE`xyxYc{#7+cN5*)O%qPhCWDUDe-#+BN?}M@@yuVfUnqY$S1moLAa)DO;iXcoZjgzxV|GZo*j!|yFOgB^L#X?CUV~Tk_Yvta^)UT zj@OmLe*FH7msQ1y=KhI6*xxqbe`MVdVKTOd7I(r*PVeExaA`o~%!XRF69!vXA09ni zk4_bc^$e!1(Vua;@gsGqp?_J}vzP*|BxGi1XSX#vtJS5*7c^lb>j4xi`}c-$+EAO4 z6F`&+XlGdf5XnwEph~m^86x;I5Qu?xBsP^pegd+MZ~@usd8&yVPQdLDeu5wbF%|q> z0Ezr^=O?HcdHRIVfgHjiTfqO}Z~zY}7zsVO0KGtbWI-vayxDpa5TK+5t^}P@8@e_oM0_aznglhl?`1>{F zBJ}Jt@+oQ!4_yqxb7xdco-rYT=A|8gGrKH~Y=>r?kqG0-fB|mcEd$^eLHNr6=DK&7 z4B)TA*o89!6mJ=Y3j)Yp?*V=>pon+aauy1}1pq(IAA(8u9fj>%5afP>P+dPk;llus zOF-^lMqwa?DQGEdj{w2kCD07b*b%}}UC1M#haZ~^{Wp(|?60Tx*Hin;)_&&-r6KD& z$=#M&b|kTDx~x65*QFm(xV|?{-p4@Abh&uD>A|4b==C}bE44g5Upm($ieKw%{q#xl zaIYy}bcW*KDY8ZJ^ei5)Pz={Tbj7)#ORlVYFgq0Fz}*LDc1O^X8YBgbm%`;PdmDTw ztMH2Zr-wCU#Ms$oDZrgr{R9OV15INYjCVa1(>~mSY#l~3an&SEaXu6xrL8Y3UTrOQ zRt2C2os=;^g~;(#qj$OD_K&#?PF7Z{(}F7`EF|c zs=gdZvHADznZTq{>!InUuEUTz9K3EQbZyji=p3$iv_out)A~k~t~I#tV$8+^I#D5X zDmnP~%})CDfcuwNP|3~C(c4}kZG~K%rgK<>rTCkxxqlYhdw*pf;uxGB#si(v3^XwJ zEbaubT`t3{MoFwB>>AG`@Eu(_PkBwKBZMzlM*jSRn*hDJ_rGik_M17>uf#3?n#lcg zjzX$=Ns3yYfts_lo1#uOP@nA$TLtLkR|CZV^b$fZrr2e>SJgzCvs7q3d)Z>D#iS*B z8O~7&(g0rTKR2lPZ8Q2`em+uIsJ!-jIDea0ZWvzg)7_l!ClXoJlSi>vu3X^I8C+w< z5G^`;KOpNs65;P+Qx2zbSo2ero&vjhGTgOAll&!&Z}g=hM`%d#&EXywXIk{q1|w_U zPmpe{qRU%w-G6ETzxv_1oRUvd*2a22sy{szpf~nuNH7qsR$YqFkb}G@`9jtO!)TZa zKb&Zy!VN7~)zcJbW9_<_OM*2-HbL(?3vl;^JdfDk5gi8nWk`SaTB3eDYhpI5Vdd$_ ziqVRoEmil6xn~}z|EPuJ$^~&E(x}XJMaW3uz98BC%F&8BMUtwD^2&gxf%CLPI^s%1 zdbdCzQUG6c{{6m~*Un2aZb^X|=uDg04mYTC$hQ-W`&N<3D}Uvh8i)B;(NeOhFXzD| zaE?|aC7wSQ&e1$`wJGX@jYwi)p?%9P`c6kM6X$M!SBV$s>+jtJ|3nA;&vE1b)sBJ- z>sVk>U<|oP208KjTZKui{Z7nRz{)7_;$GX z2I$oN_Z!MD5>|uwhuCEx1$^Us69-+iEOPOh=f^e{&8;So(08h~>wh2I=6`+qS&y@Y zgY22JbT`?@-w*KoiBQ#_y~zKJ^GOVIDn}|^D!gF9!Tx&c3z-CF51r5kys{afzX4tS z#&vT4>-?_3&Lu& z&ohj_*arvAQ+~!w1_|xFSPA5>6^csXoY-0v2rWA7?&IZHW!k<4dGeQi{HakE?O3u)!{e&oZ@vU!o%gAKjGEuoaN>S>F}C!gMXwNTIZb3DXvg{=ye$l5*5xpC z5@qS=uhPXvHL@iErM2~%$oOdOF4et5InY1i;A)|)53P-SVIIh0bI8uNnUi+AVpD%S z{(RHTYqgiH`mSuVv|6Rrmxu3#g3ABOasIUJku1~~DBn~Z0nmvB0M&B0CmaqG;4%!~ zt`itA`%HEb#%G`D=nPhO$I23(NR#mNiSmLr2t*5rCu0O^xZMNXLGHSwqD*bYm5r*I zC)XL1LU;mK zXb`Ei^X6?daSm6+0j)P0v$2AK=`A6r_b-1w``Rvc>2-BXiwAy;XjAu3-46eZfb4%H zMnVb=!<(W@KscB#HaFZY#h&}#aFN;W$sxLwRfHYr(ouND#Q7G)>354w^Lcm1|S7U%N~PFkfrsE_eb{V|vAFCnC+XUKf5`n_BvsV2yW<{x%ff0^0udvru-WTn(r3uIfx z3Jt!9d);+X9F3-g7%?XVyUIxrNqm3k!Ya&uZPd=spPbQMdOok0Xq#CvIvb8uEBjKR zAUoftHCnV>Sy53Hu0nhHV}REc3yEtoqGn-g;S;^E^h!2OOtW-&f7X;kqe{b0j*+9P zI)JQ?RrQ0O%4ZO`^_9SX*>d*(Fel8W_K`3Oh?8ra-nCsQ_Xy$`JJ7yt&8&TxUWBx9 z;2hn8yo6XJKkzmB-0S2oGo$)M7Z|qjj~mJ_PmdKAs3?M$!X~ss=Y_;;AYMuB&|f2rr;|&98;z0h!J3TO$9%-q4@uAz%IkvAoBXxL`8dCl-iT zM?PTtC%|kGLOI`_gg$!(yQVtN;H24Rc~;-__zM~JDVJ}5GGwa@GkZ$Bjha;ynzSU0 zVy%L?1ok$P!&d4U91}YO64@0CyH?#fiD$q6jbP~S!VTG*8eVIa){F^x97~qUkG#0E zU3r_ELWZ08{R)8HqcHb@IeDAR)qcL$felq))Is22Cv09~%*`qiO`%4YN~6Yl^Q|Wf zm)(0ZbHABaiamc|7yV~2Apfp={_lSxDuX3R2Q&yqUQsV-w`36RilQ>KmBOsMS2Mp^ z;*8(WxuZ;eazzcTXO#Gg9P~Tp$N!0I;(ug}`iC=R2sK<7?a{71({@-m)+B-Vz_{n4 zcY;tC4tQNl&cRLe-ekgHndeSYyWtkyDmtQt_il+@NUR~7t@wl9nP_pI{VY?W9yv?% zOQQrhP+sO)-i#^y>00|d$C#PbmGx3m*H?T|@8YAMkw{7>T~G)nYxpae_=8XVPy0F1 zURR6u^jfDc_$)_Jl<);fEg{HuvetFQ$cDgbLjK&nY(#LBI;) zKXVS$xy{sd%Qp^z7p;K!94Q+27zShsOF3yXM|JK zVNim0U2Lb5wI+J%2WOp!pWr2b?U?K8nXhXsoZ6$_sttg==>Ayd`d8Tb+nG2iGFW_b z4t9I2B2CJuW<+Sm{zi56nBcIPiP}|)!tP><_&Y$4UbtU64{*A}6AK(2pls~WP35aB|^9AUcc7dM97=?fi)&KBfCQiXcU>RB#$PqLJ z2qm0fVR0OK`wF$f!ScObPgTnxsq;CkaNBwdQYhhwmVS?-KP}mV1qO;c$t-2 zQJ%#$D?gNI$YYb5X};O}S(=^A^V}=(zg)&2elx$04*;F)`MK6!7ORzoW^GpHxh-7d zG)0P>pY3nAo(lG6^H_rMzV<1DIvchJR zxN8{WDNgj|FY3-e9)^>J`vThG5DD;f$>Q8X$5r9B_>+~x$%(3`Bi?}l7vc`%zrIgb zd`GkbDK*$eFM(&uyJps2OCW>1kU`5M;x74xhm$*Us;$<`Q+L^$&sJU1$A}Ik{^Q0^ zwzhz$WBH|r4C-IE(>4nWNLLt>m!mJR$?Nf|@)uhZcLd$>;Qh{ok_8<=aW`^1kii*@ zO`wf*9$eB=0qU~tXL!z!PPu}&eiP@bD&4f-5xp9b0TdJ;kUDOg0CX=1&gWDf7rKUC z3usW_4o8?;Rd1IHcKi2s1P}Ib5hZ1V{2v++OAB1+03o{Ufggn9J|}E*t+fi1YdrVe z5_$Z!i|3@<$nPhp={(>xPT76Gvay{NI!I_<=|dluqqkvq%QfnkcDr1j5;t||rrP^# zd0Z}HM4JF?ql1j_1&AW8@6gQEs9=HsF3&n%Hf~|6=&`Z3^KM^h>Z`YzRp;{isQuoO z_b`FJ;+(K7Ex=@z28JBO9Sl{Wn*y<+5!>MR*&d>|oDnRDm``=+;UIR`A$8EY^?xl7 zK?Pe%h;l^*WKcFd#Bv*RY?(2R2vr)rOZD_;Olk$Yv-4c-p(W}f^rC1R?6KvG;MW9x z#BI!vj1QZNTov9E<)y+ag*OZ?ch~nk;f&C^MJ`PRR5ZOh{-eSCVbJWqLHLLW0_;#5 z5QtSBhi*v$V|Bid-hy!d1hue1S8CxygbqnSGq%qQboS!?r6C$z!d+nQmjsj^m+~;Q zO>KCatFf`}9sYA0bG*C4AV{7{`}QqJB8(ojSVEAgofLY}M1g<~zi_qA;-&o2RQA>M zao%@=u0seFvhIF{*-Mb9A{%fj7~!T(a>Yfg82TtsVu1ZK%~e6m()tJ50)@N(hr0I; zYU=H}h7Zzf=pBMI3j#`&8f+910w`UYNC&|J1PO#*q=SHf5EP{&(xph~pi}`xKuQ9F z2uOk&4wA(8{GR##`R03O?tAWe=9~BVUo(eu_O-9Q*V=n66EWA@yQkYBwHR;uUn_Xc zK@20>pD0*Bg5zoBfe#`qSubuk+TFh1LdZqLe~e~`l*+syB}spDo3h%S$@7@OwnNFsbq-Ge$Vy;0MlJSg^XR@gYpeM<{qB@>?>_# zr+3#L@{x;I>VSyxBUaj{ZY^q!t|mbOIRu9_Q@Ozq$MEZwTL&s)ep~A{UY8Th#zn5& zIH7|0^RW~rL#@ZHzM$!YoFp9^&~$Yru9X$@;>nZmg}f_fju z!4|_(3M=-lE@{+|67AoD$?CHKVFbs5;CX46m8$S^O}PtjOn#3{ z9S}K+vL)+K)1ZSv(!}pyrhPPx<=_r?SJ64llh0}eRg5o~uP1#J<+6qn&ty6`c(O zH;Z%>LeC$uT1nHN!T5+o3WMAIB^)zdLOz7iG`62Nj9*z1mOm&t3ltYuoYcVEPmH;0 z06TQ=(9l9(Ob3dF6k6W;3(%t#N*6)jPd1Mh{sQ)`g?=?t=XrN{r3WgqxfA@|WyLFj zqXScLz5@EPepB5#XqUHbGg)1V6@CuAtx54he7$1A&n}wCz08zWGB=^u`$qp0M>s(d zI3kc=)?UML=+fisU`DpmeAUofNq<}L$T+J?rudno$k1A9CRZ#Z1VDk-qX$jv9X_M) zftVL2WmhlX`>rpMZud$N+Gkc26}cV!_+k9*^Xpz>#>`toiFI50avy4Suati0Uhye> z*&Lb|YSH>_GvNwHdc~X57vK)S+kf4M{x{j0ZXY&0nS$d*2Ol|8RH*rp%wXf*L*(tCpqnE7`ekL^$TGp`q02imX7f-)WK`wsZtZ?cPfzbwdj4QUNf zLaC*=)e5XQB-B;@V;Il$dYPSAD+H6sm=E|+zb7*xXtEx(D4ZT*)T|U_n$;1dJO3B( z%~&kLFP2x;P_BZB6=x|xQJuCcHIds85G3x zmG42w`vLgb*6VK6SFR~J3SXzs&TJ@tkK#-X5x++V?b?$xX>6QFo@fKo;9jCGqRI)p z=>1BrQBq>6dFqZ^wubAsAK!j9moaFtrB?OT0e72dA2Dqw`kZW*k-U&HcJzxc6OkVT za7nxC!37sKd6u3rh6kM+2O^))daJFHt0Qz)m2N~Qj@FI&Fw1@4!p z*(cjj7ZqL^Gy5i^eF@OvA9w8&YJS@gh@X;DnuiLmq*JI07IKD*QfXyXM&+Nku>MD<3LZ6$Y?x)VSaSbt-oJ78=@jGSJc1mL zS-6i^!|<&Wv%$Mq%l6kt+}%fZk~S;YvHi8}&hq7RpQ7~g{-afw|4ZsWSSHkYLki~m zKB5&=ND-SV>P^4?7XW33Df!AGe}T{CD}U?*p@jdKjwD18Xd7yyNm8FS*K_#HH5#Ci zg!76&K8Q&M*Pb0!Jx=~<1F1*QmBI?3-+YXt>xNUTNo}bVKN53)p6D-q?$(BczF~`K zuhye0GuN(-i+MO@1&mnqMBN_wdI6Ap1{F>A)MCh~!wWwYOG@ma!KRxz9VC49PFRY1 zJa;q-W%zLQ`@S^r@&f_TP`Q+8!d41+Lh&Wm?=PeAmydn#5W?X=XOCXF{nFDbwd z9$$n}@AK0U4~AhpzBoagH%53n+xhL%WRk9But3fG`2oY#;X6@~V2lh;=`U-3x}&?R z%JT~3a3Va)A5F*H7s%!t(n@|O@NuH7toOd&uTr=2f^{2DPc7Z2NU_ruG0^{*EKhZ*DwYIh_%ZOVlu7@n}; z(i2{+LFwZC_UFa^BGb&eiFHAiC97B&R~-GweFG5$Bu;o#gj~q}u^@6zVlO@rUx4X) z?iUIN_t}77lmA8q!=*}wuIQLMeh6OO>y?ySAXA<~t*^(cU@Xi`s!b@LS?-nMIzlRI zf?X!pY7rY^!a%zfTk8Kg~Lm%hK@|&i|}i7n8pfn;?~X zui`Bb*%3K_YZWXwvPUUfHqQMja#9@SK+d)n*+T%_8^lF|x3$7h#QprXE|7^@clv}SG#pP}pN(PVXcT)0feLcIU z9QZ@=*`bVrZ{{#tF1$~F3Y1`4aWfE~qwt24A41CLfiP4PVKW`AP>zZrk6t$ux|U+q z>X;OuaWQ*Tdg#zx@Bhf z6N`MVfaIymK5@2EBf?RdZYJxz26}-smcunCQ(Mqz4N328KD~&#?~w3)N(1>g|E|Oc zVt$2QL(eJWAwZc0f3SjINgQ<9X!{yh%;I`8-Jflu4vn?gvaJ`0+a>e%~eh6>p#d$IEf zKYD-iRy51^8)3UqsP~(d{B%E=cTA2fDaWb#uhjQ=P{Vkr(N5Mit?aV2^VQhn@(*|{ z@vX$`^u+KPBa3RCdvp_#^BBo){>Tr`Iy2}9NUWx1CGIy<3rfB9NgpF6I{J~7^y(X{ z_hVsFQZya|_{$Zd=?b3{m`FC7+S!xKzVytuFyTp>jG(?<{)7eV4#Sbb4j=)w?_U2| z#j>%&h4WfTbm7pK27eMN5_ap?NkRjTZ(jXQsaY%wvK{4YJ!#4@ENE9;AmwJzR~<#S zrS_jgFaJ%=^sjzQ+-O*t)0d_So`R4R7H^;!d6PI)RbhYf{F3gcMp<{g#xX`9k`v7Y zCI=8z*D~U1$HDx*y6QoE1aiD(arSarGNc*Y+#6 zq{TO0U^xDyfmIZ!W6m_d&WFaR^^XcBu6awV70W-8RRzDf^0!XUiMvx_=z?EAV(N zYGm}I84#IA8^*OEdJxI_ti;LwnzXCKz4|Aoe;oy!d*Cc+6wen+;weH5-1dScgFG7j}TAy=m`Kgmfy<> zBd^Ztax3WrlO%Or1#qYdCSg|NY57jgtO?LlBNLSp55 z&VV*Nk)R#l{Zh+sd;Chb-Kv*=j!e6S-6s7R6jK!94BG9;0a7k%u1`laQnUywf{KHb zT7O+FJX6V%OzN?axHjtgGeeUwc5W3DAc)RcIRoB0xid)E_6y`i4G;g3_-0-30LS-E zZ?JAmr(IgCc;tQ1jSkmz16LyeUWaXj@2Z#4X0dFyNVS5NTOG?-t(o8BTEdNvON<<% zDq-h3GDn}TZCIIDdDH(4C%>n@r0G($sr3}Ea>!6*F2(;rgwTz~yhW2lyEo}0rxy)2 z5OR*5L$W`Fd%xWO6s3j9z_f1e!j6nlxk1M zOUAO#i9+QY0F@r(Bi-3vh40Wmquc=dBWhfHA9`ZL@1J@2Vqa`=v>>qJNWH=59zOkB zrkBnQFJT9Ovj3L)92bxQb#w$BFubSx1W9MviB(DQ7o1~Rqu9si&Wqcf3ZxJdYCn#QManc}*<8zTuNUwLUnTwc+SEtsAZZ&L*+|*C>r;o48^(^9un--zYqVXe+|slORn%(+4?kSK+g8}l>~~y? z!HZc62mnuzUlVb=oWy!UTe3dK=kL$D;1x@~?!ghlamzQU*(NfmDc<*;kKIGWK0hdh z4c5oDMjrbVsfe}|C?d)1(6=iWoOrq8LwL~e-qkmBhi!L z#K4`yDzpwLR)$ueMPIKkatzn%()94TFZa}PTs#}8K5LM<$Xz;`e`a+xY5;&NGk7sH z^IE7|XsX0~3z}lC#`Hofed`{eJ>-p~-9}4oErq#4;c;y#B87Z|B=BP#`c$NlXG&kz zLKYQNQ)&Ek%6*fd0%C_s=)9alaL%m*g)FR@Z2UrI{onmEk-C2B_u(`6bpuH#)))n; zRTvNDGz4=%m(FT{Crkhw()o5`MmgZu)^lxt$EI^(9+wZDm}W$?Lj`0xOQ)D2nL7s5 zR7+<@T8SD-y+f0HB{9Cc|H0wfF5glQG4BXY|CZW!5*YsK!Y5jV*W#JSk+;}pY|DRnFdwUND3Yxj<9*2fy-H`z zIe`du=yha+fwd^)s(I&Hn!9CfF(rpVx?yo56GVTv%a00_e7am;%n=uwhd$Yc7D2Fdpkcph~+&MJmFX$rp3@DiA79@p zfzP}OHhqWaWZ-E|$L!^*(fw4W@Lh$8`(c1!Sjq}_aGK%X4X~IHaed%}d`&0B?jEpVZe#@a?aTHC0{keGWGaFpXT)wlAF75iP| z2+MqUr1$W+cp|;Z31?H8bMjFLWqk#-Fo-{rr(9@gA$ip8CAK69FKvlV6g*hIfj8G( z^o#2q8F^p)sx#rMSb_lps?k=l@aAqDWR)yhvN<%7gR1D8i@i4cD*J5AZ#Jfc)k?fm z!dF*7EHe{05AZ}T!W*0*UC6-uc(EFIk_}BnUnu>LN|R6B110?_H<>KyfZ?2P_d6G& z5V{!ez0Wdhu$^bm9nm&zBZ}MvjNf zpbANf6y)l+byjM7c^ORRZXdOrJ}>b^YRpzP6D9K`{Wl12kz3gfTuZw?)DSs=;a2Y~ z_A>8kEDYrOvNYoHra6y6MX!~t<(+COb5iTGJAkeL=!IzP@V|hZCVr0^bqDk7d6YhM z4aQu@ysKD*@^C*S^0ccBtRq%m7q%?5_0%(e`=G$0+;yYr>RS1&=BKBz^-qu#rYPMx z8p~v^oFp-J zF3spF=^FRKn9=0d@jyh8uzsq0TE;(6%bE+(aNLlp>qAm7Hq6<%?h_)U!cqMEM&6Z6 z=>>~2v>S9kLn-`1G$bFvmWqKwVvC=j?$78E4+JFU_wCI$_0-^(Bo`JA2DAmz+iIN7 zxKDI;w8wk8%lvlj_8er6ss>775XuzO=O?7Ga;yvfY(V%5hVZh)MgkG+bDk}-{3aw$ z>P^22n~z<~AI^voT9%saWRXO>}QkU(gY=ep0Ey+$%K`Mv`u^Cr^qgVgt@%z644%~+?V96MG^KhR| zeSG2|mY5z3B-{R`O2 zL46sd&=P9Fyo6!U=d_26D&sBJn5kt4HI7#8b4k&~-POkkZmMh16GQn?}0Dpm?n-GDRVZl8A z7idh}d4><1NA^Cp+l8Pf5 z*K?ri2is`!Ad)1DavrrPs0a`|Fgv;?_+!kJ#ly&|A(c{TokSy_1(@b~;h^kfzcT&G zO+UCG+DSS6M0l8MTf?K#osb|A?usWpoz1V>-#@Lc>ODPh4A9P)N6=Vjag7jtD&K7%6j7jPdg90@(d`s9>P6opM%J_5dD{3k!BM$T#H$G5;hdmQl*oby5` z*W`{%TSVEUcl!O!!KT3cy{F`1DGAI@@zL4AqqCHgv)zKDYnKRdvn_tbW_wJNah?I$kxgx{J01BcxLYJ7|_3l=D+|(AZ&9t@JJjbLvEOA{@jx0x!F%UK1Jy* z-xegES>(MI_{cliMMJkzipi=XGsx~N^N1)0G6He(BdWV(gs|Qg_o$=j?<$=5P(g~j@LX9$N4u5`=P9*G*a8^hy+^PS%9~oO)ygswagBQV04hH4^~qSG zQ3jiD$Jn3zKJ25phm5IX6CIv4Zsd<~Ihj@3BWbT6(B&3M*p+xgs0Jj2yqXco{o!|I zFd_xP|ElYXJ3F(sP5v!^D}M|}PJsluBu=`_y^N~|Ub-Nk`~u}?*gzylmoIMp@hk|G zcv<|i%S}2)c=ojP$<>nVxsuv z{sHy%ouQpO3#zHBP_yuX4HzS7COm+NpfDE1`bW(v0oRQ`N`?E4>O5W%|8Ta~3Q=zn zWTZxSct}#Fv5@c*wsu8Upf$<)r$J+G3a6^|>!W*vB_S`ir8PcGG1>SB-+CBze_0Gl z*y>K4hFZEVq)#%gbp_Hh4^-r{BIw^K&FX)3go^9u2u@wPGPDASfgqMdNy#@|STo|+ zIPKutw)A?PVbN99Tx+Z0Zhn}Xfy93^hV-v6Hx(b?3-l!091Z+(SuLemX=9 z<#;JtpO}!m>cTc3kz+)pVVMftmL!JpLlifErS)XG1_-~Ed(0#?WD5rLr z5D&;mBY2%!81{}J4Z2lLD9hCQH363-NZ z&vjpFllj{S6&s29sTY;wD_>dk;sMm&?z5wyW!k5pZ=!M)QO}5<((s30J5o);$TdaD zFD%-fAJ^Q)ekEFTUB^2s=PLZ5g^~92v_fy85M#Gkt!cI_M zL+7VV2-1|vkB0T_v7tD-G)r%O>Azoz3`Hp^-Jz5(XT@8yizc!S=gC&CL%Dnj(<{@> zhAKQD$TyMZEN3=biR6Ov@fSV^Ttznf`?g3`H$<5G71x2g9als~q@!eT&|^HKJOE8e zm?0VuNh$3)YSLi3%e2~l;mTe3=U1ySI#s;qKcNz)t4m=en(o8n1_JteQ1xW36sbg! zo{JRqhBM22P|`KM{U>7g>v98mAV!FAL6pV)%iukFUX7BVy9VNu6yBOu2w+D$9$i|b z*qDkXb?L2!JGSQidL^B=Pgq(t4MN(?tB%~AlHL@{my#@lK2kQZq*5J|K+{V9WII#1 zaAcAfq;I^*kv$YgK2v^C?{dJFfu#MBIP@f!>JXlY_%^7$f%?EiA!q4SUsYd+h{0E- zd0){V4W>p!#G*#*&c8bx83v}9=djJniNV;#HnQM!R|F)iQ3T_;jVdzzF#YX|sxGIR zN>3w$%eg|C~tDSm@5F={cibEN_z5)me7SJxe&Y z)%f$9{!8u=p_THik`8$qUYmr*omv zX2i4{XH-hRZNqj)a5b!Cd(_C}nTyRibW zIqX=@Z$E_SNcmuIr__7TT<^+;e-(2C5?zhYyV8*>k`b#7Sa!u<^YA`63&k3PX2|IQOS88#_Ac&+nd=om z<#*Wws(!yoW9+r*+JK#%-OMYDZ@Wyz?@E2Et_J?;qEK@Klb@T&u3&rB=quC=UNA?# zLZ`XOd23Gzt2V7^n_E|``$@n4%@i{ql;3*Q5bkyI+q?JBsvWZ;E)V54+8&N{FWOjh z=oP8a{lX-4DdTZW-&&W&iFvUV-xjP3ciI|Xw;CT$dU#q2uwaD4(Yi#VL1hjnP?aE* z;2*pz_zR{n{X=i`&4XYWj&#-jDh?fpiPWi}@ClK%95*@fquymM;&4ACT)su)B;HA5%6-0N8Kw~u3>2K5v51gKtypnt!Q zUr9pgG~s!Z+6hy$%FkwsX#=~dcXCTTcw2$5{gRA;D~g%MN`j2obU)(fnsM=&zP98s zQ4#f&cY9gF_h2~+!pRdXa;(D8(9177Is-!`>c*39JNICB4dBKo$G%8mqJFFo zK9C8-AU~*u3n&D*yZL8c;67vD`W&D;W6C$1{ZIYU^1mXXF{jWVj1}I6JUUO&XeMpO z>%+DYJa=BHhev3bX7Bg_%i;EdHwq~J38az^h$ z@gVh8w0v_n{6f*ZyraA3=N^Fe>-9d!ya5dcKBDH*v`|Ilc+e6y;39}V^)V9eVuHEr zMa``={xK~Zepg#{Sd2el;9l%ww-FI3VH`WgM_Cwsky_R*h35Jg$%KmS@6e2u^6{rw zhx`a=QE42Cw!U_?W`k8^ZBgc{%&ogq09+j74GEP%t3L0Dqxi(cNCq%=NHj2g&j#8X zaB2*-Q_JTefuXy4-?BiM!MyCi%zxK48AI`(3RR(n&P zacoh@8NLmxkgvJO>Z%byGT=y#Hbv8Hx8ealKhsDpW(JXWBja=M|zpmguL z>^Y4a#i<%=_lOUxqg2hP&%1&njvRb(O;7&<$JCE7oOV7Ym6z7;?UA)U_lm~_oMTw* zHcXVZT3kaYpYT2wIqTAl9DqfIIx{8qK;uff7f8~|0>1SXEAL+F#vZ9(NL5Lf6A66^6s$=%06ubw&Dx!GR|GQok*KIic{YkXzAQLO`oaZqhY zRZl4!uIVo@Fv$%~&gen?pYv%DgIO)Uely18j+(-TXW^Xxq-eH1Pix2Xk#>ua5u5{n z$9ZBG6bQ=0Fn;vqDc!q7b{8-_V0GN=O!LvwmGx8A3TmahEBx;FlCmcHr3oJE=$Ab( z0on%_9yH>}2IWG=yapNTzdH-gJy;97zh+m0Y;-tlVi&_;sCz1nr|oK8fi3&N%eCtlq3Fp0=r(%TXUQ(vb0(bDHBR-j05X zVz$N^dJ%N2Mje39hD}I}?7(KEUl$H&6h*u36vI0Hem7|MjQ!ttI zlW?NXg^xD5e4L0)?2?>LQy$bzKsNc}<$0>iKAeWDn-pm1xY02e0uW%*3>#QUSP}0Q zLn{;8dL%*zmnBIr$AWyj<(qsWC(9CxMm>YFv@;YD=)t861_s}Zfo~X}y_Ayk;BjJ# zB^XUSzK1RM~IOXfz^ z+{m81fi?emx7@If+iCgc9sW02BVMPYF3wLt`T=_xlKP~;zP-3eInqHinl5}TAk?VE zzP4FGama5-MOvJ}<`$3ri{lq=aYw)72hhxAq4ht2I8Cb1wP6&O>fR#>kKyj z{9gE3wi<_pk(kiMk;|j8U!p`0hOsGyw!7}7X!h42UF`}=$a+Td$L28W1~u=KC4V-z zozYi7L?*SySiHH~0CD;?!%9orFyax04$V^r=HQKbmRtyT$}0VM>~7BcY^xU8+1nka zz8;pLj0vjige2U}d8z@Z39 z!`CK8XR&NnAarmt9V@KMN;<$#_U$pFkA2d;jzIJ6HOxp&>zgy8V&V3PllOrBJm~Frm zr32?gl?`-E_U$SU6=GwRnpe1;zn%RaDk+&uQh&R)m@GQ}B>Ja9j-;`8x&hP{hC>XF z2iQcy;eV77FsUvKzYz%!2TK|>uw2FH9K$({?^%r#RVuPkcS5eMR-Ma+$)03XW~1B# zC+i|VI?IEKwP?DJ4<>;v(V&jkq}mwEz)+=;^r^49?PViYOPP01kownb z`;hz;-4ERY(0_?MzC}AZd%JZ`*+0+5NnQ?RZJ+8rDv6g(>$a-YeXjcMG~E|8hP0U; zDT0QRPNa8>QuI1W!^saSrL+4lwAlItij=3peTUdv2SPKeW5%G^EoDbApuN`MNfICL{n0UW%DVxi{I$?t46C=pWY1po0gkst z(a6SBXVKJq@^di6nL1b^!HVwkdlK1@8 zPOeLOEKLvDEA}hhGj&C`pJ+8dN&kdes4q(~0h@@i#w6TeeS%OVKvG=Kc{!01EZ)#@ z+W38T*u7|3w^{m@A)_}^Q_QMF#2R840f%m=0*Jmv98*Ta@fmC@y804xUwK)6qb`p< z>V3+X*>N)efxR_nOM9;rE6H@tXc)l~`C$b{69QFN&GwRYSqa91W}$R__ab$t5z;Icz!a+sGD^8Xw%nI%=qvkO85^eik$d7fJB#_3s(FZy<_e?n161 zMbT%8bC0^ELG`Xv1Sj)8oitNJuQUD|de=*ggmsV#PoYE$J5PN~JBvnxZEl2lnhqM4 zk01D>U#+NCxy}n~?YSiAwh=*nW4Uo+m?96U3+GFop|^ETXl_ijf^ zk6G4&NoZ$n4f_brtIzdKUj?c(LGZ+2K4EQl;hZf}>dcR<)_1HAmPE?}+@XaCEkx9c zAjTUmNwEYA@PR3i!6zO@98L&^DE^81qh4waPoFKv* z!9|-cVFLYDCu3x8f_Gmzu(s?8>|wzC_s$mYGDGACjj>#;Ib(gRkAN9!b+;_VAG9aF z)kPBd@ODu5lfhRWMH{XwLHGDqR7?hmPCDou2Ut`nL@9>dDq%1_?*V?-%JZ zB^}`%Q1k=I)n6P2%cVz30-o1;YLSh{MoiUGV#4+hIGBz_sYFUpY^l%DSB}h3Yj|90 zH=EHjit2|msU1R(?#pi<#tsOC3Z9bx9p>nJ^4uZnmI-uEM}Rd)hA1%tWTMW2mH?HR zQz)tgJ);qldCjRDnp|dBL)G!znEp7gh<$~s#b%BvW=Ck6ZjONKUx|&BB8B!U(;2NP z3xBR*b+R}4LD&a?RffbfB$J;-u z#ofE|a#YFTOt$f6gKX+*_7xxuw}T}akyF94XQ)ERSuq(@4J7U#AU~$H<^=NT-w9~_ zn#{H1=+pDlmgVr!ptqE1kYq~E#4Hd3)Vx*d9^g~`loQNSy$C0 zAelx=bcQ}{JWUvtKY(D@RbWm=+q=2R{xQ$hjpzMw_X8^f-;nv#1%G!LEkFd(giE2F z1`k#@6Vdjh`kvPyV#t?Z;wY8I!%-l;1btoN6_G|Y{WDHG(gwNdL*QARo<-kD9952SFo}2bzS)ZsmfQ+qKkw!JNVgJ(CXxu`()i{VuJkbyRK0Xg^=5Y^=*X4sHU^uX!tQT;<9(*}xa zoqa=^X)li1er`0J7O*m(`i^wI8$&mZ*7(#FUPn1cB;!c*edGnrls&a1rC>8@ktTt+ z+lK6YaK$LG^t$u{|JzYvZ9DlS82Jg9j5m7-1?p4qdO1qWuIZ!0idkF37eT_DtK9qM z>0kbpK|N>J=~6S9d>|)Ks1`WH`I(vGm-hP1P9tKueSGvP9>umT+-^TB!;LIzrh#3N zc1hg)`a8h@asf*a+P$=Tf{3K3di6!f4AoS9uVYT?Uf$s>e!WlTUpy18>3MS`l;N}f z^^Pb#*qY5-l0rY6jbc(yffMU0l65`YJw2w?YT6V8W)~BzUyqpcH!O&lax)p;H{ky- zuR$KP9m%VAj}w)+mJt`Adu}adL@vPbYBQoEI&PvNQ?uRb1Iw4HTQ}OBXX*NBUR&Wi8rX4Yyc_rVbSgkBmLrTfCOPrk~X`5 z*)_ul(xYtghRG4ckiUSrs-M1G-Yuno3U>KbSmCkEQI;F~MW}nBTvq`{YAq$;$OMG8 zFN$j;$KagJ^`fY3vs*06wZD(qyEU9T?jykPK~VkzFE9Knm-)=-N#4YcYa+@@GJfphaeme^?{hQX9Z$;`W6MW85a;|12t!bN=#jM_t{Fby!UjzRJ3jZu zay-DQ(s<17e7bl?xj`kVE%JBUUR-A@kA_P8rW&x9CaFcO?n03I@%0Hb82Zcy^}WQx zw_|<;>a8C)k#0Os&b&mRi!?4?dlaHi?u#B`VZL%?4r15Z;`F(FA8zh^U44rf)K`Dx zEx6qG>#J_eU*K)l^|6exVCR&s;0ZBkIn#bs?>d32iio@LN%uZ}kyCkSPFwEnP|eF; z{eZgP@^K<=wv5D>YxNu;d;t7npu0)FM#|W|u;#+J?U1;OB(50}L*8;VoY4+wEnM?& zy}A0*y&Q>;bZy?LJ*4@LNxMY4(*fnL9F^@f&)LCVRd>sKzYFOn*N;PaIQ!{ z{CuALbrt@7Ti@r!1L$c$8S>v$UTErb$d7tXEsPZLc2;bf4K|S`nSQ@t8{p;7;p{v% zE;-dRVTva|naL<(pzi`VYbdr;k$k8eTTlZ37U4n~Z>et0sBn2>d?J6$;wbq6L(=GL z#bXuG^z3ZjvbQ=C*zp7|PLX=J!bgv9{wTeqo<;I$86U5Burr9_x|R92wFOH$Z}= z{000Jpb@tIWnLm=H5PGfb2V+~>9p^%RJOHH=9e&)r?8a-hO4hnTCyK|HLPb3$WrUO zJdThc9KV6BjT}Y5eq{*y^Gy)ug0%w{G^=SE|J=mzPT`DvdXB8PE_+RD&I zYzI>Vv5)!z?N+sHo_ycC?i}x9CJ(npaqR%3FY8sMT#4k8IfOSHhL-D|g=|yUe4LMy z8o%^p*?(7h6!W=l&x!t>XjA17{dwU@4g)K?FW|AgAx#=1ad&UPOli`LWIdGYqtxc* zb)tar?a}3O6cmA$x6jah`2y=&GG?OwuYhp>)wsa_n{YaQ+|Vw1LaUUql&S1RQlnvS zZq(_Xb)N4h0b9c})jf~TyTUG|uF@DH9{}I~M@B*aGa(?hKWWWv9TDSCg_`U~b(MK< z_VK@CSILwJJ=PhmCBks~9w!jyveSN~1%{Guf=Fg)nHsh&B-%@x>YS}t*Lf0m-+59u zE-~wZ%)Oxoqo+T=#tsDmdjEmr-+%V_|1~v1-;47=#vz)S`}{D%6~h^6`{*nA4j*my zj6GF&+yTyZqbE$`>lHupPZsGrmq!V1;+covA^iN1soa#HTz$d8mtrPe0tFf-i1~SX ze`qF+){^IBH9^oK?qEk*`1Xq;%!>O9qj&BX|3@PMJw5!rLntSv5 zq1QSZpEL+OG#W&5f}?|24qX$x3my2jON-!yG%cJl`xGO-aI=xSkIUd^QB=3^h|~`(R5H1 zL3t=XiV(GLOGXNUuYDsoQ4y+12wNPW;TR< zJsW~gxpzRZJL?_UC;6@-5{FwG0^)QAA`o)GLAg&{`N!)yMe-E58~%!-KUb?iSq5Ib zq?}uE%|Tkk0((m1%sr)bG3>ZpJN)c1GzL_Fx+s@{`h*e%(2cZd0TD@fv&&ylu_mo5 z@gJY57!KOWFyx<1FOp+qqV_+Hq5DMp0KF8N4Ae!c6X5g|c`yYL-DO!Ndu-G(2HK6# zxR(B1HYr(V`O&*}eYZ~BRYM}6At6Q$L;k{2S|{9#4zr-o#mUYz}Acq2h%L@F9u4Q^`dUx7m-R(hoKBxDufxejE>5~7%t7R`L$jZX z76;Eq@eYN7%hm4%-t`{lMrqeCxZp$6yH(KQpJ;H(d9Vnxsu*r@wpnz%$oN%twu0`p z4~-*L(lIY;=wnCRa-}3r^dnfnj9v@}T5q+U54C(Wu&|u-@{)U4{gTohWiR8tmVRc% zyc5_b4Tu4|^-P!sHJiqWwuegc0L&$@g|uc1V#AY04b}Z_dD|DD2UbW#ZGJYwzC1{`$W)SvhW{)TF%h&s< z*}%#|#VUNTKHY^FkLjxgsdW@pXcsf|M)Hojt{wyBY2Qw(sHR+PI*& zRUX38qw)#1eONveUoTCon;6nk`?;khul=^2nTPxZj+gd!^w8hTM3>HTzLd?SX|KOt z2s+4mcwAz)E|?s#Rvc=SVE5zq;u0<3Ls-cbE2n~yGzoXcC)qO0=~48_)TjD%=o_Hc z8ZsFoR$@s%9l_`d)bI_{B^e~{=T)C`i3Le6Dn@Kq-)HTnO7F&>N1QIDTZ7FX5im(O z-wakt$aJLsb_X&}O+syc1*N2{xsyMA%hfJgDk<@&MFVO+IzA|JEA`mkt`|`gBJTDQ zH6&r67pdM3^r~|3>o46FsJ42#(gqZY(&ck9>Pi_swB1PJPJ{f9Kh% z0#e*pLv;K{v{IH(@~xXri%WV9NiHXCT$XFHPSPcgorjcy?qVBPdM72&?4V`fs&W!m z@HZl`xKU10vHJ2LJwzYJk}w0^wL zFw_gl#$F>9f6m|Ounfz)a;n4T!dGeGoLfpZhLcja=_1u=lNjD^d5SL>NYd=XHVY-8 z#h!sG-%tHUa@o};tk~UnV*LKd;|i5P&mNYv=C&HlK{p*a183FsfQSS6myn1thW|Gr zapCPc`KM9s7R~EPm3VZa%-45fyFq7z`oHjG8fa$#ppFwUqi>hF$6+q5g)`iC`hjVL z(&m6BJym|6aQd+L??k04(whPjKxraUlp+X7lMX6fLPC+=1O!x6 zx}btcuM#=}Do7`Y5KuvC0vaAAiL-z2%zD>*=bm-voSAdieeas{rz;pX&$G*Se?Ki9 zJy(u>c)uNeaC;E0^)UMAN33UHQk(OZcQs~g289ec7-jK!!y%N)X*kXYIX5wg6Z?sza}JkY8Nd<0qK~~8!bb8v$X1-pO*1&>m#3{6 zagnF#Yu{+rI^7ylL`Q4`a2J7h7hy309ab5xa(U(y?S0@YJ4Thcrk}35wuZ-I?_U?Z zG~>sqdju34LCdJSg@-%Th^zg$BOwUkb-ZMK_pMipFPUHMU)U~62=hB$H_+-DI?#Hi zhyzLXranPO5*7N8M;bE)(RWCQJ}s=EPcqiU_l0;&`HP0Kc)3&2^~{!bMFxE*c7L$I z7P&DWbTntMK8x&_v1w@T4BP>ozO~OQ#QUTX4nmv8J{6mYy3A9}=f`x|Oe;x29MGe+r#BP!9)egu%jO zq<|VOqw^@~QqIc5vb@3Gu&lOu?_=NgcupwSHRs{l%qkyTTs<3&NW<29rgchFBEZCr zI}dFvBUrt47$2Pp?9lKOZfvUJ3RH^YJo(~R>OsY)%=-gMpWvm`3NWgimHi%vokTHS zsNw|$)!Eh>yX1op_S+s3cJEowHtZKpBMcr55u}G#6ANR1I&+}q$1!4`p@}O~YN8tY ziN^VTD)$uRrHttfx9ZGySlKCgXV(APaJ8s8D9=J(%=^!WrnH5ZN7L|&eGxl9lN)6M z8&5B=R|O<>USP_z`BEIJ_4?`u9+5m)T(5I|SjrCt{7bM{J!S@xoIXPGt+-OedRwd;)WC_|J_~vvq1mD6MAJ!w^9%ohmS#)#$0pD%d zLc^hV`jvubz2MQ*2dEo0T$oV}jAko|X<+9FMbBBl=XvG4(Tjn%s#DG?Ckx7Rf|?(| zPtP7s*8w^MGdHp)iSOls-P&Tq3rYYDmSU8}3{@N?eXS z+X66{>480XrpKtyw1rC`+g_Yt<6`1vZ_PTNz=-m`UEzC`!{Q+(u4fpUWdn|KKB-89 z-7(MX;0{@FPPnt*J7Uhe{zz@0zVw>6(uU0_?V;bUz8gP_;ineqFouXV+e+A1+>OlG zwu((EG1U}utD+cAY2ON~7}2h&^5JZ_Z7gx`wSi%_eo|31UFgman8*ysSID^r8IdZM zZTqXwt=J|}E_;&>ZEY8g)!eJ@%x`m(;`xoxLCS<-h$<45qk~3U`f6lsO6+vAkT8N z=@gn>b!K`$@$$s7a{a0E-=y^lJzncIehY6Qf4#eN@#opGCaTV}D_N zN#mK}^WGB2f&_#1Zf~!`Rq2enarGP(1Qfosf3cx#Q2>4JxqW!4spA=KInBa4Jm*(* z_q)5w3imG@O}~&I_~6SII;!;^`e@Ox_22Xvr4nJ2A4aki{SE{XG1V`H^!xrZ2y5wE0HmNVLqihRH@m z3Z9d#9=ri#aTe7`4Y8iQvLW#YRvf?B^L0Kr9k+VVf1y^#GDBB5;y~VM z0W-sgYCH$iYvoeVe&t;_R@RVDkEK^wS2LM zP9a%KAa|Phl!x69hOtYp_OKPF`Do_&i-mi$RO;gAZ_%gC=S<1Bo#!G6xr4oTp}4B> zZqKRh5X-3NuLe`!+szCce44k2h-};4UIo4gDmqBo@b@JYK5zin5Dk?spEc`nbM?a! zkga}rdyf+T?o`q4i>D3n?eN#%qPfYj)ZC-cPm(mDq7T%%%P2NnZ?)-xo?pW^>cakx zYYzRkm;-I|Ia=KsT=GKzq0O_~3+rVb_w*=agh~xjV3%sUcVe=N@w@BpHPcPN$%eQucqU}GOMsaJ+;hv~1oWUTtnMVvmRPgK%n!FsTt!TAC@IWk#C zLc_QG59~%JM?TU8xkfX(wmLGnquqh6{B){Ogpk#YFjBP=G+x zmjK#bv7&o-#u%kHSBX{=HX*EuoSagHN4=fyfd#5MHo;_f{jE>51#DT_-m zkw>rb>#wXj^FEBl@b1MeVtL{`mNT0akpJizCPF+X8jl3!XsD#onOGlIn8Tm8!n z=9X)^zyljC4+Xga(@a3nYi$hG2PqE}gw?ZM|Ls-*1JW%GY7WkQ zkpnaIRD^^`L~E@C9)Ys7&jD{Y?-BW$Kq5Qq1J9%$& zh2Ot-yDawb>29XjvWy>eg$HLT{Nq^9!Z?~Jbhdb-EwA2#u=dS(CxnsF`(loX5+U50 zQU670N%WaB67Y2D6O=oah4w*A6-r-mMnI!|ccw=B{0-DtTS82-vS(kuTauq*m{)1} z{Ao#MC0qCG=l??KEnKa`gfK;d5es~O-<%PebY@pvFe2CcUA=GiWfhy>DiAQBT;>_4 zD|F?bBhR4Q8^(O3{pJ1OKZm;E%<5(DxSANOaF{)^51;n*dRtw4DD|E3Ilp6h4zAYu z62^U;4PW4EKL3T?>c2xI_uv0V;L8egmJTbKTcGhm&T3`!7w@@kL|o~;gmIszW84}e zkDPynD4z&br6X_slYkTSAK1Yeuu4GXPO@E5=wB2KPrNkUI0~4{rtN3^c?aiIE6+te zU1*Lzr|`fcUV=A84h#QM^i1qw9%Sp23fa{j@O zgVS5nk%=(v4@Y1h^eazAXc?OG(7pUS)kOcC&EsF+@^A6Jf8bwqU;e;=kfMea%{omJ z3lDSnsAlG131uzYX=^-l@a|DLuKj@V2A6WC+|_0FxO>R6mGJ5RH-98{#w7zy!(INvBTeIa_f%f5 zoK~g#8S+os{%_f%xKq22HyCu`S71SAO1g};s0i2%OLLC6AB|Dy7nlBQ0XgCS-qy#=EiJ-|Cc8nLuH4H zj08OW+-q5w@l`-XF8Y|;D2P6cGhpr{ljB?K*M^2yqV>q)$D<|w^}_$nWBn)X#{cs> z+02EZNAulItTPmkJ9N!s!R^(TEG{v=J?<@xihTszMa%{+XZ97QJJ&1R?UbR|wAL3o z&Ak6oHQiLQfKJ*jPdIKN;$G!Yr>Nu74~zW!93B3j4sfI(ZQe|eH>Y@7{t^OYnu9v1r?fquWN@j*My4Zh>4eVcAu$bXTX@WdD6wo98hWLAG>zEy7^ z@@EmN>VNch=tTSb`0{oa94};@V^IZUYL*m66Y1B7qR*JeBY8Wx0T-E!;>3vgSS_yE zr@j~M*q!fS8#p1P@k@kJz}@9-5l1Juy&UACxU`V90e(kAjF1=$%7@VE} zhb@a6SFEi|p_Qh+7wQ3Uqz zr}uBj^$b2m*F(}3b?(lb=U&04Vq}5088b#6Gb|=S^#9QO?U&H&Y?}r>5p(f+!Q
mAu>1d~c1!p1@=t5HH3Nj}PS42i^y}ELW9Cjj|I81j`aX2eh&&CRz>bv_%sHGwt z|F+IaXorC7$((brB7MRzOiPak?5HjndLAvSGpso-SsxMk=!7COD)=`r5+K?F!}n6! zaf;c;4&fEt>|P&+d90E9ru@(O)>W@6vYte8%h|RQpVlgsd-Fx_STI##KH?h8V^BA+ zi813Ra-BnmwyR5!3c_xBAurI(Lw#0n)^PVn**J8sUVU`r(36pdw&S%&0|ej0qMi`j0eKNPF*6xJIZKh>+CPSj zV09K2Lfek?J83wDwrSQW4)vb+0(`t4=!jz$F%q3n=}WX2HJ`@pjJBNmAxc_*lya>> zJ#<1GFr$xsN?BBR@FB(SNPB`nbRgz~b2I%SkB;O5ni=XR4jd40ea7P6{tX8Y1(!Rm z&6Q#Ut=+K-@=WilQtUK<0{>O?X>0}x5rRy0X6o=D_L`6mdFAWKetgvsgCTzliIGkl z_)_F?ibVy{p-JO`Eyg&rl|m}S!0tkp@m;NEVrQ!Fv-d$lJf_LxijuikcHt&(qP5e; zm_q(_fmj15I7AC`{9>1oZ3b8hz_A9c zM4YWp8Iu26m$AHetvw#yUhrBwkJE6k>Vfqvzp(C2jUyZdETmK!?rRrR?30hb`&9aM#`r?v z<;JE+k%5|%N=dnouCwrdO)!Upha^wjq3~Q@4om6KAh`ir6j#R?AgKTBWAo@PjV)!4 zefT@(x@Sys%-gKxm2dZ7%Z=t4j?tk8|G@CJD~V|s9qG-|1F2WJzROyy9&bqe$$=|Z zOD2v~y2gl#!Ym~m9=y!nJ;^}NrF$Bv0IjGkDA#mTebV4Sjp*7UFGgfNY{=z#)T17) z$bu8dQy1#`g=KkdX6-uMpIaRPKBU{IW~M;|KSh~EtN&}y zczWAf;`%#S7v5+njf<`i_=MMD0e>?^vwVNEzBoMWTqD$8?h_XJyC!AZO098XvwG3J zu7f3A@YXRFQS00|DRGHyaJ=#ANMjaIwdyYkg-G-@sI(NA6o^$nmnwYjU{G<6C+lrj z`|&KEfDAX2i+&={u53F+M@Q9=+E+XibPl7D)13+%h2L%L{Y&2a24vROc;>LupS*eN z&77W}Xz3|Kx*|k2nlli*_LI~iiX>vd46zI(MxCPH@qNJfr=A zUmD;@d%y&P_iKYAj5dug-0~!elLH{*;A+?P$i5~brR_R+=;nV96VBdZB2cK`|$X#-p$USuF<{M z_c9C9dj$qzNGIAj8P}Llfk^xjIG1Qee3nE0E^X#(Gwp_ujHY9g<9Z!(JxIpo>zTLL zu5;@mPf>1=j3_+9WZjPd(oUr*LhlJXNi+cm$iFdN@%zm#abw!2n%P z-v{<{k^4LQGJ<^_&+@^=|H43|l8kn7wz}D30x*dyK)%LqA(j?Fgc~2`9Y71MAJbT; zOG)(2AAQ2qH~q^zQ)D_?1#M5>)vpmL6PR=Va0O-u|54FPI}Yf%sj8{|*q+}&VBnpf zB+>llb*pKTl(Tgfvkmf`E|MREFKomP6hTl#YuXwBF5$xqM4u(eop^+*b6)~hanFg0@1dIoykU)P zTJyXSiid?Cd?8A#dBHznT{H3O2jnM^c;L#Lpi514Gr=3b8lvpAJY?aY44)s(Rx~+g zf5z>!biyYlm<5v{3Dvzww8#`go6h)Y!%0TnSVRYtx1!m}buFIOj?VZjhoGvg;o)J^ z6Ivzouk`N$2LSG+{Xug)7IZtdY|CgPiw&d6!8HQ*;dR~HKg@2AkDnI3ncd(ileKMi zJx|@piZFa{9z$dtrSPbbi=m1G6~Ln6D<+bE37bvrsGSLUf1aW)~9d-!p&-bAyQj<#Th;lT54~a zu6+7cX){vgK~GP5OH8Bj{b)O`G8Wh>Dj*YIzr4Tsom5E3djjSUP3qC5df(ug&teDK z66-k+>2DU@9~dwGv=JjHIG{DgAQwiMq43J@_wGStg^W(liJuRad|NssU7pQ1_$al^ zCyOoQei?)ZDUOQW&;c_YH=9a+IF0izjJ*uu`s~d*#H7b+o{cxkR9_infA?I-M5MAT zJ0{zJo_l|a2a3R-|gVXM}K zE~P&EwDEhPkFUx83fE`z5cNazeMuLyxo7FwbH0Nhqtq1*3J*8gkGK>2(-aVw2WAGH zj!*iR)laAbot`}VLf;EZ&QJ3F`tRtz)C$tsk#vt>yH#nd;j2ry%{{e>K2h~3*~=vP zX7#2={$2BGW$bK9hLYw>S$8FepXz$hGt$El4^bWp_{F7skngdGR9j3D`cUkTT|o#8 z1^J4;DwOQbme~#z$i8yjr9zt;ed(|`2bprw8Kcy7EtvwJ)#~Le;c%z-H3FhGdZ`QWn6`@7`e`K0Jg30 zjo*Cv%TcVB#OY?Qe9h38zH+&f9CTNYWwB`9i@K7fi!5{|niq=s>cFTEXj*HTl|!cr z*|r-U4$dyP%zKL8)PhUatDn94#tS}%TFSv7UVp^i7#Zb1<*bKb`Q$GT3BRk;pdt8sqM0i`7xrr=2D1USNc#{z<*7{ktl>AFVz5{oRt2+YUDs@#3h?6bl`Sf z+uxvGScmIrDKdNUnZNdaCi$u2J1ke<7}NVgP7KIk#r14;1Q4GL+*+idsR>1`-dtYSV${EKP-t)R;Rs#kvIj6w7J@-cG!}0+-{%w$>Vx3< zwYyFG80n5!_p^dR8yDCRlVG}IEttM4^zI}&^LIoYgrP%;`_ULpTyNl!SH;+f)xqoX zPQSYro@XeQ&j?#CNrSKsKPc{i9y5XCMQcdk)j3AP+iHGGKUMpI9Ht{8EI9Vn;p=0O zI9Q0>QcBv?0DRBqU?LB^vZqp#b?HW%i4#qoEx}(smb! z;b<(=bkw}fWY|i79(^Qc`CGISYN>3@CkNB}lsmh-mR>V2S#rMv>7HvuGw`Z}K^_OH zAh`B8bc^60)`M*h+3HNimR|kjP3$H72u@fizAWO>@mc>lD`deGaiD_f@6?2jkw*HE zBHFh|ufyjrlcaLgpP)Ii{oV779x>BbefEXNFe){KY^>9*V(Zn)xUk(9F(_x{Z(hL_ z$%vAUX#?C3h9bmm@E~Gfi?yqLMP0}rtZt)Jj2;ki$NYhP<<~%wO&SBKN1sNkf_BM6 zwrXH^tSx)UX2+U|srUdr_&WvmMMo6_WHAj2edc(dJ+c`!zhaDXHllb=?%SK?C@bC; zMfta7L0rmh43?_5mImGd;e&xyAAArOfQrR67)68aGJ>QqBt(uebzKf;wWERo zfZM=dY+Z+yvsQauSY9>ZNZD!*^7RAy8CJh@G11@DbH&cwM8b+sY4E{*pzuxOtfP?3 z%;pYzPk?n(h6M_5`f$N~X&5;9Z|WY6rDKX^hhd_y-O@j)sGh=lWM3f9`u31pX?Vhp zMVrf%YpQv3m^fD%}5XsuD`q}g^s1=^BY5W?>A(N8Xfs_*+Ol%<5XWqc&tT(3b4WHpN%G1LPX z9*mm2E%(zLc(P_qb!Z(|PYHO%Wtf?py|jpY%-PM59rW}USiPWcr(yaBcE5E4NtUjm zW%cTcRUBNT^|m1g5dw1*C(!>>3rt+(QeDeC9a4R|)-*k}QG3e9)|T}?mv~$QAuhTI z7NKz$>y5y_1P)Kl1cf+MV|CucCZ`t|4}xOrSm}YH@t4X6G?@V1i|>Sucs!*)KxG;+F20cz}qn#)iX^Vsf{R1hxqe zt?0}FGMKWc0PQ& z_6rWf?b%-LRUy5Fb z`}uilIbJmfu5Ms6r0;-oR-z*Hp&VjCf2UL%iMOKnjVr5}v{g%1*^TD=pQP`<>dOH{ zvEi~OCD-6-Fgmg#Q95RNdz}j85v&EP%8nOR*U&?setP=1x<-NEhrJCN3o~)Qdy%12 zg~AWD5)%y?iTi14s3QWNjpLU73B_NB=s4VT8CSkF1l-!IDpChnjq_HlKZWxk3iAOt z1F3qQQs`Op54QcnChw?d?ibZ+@`Fj0ZXZ*O^(?o$w*Dx-N6*)j^%F3KId!Wf?}kp- zA@FN6Y0O)M$ekpx!Q;HBuCgW`se{e7zdbrUBC-wZAEr>T6Gt$rbQ{jZ3nEaeyGl@ zEWprG@#O8p^uS=#{eSJ${_P&pf7CDkzdoZ|a&B_o+*-!sBMIMPh!ch+wT!INn&cl; z(>~MN5(|;fdftE26V$!=2c{VE7S`aSTq+fPc3r@wDM2eTJX@y3A}OipWTgO%Zj4SB z@&9u~OecWafx59MLYqV}2M5wYaO<%K>bXlK9-)ppU+(ujQanwqNiQChWHoO;;b4i0 zj#xX)WPV^l;UO6fjLsE0LrMh4a~B$>8%?9sWdhWg-A%21BikbigN|!O+wj2;M~I*P z63u~IGLKRFtDs>-^Ab0lTkBgUJ2Z;%-py8X?W%}VmhoQV&l%> zXkuQdZ_|m8g2)ixiqux*+V#9a^a*LGeT=CLGY!s$OpG?g0FndE9%f%n_VS9}CjpM8 z)@BzXO7p^>MxIcDWk|ElHs&PAD36R@gGFiRBBe1C&b5I_oyXB8uc<|yFp9ny9C~s+ zt6%b6ib47BlKVFIOX`_s&->?QIr3hqxdZzm1p$zS;UdZPS==ii48{@`&SvHZReIZW zEog?h_ZJ=ACzR@}l*1yWx3EM6ph6=ye<}22GH=*lwEvW^&Ry`{{;F%NxnxYJn~-r= z6EocVua^8(D3CC+$VAa1;_il=xSqFn{-a;;Mq*5IdRYQ}dI}#bSMriF-JpxEm>^Mw z>_wUZt(?T=#ew&cHe&&*D6z-io5(ruW^Q4ApmuUgt~fCf+kx-!R{z1-@KNBPqQul& zbI9kBTtYctv;T9!5byF7IM1!&Dh&gX^mb)7bj#A7u2|MG_AX#QBoYIh<2q4|rtj4c zj1n()?3$lkOX0J7n8=awVy*9{^(8F}rnjBEKu8j}IJZ3=$bg1J8w5GGZl!ly&+y?1 z+^cw5u094YiNng5^i3MF9j;4^F?IgclL5hVJ}BVc58@yH$DQQ2bINw3T~W%Q*?94s zcg0=LQzJ;n2v*jzjgk~b`#cy=l*l@NXY3OTxNfS$J_eq`m037OG zoQours0Hk!y^D8o+?`^J5Z|h`M(xN81DTd&VXv&a7*m0{Tt*A6SB8enQa~-5iK=57 zw8MFUST%3loWrtCdQo_eO=HH4-Rr5bUzZuu#r+e9vqVK*Q*SFjXN&Ls5?v&c*sm~~ zPj01_QIsN4gQMv*!5o{3>R!n0+_+n|=t<$GkVUnLBsllih$O{~u9jXH%WI}F9TY^j zg5<^YkI_Ed!BmtQEgMrL%hsgUy~}~$R-@@H@j%Xp>fEf?e?@$8=jak0+}t3pIzR@* zbKN+0ZIckUpyyEJMN`@5RgRyTZkk96PT6HdR_aXBO@(h50Q&V(;SngT_xEO-)bkhL zZ9{D|9~;EYT{C5s2zoUxEb~s_dR#6r>w-S?5&|i_wmnDJe%lUWVNgO3YN0#i`T1%^ zr+1svkt5iS(Vu`MP2r{{(vDzmh5NU0ldIqfnStSY5cInaxTm{`0%q zGUMV8o3%w8y(j^nX?HFi=v*z53w3-9rIagM$xm z{z`rV%}7i?s?IZ22ZrV+VS8#)@ELJ0GI`c*xlO3)w^Xx)uT~$A$$Dg0dMsmbt?YV6 zBn%-5MI1=c7Jyy1-|vc8oabVi6H`0U;OCe8(C0OMG(Qf4Ue-N_x0b{ghmG7N0`Sbo zqtyVmdWdoz)ahQx#J`I5G@8O}IIb@=Y|j}QSSAi}Y@QcyP8m>5dTwpiwWE~AfG+}Z z0s9nQ^IjVrRVWKsJ7`^b8t&J)5M(tWFVTGTLYBEszHl8MeE?muAibxE5bXPQCj++o z!Vd(ZFrIdVTB0o&MQBKxo!iL&IK)xSUL<@)D>ckT>NA`vjH&dmNV=QUER=^o6WNZ) zJ{`yS3cr{XG9UkDM(swXXW_|Ld@_TUTpr|YiR|q`;ihOrD9C#|g}Z?jb-*m}gFj(@ z{Wml5&2+44Lj(7FGD)#HL;L=2$D3CDpl_Z6%hAffqfV3h#;@~~FGdAoCdKt+L#DQh zz=)7U@181cES}!y-UHdVhiSQAazA4gQN&^13(|E^z)KRlymPnn1Z@V(O*s=)0SRq| zunYHo7K!gDJ?)PDtYR3W7hWPqd;Qw1)lye1{y-P=*OIRWVk$AHm$2oa*Zi>)>6rZU zw1VeD`Dd1zCP~)E-QD8FzlD5@*2()TwayhaS32li@07R_r82156o*wv4%vHQFDeyL zekJk7CzEgeV!~$=jNZVa^Ez2+*H9@hB%hNY9N3y?tDbgK|$;?YCV$dBF z$u^O=hoBp$aRR85%@oEh}%zzKb&9++&ZGV_@Un@qTgn>PeYmZBSOrHMZwaH$DTs;KPBz-PgZ3xpko zdeq8JLC^%`uk^_*UB`!vuWPDTX6>jqpDe#Qa?f7f=2)~iWOo}B{@zo#8->sbWFO+N zM)#r^ZR{yWY7bZ6shSYhkT6_FgMhGL-ke0vQTFaK{iS=GU5}%q;)p6sxVtD%&}qD5 z-ix3w8|XE&iIA5*`abODYH*IRE<8wS?KJFU2H-2?Q5+6LFdwnZItt;b%Qc(6BrR~# zq206`eZ|D4$Ec4le7LsBc!D!O@{AMbn36@m2@l;HqAXbv$}1GICEugI#E8#A69dzn zFTAE7-Oh4ziT7ZS6LI^>7FuHXLZm+X-EEjoUl9kqZxIfHs@j(q@9L;;gz{TK64pWp>0{_mz(`$(?Fr@K7v&xb2`0a#~f(I_s61tmp;7EtkU@XCf?9UVR)I z5)k!*Ygm9SKhl&K-`pp9Zn@%aF|mK6!KSTy+W-BN-j_V?Dc27qt`86EXX{yM3Bl%w zf$5ZEB!pgKzYYUQ{z;8DKihZSt@xhztaVSS;3ve7#vb1Eg5RG)c_Zfb&d&(<+7ANS zz@a&xq;DtWJV^{MOoZcNGgv9X{c&P ztv$Rv=8Xi~A6UE2dm!ZVQ)6Yf8D=*(eG|L;VO{MJYib0a@mE7Z8yl;0htAO7(iNbo z0BV~;qULR%#Ut9j#ZbVo;R%u7O~t$=#=X#$x4pN>f~s`3T{vEhAjQ1-(FS3(4}l1O zq~(wEq76?Lw<-p#NnCNYcq{_kR#ml)R-753GYvmTVX486?2W#;b+3hTDK-P!gw9Q+ zh&1iNv$?m;%V9xIa=?L;0Ycb@c|I1BKNj9bgaX*qt#qSJ zIw`my3adMPDnw3s8>RV~{DB}9>-7iLk$vps$To)z{A;xl`76;Mw-v(Q!vMuC6_~^9 zpit9vbBFdxmkiX_^)E?WU&}bzpTFk)vi<2aczWM6nZsAQolxTijzsBH5GCmST*aaB zTUMn(1T0tgRD$ut{F`iLX<6M)gc8B&SOeQfQqes?NBpa09CDG4tY&BZ4eo3G05|`E`Z`b_kmb~ON1M%=D{lQ^5j%>WVgxLNH|Hv6YYB4{K(oS5kPXb?=4Fv*^>{ z4wy%Ms%=u;Pk0YsNsI|$zeMP>aB$clzuIeEs|DTZ4!5;dRk}5yrxnlE+WbG)SFQ6+ z3q{@Ky0+kWZMgPaAj=8QY$I1*-YZ;o_h9soU|&K4Gmwm`3hZ;?gLY8;(t#Q4Wm{i% zL&tzw2Pb;}>Muh$zn1HIv*y|sj1{q+5@;PL$3?1wxIZX_NcML*=8=D4@_3uwgCDlL zvs>C?90(q|k7G;&D58z|FEuaRrq={WYrgh_?kB2uXsHqMAQTilEKLw-kd@9h5ja_r z8Z40=4&(SiH-y@c~ zvmm6oL-*!I1svBN_1zf59~c~c3A#m6|KNX)Fq5f@;C7f-G+8v6bZt9QE66IH*H)Y#HExJdCO;xc}s;_gVb5;q8$z0WD>ZuV{2rtGFx?WT9{&X+2T z3Uc%+9ixY#XN2zt)9oWR3tsv>+S&X_oUohzDO;^i+v^Rze9h7E;`n>j1f{$%*Q?1t zr9r6jB88{4&q>YEc`3g)5Dx7)5F1z1DV9=&lG8bnavkxk4(D>dw&k+~=;1G^qyQp{ z=oBfCmf5_VNZkkU#?XDbpJGqP&H7XZD2NnD*JA$|lic|JZ>xx_sL6Z=88 z7W|)Fi~YMI^*{ahp|28+Xi`|>6$D<|5~=(h!#}3_GtgXldR!l+hRTISRs8$P`5zt> z-6&d*$aEKT0@ARCqE;3}eA*bB5pz46@5c3|GH?0?rsvM&GM(%q>GF3d5#I23cM6ZA z#R(&l_2a=>0;(IV9ui6Zgft_eLlLK~;ZMVh*fsw#gq_ih_RM_We76V@hg!-+?&?#( zwam>I`(5-6ECSJ+8I5<|Vm-i6yos6VYV1a|BX3mf+NaghPvUot#iyh)Ry`W5HpNe} z)v4bWxE?QlOBGmQG_P*@6G3d(=C4Xnb677y6v-M$XS>I1b>fwzv(A}-k~rT4w>{Yl zuS8s8F13ILejM-s=Uv7zg4LsQO6TVzw^qLs{QJpl<~NJj0?tWf$E11a zYFjWBV&49Ny;kT(aiWFCbYv(#M9OCkQecLk+svzgM&X0l*Zskrem=#bVM^&)4;O>@ z`sT+qS9KKR0D=&AWXgQc6E;_}v$VbY1$#uF*OY z0GExR4T%X-i)hG-K2%1%GYDVO5pHYws4YjBZqF9B1JJEk>ywU>s`>H)Sb*Tk`DR#* z?%6^zM3OHAEu!Jy<~^A)p;%&yLwKQe&ZQ*0;{}O^M%R6Q!!v@ZUl(GzpYj>PMXtkB z{tC#3Bm**u`%BS34&ywPrjtb}MvMj;6TP5J7U~D!N z(8TcU9BXe%P7uBG# zb(z#NzkIP7pBUF6w_Gs)tAVE@PuO=s-(jOH)q1!gFaqw#rDfC3o z&0bVAMw#{@FW-rA`rD-EgrtzcyO;Tj+1W;UlAxU_pui^J9`crH;xlLs#Rkyck)1gv zdd1|4K^vXNIsnZfxgu$`^YQ1AJ0lOJ)0+u8w=Yx*bO|?$WYYn>saxh}y3YhKspvD0 z$3HEX{8|&+VMQhzbZ+{}Twjl?dwAS)@MSG4}*1#3#B zM^Fn1NcMH1yS4Me*SFyW<6qX|`npH8v+ET%b!9MPIfvK(m3YhJrVBO`?80OnOeZ8oLHPx838k?AkHi}}T20BeDY}>DIlLl_&3K(m5 zw7RM<{TRA2as+VJxjyg95cQvZ2Fth=WlTyF88aI;EHBDV1fO*5EzRkf|4xZj`d(1 z+T>NZ&^F1sdVBQ^N#)u)3)FqeHD9sihrva$Njd=Q&{t1yfmg^)xd}NE3OR?WAQjyi zmlooWulwRXy9)-ZHE^omO^0T^{h$mlxGmWYK+`#xF)SH$@9|Cd>GsoM=*RLeG7*-Db5DX?l@eep9Uxnn;W){& zx8?{w98MmB!8*G`S<~8>Moo`Khs6!s%YRkwNjjd+=9d9LrsTTf~vST9jA{Pd?#WJTYEgHIogk zH}390J>ltnoKf#)x`4cCgsbuO`-j4>>-H>JFwNo`D!L0}u>urf7pB3~mMY3w!h&AN zdCk9uw|=kw4rIL!Ujk3V|6lti!xP1i?6nSUImRDO< zzVfxUw(a<&u&d{!2ze55u?UgqD1P8>x}kFh%qsVFkO0N-rTI?$V9y&L!EGu3@rz?~ z*Ur5ucjrtMwyAc17;|cl*z&z{4Va@`*W|1l7)1E@@~b=aYo2H-3)Ec^p2SuJA2K{AXFNlaGMcx_EIhl10G zso>N?P9raYdT(E^`P7)9^c#vQp>$zlJY$T3U4aRO#EUg}c=w&a1npx4GtQ9wMuUQj zlOfDoA$|E?pTi_u zwzPNeFncQXN@qs^uL&g5>!93fz{KXouzbhyW7NTdRBGwS`_`*pJE-YqNzWGak zRxF5)P+qji#~1)ZGp6j9>S@?r&czfh39J@r={&h)P4aX3#dVJP-jdf+7nDWlrGsXOIHh zMa`!!F%1@`{DE=maH91gY0o0ZPZaG*=}G!K)id|AKK+^=@7t^pUU~nT{=~uryal#Q z2_i+(SR9DrMlDd{%FZFQ76}{wqnrH1=B#*lHhwN5lwPwfS|xw(s8iW>A=nm8l@d0F z0VLDzv)X~A%iWyj0gp10$orwU)!pkKn+aXhAB2aFIk3S#m5Hz|M{D6`ImYoLWEAAh zUTKm3?7h<9{^xa1EF~fx7H+ilUU3}dkDLp$YnUg$Ob=x}p|qc7R_P#xI#D*bqRURi z4`IQJxkI{&yPU4kQ(GX~6|5Om1dJ6us9x|*T$*|)KC{PM`1^jb*_)X}H5m^!Wp%ZN zn7XryW-#=7u*H>3O247r&&FGX&W7h$G^bL>kLLgiIp9@pru<{|)PDmyqDnQm&26wd z?e+0#@;wQB;d3%MB8vBgWdCqc53>Y3C|*<tO{{-PW6LT@eAG)=SN_CQeM9Vg^ z%~p`ia}Mi5)tPf1b8OrvRsC1}cZZJKeOQ?9^jm@5qg(h}4e9@|tRBKm4$;2TuFw%Z zDV!@bW9Kx`-S7{jiZ>TAquj8+z?BT|M#BLR{sKPwtS5~HG%loUN&GYd@^@%`5kd{h zB(B&)V3(q=C2l7Z+W9)BPH77lZ}1W2!;G6o*s|l{*`JN`eKJhCc60$CYJOmi79&+` z?k{a2u8?T)rQvZs__IyP&%N~>JZup z;w33q5t&kd6mnku{#&fZB`D~bWBEI!PjWe0MsJv{y503}Vv{VG(y-lN#eNxA9X()Q zcmAzt!K4LdxC@9t3a?m7h+El0T;yNr`aO!@S&_wgJRYk3R6~-o1S2a%ld*Ng z)|qXrryQzlom^+&S#=F^{md!o$CoZ-moPJ6m&U~imJ0we>6iY%viAYycPugpCfB5-TI%O3VaAqZ7mc}obCw>y@!q)yNxbC9wcTqz zseP7s(a9%Y$^NQ+Z=j3z-iY{J1LGC8)l7Y7t_X0=qO7%#5g&5uT9aa^i882FZKasAP}{klfZwFi@L*{pU#@^6U` zoo6r~<`IJJ1wxXS+RK0)-pnA5+e=9M&ZYd9pXU~5zmxQ{B+EORHEKjoChrzmdsw{e zg2Pi_8@L;&UnUw`OrWgUp}zD6O!YCQ$3BY0+m!>8VJ2|L0Pcla#wF+Ph$a7Gjr*3WabtsOIEbB#rnGtYElXpZyj^mKx{c62b6w z()Ryg@7?2}47>i(kt8XTbB02)ce{u>y1V+zna=S|Y;fz9FB z5;hFv$W?{ED2evI$JDGg^5;^2Uz1|}&4KY0nWja#7)_gn&F3qwd^gzCLoJn`J44H2 z1az0F(q=1(Td&a5zHGGe(`;8#`sJB!32G3;owvze%gd%TT}ef`B%t)TDJra&@0ds< zF&w~YrzTJ!p^RpR>sO&XCsmpfK;O+eajP(rtM>Ee+5++De!Hkr(_Ry4xm&{F@vpW9 zCZNvxUPD!~3-x!x?u>NyD}6pf3znzQ4lA&Za-Nl2u`9`Ye9A-QXY_M??bqU@jjB3S zd5K{5h!KhqZ}Fd?c}rZywdgP$FEC;-XW^t#s%mwScy0{w^}vsrL*eQEp#?+yudw^B zINeU(>D7~o!sjus;5pe5%cM}pU2tB8Z(>y%KmB~jqn|v)A{<;kKei zTVvD>BLU}MgMqG`b`{d9^x;_YNOIb4x=M}NY$@9c$L-#_huWvC*Hw1hmlEH*P=A@@ z(?X@8fN3LY$rF}qVLbsxI5TP;HQm1us7qu3=R5Hi`w4k=8CuF!;G9y4!R8fL5c}n8 z4-*c-FVbg3OEP29o;n}0o3XTeQ*0v}xA)vm!4hk&*%nYX*u&DEWgG$goLHceE2%BO z5HpbO14&AFkb|yAa`ZR4)m*A1`ZBchZyr%H>kws8IxagUW<8m9Ro))5cCJ zb=fTT>7=dtQr-*nd0K74z*uG z~m#0!_59*zzxnef#X_%IU;N?HXf0#d?y$hy|YE5949g+7i1dzry}1dpnc( zf|GVqNsj&I^V<_pAbimcUH@HNGkKrv5-Wt+$~r;&`8gNPFeRR7^SG0HLZZ>x|8YMS zd;PkXxT2~kcK|Z_q7u zXVA{TJ7#KnAxQm0@{kpe%<7@~6dM_CjL);yO)-GCkt>V+ICt(B(G(DY`wzVNurtz* zSHbD=vqduX=4MEgdy~%AbRxonG|01R$q-HX&ooqu8+pQ-+%R! z_dWT0le-Ls&1JTPr{4?H`2bRy-2PEeaIll{D!3iN4y06qKsY=FNYAevu%u!M< z>q%)0R^JlwrAg-^>QGJMkEbX)21BSIeuQeG{U&SHfa*&14>--RRR7k0Ww|->+hp?9 zNHG!J#Kw#138wPE8+(D*2OO*XcqRDW8{kpO-yiMJRn6f6uA)-+%suh(mhF*N_Z+^Z zJ`uLw6@=RAv>{MkUzoKA4xm|99rP8Xq=W%Ku0T^hnW9YS(yNn=n;5`<|8>j%MtvsA z+ed`vP`M|!^I-GV2B!oqp+hC{!LOt@%RIE;Zbp9NnYWSpA^EP%?i&PwdMDilwS@XC zP}AOfuIur*EPXlgaI1ux~6e&I*Mo=+FjOr4)sRJh5;Klu?< z(Wsfe8IiSGu%8BP<9>DX# zUNq2EQisyAVF2X&Hl>tx7Eq76d7hndF=;IodlIf0s^n3@#j!)Tm7C}a(N5!fXAQaq zZHr(dsm{0t*+^ch6=P=5L~rZa6IMrNBoBZz)mdaKtU_y|%tw`-8Y6A;wNh5Gqu)2^ zZ?3TxJoMwS;byv^R*dc@&*f{MTeQ+Q1BM1L!TSVRIP7<$C1nOy1b&BAwC*Gf+?MaR zRPdV64VC?=#*F-;U%)YB!5zu6XGcRW=ooEQxe^>r8|59mV=!59o6WY8@F`X}hmfSvnrqF}d^lr!=_Ywpaj8qHkf>5D-aen*; z(j8--T)r-&Di4;;IeqI-BCT!e*>?-d3m0ESHAj@Zhx;4|?~bHzF2}%}RajoU)N5bv zil^p`tM@bN+H7p^Ei_Lk>S#$!T)n&cB=%c?8m;EZZ!WEyV6y{R@gwS^X%&x>7jC7a=S!#suU?1X54dfx&5=z^R{QH zMDQs=#SqPn@9!pTI#Q9&hpia?&@)#^`_*(lpEg;Qs@LEB+$+=Z$Ka6tkME59y0&i- zvI!z#w^aAK7D);=qwL}85fvWR0d`Ee49r{658gT?1Lnqse(h*U@Z7sCshYps%Ej|Z z#Oh;^0-lP!+|uViVol{@V&L8Lqs4*B-u|<^(N;tKYg#`GKgXy-s@E4UdAnGcZO=Io ze41M>Q2#xw$~?&0#wH>i@>N7szw3%mUif=YN4Fa{G7WAQG^wh-OAJ15Vjon$AyDsl zpk_-HmYX1nzeRfk>Uvc8Adjh+GdgV^jL%duJ}9^dmcj%!*Lk&^K;_Z}!79c^Nplx~ z7%*>TcYjM*!J?q8t(n@JZmQ%}HD`c@=kP!A+4P;KQA>?uY)zge=+4k-Q`am!247eq z@kYo*u3RX4y8TGQpje7~tjk7ea0FKzXqX4OrX&6>q5bbeZs1j+ti$n#Q*YSztK=0B zY(jKjzE8Y6KsAf| z9{silBqwXIVdZEzdt}fIcs{WOvD8zX*2iKP9aMqJpYKdQDhvxMz84c>f|aTly?6JiFz}83B+E!5)SY?Qota$&Jk8F;DWq zeG37V#;LuJ7~c*xT)yrt5eMqYB4gAdL=@uBZ09o&HsrBM`Q~8Iw zKD2Hg@6Qby8bunn`%G^g-*jbrNltX&rk`9NP|YYGA`imDQiP>mm{G_fj$1#{*wHn? zHZ^<3lYKZjUL_xQlihjeN4=OPh<}@nidI7#Sk|=KmizjVFgWoTZ5X6@)( zM=dC$Ftiy^$uoe262md@!F-@BY=N@q0(EP}>}}ZXSelc|I;x=n1yyGQ5(v*Val%e$ zqz=Hb@vbNUAzLHghnLh?1^}at2C=c-t-!G1Be9>8r`+J6b<}+cEMk<-`LEae*XsRC z_Wt)d8l02l0V0C~_N1AQoT>*QCENN)mHkT|r`uE#&y@tbhP?WAS#~q!s6?}`mh3~p z`kjI3mQ&nDow}y)-G-#vFD-f&nti&Sn}M)CemaJUI*^X;&Bnre%;y$8-8Kq@ZMe{E`_>bHo=soOrAb1Vk$8FRH)iZ#VxNu(d$IMg9 zzMO;CM2p%Ds_fpp^!OSlgGn}gcwq83HF6t{X40~HSjrBmmzZbo0cjjX0N(hMe1j$; zN}y@dZki9(l$OZ2O@qRgsAqAKcGtv{zbH4Ee4J)%e{rx`%(7VN(&rr)kL*j|ehL-v zpA|^{tK8`XGlVOV9(jI!wW2 zRAIE-qVZ>1VHSYIU3eRM4x4OIuY7cf>#T$9zVk81er?_X+Hv{)e=djY@92HO@}*C* zGB1JTtUGyt_|EqY=d!>***@o_5PN+XpmA1+6Y~txTEZ&QH!4bb_dYt~`4{CI!_->t zb{P<-Lj^SdJtB^$fJiP#a!xd%6WMn=uuBwA17Y$=JaOT=b}>6fV|dNX#Q@{+Hz&0H za}V4<@ht!M9?b#M$68iTfL!X{X^^b(if+L@vj-)_QH#Gp7s;cb@9XQ(UBW!|8{bV0 ziFX$Aim?h4zosd=No(|vWM6tDd0^*Q9N@+3sm2swsBcF-Hq_(pj}1ONIDAy@!f^G| zd;8FrjlSStt42)kRHjE`Z5o7EiRSahS%A;nMQR;&#De)4s`|+;B2SHg9C(%-_<Fo z2gEe0AYCc_O~5zozkggCQUZj1vV++Q)ri$Yt>^TAVrZYq9;7u*Wh@{Z$^}2-fg9*> zU>&tM&7Q)JZXF{}?Zezg!}h~PJ#0x4;C4Oe20t2>p!biV)g@#JL*L{zyd;eNoJM8x z;IF{-bVGYMHJxQof9V^E<2BE$zOvhm(fuyXwPbPg_PW&kiTaiN=rSjH?D^nYr5n;a z!n5}(5|bvioENSK*mIfg)xObVr}~5J3W>NQT(Hy_v;(#RC)Bn)_T>+UKKpy7DoOj^ zpD)tf*SoYcaz?j$9KFdmn=wHrJ}7`h1_U(r$E?FLvd$;Kvv)G@*b{; z3qN!Z@>`uX9q3$sd+z+j*CoUg?!~Hou71%I`EyTsO*5lE@QGQ(-SpLsoKeH-B7&@w z>|#X8SGuWQ9HI)ozHr2Ead)LgXpUG(X5MhbR9JExSz@;VUGB=mWr^03C%0;V;3lo};M&g4OhE z+ttL+=f31|PLMos@YutfANhk#7^K{naI8IZ(5iH~B^#=STAl<g2}cpF z!I)b|Wu&%2v=ne7jr(DHSQ@O=l%e~~Ys7^Qi_ndhHLWk8qKU^J^ZU^M`8M_+|FR}I z3MUMCkRRv^Nvm{WT^%&e6pP07Ki)~}3O;y9KsC`brfbHTWo@ik!?qRr;gDMD{-Q19 zn^WZ5xz6i^SnIB|c)AOflWxw4O=J*1!G^Th(M(cEL9gz?O&`yW85fkC{dp>O(Ns|X z^u&COV#D`*U*}%$@V6k}!OGE%-1@V2)JjB>in9}w{tu^dVIB47L>WK-Uo2IPtYdIC z(~Tux(B@5r7=ayyc8b&+9VZip(0mt;*POVJ_-SCNioagBCpXdSl;q@g+S2r=09(H=BEZ11VsTP` zJ%9J0f3j}+ua4)l`WiBZ@B_O|7yS+%WilSN&tGhhml}zxKD=eV#OO3nalKUw#i7r0 zN9(sdi>j)`(+)RE?fXOBwk!*)i@XmQr1e;L>K>uZR+TVFUX#PuZ1+SDc9sg`8=Zs1 z2CfBex{w%%S|a7?8D^xTzhgswi!U)MO@dhKXH4aJJzdlU={>E)hR#WaB^zsru(e1@ z-!X1_!Qb_7JlnFCLsSp-0{Uq8==&~W{1nAYQP{j6D7XIQ(}^{J@0JuHeGoPSOnXuF z1dcB2Q{8iNd1n5}#W6R7%|2y72}W16tS&Xo&w^ql8(+}QVB$-DFdgfO9V+gQMc#3~#;2LuQQ--4T8xUTC-hl<59Az2)@9{yGazZs$ z81OCPVAE`ndvv$_X&vd(s9m%24S{wEukm`Jjr)7!t><+>wiMI)+jrdruQ@{V(RZX1 z>el_H?3u|opmu40;YiDUX>Gbs*#6r0*)L-ZTjbsslpMRMQqc)z7+2WRkRy;{?^ zw4VFr%E-XbAzJy9(NyzTZRIlU=OsU6&-&hN_{iBiIxI$dQGJtYe`-g!rKvh0YrL;M z>cZAkj5UWPTI3^qum=BpF6@8t$69RJ;pO{Tm*Hp<%beCzJHO+|C~xfM z!RhYqM8iv5??{jLpdvo-`8CC#PiPU3qytp&e0&la}&34 zPf(Sal!HZ7z7I>dg*QTV#$G+S_ ze{)~IyN_Z4lh1%Vh5SGTf9VbWPTQ}MkFH5C!m7KE4B#X0ZyXp4?t6dohM_grk)B5# zCkEF_r6;$+FM>x;eoeDG{$OnHuvJ$qQmwG?cTGr7Hj8U5!Tl&<|nbjQS%SG5=_BDB3mXo5UYa-0xl9w+B%ATPkxzHe|va! zM>nY4WZ$mCv&?m&%(Fi<$4o@L<+srRGGsz}=sHcoRE zIiZV#hsR@Iklk^shhrmfjQxW^Qwh&vEQl?_F@n=Bb zuse4h<){iqDI#5%*53}2LdS=fFl#U7HNGBWw!+bF}0o5f6t0z0yGdPt7M*r&V zmOl`aCZ?|xF%KKXu_HxrYr=(umP+!1Le{KGGx2;9;FwY&{tU>%o%NHi8I}M>JhG(1 zls5#me!twuHHQ%+bC%h%?zaK<@Xy#|r25#DKTp*aWpskqSAymaUIC$0^#GrZ;O^^! zLXEQ?0nQ)RlQj%34`YEpLYG0z*HK^|9|O?Hqf9(b3TpU>9m73}Rn$Z_EVIR?&##It z?)nRU#2rHD8D7{8z*yz@G>^o~tW9I$y$m#qM zYqsAkc1^C_AJpQGo7=TP(C(Fd6v*HoX0D_LQa_ za(qb-ha`OeQOcgd^cc?0-zP4Zl%dyAmxrDJSS7FL7k7_T0E%#rP={GC*^iym3t)AV zrN^LEz&tmQy{wNrypHl+l^IeTM<4hT@Lwd}$1OllQ`m_`V3<$t3_c(a@i=xF2rW-9 z#I8W8`U9j7dWJs%1AhwZDK3gUI|l5c_9+F>G#Uw1Tz-nJj(v%}PhUsP zpNDEynp0*CMIL!dUO0va%j#4IfR>jY*$aN_sM?flZ0W-MiZ+fCJ09!8ehpUS z2O=ln5my!HcQ(Dgl=HhVU}`l#!#=euJl z4ClrjtIm~?hj_%!mhmUXp^yGO062fR#f3CfV>((7R-`Xde5gDW=DZ+lZ_h+m0acsU zYj3vaQFrat&7|V%?_ROqXYF(X?U()wl9>&1rG<7$`R2ywu&#?!df&Hi$+P<>(BHc=Z?|Y^3URc;@^y~VfDlGXre0t0i~AP|`+}IYy1NmhhmMZz ztCig3sIz?=>LSY;9;RiC+zn6L1!s>1V5=DkMZFtqU!~oZI%ZKPZhA2O`8%&+A&xP9 zF>pI15*i5dJx;AoaO~lyi(hv~&RTZtSI)hV_5d_4R|f>Jr_z0DQ2|<8TYnr)`9FyS z_x>Rg1Pw60Fz^1(A?DIGy*)B43jZ=)o|QXa`k_PVYN`J%f0LPV+Yi4|SNML{^@F3b5F(~p(7L>Ot|Ir zipK5D$~V+cX}jCmJX?}>-`~49D_1%2Rt&Y~p8NkmG5g;Z2P5eEE+S_?0QXJ%0#IPO z=Q!3PfEe7Xr^6cHoOF$)%m_ByJE5yc|8GnAqbqk;hRI3M^^LaZpFu_`m=+24bl z${4j~M-@YBTz>V9X`(b0#H2n5s80@nyS}s{Z*0MiyM(W!B*};IH&oc81k$3;(}kFe ze+GKQzE2)c#|MClqay627w(HQZb1nE-@({;#s%z(ub<4!un^Qxm<{zjTt}@bXLoCZ zBr_-6ss@`IDOZ7(UY}s$9zx@umhITJ6Z)2F>nOjJrUkvCa{I-qzgR1L+>9F_fI@)* zvBu?)6ePq02ii`uBKNK}ExL4Ffqpejt`gmPI@2QP8o*Xw#eU&M8e74IQ~Gq3PXk;{ zmY@!lZLSJ#GN09qHbwf_#Gfalx!8r+@hA*)s|_@D3g=LXT~NnS;-@FD^H@1TSkIhs zCuTbySRIM$s7`2(2uy~|y*SZ&FvVE7XF}%dcWmGn=(yi`!f|}VUm`Lb6kOp7E4&;a zf_c@akh48~G!@pv350vKX89Jh5(d0C!=-&>2mQxz2cr}`IR_R78(PwWM3B-($*Y3h zOSrYOnzvO}Y*=KqyIpY5!jV6ALn6`zEqEd-paYaIcIoxLH9nEwR?%;6%2)Zt`@zD6 zEw`>QMb^j&{l!vNUV5_ zAb3qWfLW(fud&j|^eZzhzgVU0?)F@j5HpAJPC#r zEOV0)D|_ijHv4PJOKcQ832atRHa>pPIvt<>XCpM`K*ii1^Ran$$P#J72lwRX@O)u_ z`P?sH#3gugBO3#Vj$X@I)x^SjR+@Y@nkk|}eqQUZi$y?kK5@L|I}4ao33mLmX!ia+ zGwkh%$zK<}LCt6X(9)H?=_(yO979E`!g zYNK1YZ3IPd#aF)^grg`~k-6$r?P`nE6Q9$5X!1MM7Jmy5LydD=c5FN#v$K`k?ti#O z{@YH}Advw007 z)-7m6V?2LluRd5Ger0%fL*=oZb`Lg(iN!rF;CWY_gU_kPZlQuRKW25VH zjj_QJ^G59$TM?Ocr z%X!R?dN+1^>)VUybi6Xt(B2J~Kfv+^{b8lNXq|mL%}CN6=Vafb_B$?f*a$bBIQqz; zY~J#jWGiG?k9*g7hZ?1m2efImv(su>+_cDEbQN#gZ#y7MllwShxM^UBWPBU?BBLUDgr z1~>144$wn2s$6o4$XRxTZEMznDr1Z*@-CP8%3cKpUB!$YrhrMEe!H}n~J)@v71slp9(2e^-B##Uluhc~H{ zLiG1_Z;?yz%u?+iHksTK^i=b!sAtmVq0AtM$`6~l(+OSV!sDq-rRg;O?!wqDQdg^M zp0-c>oQ)d0aeWu)xZb}zuBq_W-EE)iZBcY4aEVq%=Oac-=o-MU?wzl5JV0~EN!a1t zmzg}kJSrt$Srv>Z|>Z71HhNk7t1IF2}U|2r*Jhx;;o)bgPzqnySrFewTL}6 zhYqP4PkgAjb1UXu%`T;fTOQuxpbPg@TwP8Yth2>`$6Pb_9qwRhqw9)wtQG zF;8e+*nGkHiCLWBj*A*npLb#cSjuqa7+ILL7xblLnZYveeI?Ge^;7G^PPGpWIaF>` z-02%(66D6?d|p!`W&U;rcZ&>TiT$F??2<;Lz9dpDt=h>_-L7gJaL-3p)xltskFmhO z;nUbp&KO(Nib@}jkp*UOW~6kOAND^{(&l|ie8NLLK>C+af|;q9kBnlVMqTg4ob#p% zBX0mb*&-9T>R(OCu2?2FM72ACV%7y>)yeLgS)#XrNXo$89?YXV>?i}St-Mz{O+_Rk zf6NI?PAJ@{@n5VPEJZ-izB{v64rC)tKIWi!0X7?HHawMZaAI#;Zsa@u%IKt9Wq0-& z7ov>lI<@`Ab5xqj1-fGp#r3@!dBap-pDV3};e5a#GQNsb$Uvo|?iQbZ#IdNK0)_@$ zX*FLI&0|Nn=h{XgX?KsT+S|GHo;RnRGVIHtw*IjO_TpCWfgLh0Grn-s}GeC{~7N>aVHA( z)qGsVUJq5Y`gEa5*h%bSsp-BG3AgwT8ekM*&=nMy%d%)wgY|g`--Z1sZ=gc3>p?G7n#Lu)xEc~_IYTA zjG6GjgZ-AbjU|rPa$MP(dEw6G6QQWJ^8b{>4SkW`Qa{XzG!N&egq6$s&i!b*UUD_{ zii^v&cO656dY|?R`G;HF&B&7t6yCA;a66h0Q>;QY-=BhsqCyvm!p)Sv48dcKZ%_89 z@Qj;&uuZ9SB>X)1-3w&kY#x^WCCG3sXTQ8rj=ARN-Us{5Is&`c|Ll{Cp9cg=m2OZj z$kBp=`c16GEN&HWEDnW$jhE{t$W1-Hj@tl2G=lJ0O}h;VhD>Je`05B)tXYbXdKg7AZI_13am^s?l;DLtDmyl2QQeD)o?b1Ue{ z@oua0n)s3JXd}3fd5OivPAfa(HZmXSs8H77Z8p2HlM*R)Xmnh?)KvD`K*mvyLBWSE zy8e-<#bNl~v<>cL3M?I!YoP?E=ey8-Wr=MN?+yg;Bo#Zmmj7(#F^j^a=rCcDwsW79;rFp zG@^L#26T(OJNfd-93K?A0P7UH`kRP@p$NR+sXPO5+;-ryng?Tqk)2aSLBcMWuil}1 zNx~|>NaJFez;U}<5ihW^17gqK+%#C+G4kvpXVW_BMoS2G6Omy>FUrjI)j9GZ@kqzE zx>z5sM;2!8zEa0{CEjouHhc{+-n!$VIBJ5x$vQ>%te%JEjEFVSXWguAT3LJF9eyIa zsWs(o!Yp5vHE$MxHRwo&BM&S%8hoQnVkB3FX?CXL;RD%rHRW5YoQXfL zwjUP5?9>ER@}-1caz3VN9)o-@-&fS3%l*QwgdAGnA8l#)IWI4=-|U`8hTFyy4H+vp zgyUYU5-xhhZT>g@mUR@I@(y%!Ou|cK&nBw%$0Q1sNY#9sczWjAPtC!I@w{;3gnRPx zpnHtq-(#)`{kILf|L{rfvHDo-gd!SXNtv-oVyyw@%_Ecr_cKB}v;e#;LE!kE?uM3P zY*DOv7{FArW7X@Z<#A#+S*isZcd}y{Gtis>X4JV4`U_}(T`+(5$12uSL3rAL!Hpgx zu_0ITR|ffV19{xexD_!NBLVCU8iNw_L)p2|c)$K0sJ9dxfOP8|{1Xba0k{_r0ZVFy z2zG6cJMH8ma@;qT4t%>h5KS182M18~GfCm+pK-Lzdv%Z#-1Wmu2`2!@F}sa`k^Kz| z%dBFqfOqlrASJ>yhKjdNRP) z&X`a&UwMls%M7?rwCMi9@J67k^Q-!p2#BEoTtllaCUz5meqR!E$cF>0tdY5L=n3Qb z@0CV2onuT4Q7 zbS?h?t1WXL5tB}{L>3oZrUD2x0Y?CRW9%xHZHzQ7Vb{(k(ed73ehurWXu?_DoH3ZU zXUQegdfeq($Y0z8X<;S~YKWphOi8uj3FQDHSj~p-xDn`?xi!ELdnN}f zYb#)JPh2LYQ#@BCT9=_UGo4%WD{mkNG^v~qYTQl3vZMT5kc4{NCyc%K636NUa|hJ8 z^Anit^2(wu%Rd_IvO7>9q|R0R0DEja)|V8gTk!`)_)gdcH6qt=jMD>P6bh+V6N4B* zj72s&>nnB;xz4;%Ua<_g3`Sn!7TgHH)g?u%0#>K=boBmKXq-C>>Uo7-Ly@{&LHUt> zID)QUM|}a_n-KXhE|<^X&mb=WDsc^nEwhZh7~FAL`oePAHsP9WCat7#qx)4CJ+K4QKqM}tO;`<5p$wsM3Nf|;fWFNP_Z-u+37BRSa2N7{$^CaSQ8^}Y z1uivKuo+FFepq04T;}&63Ui?JsLtT##|C=&dyf$PP>U+CJg(Z3R?X0Z{a^p6oEV-> zRvuNXli51&_tUq@Kip9W_$wCN?dSdub;Sw9&QhM4wQ9j#Nu(LauI?g$9nUKyFU1T6 zs$!izw@yDz0sVCu!F0Wef?I>G`Hr=&-w$1O`_-vRdeI+pyJxF0=-$!+=Dh!7VJZ6| z-bb&7M>hG^v}(n-L!d$3!0`)yQXe(f4yL1JVCJYY?lce&bNEuO+%x?b6lkm4qM5{Y zs|_A28;ofm(+y`NxN`yA2}m?>MzoYUGL3w))#F7dZ!GWh+N0OIYEncklon0XUKJ#K zp6m|PCPxo17_opWf4Tg(5G7b}ewVBDf#5_L;AvGAKwa6)i;N8Ulihqy@moo zF&>=IT>+M7+x1qXD^8>1cE0B>Ibq2590L`vNuV+8Qw-t4lT!;pC;0;p0o9KgjRhG&Js(wXFfF8wqoXD0*7rAWO7$rFk4mE2LRFf~SqVdB3pkggRs^D4l-B;>^GoG(6oqGzmkb>ohdeew&Mji$jdzLHU81W1wqrEW^xb2>q_su#O zFY7n($bWwFU0_FCoLKKtBMN1@bAthQGXcZ?fOS`i62A_(QV%jk9DexvEv_Vs$CQqm zC?P#O6IaeXI(|4lcuW4IUH(w2`E5BO?|9p4&VXlfZ0hz>8`iQmS4N4}^h07Vct5Dxx;ODR9& zX7^`OsF!$o?QT^#S6s-DGeOI4+Pmqm#J8e{Jt7V-;bmf)k3QaN8ZysPZ@yT!)O+}|Dc#W8?jeTjII4|F2O6_Ba% zAY9rvcA<^M<3c&*R5zqc-@>ET!so`h@;;|_djov=MW4;NeL4bJ6ZN2D2$6b(9>u7k z_sp%1*_n$Y4JA+rOUmbUXnA=VU9Vt8#4Cb7`3T>{-76Pgp;fe`R1Kt~Xs>O6J`1*T zEPL+|R*m6Tkev((p5uT~+!O;raM1QGNy(V8A6+lB+w5fb#l^03zvrtni)s-&BRKTi z>K;pJfi^V1H*-tSB{#}^IPj0u`K-1TZy7mF^oQT2KH*=C5rig%E;Z!!>Rmc$LKL(h3#XKil5G2jPKo0 zqgVqxisXCg=yXIMG~R^-q08&2a7SI*;e5XA7qj|XI)5qs8r{f^+ih^fiX-X%hMZR$ zUTrtn9jHly<!t)vMc+h^(n`%n=K+sKON71AX-lmezaZJ3$!^ z((6u& z@sn|DI8Os9?3@=cn@BVgQtOXlivYf~LMdbOwk76sJFD{GuV254e)|0RQ=GBTL4`ff zdui`bvWXQN0{@0!I1q{6zp zKWq-7b^BYLeVb*@>9LbCR`O&=fY07^5CbV0_b_C%u~Y8&^LZ*S&$@r!Kzw}EThS>& z4;5+%i=$GH1l_^GisbO|(l{4K+B45$3bB zl&QK}Z_W4K4(Tmw7bIQ>y#x{w9;~~!Cv)GuVvYws`g=O}+}GQ(F@#UASOzKJU^x60 z>HZJi%HI}E`|p1JKd)bt_F|d>>#&q%4(tpa8(?5*feI2b1w_|k(ZI+{fNWVlKv~5{ zd(ttzy5LW;ztF7Bvx<%SZ~hqgOJrEynDq!7LtbVBYg4hG9km;L!2@Hs${9l*&!`1v zJYupY!6*bG;|eDM^ME>x1Ucp{xP-8HfdO}L1Y)8tfa97(s#I-R6&uCYgA^8tC}7AF zvsvGNmSsHtGt>n3;{2)t1_RA`!~*Ma-bR!a1#HQY*ztg5G$s{rdcYM8w@fh;h@DCv zhY5RtA^YMA$bYf3o?}S>XC#vFk=3_sgJXVNMO^y5m*rpLPtnPU`TS9k0CNYs^!{hL z;csjI9B9@-w*1yi9zpCx$8C3Id}itZV5#tSA{oebru(;t*R+bj5zK)V#eGB#Jc(N5k$rnd}F9|T`mL+|n!tdZ^KDC6&*{OdW z=)nl6F%vFD%GFxf`-1~@OTn(jNk0cVC=N)^2C_{COLI*I#$u3VYt~s-AE7|&&nSFR z2v0V6CF2_KuDaKtHMxGWJc(uFm(L^v$Le+_vI8+;{3gIY)iUbCj!@3qQ?a86n0G=f5DT2F&-p2g!k-N}#oPcjTEfhA)TIE$RfnFR z_llWRoF!8y2s@ryxsD3Lxb!>=aHCC9fvS{`pj9KaQS*~tjxFF=k*g%kq~_O<|fmBYiCIYb@E_N(s&(Mb+lP! zlsniP0E|QS@bmY(in`VMaxxQCJ{pvE?K!{a*5-yC4Kjr&Q>FqQOP>tGdZ7D7YaMsd ztiEJUm2?dpe)87Qt;5jWZ(GOVOG%xw8Ir75bOUK1s$3j2as=LmDdW3X?TDuR%6dPu zLhOTFI+g}Z_rJf|_qM-uF7klQhsbX?HMf`_F%Y_g4P0c|vhV9^(Nt@@i~XcC0!F?Z z?n+qT4t3{+Wg<<=dK>sk<)SrSphgY>Jer8039ZCJ>|E*y6AA`hcCHXb;kx1KJvjvd zw~v@!4Ryb08iPH!J3$M@y~R**(9O*;fD=D>oMrFmUT zY>Z*4jdNLP@nM;!M5p61Hsd}9FE-tM=%Te#5|q>lx>pYEr>V@5vs|S$XhkFn^^AJJ zQLB$#*RK?OI_Jh=b+F-Gg>*fLtDRzAN181O9m8BdX`#{HT%_=&)Pk-%rZ*&Wl(n;E zjCP?Yjv8LiZVsRgj-Y6sTu47S-=bB(Z|V0H9mCCm_FDcmekYGM%!;2ARI`nL=pk_M zf^+n#aeRQQxO3;=rU#k+_L)5UVG;cE6TA6DUPNI#Ko zD)v?2zzl$hK}Itt1yBd-@AT!SET;srgh{>$qfNDDUbjW!vrhTajK)7LN*>EOf6L&< zO$AP>HVV#1cmANAGDUI^3pWl5*VNvua?#V4J z=>m?*VVIKA(%-Qt+ka8#q%>o;STfSvhrX(c zxipO`s%O39j?Sf59p3&9;-XJo8-iZkxfxE~vE{?p_tx%*=47u1s*~N6bw;hJT})ZW zYx?}K>1ZsEJ3*3e?^EQrsZ^}Eq|1X+xNlzIlf@CH9d+6LE^2%ZG_WqAWTyJ6?NSqRN^npQWVzw0TwhB>wD0@iV0^sI=ff4amh4@{W35U$UPpeXmkl_DU%kS!tEza5`5<6uWjsgWpVONd@7OPqEqLp$0yC99 zYg>9x(1rmi6`MA$`lkcw(XSUzdVTwBb25_S0(_?DEg)EoA{Lla&oHfEpvXsxt77|9 zYY8GnbTyoFXW^Fs=g1w@2uwxd|H1jp=QeoCr4 zncrPRTJ-b%rVi?>r*=(!nQpJQ*QJ+o%ElHr>3G$m?hbzdEJ3S`?T*u+i6yLYob( z3OimWe{U`uNQ`)Wey>=y%8<-F+7>lIb~oRMcL7MiMPD;U8SGChwtyotE=&_ImUUbg zo=6)qm-F4>5iK=lDOs6p@bJa`O?AgdwnDK)5%M)0IQ^94rL=j_hpuUq#0BGf^)qv3 zq<2PM_@%y?8kSaM<+XaE-BdRIWnfU0@blOi}LFLd~w~y*f z)YGT+i5HxX$q%3c^xPMq%lH&TlSVft9m@xN&TJ`&unS!pr#B$im?!HQY_ zPtWe|TWfXv(&x?BESmx$fyIfP7o}*#0mP4u=GP-5J+`IhK1g>YNuZzkDpq|k_-RmT zL$QRa+vg-$$lYG?fquYU`vU?NQ5U8uaeb4Bg@voQk88Q@qqsVbUElilys-{`&FRC) z)B8yt-e2WYAoMf8!!Kw%E}({wmI^}jy38&Zf4HsbSV)x%bz<1RQftlm^()}TzUR1w zm}=LdT3u;2Ew*9wdwumuUc6kaX>Bu5d>;g)HtkHp#UIkZeW)P>_nKkhfk@29`p;V{o~&H>#Lh~9LM&h$#pah=%1Y*Va3Jh!nf zdI-@d#F_-iAm{C`WOFQm?&(Fq@Xc57=159ew7Q+6mSR=%LXMBTJXB;30k?7h3P!7k zcV$A+6Y%=^gnV2YYLCQ>{m8J6HO4DfrIC%}v}%3NDo*c$IsUDQj5o!TngU>?YxrFQ zWCO4}i->skkHpKl`rqrTR0sV00zT3|-^n(Wc#)O<>E?{iiy{R9HfrWNg&bu|<0Sim z3!O(*J$HR2G(Mm@=TvHHx%9IF6PVoT47%wT^&l%LLKtmpK-|5k5bu1Q~@?1KTe=8OyTjO$7NyKcWh4PypiCo@QF+ z`tQ1n>@c8Q3(5I{J^N>*9f<_{iFFFzi`-!iV{%C7Zct*;lv%{M4P=V;M9@DT1pCo^ zbQh4@HlR*{(-G4JbOI@kv-w&N`C zTkiHB{(PsK*q339`!FDv*Y0I}SJ|!vg3<>Vvc(VM0Z!Hi>}0enG4t2H#=sp4up=;Q zjO?`{C$9Vb1A$Zs^5^ai*8P5DDkYPVz8Y!q2gJ(a{Z;IS!!NW&)8s%w_@C+4JD?l* zf%@GcW!*X1@!fu${OYrP}?0P>F?YdHaYhB@q{T$)YtYgQsfx1|R-OMH;G5 z+rUTr9X(QYeR1zy=*D0`vmWC>8mL=PZObAv;VbBP=e5#NxCD9<^CJYi3%522poQUM zo3qX!VU7IlU~tNU-;1>dSe%S$oPe?Ap9wywBB=SDu7ipAxxvZ0-;Ffw!2i%G0_Vn7 zZs-zz!;bX^jzodzINJ=M2K%-KnTi>>8^*8JR-l^$)WhKCrXaVvw6^iJk+v-f*3cwt z4q5)=S2Ml0UxtE|jrg+)g7Yy$0>BZBF+gE9vvN9-FK-|iDStpn%R6w}`c6~+8K5A>!Do^uV(2o*5KLiz3iJ`eMH1mr*%g(w6Gq{j!ADF&==H84C@ zS1L0)7YA?SepsTw2Jz^_<)9zsU|u_HSx_+Gi}zckgSXB$c(Db$|L2q(V1fj*{5SFP zvMq;Ey|!sM!=%=fGoJuX&Sn9WL1osjeKKyK-J$|!@?ID15`bX3w>Mwac!TYRCu}kv z{=%o=epp{N0>63I)rx!y@Bb5o;=IQi10^XT8#~{;!S9YtW`bqj*9Pzcco7ZEsiKqQ z2xce1Vx=Yy^TlomEMQ50fG_0Y^bys3DUe^CEFNj_?XngTJ0zgY?U;`N$_07Em(Ktg zlM2Lb)j2iB5QyTj@9NoNrQ>%U7=k}#1^ju3Z zod;a=Xvef6&Zd7{<$nj$_V;0IY|Ds-^z`rLbOVONsMdhq_YHN1XjOY);O?td2eK-! zo;!K2|EfX^M}IdjH?;8~n@P=$|XZ8)Tdy-$4kVphR1?+4yvNa`aKMA+uU`*Q`vZvPH{lF+ z&Yd~1q=&qa41OfAq4~~tf|cRjgQ4*wXe9XOUl_)(OxCVAW&prlRy1o^oEn=jS{O$W zs0=^4h+uHS8KM8}vzfA*!7kYn2iq8$_p6|tu8AIS!qkWKrfkSjz7`XsGWQuhnzfD! zsBO85|J2@hRqgGd-r%-bm}4o=VY}y*x&1OS^1`m7PUqJ8-`^Z>`i`$K(jex=kXXtn zeTr!3K&!bm7wSamfXDWc9_gq2M_wJ26*4;NbQ#rIifK1)u`N08`YjSka$TIUHn5x! zla1%G5V>~v#S6uEkQunRE> z67aK_OH0@-7>vP=x!YEr$THiO{hMdlGZS+$Lxu?QUY6<4(}*7$uQ?fqk?FXV?`Sy| zjuDx;ZdlK{4U+konjf$i;84km?n4k_8Q^HJxJ7~e1QRv(7jF{n2xFvvHwDL`=wjfG z0X7L!1HhtUI!qPJUpQt#g%P*<0P_WT)5w*2oteMDe6mP60h3;JKlqk%ZdRw+*_pii z-UpL~>=Q>W&-5!yK0EVRx!n;AXR=yW%jKu9J!{t`a3YTf*P!YYwm}~epVd|p29lDaB@|rNusGryADIP zs&~X9&!0bZ_HyywVHJ6w<64udd$YFoUbo*Y?aiEGV`-sm>uIIWQLd9H>GG+S;gtbq z9z=Dxvaat8b2UP9Ap@&ic|`1*?yD$$ibLkr>ZB*SV;Ae4Ex>5~)$~@w|6-A7?mUP8 z5}l1>iD~rUx55wt;JRfQS^VX2a*l)By|3*HoZODq2ISRiTH?+h5Xvgz<&VJ~&P3KO zT%Ym3*0O!dQ{pRHLNgIj|$zajVrSDX|q+j3e+FZx>>5;6gHCMuq5Br4BBk{^T zZX;GXdyEbR<_lWw0EjoJb~CJJ@bkQFDrPU7*p@{}?ULai!=6q(oi@YD zSSG^P0Ekw`>Ks6<$iQhle?Vma<@4dx0ut|tYxy505afI45!TTO(shmfZPuv8fOgaG5WTW(^gX3jgPLy`2{e#8SDKw%68YhIK`{E z>j|<$KQo^2I$=6mA2I^httP^6=IA=;u8K{ZD;w=H9jV%@+&VfgtZVLOI`JG5x4lKR&*X_gLcx_bUz%Q?~m9aebWRDxW3Y zJ%-~hXFbK$@>5O+b=t>$O>;mePRt7p$&*Xtg9CX_+?*6#nT~ENxTi_dApf{u`d;{|1XQV{pNF9pptEZORuJXUC1z*XECm zRcj?{-8^Q=v6Nhr-i~k0#fJ0F#Hd~NnTa4%li4&sWa#gS# zJ5*Nx4iXIA9-pmOz3g<(YrFpIv-TE*o}M+kO*6Bbbpl7g7ifZG>LsO=wF^UCMKn-w zn^a_!=uTcIZi^kDRSrD92kwwx|l*3ZuF1m+{;EQkKp>R|j`0ml53;ZJNFhmfVp)bLfbZB?+w%uC0-T zgSA*Kh;PjIAOycTySUDlT$&ctvJt;gl9u&Wx9^)}@e-d7i+h}CB-6up)S6jt=;rHt zJyzEvq3e)&aUsO!U3K48u@^yh@#fi^#u(v_hccR#6~x2$m-Gyph>rr_dS%kWmeX)6 z>xRA{xWF>sRM2ffmWKf25C5r=h*{yuV?l5r*7&CfANC*goS&IfSn4I)sgdRG=0VCX zkl%)()pAGt9&V_eJd+N|dz|Rs-Lg0d4}GnqckbTfxfvsdwnPShn$IO-F;|5`gH?q5 z>_9;tah&Aa-fdma!XTyyN4{Qjk(&>*!M;Vk<`( zfC%O+j?B&i0U1AolE;)zOh=G~|9~{x|Cc9z#{5S=%}mXk#ob;No_Z$FauVl@-;19* zeYQU^$U{cw(1|^9+WcIcZ^{q?{&mJMzwS|kXmTJ#hd0#p;xn+qTS;nO zl3_~fOQ$>2GP`m7xInlV>Lj@*llq7xjBIXe5ZyOo z1${Hqo!)M#06av+HDeiBpjC`vC-phdcd2xMCJrEl`H})!zM*(81;NCsEdvJOA6nKO zW|retPLVTN+<}N+`$0pQh(wajACTkDcQ9mYkZtsffJQMN5Z3~Cgj(c5c<(yMvIOm4 z{_}(V=ZO918vD;(`Jczd|1AfsE-nqx)Ca2(Sgxn0@cReTiia#JuG~D#wl?-3A0z#r z9$x%=jiCMLv5%rIekH>{Q&p(7ucEAr{BLAGP&$61hi)~!YRZ0wOFkdc`kyi1zi|lv zzi4@~S-VcC8X7&tukDnyoy&dbr?xZz)ZofPZ?W&7CUC%zwW|b2(QBp_U1-f!C(><+ znQ4s}p+UK&ho{@l9G&Hsdyu&e+#K@$&IyOQHOU(vZ4fqrW6=OWioM4`(VmrF^n>#w zJf)qxa<%U<^P|#roC?Ah^L+f%2K-`XE}SvqDulVvbm)$(5sUy@(WebY8C<_b)Zp-m zO#RRtxitQPwD-Yzx3QOg!JM}am8$Ap0&YVUza=_G%pw%Bnt`B+E^bJLyMI$_jOOi} zoQ_W$6zZ!T6jAN#KEL<6fUdj<+cKtZqX#B~I>XEe#-_54j1k^aj3ypcU9=9)9qvk3 zY@Hhmf~!GHQk*VFVCQk%!Gtng2|VA{$c|M($(KO9<(eE#UE@1%ZDeVOa|^A}Ez z&dyzYUu(AKn&zkPUF>#j3iocCz@@&y$qxA4;fvHvKv}lK5{w1zT^tTcHw~ULv_3{? zc9i?%&b8-Cizjr?hxm*eM`BHkhYDeLSw~p&F1hKHs~Zcpz>6jplrm|Fk+V&#vz zab9+|u*jpT_IB{iN*#`lmz-}IA+*H>PXs3#sgi;RaHJXJCNtphs%MFC$X3Sv~rXk^E`a!P~v|InU?^mr0V0tR(7__771< zKCRy}fqO7SXuS*gOeDWnP4YbZtA|@3?azw^2q;kN`BcHgj+-z~(Y09{6= z;F_D7Vcq@~3wi6~-Ayv>nTQS4qEQ@r5=v-W<2p$~3#98uLy(pFM7rK-VPxj?#+1nCL$h zK>yc199qqHe^l+87pY!G0IF*47nVX7JLFIHo)*^QI+IF1n5X!S?)!EECN$nWd z)w^Y@p%ubl`)RU1A*9iV!ndidv^nzI?gC|cSc0LPyn4DPRWiKoJ^MS`pIJS1h=JZu zA5CP^>3$4gV$gw@%Iv`kuw>TYQ|_`OPVc7XqYDl)iWQkHwm(i)rr*!%Jrwjf!$ga% zC%m|OUXHTO5KeWX)TD;Mn*H?go#GpT-%%y0pPsBgc)Ol>qsu znf0CY)i1_`P#+gZR(NC(V~hc(6Ptt{h6~}mHF#;9-EjWK+ddkGW=q$azD{$Sk)B$# z7S5>#E`4v8ytn6Qdm+EWWG51wU}^bM7@;kEYJm{VFYtvKv#2=y?0JQ{6TH8OnPg9K?nVf=*aKOe>HJp+gN8Yv~OPT&ubJ-bJSKr|JTz&3JhHc)5yX;&Q9p8P8 zJ?^uuz`2a2S$KCYT5-tJ>6yg#l(yXnoD2R}qy9}T2dotfWz*?=?0N?aT&nP(8g zSrm?M$B@s$cxgc3OH88C2?lTH)Q5laX6`xftG#4+-)&>hb?WD=ce`(z8tvUof$VTn z5zL}z%n~&FH=NMVN0M#^=Mo~eNSBql_kFv?&ubW!d?+m~GeM!6s2P;awi%yDg3?{k zQwTwnbqV@?o9wV5CwZe|Qq5B%S%c(2bTdshxPa>Wq8H+}np9D1IseN`1kwx#&DSo` zr5IM^iQ9DNwP}qEv`lz4;d&wESbCjnf9Wf)J%e?+tOft|jLHk9%3mtfAhtj%e_@n$ z*qWTx_)(?T#SPaPsM*2G6f(FtH$SiSp=P?4RM;W>q?$$!p7+gl_XLaRNj2w-pezlyvgAT zUY3P>GG{iZrL-#gG0?Qgj>Yzfl7S{BUs*Jd?$Kc{9E%<7)Q#rb6%u|r`dlBO~|kpS11;^a3M0lTo!UXhTp)ES@s9S z-_`BZ_laJrM9b@@gb4YBAR@cWH4ZtX3{qSSk^Z|vGT{ZY8R-5xH!$QPBumExZi;fD z0d{%X007P)j7iqs$$jWwq4;cR%t_S!B6YOrf;LCHQ>T}um+a3T6ItjYE*;O$5G5B= zN11sHb~3$_`bInPd1WlGYshNP)!Fa8U&F3@JxRVCp)ka`!tu7eI`!!_iO%OsFB@~e z)Rdi6%LH~5K&c#S;bRp|#sLB3-jLISHQ{;PH(loDbS0N8JyNbO&s)Dv-tK({Im+T` zzcRmGq`0c4e7*3(KF2BEF%3m(ZBI(_b2**)zF=glHvq2dT z3Somtn$mKTy0LKPG=@<1tSkLFN^kso%~qd+f47T9(+N4f?@pg);m?4_64^?F*VPrp zL(%BseRkAOMv0=eb}N|CuV7~;^Uk%L`Q={o_(zBr2tw^IXTFz&pIC~FFdBx8?Pl#J>!el( zI-aAYm$%(3>h-^V6{i0yD(%kAB*U=*kJ|;H0&)0H{r$iGf$biq{2p_ZQ3m~iVR28< zWl;Q}CQmf{UN=%C*v)*~ZZw2CN}8Ur&z*A;4CGySA;3LL0(tNt*^dfmV$sm>2}BZ{ zKbY^#=!KMNSF`NJv-!b-eUMAF8ApNrC>b*)n5#ikhLhw1QsmG9Dr&mNm=*MI9YTzhsGOSyTn{kXH+9Bbb z6x?eN-^#3eFNkZq^CO)#a@V2?g1*k0z_EESZyQ<@Y^Rl|*D1!OPhH&oxS!Aah>8Yk zsfQZ2xqtQlG4rNtwO{Ts_Z~4J?U+b!#@>*sMnFlyxn0gbFf9*%Xf3xYWU0rA=ZG<8 zK_C#ZT`|V|%rppc<`MNV4+K7aCj2M;%6-TTEECb_Ou9wM>uG~BA2VEw7{2ZH@7QD(XXhLu_nW1L5mj(Wa|U~hp& z7i&t2GmboKZR^?1^|3O_+@f%iTnI}KaZL)*ps9RMYLm&=494>rMox6VJ~AKavDFR{ zT?5+2*j7Q|<@!NCiUEVH?Y03-)+FFFKMW`$dpy`KyqB z0<49JKOjr$k(|tAAVDt)6JcA!G;FV169zNqkH?iD)(+8rD?twOD1X16JZyAn!Ct{e{+Z8>Sre+JL#bmxi*_jHfz z9A&qm>y!s)LGM~QN0$h2gJ`s{X=t7`j;L#V((ye7v$t2c=0#I+X)Hn>oWDby?Kraa zf@l{4o?|W&-hPViuPBl_Z}W9X9crSU<@IhvviGitP~0{cCK)u#bA-Uq^05R zWy^i8KAv`+wVP_CaI&xoFlSq+-g$~=9Hxbl=er&@W(tJAbh|KJqo$lNZ+Ql>5SEom z+Y31dZrs~0z2xo1!1Y#&4DJ4iA%e(Nbxd*ur(oZC%S(1PN?7c1=CNOwQy*r9O3*161lKr$BAEHXES*R-4(O%-?_Z+uL}tIp^Fy1$OKINMuM85C&ke% z@6YsVSlAH=uR;2cVNb8z^IdAqoSCO@q9 z&rT@+{LRNzRZR-KB&r>D#8y~ncb2Y~uC8H*{?P6Q;OEIsnISg*u0y+1pg^k%dD>sm zO;`(l<)^$X&@l7Nk}OrhPXDzMX~&)kFe{-2S}(b#6Cq4$NE{(j6jP9Dk3J~+K!vpP zAOYVdcf}Zh+XtSCX-2LOkdeq&WNd1iY*kOQ-_eVyvu`vVphO47xYGhk=0?lHJ+ z0t3a^H(4Ly5-k>f;J(>SW8#`?tX%z#cRGmER(%d#^7_|3W6nUmGw!o(1ph4w!I=;) zPyHS2!FF0)q zJ@_Hd#S=WJ>I0qqOXN^D5LCfQf=0Fc!@^yl2}@up4L#yC19{MJ?^9=4idd)B0nMY0 zXAFojhTfeBvP0k$tZQdIat}uE>!^FgNqe_zkzU=66~aRgFNF5sNLCH}v_e>^T)t$I z19ARr?!mMA`iH%|^(2iw~P7Jldt zaR!k+shu=7x=Hx+<#sHu;iaG#s}s4fa_imsZsRZaeNzak56C^{n2^J9v>-Yi5()1@ zCcCrKtO*7-y?RnzU^9$A?L9ZBH1bac#2r240_RLErGB83Q?V!?nCJ`rWMo0}z zy7leH#Zz*zn+~JijSX3ur{Aw7_0k=#%anlftQYTBrNuN~_JKp$1a1Vd;Worocw8GW z$?{Gyj8L0@UD~(5rG8}nz#HwsId?Op@?n-D{{`FKP!OUZ+Hsk19<(tn)K-{rh!WD) z=o>&(DeExQ*;gk;@B&?4j>g5BOYSZQWYEUSzYf_mHl)=OkRm7$&ctS;q&FElFA(v@ zGBj()$&5nkP2K}-uQp|W2ToC+i~;TATMB-(D5xv#!iymq54({QA#Rx-S}Q~YvT!Uz9%yFI0-5V(yUXE$i3 zg7CzpQ;`jOdr8OWhfrq24KTq~>KaG=^EDnKZ`%F}(j}>sHQ9{)o?aI<17VNZAz#{* zK#Q&>IPkjgdxi6XrO%7f%^$I0JM;O|CA&-23(pK6n~jBZaeYY9u1wQmR2JXxfIZz0 zOpZSTtKXw^18HkZV|ioFK)&dTk9q5eYuY_(bO!F|z-;8Ns9J_m85_o1<2Eo%J3&AA z4GT52F8d~`|1&3{%1`lQ+DL0rR7J%@Pq_ju6#Ek>D(-Gj0XGCxC9wtzjxzJnyaVLr zUYCMYOu2DmNLh05!)LiL^^?hG19Ej|LOu0<1@(Q6F^(;SC4Oh|W7=&D1*?!23&6~# z!(D#p>g#MjpT;qKki^k!!L+7N3lI~cgYJs*uIh*2^@os>jOeGdLFy>2w}h5xMcyKU z=Dd+2CtaMT)ZPKQsGUEVu&-!qa5`H7ghMfIjkl9Y;Q@~6q_RfV6e2vy7MF_`=c7FpI;Zd$Enq?eH{rHCh~9fpLc1S{=9O*BM>rv$J#f$L!lz#|G;R9j?jGA@1Ymfkofp+U-J>WGYkEjfH_Fe18Xt!b-2M%I(g%z-RBv35 zi#k0LaS~19fCi{q%0fZmL2MnSzrqYmoH;1Sd%9f6@A(b~!3Au@8*`5^jOYfaAadh& ztVSi_?Wh{cKJd-C&#jse8TXqs%7IePBf2y{536P{XwnXVcd3 zs=%u)+b%HUvRP7(GgBSWx*wbKYT*iWh3PV4%{=s@=fJtB)QJ5Kb8KF+-ZS}-UmH{b z<|9TRT>~XI&R|<4IiDCiCViqSOrqw6?#lW1*%uHT7y4of-DZotAyL#Os>5G%yhtOSR!wFverri)LU`X7& zi0j@+$rtU`m_ZN=(updzynW63774-1Gwa7q-qZ4e?aLhc-oA78g@F7##>6;-`OKK> zJb-XHK!y{q^QLH2VO;2-zs5KFH+O`mzt2$dAam_cEb7I?U$j?H6sOJH7n4sPb{cDlyN+1 z@EL0~Xmy&4+?pFReK9xP5Fe=dKrvI!utp=`3a9ba8!!JwzudjS z3*)^O?Ruq2%H6hGVs)vSL~{rGjCkq`7vGxIJ$lfeAqgj!U9QN#u@FTopp z(zrlJ`5~G1z z-mz#7Dneqd^XzC(p5cgC5#N!nwkuYRxX2cDLOVEA^8Nw)_*K6cwm!LtO-44tI&9PP z_f-{{7SVXas5z~(n`*PKvmQhos@2rtq&)#H8Ne*5l$f`Ei!#%Wm2AXjVrw_LT9rEU z6H}(1O>Wt^A%?wk9tCIM-ew`AmioM$oV*YH0nr{})9Ao`#x259SX#aqIPSVevI{ab zq=rA~%TPnKB3FH5j#SI_sgz^8v^&{P{(8*e?f=RH7R7vy@gC$uj{^FuO(HynvaaBp zO}82IOtPfv)FyB%A2%m7&WYdTcJ|=Xm4z-LX;(LB2i-Q}kPF@0B;JKFa^}>h$J2M! zCx4tLecr!m{9aL^#^Iul{a3*;=qgw;IBGO7Sr{ev41xx+&vMtUS4EVq3M*d-tdTxz zSjou)3SL`C=j^j@T5^8E3V;>>nV^!vSZfUgHW*KAa=vlQ{Nb$9$zVQT*i=c*gX)(F}4MT~c z1=2C}T?0V6njAJ`4BCfnwgJ{&UkX-|5%9*KXh4V93bUpM3_k*+; z%IE2UN;lOF_JSu-3f2lumu(e(4T>A^q8=qFnqCxQn@4{@HhsNES?kH~?ZAjJ^vBUM zps30$dHIp~n4%`P=kX$Hy+VM3+xs4E(;tsx-)Tb(u)ut!n0AY<9&QX1qRmwYY&7a* zODkE`LuQeuV)nHo4(%Y_+vXQUGmfNIH|hSG8H>8sCexaIEalvr*GG3> zhTQ%7VUK_*du;jQ*Ja>@ifh1A_1tu7?(Ne8wfuLrFV|Ct`mRNmgz>CnToD(Fuk7vYE z@O`fG=X2)QcUeMr$3*nRfvlL10*f~cET$Py3z!ARHsV9y(0_$WjBVQl`M!Dyv3jF0 zU6J)V=Rvipmwg{(+ZK>6G-iIAz&z@l47h~u+;y3Eb>S7nP;FSZNONC*-Sm_WYFM6* zF+2e7FTTYMiC*;@x&iuqfjTXoFd&P&Ho@rk_lVe~V!ek)HZ<#g_G|i-_*ASiTfQx0 zFO1NegyLk!C$>q_?m5uzg_UXN`7fw=`Ko_;uxCS9^-9EfaEZ}xOCIL6V|N7r94?K) zx9AV+L^kxUk5;3!M_NuOr_8wI{(6?8T9^K*Mo49U0LSr~*PI6YKG5}=y|kLnc`kBK z>7vHoYgxWK+L^wJB+~3d6%plU4_|89LSxtz79dt&-5yP3@xUpry@L2~>_XT&AkM^S z8fWs)vOfDY$5)RZeLIk@_C+#5U&Ye=24FQe@bz$#*@DJ&VuvVj1$@#v|L}#cP*qcX zvHK!v-_gSc5mJW!jZJMFp)chuWg+3)=>Y4V7bU~H17%QhV=PfWmxQ;3M{lDMCXFNB zes41cF7~HA?UCEXS9F^`BBZ}7TvqBU!2H?w(W=NjJDm`L-ZFHdHiiK?<4zp$1Na7GWtUZt^~45}ebJ53_i_aZc##4&M3Y1GVa$tNy6* zF-2X~zOtEd6Y=Sr&Q%ap6`hAQgqLm+2hMHA?BvY#hc)4EJ1ffE%nfb*D}M>p4MfOW z&O1-xnHR2dE-zpD_8Gc{pGH#ejiDhY2y|&E=()m@T`N$|4sTdflhBykxFhG4YvvrR zV97(&?sNJENXT?+7SBP74k4lHnJxV!tG_^jtdkIudOJ|2I*)YYl7ct8gov{T+vP{& zO%(#ady!pazOAYJY5Td8C{tQ`u{X#sti)w-_i&Sg<@8q$C&yUlWbz4p;D)ztGCFYDp&N@{kmxxtGJK)!;i5Gx>5Xm2#-491 z#nIhQXB_KBFE>9i?3>mZFy|oJPu-k)=a~Ze0a`gz=fMaLQ*L#_VJ#tqWc_f@4v6zz9Cm-NuO#q!~%0PLtzycXNi&Nf3tc(Lc+(|!sc;xbIMcj5egy7j7a z^N$l6Q?=LzP2Q`5&*yL6e6@_$D}{NCzxzsMEd+5CdM~tC05kq=HL7b%JRWnp+2)Yt&v{#fR{lFEus%h!3I{aRx$ii~T z$;sJCGGEj5n+c4G#RY`Xo=`3$V}M_t;I%&>e1>6P3@38KnmK%`sY(yt^IAw2i*@QV zDl5ynjBjuOUnxh{XMNau6%mn`AC_XMa-gV$HvRp(Bv0*0yi#lxkn|i1mCX+)rS43-^c;cn>iYbpv-# z3=Jq4v?mRv0kix`AGqMoVEE5a-y}3Nn@@Z=Rq4!~G^?2{>Fa&G-HOLTi){s4k0lvU zlomPDej?dWmdr<6|Hy@@#Q z8rnfv4_16FU%jGg?wW*%gmzYm?+x*Q5XoN&xx*a~cB?xnDHIIEh$lI)twjPOO%ZG2 zWT#c-&%ty|ZA~gli6A)>pGdc*n+&Kzw)SN1lDm~5Sa5Vw9JF?QR0y^qT8aE5F(cT` zWj>nM;bo>{bL(7_c2h9Ff&z26;mdNV-Zv9C9EoW+Yq43x39aS>i%z`SY_SyT;gFlQpO{G#mv+42JU>9|TPKA@bfa zYQ0-^g(~S-lI5gAy`g>DKIJ%1z1c@!*ExZfP$}y^t(cS&Mqw3t|p^^v({EJe2uC=2}oYZ4WF<%zG#-xpUk z*0dyi9BtKfRL^eV_SYlry6-siSaVn&G#_HUYSG`?g&Z1p=|J3b$#j|Tyvl{S6J-6Q zeL8kLiCH;38DdAowAiAkqLV*qh|5Um+Wesi7h*#FWN4EnfX{bAC8vdZ0 z)H}q&Lt`%5!DzbSH3@tdleb+Wqr5!4n%u74>hG&cRE(WEu5y~!f4)weGxT>Crcr`z zBAl0TFgOGU4d?BcHtOy=ARJNhEtI;Ua5B~rb&A{J+?k_kWk3^}h`UBQ2pSyCY!kxP zmudzS+60ES7AvoX`l;;yC}&lAUwal_pe&mv*vJ1_{y>cJ9cCHM3m=aVG=|U~bw+x3 zg6^W<-ehE!Ty*yl-RC6eYV_p8k@~I|!(}Rxf{l4ad&2uf*oYV?&KD*)4Okh?keVb1 zWZfEzdt}2<$1mcoYW+dW>tB{>SpjOjMyI@6A;Ea^8RXO?ffY~JU%?BYEC^~UP86N= zuAO)i>1W-=KD7_kcRQ+N4bJvJ4@+P8bnA8uKRfE!GuHdx8*78#*BCNdG{R7#+%gY- zJL>;wAYAE|`pKXSQQ?`c1-5u${c5&%5GK$69xOBL2`*qThNZ`R+QwPhxsh1&y!yqP z@#;bwW_yChA+7s5ANIZ;$T$LIL5mBdLu<#$8HrDkB0Bo$NB(}=ojG`^Q&n-ZS0o^x z|M&Phe@z?wkN@{iFuFt?RUT zBGutR;a96_tA|sWxAGV<(m>Z-&=M&1n}S^?(9b{Z2*L22o_uD$r#qNwvTMHOga9xV z;DMA=n%>X!;>qVQcUXdylg8+*VxQyGiRqiCZ4K@4ZVD-_N5A&&i$>k$zrhYc$JznE zbH-lV&ZYtlS{7Z8c7~DVSGAU(`$TVd+|ozmJV}oXi#U9m!QYE^R)~;)|TP;E{d! zeQ3gcw%2a=>B1(`I1=s>F#=Yx24_aG|uRJUC7_d;#u_a3!)$S7Iw3v>R3%`Wxi6c z%A5IbXJAE~r7yP%d9Z*@5L(*@e+kHOVsmz=jO>_}tHHLp{1Ik>bsDZr(-clLgV@n* zDN)o#GO`oSLvwD&*1BgMNPO-?&7}&JCRGjDawP{HlnI>2wB71gwfY((OjW1Z(9OdA zsMWMYU|=2CPFV1V_DZ;ocQujmGaq*$8^?r{?WNMXvZx9DV!s|yARm}dT8=&kKm++K zGTnvAY;d$G^4%7i6217qJEv;D=D~GTQ;u=A1Y{fZS@(~BUPPBhnOncwXUbmnr0YtC z!^zf|3fKuIxAY41J~?>vBBovayF$V7bz{NRpjgbCH>lNOG6dEUqXiCZ#E+{ie&FZV z>I^L3mkasez%q9hfKBb7mxwl&oxx4syhzKUNc_;c?-ObDxrs0tnf#1rz0#BUb)R+z z>cd$9OU?mc%>cxb(}TV6`cJif6lbd2sG%r9yR|l_(SoAbxb(5%X~++XknYX%PwuQQ z=m9J90NSKhGn{OFUa7!5LV;=01%f`;Ois|bFEzhTd}pGLA*xvOoBgYSA--Yu{;r(|2Hy?Q*vc@L#T zTcq2d2XGBK35cthX;1gW9<&SvHGF}F=`uf|_)Zg%1C6NMoee#BHtWDZxySn@lX=MR z%sO~b4AX9k@W#W|w1;Oe*zz^hpOAfjs@DCZdXIGbRII)L*iJkh9yd3fJi}<-#WvVr!aacxs!{OO^)O#7r*^E#g2zf578zI5? z&29_Q)NgTVsg##YK8Ig35s06YxzZ?2ZR&CHXUOOVaBif zqlD>x=wS@Z@Th1Gf%g5%>kP3Bx!c7p9$B|%d_hCq_#r=Qbh8eQv$$#d z2Z@-DR*YS=gWCa!8S{B0B$+<{Wns&2RCd6)It+*B5>&p1GycvypJcSS3&>c>|)S`(P;jVjt9 z{VAUQrLG&u6LmzuKJ5YtLmp}Jl6M)a542PCXuYjyx#fAIC?QDUW^pm>#(Nk65NFoJ zXjcI5$C_4Gd8>ZB$r7{Csynhr_L0NBYTfnycjMOiAW?(soP;O9MWfEP^YfN#_#$g$ zbjI;*u++dUt+Y=cqOv;ZyP<`JFgEZqcU)X)7S!heB9W&>%8Mpx9K*l7*rHIGr%iUh za6fIaZ2xMoaZ%qFI}XtNV1s1=n_6S_awE3pQ-OB&?GC%!o)eJ${(y9t!^nkv7*6z*G23K|+f$!VTy0(#-ywesho*)M z7KW!&Dl}c;#Z&JanlJ8rS9&R42&?-k{lBE0585T%pwmf)KlR-{iPKK=@HiWZlTL?o zPi736EpgqCySwWK*Y6W6kMz5qA5*7w*m_Ta6lKqboTAsgix0Fs3R0VSZ;#%M-ZYU> zS*YQsVW}`?14EMR*)gv(9+H`W5Vlj*4|sO)?(|2M29KCR*_^BAL1&p4Iu-{e#Kj@f z=#u1OOuJifbIT1xv!O!IOb|4u_P&4-W^wKhhzo@7*&8-7?R}eJR_-pckD7mO_D7zs zT`KRs{;hXfY`^@-2w<5FqbYLs_^^U6pw3JVVK>a6z5!pNu8P8M7sA>Qm#t?&N-mD>GY&LC?xO5j_nFpW@l+~l%ZyodFF*}pTy zi9h?dVuAk=Wy|j7>{ve>{Pv@#{e3gvOM!|XzGD)!96n$`BiL~4`nlf(MFaMonhFsx z9EL{ER<>PQf37c3_L{zs$p_x}zq zfoVUB;#|ZgVIU~I;&5nt{zh^mm*FM<5Vhx-r8ZHT$5m@Y`uc_Hckfxz5lnk24H^M} zi&^we%Q+(cb452=gj|!*=;rPEs$tkkTF16No49&HrpV{}J_YvAFN%-6TR3P`1|7nA zJ&|-KZ?6-`lgB5wDn1@{cuDTT{-eP<^+)9)afS4lD<0vpF~DK{=OR9F-l!EUo)VM} zE#4crzF~OJhhGj?Z|vLpmfyi`lUq_Gv!Ib%>0;S4GBr?BR>+HKhOKW?ywB3&7HX!k z)W~|b6Fn^Gx!5&pwN2!OlJxytqq2ydso|l+-d2ZO%63ELFoAzSrg!a3s|i~(@1W(t z;bBL=93CLFYHc%1Qo$+MlD#|X{^gRBkvlT`iDSDZsR^H1X{XLDWn z^C~tfd{gvT;AQ<)K9(Ucz`0Teyf##k(@6MuG~4W8UzH2=0<$+cbHLE+sHFC_Au*jN zK~vIpVTd(+pt~R#>tc0GW+UF@45 zGKSeUXCw7FzM;>8(=PLik=N0OSKe~Kmrk{n2%J50=rCa8Rv*D_!VI!fv8e6N-_>rs zIv9IhilrxAm6nh|xPMj%5X!?;!U~pqm!-v7h~R`*mz& z*JF_vbrIcMITYV~%JFvT;Uej4vMk%`Y`@b%2a+*e9XzwQ41*6GgNW6FEslK>sNVS_ z4jZL?Daz&^k>7{JPAJJ5a`C@4@tXlzgUigf8pnu80r(?fCr*9bn*D5_o7IpI&tNmA z%qRTe7wh8<()NHm$Lp`gsx1s{(GqrizcjCy0VLJ7;yu!D4mJnez9$#%d;IHIQul?0 z$?&zE9xPig}FuF(q@v@_m7pg zS@%(Qi6B;-#j&)+QzT5N8)mD+xto5wio5d{m6P}Red{G2Z{sjtR^Z%ahCVd5DsIm@ zp^l`V^t}{kG6N)NCBoY%;18Zp(ySIm{XLYM=c6lNSb+jYBuxHgp|? zttY`*cxf-9`{L^fv){x^KCPFt@t)2xQ3^I)=Kk#6$u@Y8h2+8DjUSteF6kKXA!6%0 zBxt4$5=a7h?<-%&O|u*KsHWSddeR@94wua3=dAcH{&@{F8sm#mjd*Y>4w6Q4Nm5Pt zBK^q9Q(ZUv1^c`cLl$l%sVBsqyCW=l@3EiuiQH?O`0~yD_Y{G&y%|}C+6UNqCvg@u z{g=}h&C~BtsqcL=dA>rI9jMda1gK`lZ%U%o-yY_dPF%NO(caq3;K!fr(=eLA4$$jU z#VezF%zE97FMezy8ge-DHxxTO6iqmL%KuYl%yyv}ZSlzdg(|wKr$v2p?T9opjN#xz z-Z~;>U!Gvz&+DM-vEv9co#Wh*>E&g9!);QLV|Nj1qcj;XDt(;HHm>&+IO1@u2w(HW z=tIG}4az_!7xm3;FUTiqbCHo;v8;^!c`@-Cgew(aUeJtUx4vYL;sf*B@uBBt6S^yn zT+#(p#XvJ8CX7>Ad(1;e%fYOm|F`VLpMS>?K6I3QC>M&`&1FS;>idqsp;z`JDC=oQYrMB6mSY7xWRQim#vpR66Yy(8i3$GpNG zX?`go#1IKpkB!Cx=E;K6IDU|QBBVZiG$eI1&3i^QWQ9{FGm7H#WY|cwNDXu}=5be4 zNba+YEDZ83#Kqcv2i*vB)(><>%R?MS~xNDe(+ zTB8MW7Wc0+PIlij-mkEKWMuE>V~g$c_l7G9Ta^m;JOs&(4TbUYW+-x$UyF^_->e9C zh?ECZI%TlGRYq~q{6C$^I&5_yhOl(C%qddp(FE&$zgsH^a+^B514%B&gwW-hux~mV zo~%wc)r$vJwVu#O&3!*~_tnCE&@SBPFN+hP8Pc`=x(FpB@pKEn&XFC?o2FCw%MQ`( z>P@E>)csvhPa;@fnkN_ve?9lK4w24=5cgQNEApE@`koxfeN?2J87B7jFl1ado=jUGZ*xY%2I zEyc-6esXOe);fo!SzaMyalS-=a-QE4y#cmsW)K zZ~E?2_1)uigg?iF-5NnB7iC!|qXY3g5FkZ;!59;0$O+UyPXnI2^4N7B4Op}frzw^v zDjjv{#WXR-$-{|mbt}}GBE;>_HAS_j9V3@dC@d)X(69M*R_~0jodYpRbWo-0z1QW3 zc1JY5o~=1_KFZL`&D=}+06zlZ@7w!diaK1-qLng42!#_P69RC;zSpnkdq!&$x^EqH zw(j0`{cQk|Yd-ALrlt1d@y#c+h495}b&eeY5NPBLsOFu$d$HxEr;lM2`&#MM&eEZH z>2E^*r`Gdn(wp>;^0CY+|M$lo|N0RaTm2~MEMdE6ZUbHfy5Y0LMYA*Sl`kRi4IBBY zJbo0Pl_OepDEM*APX7n43C%l3g8j9||Mlt=|9Qv4|95@k*XRDv#5rUfAY@`fZPZX@ z*!dxnM-43BtDJ0YK4H;^({*yV+tuuAx7kQi|8QUkX2mBUcyZWO8+Gx2U5esg)>N>3 zODdQr1HnFOL>J`X^A&3W?VQ$qXP102Jv5ROLWi8dl;(5s);FM-lXIs&5la?E8G8i znX>;9OR%fDvaIKIz6yKGNq1_YM)$kvmuU%T|E>O7ssL=Cy;Aq!v(+lg|(<6#JoLXm$x~o}@~`{*wSK?$->>uWU$ZY7ywm_Dp5Z}PrRCJuNgV4__L*zb$5@|| z>CWGqu$wz}f+eD`>BNmAW19{E>M)aahCty0X} zu1>C^y2^*s?!^jSdHjf^vl+~h6G40;g#_YcfH|!1tL6c0`IWD%CkqdQIZ=Z&MxKp%KuNz9NYhkmV>RCKr%tGd*X?x zkZLX*D9s(qxoPg*-zY9z>acKNGD*%#cT`k)KJB%`)6a{yknsfo5!d}nWBrGi=0DjH zQ(DrVwSE!|O)@Zptv@2&a$wbRM!bI)|x4Zkhb4prP%3XGTn3#!orKyH?VeDAIF z&-a!FJj~vfy|pHvN|oGV?W?~b@73#ydSdOVj_yAr#tgf%7F9F3ZDBP@DPttqfV)8Zu+t=(%!V%zeBj`fD#pU3q-jueCEUiE?x5 zofBADIbB0_72l_A2d-~Z6*jgOAdJ7$0xEZ1$)3?a5Nseih|NLw9ms{D<|V=YH$M>V zfv{-&gaT8zJQRd1IP|(m^lJEiAT~y(*Ux={rgT8JQwuo}ju>iyXj&*nr2a zl+Y1{Zpr>iMfzSx?*#m@>rvm4JGbiXZ?t5k+B9&}ERr9#9}+qh_1HV-u(jy1*{d(R z6Aw(rD+9^cKdx9V@~=zM@@Jk3{Wli?lMF433f2LErc$da1iptN4daEO>9$IJ5a@v@ z?!#y3^#uzCDDIA+Y`{BGL(*XU~ro2V6w zb|$Z7^BpRr9{tVXC_w*yI+jtty+$^Ms!S7hL&=#NSWOy4Q9;cKm;q}xvG$K&ye=aX z;C%x<*XoNLD4yk1i@tkSa`fq20<%+CQ2|3PkZ*{-eTL;IHUHT`r~6GE1LL!=jC>9@ z_B3?otY48HphkLco$4TzlrLC7bTloaZiKhnPw*10iQahVQX9ven^${d>MZuNJ)##0 zrDh1_?K4U7%DLA&G<|o%l-bfwRY($K)FTN4J3 zj;^nCr2IhG(467G-D@EX)zL=uhO#DLYO+lLGiDFBfbG}u0yQ@Zs@I*@w7@Y5rqi;4 zEea`s#u&hRpnkhZJ=15Lu)z^Bf+CGVfS57exw^zaMZ>xobQwS|)_MXJ;6P?fCx%`R zi!KrIOc4n{*b5|sV_5=Sn}DWzkgH3fc}6Xn#lF=Xk{YNR4t& zD{&_`%PtOl(MAkfq&?p{EQ8G?>qy+}y|K3yQlA`~*@GI7yC$?~VzD3Sg0=82i36mB zhs3OsBN-QZ@=94)d|`_ep6a$QqJ=wi$i~T;-~d+6={^Y^~4gA=1d5KR_wjkU?d0$B)gdLk-+w!M)y~PrAhnX z3yjkrFM=*|AuP{?Y^>EoE5gW1^dDZGKJ|<$?0e|H@Zy%s!SpCw&{iu9%UxpZBeT`! zc5EFPn(m&HxZeKy;Dy-hk(=fzrozkH+o6zbm6nS^r+@#{ zj(Pd-QK!viJV8+jl|Uew6|5iB5&)r&i30P$yENn%V^vtEkFZ*cUg6dj_?685zmEI< z6NtDRJ&1vgM~M*g;a~?G+X}kMe;@#cpM2&C!IkkF{S)t&7!i(8C|o-Rt;^hj#`G`P z7_RF4B{RU51-7UfdNQ5i23l z>pVkL(Wu^#i6=!dnA;JCETjZ*+? z5}eG)DkvVqt_!y^kjoEeL8I>yvI`EGHuHP$t4LN@C;*q1CmF$2D$z^S62Hyg{?yN-(S%$vU;+MUuA2>=m0}UJc zozo&0Aeg5?H3!*>rS0!$B%;`%FRujQgiD=l^LvwlxcVG+V+_BsEsg!p^GiH!@N~VK zN!bgrIex8FKcP}zv_9#g=kp~CRgS%eNuCosy*vkXc=cSLt%CAs_dhev{-0mB`cG0* ztnFwA^tvka@CSk_N1a{-QNgMyjEMokeYig~aUoegZA$&^~a8WO+m zCDZ$RO-UjV9PMfFfe>;(Oen#A^T8Hg!&2lkrlDOf{ftMR1Tvuob;7TYKyd=92J_Yx z;9SY|X`Ln!0w9W>2J#2uGCPXNQ%c+G1;Q5%ecCrL)yV}~;$4NF4);HD{r!8yvggHt znPQz-Vn%{oih3Ax4)o^5-as+98PP@Ht}5!gQ0Cwe=;m3$m39E?IN20-<95?UK!X!u z8of&P0lW3E1(9J3zQ#KhoZ#+rl&9JB&(+x2g;L5?D&Jq2PIu779t=FPfk5XwKvQ!fRb=MT>A%D~%RN z)Mwz~3@~PG1I65TidGF`@z`rn6Jei(xeH^<+=fzjE=B44U(fyyW&vSRmqcJji8xad zssXKQ(oTVdZkNf8|ADAP>u;da%{FFMni~jZ1je)ST7`|3KJDa2jWU@$M@1T#xeNYvK^&70!1-F zZ>%9dQ%|i$xBmVd9O!`_7?W%Ivv*kreyoFKZNM)P_H5vLC6G|du^$MRJ?kgwO_i*x zhkzYI%R^(iSjr`wJYWH5VmB@TuFIPwXGUNiGpvY}$?9 zIHhH(`~%^g*h;tWDc zg07_aBK&OqJ`yOW7om9-FGe4r=Y*h#t8c;PoVv?X@V@?v)|wE1vt=b-wGJ~j<8X55Mlm&I2c=T z(4LjbYy1Nd<5s}DgJEtyyXDUo#y8(MZ?afxre`RF*nwCF`XN1J4n!Cw&KuHH+F zGQP1028nGv^1fGObz*yPosvkifTS)5{{y_z^>?*0tO(TEKMi*NcTo9%nVDZ^=9iiI zLy)rLV&AjD>PRRvt7k}Mp`?s>P1rs>-MYp@>KH>7WC3oV{gW)_+emi0H1&+R;<>Z- zl&_zhrvk3TxuT`B>JO!6MnKh&f*nk=m3yiUddlm$Zf zK$wqhp&Vp#%j@v8W59bwpMzgqK~GH*kQ5{>;%PbKXN@NwqUikXRbPUxzC-wB3N~b& zjiGmX2*v@p(Lz|?1xB5-(49cmMV^!mb6o{vDnCQ7o?zD373vv*u+(1gCoPcYm>gxq zKz07iGEnVFpn=b_DMI8~^yg193`XKO48D;&9v~Mys3K9EVS_cy7uPYUap}SP{)Q9} zVtasX7KZq)q$SU!o2njUyY1bw)`a3EuUGx~zPU3bPs_2F`bai6ekaVQVqE(R*Qng+ zoVdsO?g^uEsY7&a+6iki$bzFqXma&OT3mAGE^0QWJi|yf70BiD30y^NB+Y2_Orp2p z)R`4H(QzlKI(O7IoS;fs^mWchCdRK{tQmC5%0H}WVSQ5&*+?=+3Wugjx=BAv_O{Be ziawZccIx%;aO(S{pZNoL6t$vP$qvd~L0~ObJK*%o&=dd`NQx5Nl)>mAA!z`EI7e?t z*=zy#RQCfx-avmofMhW6U>TT^`s?^H)E_Ui0==|Et*@iOlF~(@m(exoaTE~Zn2z)W z0E`WMSu+4!XWyWgDG+!+R29^B$RDah({WIj2tc}A;D1GT{rjCD=Wn7v{Xnn+__DXT3QRv30OPsG85SMIaRH#Qal<#LW_B{@G;5L_b%JI(R)OAZPuQWc z+r!}+<-L*H>!xeA(h*bYH&}BedMWR+w~xDCI$daHC;hdow8VVg>q|sgs$|Biy^HO| zF!v9{&;%UX9{`c=5Ey=yKMLlgF<|fSc8MXb6NgNmN8D)<)t9iw?aBc0Jum2YDEL90> z_W0Xp%Im{*-A-A)YrVrOcUGi=U%YCZ_I0%hct@UEkR!QLgL@$dPK61!+734R!LNDQ zLAG7j;I7Y70VfIeDZ@?&U*E}pqIpc~@%I}8)*7S8)th-)_f|$Rrg+|vsJ#;HXPN;* z{^NsT75{xFXn+Xlx4rOBTqHJz6qpUu%;YZE1j4s*h{ik*q z3^^@@`w2;K_gC}NPJQhBT7R#5&{5ILzsZpG`;!RQ8Jh&ty0^D7xL1=*0Y8DPUD`68)cXh)4b+o4_u2NrRvu^EzE~@% zo69}%Hs&smy4&p!XgVmzS0_v-&CR-Aebd!=BJRxKaKqnjp8{~w|9aZQ{;nJ7Ppr+K zuph1xnzTK1N1Rz%R|%SragdZ16ck+S_^PtcyTQh8@!@7%&mk!;%xRL`=Z{CRobgvB zBqJujCiQ)F8t@W7daF8B07rJhrsv5QffaJbWLpsU``GP^hkqS%IF8^;G)lSAMgoeGwRnJ~+;Yz%0Ri zAg=Ay#WE5>kc_06kCxjpc@(xnW8cv~PkGlv*Z@XLA(FY}L3MU7~vX?y%jWgU@EU>y`c0%-pR8CZH zTX@GDar$EfG0u{I`$q*qk@PWN){^{CDIh$yZdsk)x}GgG?SsAL-iGlk%U34X5(i+938w2O zV;&3J$Cm~Vt`R#Lm$o^GXg(VeO4L7b{Mx?;HAW`cV5?C)IXyc=~LXS#Te6Esa^ zGh#D&+jj4liftM;r~=aHVF+el4>Tzb0)tDuAnM>_x$!6ABWo-OfEw{35PLRhwICX2 zyfs!KY}N9frnq#NzN$y}GyNU&=Cv{vm%Wn^b1(l6DC+MD$^N@9Mk0vA4XC22h7N&Q z8#Qus-CH+J4~s{}KJIQ-w1|8@S#_zDvv<<&u$}rjus+1quL-@Pdl`f~BU!-70LAnA zcLZD1hU6P)Y{>+8k1*gphDQsdncS9WlE*8Eq6@sjqVtz#IP^7&rv90*E(zR_hw3~C ztWKIE8RD~meGA-+dNLJ+w9n<(V6IhAwEj7`pp6bVvR(XvAlmf?&Pw{X0SDii5zurP zxRt;vBVbs6>qP%`Q@>W~*Pi-cdHUG8xtn5|3iaNeJe#A5n9=6?v*9$Y_-C&CVRUo! zfI_{mZga!SRmtk-eMaUTFj^bs@QZWsPmDl+kAwg3ez1tW(NR31yl*!MFR!yXa!qu0 zZvJP6&;NIK{jajR|M2_t5)4Dz=K(p862L>h(SkeXUoDb0IL~my--8E7-J*edQf*YVjHPVM8e8AMVbo{ z;_y^z8aWVX)K_`^-hmQz!`+Vusjkv44@|<$&aB&_6j{G-bZ%mtp>>d2$9nUh`jC|x zW9;wDsX|u@y{;r+)Npo}DZPcwt^wNOc?dq1$FDpK>Ne!-F3d?tcfYoae-QSaax|qn zAat)k(9~gnVnQ6emauQ@xvoGth>JZv9s>0opqqyxXJmt(XP6y{}id zp{Md*&+YfSX^_TV!t(T`C%*oo@x0pYS48o{?TZsk7uB)5V z_1kL}iQGDyPq?Rw^NI^k+mG1o!hO|2Y)Jme6w3b^uZj!;T#PqpiW4jd zDBUAHfQ_&4a#J5Dx^GTnCNA9zmm63`ig_gpbe*F0rn9=r%Ano?Why%oTRjv2y|Jr9 za^K{3+$c*K-?r3a_Ib%oa;-@BK(%qv_~mdflanV{wzo)=J6ZcH#fhN64FZh1_ZE?r zQZJbV-M$6es>1h_R0TCbVeMwz)f}7~%_0MLsoQTenCMQyx#ka)!uTxT z;BS@F{Cn@hcbl$Y6Gm)<@A%m~i}1(A(tV+p0l$wZs_Ka!h-`T^e^ATbas|}`v zl++J|?pd}GWtQdU1v1$dxZtgX8(1FCf|^8=3y!KvRt_;({k8c%uT7P?(?hEJ%gt_> z9OGjdgIEY-NrcT>k_>6mR0BTDW)B(jkYO~Td2qq3uEiS(UMSJIbP$jdxVv5jPJ+B|>yj3O84b_5gYV?bX3 zpCHwM-2ZSFDLDkr;C)6eUjg)VO$JcP3kFVilSvjy=>$o}5!c?E* zP}Zli~9 z03{pR1#;kcc2fr@QV=Ik_Du{`F5De#=(HwPzY8&+Aj2+ge4nfT>>ZtHd)Cx`=qA!X zRgfGKrm=;HeW>R5IFEL)P99@bUKwU@(l?^BJ@Ucp`^VUQUm$pn+}NwFm@3N}L>guA z$x)A!EkZL$P03{MK~W$!v@2olOhrlgpJ7o>V~0c6NV*mdUeh{rGK! zTlfLOM8ex@0h$MSf8Uaon}S4&hok1BWyE%^vnhP1v^U2Dg(BiWuk`hI@5+kG6DSno z(Om%|hSwk0W!i5wQq({oTK4*Dx%5F}&%vSW3&}eY;!5@teJB;15l3t_s(M#;oa^wO zD4u-ejpOwr{+b=N*qs`@WYN05ks~D4hbv6y6BXv31$ldssMfVW4|AJ{*R1=-{OwtZ zKV?*E_5H-_>6h%yMWDuW$U07ilkY+-D*}y^Je^hac;@%>2t0l4vW+9qK|RXgyT4XF z4NYzy_h|0mvi2+&bpg=Won!4T_ZBSLWFj-MU{aAj*4E4B>=339^t7 z(V5N)8vCb1(vXkPWAP}8qacc^JPLl3>y21}#K3KUlGrsu$i`}Oup2(C)B2`BoKU{t z={N7HqV}?F7QPypA-z*)M?=J2pbe$P1$T~ag|!B3u(& z<;%%Wo4+M?dEuxsOQ0+LvR_--MeM`U@E?eYgZ z5woh(#2s}|9Bm7Pc z-0v-dz1U+c_0k1}hY|bWP9VCu8}wdzduD@lcL_94&vWd9#eHqWtfE-k&-+&&}qg!=<@4{J)w~F&LHA%?~rg@r=`WQam zUh>3B=&BbV%Tmd|LC{|T3t{@GJc#c$~NYH!bgyMca(Wl6GZ9%RdY zEZxYy?-;torRsRZ5smi4$I>_z21A-qd<<6rla^=!1&pgSAJlz@9%Z0$@1uCH^sCk; zol=V}Ku4>DWf3ZDW+V~nI_UiK(28XmG8qPhxsR8sd7&>EXgGHEGs15aOL4~*g)k$* z7N2Q7lg0+Npd~bEO2YsOOem=NoM%~!M7aDw2xNSFn)=Qr73%&2Oi$a)WU&JOPRbpM1%;qL2Urxl(` zoWFl6cVErr!&&}XIy7;TYAqy87|TEjFKO+vFWYF39XCEjKWnTtqdL^#zP3Ns>!qjL z*=tUjI}L=#c4}+l;T@(Vx$)CbQG3-yq)d&s#N*dAlL<^x-4f=gTn>IbRguEa-&rfvExkN2F!z3jD5pk$DCC}o1RbXLooL~;iP%z9 zQ!Da8s|(PnGZ|3PJlt+Mh}vU5wXOOBS{9?Fi=cp*F_wr9^y8<2R*NlmWanUpbM}jj z^ci<^wZ^Xz_}RtE6v-Cb$n|d;EmG3=E@xpjcd~sREe)B^BDn&ZD@&!B;|qp5i5mOb z-dndl(s1yaV_PO(Rx zVtIbxYP)-a##38G#)Cub_bV@R+>vD+L(-%HQ+$~;)a1uc#xx6PpIHsJWIa9wlR~ zjypMQJFXut5ne?^J?Lbs!f7z$wGcpi*vF1U85(yER*|z@FlV1jIdLkO=zHAwt=A1N z{-;l)&L{z53w*~kZc9+;qv z;qvsUKT=>jUd4W<+bJ{VBk9Xv=C*e2En#VhX9%VK0BvE=;ewvT(~{wue!|A?Y1RoO zZEs&ljwhZf3ieZpvCeBZaVwMl15O7Ympze5XqPr*5kFGQ3TEF^6b9soYc*iz(A{OQ_qqXv};Mh#9&0io1)?fvGARL#XsugGf z<8#LD{k7GSl>+X9Zyq{D_VzOS2CPND-Oar{Cb#JwZeo8X!rP9)FFICGZ!I&nC%Xa9 zK|>pKR-Q<6GARH4Tp=*tuNiY@j)w-dhqzVKcgWtKBbBT-0*rmEW8YFR|mnH-6;?>HWHfrHRGvf zG*dctaF@3COsuiYI7F7 zOL_CJZ>9T$CnrDL>~ex<)2GGeptCH}so-$0k9&ZMe60c6Ay3kY=i{tG0Dgyp#z4V~ zh#W_Y_I>d*rjnn z(#X4U>tpv?Th%|@7@p(aA9wQHwwS|Pj$VqnyKUNlEz%P?aR4Pj7<;9) znHHHbBNB-ZUcBX;r-4>kEw1bDJ3Es&efzxXd54BE+ZV%Phb|*Dt*9FS`4ll9q33Uh z{6L(!=!QVUJL&4z;5@tCE1U&0S4+X~GZ-V;Hu7-H+FLYI^$c%w@q%ls>*k;)E_RmN9eWS+XE$M# zSxWz?TR(>;nK-W>CWI2!K%-MnX$rz~t!d6#z#6KRtvpoCYn4D@$d9kPsg!e8evu7r zUGo||ltsVAnX05Mh`ymPSWXq8sna!ScPwdvhMlWzX#H}LZ_gb7XTag>&EvbL;_5U$ zT{IrLq^P}3`0OFX(E^$bAwmTT#g^GMMN7_VOme5Y6 zCnA!f8JmRe=pyc7d8(CkNWAcllnHd`MpGi>H9lu$tkrK5*Z%c|kI}x8WFfdyd-|1< z&a>6_u7vA+EQ>!7rC{v@$9B>75K%~+EV>TLAehuCpVOk1|ZF~E@sg6Onf$)YjrmB zP_0zVj!}N_gh-kUwrOb-=mq4(*^Nhfx5$5KD7G9vn0?#e2O@#BRgjxNe>+vc$I=MZ z$}dzJ1NrQUfo$V*I4P}C_Yq-{kWs~LE=n?_g|8(%c!ei>ns>k4w8r3Jg^jZt2vm&` zXr{W(A9R1#S`a2L{p@?R^#>sKsy|TV-AAY`5MZ39?Rnu@UX!3S>S)8#t5-mEyr*y= zA!x^^u9w0$j!WW@4c#kOT&e2e;YSX{wiRM;7znFdVEr3(%{X#Im+%}OIzM*J zRa^W!fo|vcl7c7rrd0!@aEk%%A6g`4IerhM;9 zR^YhqX9Ln&=1l-bdY~1DG~^XhNIe>XGh^cYBs^V-=m+;YxE*4EwDF|-`v;pWyWNV+ zE+Ehi{KIx#kqA>TA6Ak17(E?8Qw8|ARa;!fpJ0av(G`6d1(QC%F$|8B zysvt>B7o9cW5W?0$D@YWuo&-H0Oo1)2F>_4u~QaWJXid42SqP)D%sQ3TsrmjV$;yU z`MaMQK+3ao%hFI{jrq?*HSdQrA~k0BjdWi0Y}T>d6)Um9ExMImR5sqFJk|XCexah5 zP4=v!V`~h)VyYW~b}S2o|2&mW2P279N@v12u|{!q#0yjJBaalL|r19LMOii+la*SRT!$^1ns4<%hb^f%U6o^6oh>VSEPpvl<+|C8ds{NFN^KsjH$y*f zGTepmu~=B6YhY>hRHMR99IK;;>r2MrBNu~{oo>BL@^Q33%_AAma)nnIjNA!j>EFBw z9RiU|od+sRY5KNAg1Bd$wOpe>*)|nT_IchG*FD61;mi9|t#XULS=~V#8ZCZ?xQi#9 zL2siqH&v_AvguZ|e!aInro=tV&FT6_cPDjaJ6w~CQ9b##Jkp9*`2wh;SJ?l3Dl4p$ zqXTrJU=+@Xei!b7Ir?xHa%@2UOwpb_b0&Vcc*wG(EbYV9h-bTRY<}jZ%(D9U?^4wK zn`v_Yy4e&Vn?od2K=tmKLi-Uk=|&LINo#xKXY-_{6wdj5hs@7OX6AeplY2Y7^-hC_ zGE2FbOQT35&wfcnPR=1dE`P2%e{pRI=UljrP&`5?K!>Ri?rUwW?!6Jyrt-BZ@FvB0 z_hjXR?Dmm`cx~~cG&d5>F$wdqIRK}jlCt&7fWyUT>NS+ZoAUi1uim$=AR6aS+}wHS zlVVF6nW7!pmO2Lo%4XxOX^o;5gSg%vo6$ z+=VgJCK^@F2oj^o>5%!%d3|4WsW=@GEiF$JT9)F;y>$!PS)0Z=i|FXDR3HC=;G2n? zu9oR6VPY|Orc)L(=hAU7kX7n_c7Y3cAmsd0`k`7k>IWc#vJ?W_WqA7a#>2FIVV=Rev1&_*WlOGA4u}m^PEtS2k3t zsn=l~ESfv1rw>GubtoyQkM4T|1$zaF!)@)32s7d9ef=Kk72#6QRX4j%_g zBjS!bwirFXknM;j#e89~lf*LUM`@?G7pAWBINKw~Q9^Sr9-v zo(|?=sTYDK_SIL~zVSD@*3YuN)UmzpbS=rP^cnX>eW9RT26at^0siUw2-}EzX&iVq z8WM!cNadD$ScIq2aZ)>8dBIKoMRfLw>+GM*&VOj($w}NF%Fg|4e^i}6Z=)XzNg-OQ zIHXyEy%VP^fma=L5;X6*QX#Mmbr>|{42d$+1GPmzvwR;U%=gnZJp(VrH@u&!MY3XN zeZ6#TKZTS!q|O06OKODKE!XPy#q0g zXhdA~93AiQ*V;1^J3YFS!T;Pdp-E=>YM|Mhuy2w$TW>Zt$IG#?f!b^Vexfwi@un~J!hGyK)4M`Wp@Kroub^U+X`|^0G_x=BoqEby|O_)wdk)}ejjuwP8 zl_cv_6cQ?l81oTXvM-@1qZCOcd$NpOlC0SoGq&tAmNCrCr{CK--Fxmi-+LeT-0wY) zd+zV|JAdiXV?MLI-mmxT^?Ys5*M&1SgZ*5MYZ2sk`T2$UvCUJfhkb19hJv)$h21@$ z&Rw=rY$Nby6%s3H7#~Gbb>;pBp06G}k#a>*=j=Uh9?1JVg}Aj(=Lzr-CsYuZy$^ae zbQ&-&lUlc)DP*qd-?GOtxDYyTj+i*1cWF&dlt+|G5>mHGQJ%lT%(5haQZG@X-{Qz$ zm!F-Le{7o{^gc3aZF0w2#Ah1WTYXLcILE}8769(YQ*R3%-!i1{(gqqclJpV8**)f*4&$2r^?PF$`@g*sV0I`!i zp)oPB374Ex?b{-{y38sg4z1jDH*9As;ziIwV*~=x&!xo=x<(|fyRnqu&v6SzI2?Wj z^SX}gTe;so+k&~P zKlI4Cc&w^J-Q%@gdeYIXDSiAT++x$nn`CBYX3u!r(P38M=;$n3wQg_H(NKX4#%Hb} zxDjg+)pVk_`g;0SJ-D0>I``+CH;q!*Re4`_TNi}lvSr`>U=GM?;xr?^!J#Au4+ z)o3IBMf2SXqaHQ~8g_oZHW9RCX~VNQ0j{=_ypOH`3fqarbf!O6aE!FF2zfibPUf+6 z&(iwrTB(Ph86WWtiEd&A7e=WZ)Z1pznu${>#Uj*{3cp-*Bl&<(6-7f#z zEC6yZ69R!1no?6W9SDiuG{l+_EtvP)4AT#}DH=NPc3SU3b$+onvG7;RFP!1orzJuB zn@*Phdp+uZ2psUD?P(-RZ$P*+VV9}9f?hyXSIoQI!nUYR?H-z+_UdL=$*ZTrpQ4{5 zT>rRMHr|Rp1rS_3Wp3?{tNw-%#cyWL_iNZ15-Z*lV=knB4a`5;)2Sbx{euXtq!SAbOM}OWA~>K_=A*s5 zX5_96LnoWAR8>+(+B&ZMzCuU@+HpWA6U`!hofv@M+ZH3dGXFF%e|rpE%sQ5_d-njN zkmcZBK?$gEzL&TsH)RYzAwd2X;&bb0?$D^9EamvV&DD~(BD=a?9!w9$ zoo1mE$2VR&A{Wy$DH_F_V#ht`;CXv$YI{!Zy`gy9a|59~rX7zEyf+LH1YdVg1{hiP zxyQ6Ul1YR8-Y}yL-@d(bK0hai-dwRxmbdq)f%i)d?<#~HRG|o!=|f+y)7@Y((4h6> zob~_n|HWkudIY9xRzpsx^=w6EFkP-H{$7i-QoWi{ONQ%saG{GvacIXS+2;M4osu8a zLtaScA_x`#?wky-GF_HRZJ;y+y_uAWTGQWzein4DBRx4#%>T5?%1sg42!aw`tpK#c zqIpJPsS?hU5}899BMI*h7U{X<-un1;U!2U-Am*#?d-N@Y{qA;$<#oErFR`SDEZsFs z+c-K8R(D^9+4cL2Z{p;0?k9h_EiS_luAE*vI2{BrPg_8A<6*Go+C5`&VVN8bbif8G z#XybP*q?aZCeZiDbJxkB-IDeU)Gw$n06hX7H#_wcIozvsKn@4}$PIS^e*&z@3|po) z^^>moh7ij2TDL?i=uZOWbRMF1ys^V4}_XUGJQ==I4jQp)r1!Lzhap? zrUUUV=o;L;l8tmsB~HLSu};6~s#Kon=QO3Zn#h=k-nV|he$vqrTg$RY-l|^1yY6`N z36t*H+$B`V^~;6^0kc7!u4`0u#PXcl+xEp_jplTUB}e|ZG5LS?xqq;G==Vwe|E}-C z4SHkkPyqwf)H0M9UgO-+>~RnA-r|>$k1TIrN^&kYKEed7Lnaf3v^ll`Xu%a{IytyV z=a#&Qf87-sGjp}ICo4bu_90BamRk~%Z1xXUz;Vy87#F`G?!N^6&mEKj6lc?mle`oE zHo4Ockj)0_`+!M;OH~8l=wbvxlg$UZuT7qC_CRw__1V$)NP`Y0FyLBjKE<)7WU!-{ zI2Hs7{l}-}$~C`!@0OUyGG&z6k9n)UAy{1Y#E+^R2Y<|sr%o_ok9QUTx5!3>tm zH-up+M>NBosg;COWpe0+6~yeC3i%^cV$fW%!O51K(btqHGWJ~8Ed~9|v=utWrMuUR zJV#i=^_ZLnPyoJySw(hlDQ)wJx$EJS)aCjm_F3lZN~cZY=K^~H<8lAU%eRwS5`|hQ zbOtDbQ&xf=UoFm~n;h;+xA@w2Kz*xve_waPrkfku2kS_yDG}_)XZl4EOFqjpJ>2{i zuo5ji7_H=msnC>cnKyX6f}X=UEoa)MlU-nysn6Scy+^{1+iCg5p!Ggsx!iFyBl_4?Jl64khbh155?+(C$?!#{q6@DX~(-6F%6+PK3-n=QlEEc<%(-wJGrm?^dbM*OZPwT#UISA_pcDSu-Tj- z$hLv01vxf^n^opn8OQlOKVWt-i*j1y1TMEKzt?fVShLx)k@u_kKd9t!Eqbx`K_88s zK?lw?dN>@|x|5snZ!ce0fr6l!<9HlcS|)zv8$z$o8-pUO0#;tG5st$wosuT#Rg!0v z(!U{`K*3KH5*S6=;TA(d-Ta<;@c?z=1o(t6^&5iYozpNRPROl5&B#6n z!w1mlaTp8s8tOkX;S_vBr1TR)O^Wc_PV#=l(TvE83Ju_vx5IH@J4fV6-p?d(XGqz~ zlb0^h$|vhdegs8#HG_7=*Oh+4;a6l%cp#)$H@CU3aa!rS)OX0VU6;;Kwqj(0z% zCXP6kV7;1>;xx0B5QIcRD=sJXF4&zq?SI34q4L)0F_SojU~w4(f30xOo5l}zSwD_^ zaoUu2O?AbwmE6Oi=Pew#>eG|_<)`shUqD$6G}z1dJ5Gjjs)JHarD5n~9-#ac*TMymW?DMb*2u*OYL!$6}QV zVllM_gM#{>6W8F;%2iktTZHMX>_0bTnrpsZ)5J5_03Z5UzgI=%t0}&e4d7-DIl zKcgCn#5*vLfVyLdsQTra6Kxam!^cu1pr^H46N#m7 zy9UwmF6W8cG-5@*#75d2wGX?|J>s449^@))g6JE{hY;Rt&rO4z5D0hWNlP$9@(mHD zxeYJPyaXxNwyNj zv*Dgt442fd-kKg@!Ru%(jBtAXBsXh6GMKnh zoj(OTpd!`*NXM_VSIM6*3Nto!(?1Y*XzT5lnx_vLOMzL|N>XNi2+1*xtw(n|3mRCF zi1GKa&#AZ^W(szrq3ojD#B9{t+isiK5BIzFHeIvhp5vo}g~O`u!u^_$sgsfv?bKjc zQYrVYS*&l0MbRy~i7><$u15s>X_|x$^M!Eb#ZhB zS2A$M%!JBQcF)El6f?P`9CO&ORxkPt&Q`m>N^c93xSx)-OCi@0Ad_boXhfK8(F7M?BGzFFz)As5 zX21kI&HMrM1RY_(0DqH!@mQb^htH4ihD|wj0Inb>Lol`rP#KbGhRR=HbJ#N95F}wm zD8tltreO*P76@Fuz`_*5tD*8n4v91}4dyzT1<4$bI!r7CK+vEs%Np6qA=C6#N2;Oj zmWri4Pcm?pcsU2h18wF7cW*67f7J0yusc{=dT==RN%g~9+h5sh>(AKB3tOtitZTbF z_%ynGY068DrHDUD=dB{%N#th+P@h~;vb@q|`bPVe&cXcMGuyhJMIAiUSMqMI0mI2G zQO2KQYLy^b2!iYRS=egP7JI3&w~CP~4zI+>tw?s6xjm43;a=sDH9D?ijso5JHgB}` zUS4zYl*jG`@yHVv1T~OBizVF!C-YZX=_dRqg>?iL3Yg>oQhjI*Xn?4yM~%0cva?yh z%}AlTMF_|@Dbwag#Jc~zmHE@H* z(NwlPvYQ1HV1BSBH#nZz@Ut4JXZ&7z^jk!O`o*2F1M8)-8it5HU^52XQNe)R*r3q- zx96<8G#?A?hKB%j;AK>!vRR~lu%bc?Z%v14a6Yjm82bB;J!uD?xLqm#$Sq(ld0u#E zchBrN?y^VfyOX?Ss_01>mJM60y~v_uMI5Ea%p{;l+w*BstD}r5HVQB(tLgC$$u)?L z1kn_%B+CZpjl~Yfbd}uG*wgOXL9HA}yI}J~E#X%e8C<@b!{s;!lx$#MaOSDCeKl(05j!c`b+XJ_??C93yIS(#d3s+lJ5{-2 zMDe-#hR4x)FMH9w3u@OoUR-=Sv>~8VX-u5ARC5F8Imr=q&*Alz623S}rH9kQSH1T7 zK%{yun|Lw1&g-;WxqzU*;c}6I??<_pB7SnYUYk9x6uzkul0+HsY3lWj0Py!q16D-- zS@l%DA^w}}pkc!uR1WW?a5|ysEQzONzw%uPz6@l;t8}}(X5X2T%B?+1={AmY!zI45 z^>(%bmUz`$C%Q&oM^CXHzq@*T-^mgGZq)hql$$7|doUw}^Gf{?=xs#bAqds>ADknt z8VkOhdN%ig{;7(qixCku3y#rux{55w8s&pU!J89?i(f@|^*I#O-m5k$H&0w*Nd596 zSSkN}6U0b=u{6KKb^7Nj^-p}%c6TF25GP#`8%o{)qwvPjrCIg_MN(pKSfsieTxAJFWmlmVJJj~ztL689Z%O(Edw>rxI&?Qf- zVu`SA=_55Lf7P0MCEMRTP;ykLjPt;-4U+7h?yC}5ckr^*<7SNJ18Pb6qc6vHy-rL` z6aEyf4c5Q#0bCnQ;9+lS1>4u9ubQ99E6&M3_dvin5OPj2isu8aATA5;64uS5TQb$>VT@w>>BzYDU)g(NC4M`<^i zfzBk zD5)O5sBW_55>tQn$tiuhqnh;lmV)8d`kh7zz9+`9<3>im65oZtLYAZdZ5o%r6306+ z)95##Kvhf?G=Z5!^ImBj4Z4}&7PkHRj@j6Y=PvPV;B79+;JfF)|L#5kMwB7$(_C7; zvCpGhw#|Z#rZgR2mLA-R+n7rf%=aE*x(UT4x%)3@A9~SsB8w;u8_h%^Ki2(?r;YhL zo|f$5@GF_!hJMCZAd~hR5$*4yRR0`_#FIcu!w%hbVwMrw_@lGs)w53@6*fzm6zV!z zs*`nfYY)v$=f+<5X)D+$fifF(B2QdR#?kn_%#Y6Y!B5UsaM{^P@GU!A%)Pf`OEtdI zOlP5l)UgHCi^1c1%@&oq2TzxV?4M06pq6+lQY|D-Sze&8HMz1RwLprWxPEcp+}oDH z$5S_PzW`Sb^*@`qe+j2t9_`!1DY*IC#JFvOpCV?`)stxm?e!VIIVona+4gH7;_t)| zhUkAc3Na*mlTbjMuWcDY0sZ7^9a?Z1CLpCPCdb@ffWD)0RnJnuhwjmOuMNiWjh}a_ zGH*^i&TX+ox%Es|P%i`ps2}KHttggEV!Git$ruOJMz%FGxF7OscHf|ZF7tSsiJNpV zz55(RXn4409YLqIbkb^%Gx5R=lU4T&XRX@wCY_af1}g`4QRExEJU9Q+S!8VeIcf^S z{deJITKJ?O%lIoo}le(mcP9JZqVNPS(U9PO!t*dTeY1^ypa9I z&EN1w*Mgoiq}(xHVZDnwJu#i?GTFsAkpx(~Flf?c^rHT2kZcQ+*av2hMEwwi@_)NB zhNfd-z3^a=VdQ=RjjA>B4osJl7NtE<%_H9s`cfIS2NPhOTxgQakw&#?iUO>U_5(Yw zz;2_#&of1mQ$MDvLG~GxiF{n3+Q#L);#;AI?BtBLch<1>8Peo5XBV*+mIhB6fnrrY zX08X>^l~BmE?5<70I&4~3;k|WiC}228N6qfEP@Voqf14La9MGs#L|zTGLzP0yRypb^f|?(%Z}(1d0~Cc_1p4&M5=`jHV?&|*-(n$rD%ze zXlFK2K@<^&%kGP$kSFE};*1{HW7BLp6BdfHPK|gYr^+@?^|PPRwIt$~z;x^3qjrOI zblF+mRi?*{A`Xoqr|h_sfG`L!m_-u+b9t-W)62o+jq{vA_@viqIhJ(u)U$^s8}CMB zy0@&HJn?W8H2Q*U14!RTVyO9?Fy0^OonFIjnB3v#G`UJE5vCEXyA%sX?qJ2T7RBNF zH7hq|KWNrlJ+z?ucI{g4O`VUmwjgGZp>M%>nvN^>4y@LuDS3s+lY?3h?#jLwC7CIv z&s%yEjIP30Pwg}1ub0(No89*=%(Ic3bUESi;cj@10ToOtUM~nPhxhD@xQ$DkmGZn3 zrSX-!5$cb-C2CBZ@TIUqHFvgAS!Y^DN@SCSA6ev_&rh344s_{2>LX^zOj-HZcugtz z^HSI*=k7}7y(dkCns{` zC3dad{mPHKFe_b*CsulOvP-u2zB&YQPircW<0EE5d0bat4q>hP{!bnCN}q|>_B>DA znYa04)zQ&HCzpUK#7Xe7!%@&>NBrg!>W&_~`AdSK%URhnG&HD2pM`6HD`=d0!QDY} zaPFcGFz`jR8gDkT|F*g{ox3_~QB^Z;ScIx&p_6q?tEcu5qI|jU>kb(E6Tn9d=2IAS4}i^fY|(@4;YZd;M|u(a#p4LDt<#;e+{o89Ix5L z-m--GFe#kLlXkcOKzs+IyC`l*;K85& z$&_(bvB8{Ll-fWBOK2X=*%4K&9tZ23gXlHD9O>!^GimdrCeEEZBZg zqx{`F}V2SY=1WOeIDFxBCyV+~Fj{_5V}Wl+af`-YI2rF}!d zRdU}DLoT@VYr?EhEFBJ9Qcqy+pXVS89Hzq0vjK*O1`X*DC+s~D_FYfV1v|(U!kNM& zlEW_H3w~co2RxtSO^3dBSW7t27GRKy)cuCY@=}KH8{l5kWZ1#BFlYvSR=H$BWFp(= zbi3ynsju+-pdNF=oB?u-Aah6{&giNG>u)`k21x!L z)0brU*sDFzA6P-S6?15)Ob;4tQwnFTwI?qfJUbNYNE+xuF9F5sV z>)njf*@nd}R7?%Jp136#R_EY`-2{goQGRNb<&|Ucd_#O#_|YK1F74B|h1tQ&Up#!! zD;CM=lL%abi;bSKS)90%=}J&I-l>`aw`9~Z7hoMwt|}$XrJCtJBfziK)9CRK$b^I z<2=@fum!69Kxl**`?{gYDuM$QS+%SXq`%Pzm7n#Y2$U0L%vl(y!vK8Erc=Mv2Td|i zhnZz{SgtA8O?L*vr_&*DssSO;oX!(1)Z6?2X6rdu9UJIjlr^~gEW{XU>BJltlLc10 z>RAQMfm&hnJ-L|^JHT&!jbpii@}%y3xIbyefC5bdGo_+Xl(Cxi7lZT#)BlJ(a}y;D z(95b6Xy^F+8xioJF`5NXg}&_!bPt+2LnTc%aFn^{t`rYYR(NZAaSBU>+yNEoM0W<0 zmy+kl$;`0o?J!uC+Vv#AFS94PfUNLzoosa7x`8k|?))ZLsEUg2tUJzB9;idhOeSu~ zqrqvbU0v1@tX)V$MCgv3JEW zi;PccJ6fgVq|FqsiV1F$kbAnk-Pq^(@>E5+N5jN4#aS5FrSn68rE1K3&WjBzqwc=f z-VODJYgFG(LZ*OXrhN&<(w)bAy6bTJ(Hy#E+mqc*oujLcABepqIYX5;SW$NOp&Hpo z)4GFufVhG!%gH8L9q}0d>zOP}yPV3YpguuRss4Ag{;E(5i;AHPhY1mn zlJB@xukP)~y}WsF6`2QbL>p%qHNl(kI9lZ`GvD>6*JF=+K5!CDdROpZ63v&jvFU^A zzHwhL_NU>_O_7-_X7M|`2Tk~ztKB7ana^otlM)A($dvQb{KRG*F2eblojiy_r6KEnX)+Yj;+!dNVe&_o0?- zsHE7Qy#hNC3%KqE#uX-!?t??HjhIr58_X6;Nm6@Wp77~shbNCm>idZv=J(KB5%I?O zoZ6*3h>aUCMkU}Z!{A1On6qDrH1h%l=VzvqMZD0?))OByRPH=?@lxI$C#h?ud|d4+ zH4ljH2oGU4e}2DPWq2o0&b;FO@F3C6qaKg?L&h?T+(uzpSe67>v6iPAh!!VT3Dt zTnnY$__u=s|EBT!UpOf%fo9|@As<|{3LPpXCbPp* zNZguxzxy02tOV7H)7v@09&vEAo>ij3H-zMF408Z9YDgyYvE4l~J~5z<1=ui{LgIKo zfiIK1G(XKcGj}<&6T}-h0XKn-UX!MAC;kh&AGCXX7d#J+&5wl*Z&kn*psCAWZb^%k zJr<7u9WoC{?aR&}%P$>L%=mb4fddnn!f=e_@GNmaw`JZo-=so9>$?DkU~zy77MZ{q zU5o*XZ3{BKyyN3kr!oN1p`I96EG8VVg zu=!>qXH{xg<#V_MjpmZwGd|xtr<9n$U&w&gO2A8{nE+%bozceRhqbc-GZJt}7rpji z*}UURl9Nj;qc+VoOb764dDda5qs*55MsC)5qgw???*eyb-#=nrF|s64fqNwJ7dSJh z3C=z`l|#f#$OB;X^_|=t5t&yBbrV_Fr)`KcvqMY7+wEUR-s8!DZ~_Tlt4ELH+-kwh zQJl3z;4Q$RIqQ8xWQbtq%-U!3WSFz_&_mk1rt+A5+sg0oQ3RqHHY`deq5&pcSa#|M zzHs>c;FTJ}gI-nWJ08M-bA&YGAX>xmQ=I0A$=#?sK&DR$vx3>_i`6)OO~I(v!0#&X z0bE7^z-!;t0hQSSB$6|}q>5UrIeOJ2U=qZsJL>5&aMgGQhv$WD03PU7ltMqm5}v@! z9X9E|J;Dy5qXy!J8$zl8P$Fc^ss+t_zV;o7#&CS0K~Lr)cw?V$2$k`LqF`A20YuZC z(X8Wzr;$g!Agi=yf)`vzcB4X^`{E|c;BSx(4o7k=)Q?*_6Q1xH3P)kTB$9C(Zhu!U z-Usg1Fcu^&7&16WEuMq(CiTcd;ed@)x7mhV7MJlgyg+UWYr&r9L_mYubX`uEYTQf^ zY5T8rB5f*{9&rDAZ`1Akitt4YM-8r>BQKqvUm95feaOQNK=-jJwP0T&2RiCijI;Ad zEyJuM4Gv=LFf5Tt5`#)oqVrOhm4W=Xx{&x+U9bXpKKeC>NScs-34MK${tH!bUe*K? zpb6kR@D43Z01@soCV?ZZsKzWwo=ie^=>LOmOvGP;obQnIAO0DFcY(9r2f`9>5CqA1 zAlP`O$e53`||dQ0olyH+AJ4NeVYd}ci$FG!X4s-a*w z{8}H8jQbqC_n-aCWf`VDLv=yaMP5l3-*Na<`?$IS2rFLV7z^~~Jf`X} zWIIq;z{DQ_2tiaLeO6;D1K`W()64MF3V6iPGha9>z0!us`8DiNA{S}G4ZxVg{zXO8 zu=cxfn!${%CYs4V2Sv8F_uE!~$9`hC>WoFn5>D10dM}Rlw!RK2Sfd8BVF~bujnYRjNq(?|loB1w& zfC7E&!IN(~pbp!U;8#~c$56MtjI>ZhVzw}-+5|z2lex~rF^jt314u1^&m4S*XZSuB zw%rop6=RHP{o(tYsgo*5GY|&IZ{jowHc?v22Z^#)kSO~ap8R8;tdSL4@|A&XaPAa_ zkCH)Br(-+~0Oq~zgUd{Fk_SPum~cbW7W{R$3ZAunZ&H=`m*8lagoFkGLV;K^Ezs!E z+$HidUV{dprQ_J=wci0733tV#rVO(l8XO|oVOb)(zae6(2#(YZfZPKF6^`m+xx zh$9^iv%ckj`O*P3#2FCPWRGzCaxom?RfK2?qa%EIa>PkvA{w^BlZ>WSIbsXg1Q1|9 z0Y`4HxgX4Wb=ks?!`NnS!R*n^knRzRZm)d!NzNP0U}uRGkZLiIX402+wCOvf z867-0kBgiDI8dLB1c6Fqq9pqUyOaKaG|=z`w!Lm;`uRHyyFq&E%npZ8n2Bq+5e>pZ zB=jL039p8AJI{YZ^isZtXN-SQ#Ws*SJ*So<)hyqa-w>0kY+^VxcmcaS!=oQLd%J-? zFuIi8Jx*!Z_Fchdk?WvZ4hZ@x$EQa?Hk~HS^25@>qc4_@|CmqTv>@$OJ5Ed_Wg*#t zB|Cwd*PMn=nG9O|Po0?8Xe-(K*Qo9Q6ND=us!N4_3hJy#7X8>op*ptn%)Plz3=IV$ zgZ%VKod^a=*cn$+_5$Z_E|w!G z_)DFbWCN9u1uAhEgqQ3R5aSNX=7WTo+P*mw8YlI;;kODroR*B54aY?F~mQ(ByItLiAqMj7$xZ@O}zZ{dp!`4fHL?Z*}x z_C`TxaYJKn5bd*SK1a-?ip<^x;NHCqIgS_b1P$+?LGrK*bhQmO1}V%3P{qM9Fb0kj zhG@^6;T_@g3#e}hO)zE^4L$gSUi9!Q?yW2{p@N)~Ulkq@Ppm`=xW^ebDOw&dFnT?9 z!%{Wpsfq|CtntR64Ogk)(gxk|wD|M8sAJEb%T4A*Im~GkN!wrCS^X^Ejom;>Z&O`% zh<_BU|II0~o&P8R0HFN}v2)gnlf>4byjHGARLG}&0d&Fh&)O~&#((Nr@ipY-iYe|N z0@VkvH$Q!Su+2AB>E;(=vcS`4kE}k;V3{^pKvW!7-tY~v<^fb)gqk-3=!7UY9k=)h z*ZB@N$i^)Ou=mSM8iPK73$~yD^$9beyOb6UbR}k?j$Of0=3^6szac=)+W#o$CE$Vr z2$uFRfDNJw#Et!H#Ee@gAumI|96P0#>iQ}lw?8?K^0f87tCcgm;ko7t z{JsLF5-qp_TK8t&eO@m|?)DE3@2LBB7pC0aFA$|muCe+uEgpbm8Nr`Rq?jDuyVA6b zFh@4}g}Z@~*9pF~SD~M6$|rtsz!P-Y>r~~p3hoE2m zTzBZ_-wOhU>}#Z2CJ9g){PGOKe?Ec$hLi9jMi{;TmTBB$8ADASreO9^0m*EFU)p>_ z%no_Y0LjDke8B1z>81@eEj40;IaUzV*D8UV*{*|-bpOVQEf?HJK zXwN4g?VhU9zf+r!&^wc`y~o=vVw1rN0Xqx0=9v}@*YQGcyB_=TS}@H6UP(HvmWwiCYK z=57f=%;OcXBJ}bhhP{ZxlRjboh~r44u=cQ##hk!>pkT4``$YUv8S%bCcO`xXp|dCwmaFKkuNeOoXAw1F=HB*s_-LiWp_D?)Kr% zCLKR@)uEJE;puWY9n4^DjvuVye%SIfawVbpfT|`j?TU$Lv^hGU%|dV27U{?LKCcKr zp=uHnTDx~kDAqA3wO-C_0f^7?^OX*LrJ?%M%8-J=udUM?%+I&=;qPtc{<(C1gYB?2 zq#a=xFym;f0SlB!$#(XI9J*_LN%(NEH4;AM+crhF@{jMtQ)-HZp97O(=QM~;P@v~L@{cYLr z<(KLb*KtBh_IBb;X}C&aAW3338!POaA@x2#S9E)K=H{IyCqIB%Nt-*=UZO9mJR93HJL@_aL^nw1VeA@dOi-2PZXFW~KYtJj} zCKy(_h$U@%6i9C`M&MbFED1tuY%JZ(9R3n(FR(MSnnVk!y!AqaJNm0!X&m1r{^|LdU6KequI`3!UB7Ko+JMuQ1JuQd0l^*3D7EN}_$96zna_mcKB*QE3nOyjg- zVj_E7i6ZP(m@1TCi44=tF-eN zM&D0PpWpK@r#Z10TMR#bg~a|sx^37b`_v02>16P! z#Nn;z?2FpScQHMOdT{FJ*DmZlmFk}=su+_jXj2r~xmxx*(8q=N)?{k1bBEhNMu01E zW1l-eBP6qFw-V>__&Lu*NxQpke)7^MfA-Qb+9rb7cs=u5Ig3g?Bd_l!Th5*d?R5Dc zUIzWMmwE6F@jIjGf39M`#myX|xiGYuPx^_WUD#?&AZg2Mb@Kx;sTTH4rn; zh)Y+`D`XDZ#?S9nxwTc^H`Omd)`??9I`BDOaTldsUdhJG#4;tdQWZ5JPaiL zos_nIX5ZzXtohV2uA**_)cughpDRbWbW;U~tZ%6ESQ~X;OqM zhiYzPt1%ZYQC_H9r+Jll9(nS*Vx3dR^T%P=3kndf8EUBe{j?z$z-WWtv2Q$Z#BYD%-UnbWpZ_ac;={3oI0 zqvp4^AMG03BC_FHY6M}@)-m9`!G_-@r5b+CiuK@D4w*#Ep}6YdIW4J^~y@kuaha5T?PMEUTVcL*P(wH8X3 zy_%Dvi6k^FX?#wn6(>1QX3H(`oGAstRrt` zMjxR;0l8>F_k_~{*yfwTrf9TN%2lbo)z2S>H*$k{B`}j=i|OJ&%z3UZ>^krbv6iV@ zsd^w-d@lBEw!O`ub z&Nr%lhHL5G@Al^Zu}-T0S1#~O99v)kK0aH8nFp<};5=;3aE+6)75DbQpNMN6r?%@| zj4O@btg7mN)+n`n^)}j}BxEqW@oS$b4Uql2RcWG_8jh>-13(>g50v-%CY#UPe*G)# zenhI>DYv0Z9&%Ah>&_#*Ff3g>mh)0wp6a!=jGP@;%2BA2G0IU^7M)6WKl`!1?eOb$ z_e!>H=nX+8#P(<%$rS0Z;K!601%-kMU$c?AnIkGi8P-5jpCh9lLcaxUt?_)e~! z&q%Q!o|mdw^!Ia*hIb`{5oqWVNGL9#HqCp<&zq;cNp8t0R*{~8+ugQj%VzgaXhnV) zdh2en;aCzfQ7Jr_L^BGNqYTwlZy^Zd!c|#0Cfvc;dGv*lj&({%&5nb$g6})Vq6p7}>P-23mM1f4bF0v(EqTmOiQ^&VNZ<>tS%QNVDc6Zgs%rjku zfvbpqP#&fl!-sbQbE38xU9*(W;S?|D$_qTcukB`tZ+i=EbVA?jam|WdUH5ABjh82# zJ|I|oyfr~nqt8p`thp~G!?%Rj-3rHih&O0I-FK`^CAP13V7o%toz(aa)`Vo6w`MjC zeU~jy!%{9sQntzkUd=)Cj;`VE`FL2iYTfOq+eobwTRN7a?6?O)l>x8WYsik{W&(Bc z^~VOm>zm(b)^duemv^=t<8lAg^fYSq4yDam{->+lK&ST+xL#r}CpDL&P)VeZ5G($0)GHRsoVqFd`D;(Gp!;~;k#c9O`_Cp20qw&O9ZEqKW$CFD9L z!NuN%3T;+7XB)AZyXu~`@w&YqiA7$|6l84?+zyoR$z-~3IYwlT=nck1gv%$@$^k`% z<@Uzh;_ZiTvL1>|YF}*ruz}H)5IOVa;}!ui?+q-*ASb2BoF&iXt&mWpC5Aq`qI-re zV0LNFEM$Leu+h3KmEgK4bv@Qwu8Sto5q8{lWJVwm=pDR4lQ|$5#dWgE^9Vc-!MKA% z#&MkSm))87<1Y4W+uFYFCE~MATg1j=Ji@+E|5K6>tY#;Q_0dG&k-;aK={w^MZ^$1$ ztQfkd?*QM(^nS^WRdGYNEML#Sw8)6{@+d|nXF02RnHE=p>KdVB`0t*BwpzSSeEj`6HplLPbyeN7Aqg;%)a3~9&T)IYDIdOl(F8g3l8 zk+Y^Zf$aLE@C*Fz5}IRT!PD^d6}O*tb(+_6QoYhYmtDx8 z<-UKe_Ve{kP6vCgJ>$;eB;xH^x0r=Bjbyev)0%==%aoY;=;;^K^d;j-^w5mt1xDQY zV^Vo0q0cr31Rclg@9L;T93WJ5$!x1~O_zxNoSY z3tb#K_tMd_az#TqsT(x_0#jvsD7$^~Hf=wBmz=Qg&(sW(efr&r2*G3)@( z*_A5BJC$t^pamvZdD42I!5_D|2+hLa*D^irsSWou<#UwV=wqAj=j8eBN1Wp^91N7P zezKC>Ca6EPPq;0o4U9-*109Djnf^FF z#Rvz&wzh|%&rT=b_wus|6*Mk8Y5L&95y#_us9duRlkM~xM$kK^E@(IFa-keZNE#P@ z|2!}K>3ICQlFN_1#JJk*Iuyf3C}@b6#ScAVESsF~`{--Xe47HC~reY+WIj zRwNYh(bYb8&7~s$K9O^JLwD`Zdv6-bzR(j|bET12X#g1pc1L2@xL+BkCZ^IPDf9UO zH(4%TZqIgSXiNwNDXYCRe_a*h@;F(NcUKvho+Dj`!p+c=(zF0QX7V%pj1;|FGhJl+ zX_*I(d!AlZzd%#Y=)H7xUHTg>6$JMPNfhpaww6e?NdQ9CM}x z?Dq59MG37nyIQwhaMc~Wz)gz-)hkb~1dIkaeztRS3EveDle=^BTQtSV=AEg5N&$r! zXMT+iK4$?f?~O45F6_BV~5_!4Zq#wBw?lE#Cv{&FeB>-O9k#Ftp+wvo?j?WLd~_2eH2l7Lss$Z?6E;t z&7-upyv)~k=1Y0M8vX;e+kbhqii=4fXjs$-b=i(l5G+2UpX42r{=MG(U;Z}#+Xwz* zB)@7T8|gAPA8A-q6ejrDxahhFROb+_?bvzt8I%#@@~e!`R~jaXfhB3+2fI z>TBK#Gm`(!x^zRy6WNamwf6F_uErKdjj6wsT={vg)@YZvW)deJV}tV}XY&y8eVbUf z(lf@!($mJ@C#3gUAl$B6CaoUmaml%HY*nz}O2lpDN_6+u0p$=_#Q}J-iHEcb-S&Bi zPeJ$1c;B|iZPj0zeDt4eilB(y?Xk$-do44fFp#$%M>|=JVN~Wa)t*?uO@tjx2iGU{ z)N_OyYs!nMRjqE&rGg3t1JUr^G7na-yKTcpXvPC;6T+7GLfB3CnAi0t6tZ7M_IU<2 z=DqGMFOTkiURbj~;D*4#%{*JwC%RNFXo)$7vjvb;WWfgR%)e&Nwp~B0pnzE z*J#F`dn4~3#P?fpMQ^;C*NhO6wK$)P@R**kr^!@9*|rmf%6_?n;ef(-xTM|5QS!mD zy!O*a)5W;jWvkALR0w{YlK0VBgJ5RBYZg-o9ql+i_7RAsJO;YFCvQGf*P@Ssg_*DX z=~JlHWa z5qt189PtAoA!18_QaBVrfZ$%HbN3O3RTyB^w1&w!yJ{qC8ISW^CL<-hUT}{hyFCU@ z=5nG5I}K8H*(Yel9zc9U)X7{j#d26^EfEy$ME81$8ipFvRH@9*C zGZnPVinHn*mm=BMWaqx!R(s12juo71;2q-~0$Zfng3+*{8JZ=k2%5o#H4t9=Xqx2X zIZXNft&W0%S0!oHv3XClxIeOXd%8<>Bq2kU>@Ku`ubJ`w2in{|-xu!oWU~&$A}L#@ z?5gX*u)WMLM44Z%UI4AQBu=V~R_2B#k7VjF}?|x^X6{7rL zx_+QnzDR1L@zw=T@#_tJvpc!l;2nfIR6eN&Ddx^wMG{ayL@P)rjek&*<+cBI|20uN zsc6i*#qcFn$1Nf%^GV1XoYy29anAv_du$bY00kZ1T}I?r|3Nr1m0U8teR$*NQLzy3 z9Cv%EMW1(alIfBw_Hqp_nM`J|vndCR-p$a%7aSZ1G&Y%hgqG~a?pDbx zJT&^~YK6~o-zL$(GKXtt2+bSaZL9lM(my5g)5*R)Jzvd=E`^+UWn5l>eSGcXtE@e@ zhDy%%mhuM5O!~6U5YM!ah3B&A{! zx?+dZY6O27BWZ&Syqe=+EmQ7{8o%)}FZ(t%QgHB9@+bGGt^?uM#}|ek4YUF-olano zicIaZ3H9;o=Nq-(=h6M1c9k@}i_0nO^Jc8vGXVDk9GCGc@W%^oaoHhfrLCxraq5{R zGogpC950;5jD*C!l}e?BK5AS=>d>sy%j86si0UTmZa%Uj;&NOa!s+pFWQvS9_5n_j zYqp&&a9QLP$sXAmVA^y6JISE`5GZ)6xLIvdBk8Qh^%A{nm@ena6)2{xMDPVFHi(>U z65x95thu~Vr82{0-`fxF&kLHH$|NYjy4e>}LdgE~ z-4;R5Mnm2?9DCi@9#m_>yO7}eWW)REuu>T4PCyhxOhAuyaAltW2~E6~v&Ri6lEv?j zojuLI%x!J+AWYjRd@U(!sh^<_-Dq|7W*m0dVx^L0M$Iy7-^(B-Cz0F5vvRw%%Z2Xc z=RvN}hpUA%CjqVZOzpt^7r95{rrZ=7wf7%#w?j`_XG5RhoS|DsvA1y~Nmspb(Ua_yaHyG<>7o1s+A#)l*C|mxXBBhUfFgB3CQ0n}vU7df6Q}@^8 zvz6(r0?)~*ADR&C8#s1ywGDyd5u#MxckSBE!r?=CM+`*@MDE7V9D{tm!aCb(8E{qG zsK?4X4t?WwrghEr`-3ODjsVQ!22qA4ift*O9*`;l1>V6WFX}dMnQ;lDm0N9D0W#4? z2heeldyiZRs=7Pxqf3S+x4r=noFV(L+yAz-`75IsQAdG~?u5M(KCec^}k z4XG3P1$hVEbS&T=J<()(kj~3}j*fFg&-MSXrddwMy~lC>{sMAoA%K_P^^A^~g^jAB zQ$qHe#L;&Jlr+`z4AdR>1f@31R8Qrzuj>+|h&wZ^b7LMcMm*xK?72M??<~o6Hol;> zH`)GHt-nfM#-~U9c9uokwIrnh)nGKqksw}L;+~eCLhHQ0tomteS*170=fJMa7-x0J zja#fVmCY{DM-v{pAim!gdpk>%uo+)D$&r5tb@t|BZGyzMvD;}gmci#-wIVT)&(5#b z_ndqgVD#!v;j0$hYrFRq=QFhC=t3Wwu^Ttg{5V2(GVbVagOLUOH;l$UKy3Rm zZL)m~5r-t38tQM;5&-+~+z;!ZD!@FGrRx6f_c7ln0Ox?`{zOf^`U!AUAHWf0nd!;h z0RqhqvSqZ~5ZO-eTsJF!qQ2rC>7iBL(^IxPMY&(8zo=stxl9bGxe#a`YxdKKs^(@` z6qp_MwyD6B0-Ysxb8~>TxWJ-Es>+{@BPZ=$U$$#^_IJvcmCDF}Y)*xL9{j3b7r090 z)ol*hN5Cz8@p(>mbm+es)KfIns&G{yYr?dsGiBnosQ8S^VJ*ksT{lklw{gQB`@b*^ z|L(w>01P!eAOO=0KS;~C>g&a`IFeX`8mU|p$)M!Fy8p~O)V^I^EXw1X`Fz*Y1B~m6 z4dQ!vM-k4k6iNR34l?qXB;3J|8n_tDJ@d&GA zr+e2|uMFyc4!l4`5mI<1?$x{oyU3rfFRQLUtd!Pt<9Zo-X9Hqx1}(_IND<2-i<7!H@t?qQXBQ;B8v z+!{~C%tcAbVKG}GU%^hZLYEEr&^%(%bz4F8 zXq?TbyjSNArrbr{UdHPk_RcyJT+QM9&F7I+Dc|LLW*#O|Sfu$t!?aWE(>ZC+tHH|- zTO%Rf3vgi~wgodHGqeApS}PVd&+i~IC=vT{&~P9xR@(67L(|)LvjZI=4E=-ey^o@i zk~dQX3w+kSLQnwJZ)`KFFd@IaQNsICgzS~JH^B6OQL+XhvzCuJ?B1usVo_w z(V)hPtz8109m$nA5o|8sH~q&b3gwo^$Xpc83CF((7aj1bC_b_K<`my}N>iiCrIKvR zk$K%c;l~?4b01)wvzH`=5aBs_*2oVO1L_5<*VT#67Q~$ocHQS@oKi`Ine%plsZTY9b{dX1_l8F)ec9^COOcu3I!Sg|EuM`btoJ#2re{w70v`Z@t3TF$bZG|%uHsch5Ay4c} zI}c%}%8k;N>D_*k?ZTyO$!8g~7uT;gC3pvD+1qKkwLEKo@kXpRiybvmESnr2+f#89 z@n6DT+n+{H>F0R;AD&8oJ00Zd2ILI3RUUS$>CGsV?u>R&vo|@Y`S6f+SK(y4k8JDX zON}@D=3He2g-UvrmQI&GFbaq%Ja!84M*JIVV)22_Z~B9uM%+W3P?`jn1iIoP96sQy zh=FmdBs~piGV8lo)%R^URXRpg3D51FKyH|UShWykcrGj1opv<~cWd3AthI+FeQ_kN zzddgnYhs+4|GUi%5b6JvaZUXn7%qUEFjkZZP&H{~S_kpAm^g72qC(0u@{LW?R51_B z)G4io@&jADN4#HhB|RD(CQNu+X*3yCUU<@9oJWxv?EiLMu;4O&PTE6xbzxib6}5q^ zjYo^3kiFF7)1J07tIcnY8hdde-H44&C8s60axCqbWMC-FX$B-$v)#En?WG=2BQCcV zZLd%DUGLVshi?-q110Wtui}1xCXj3VEuVXY{3);hyFvV4%sT$gEC!wkpoSw50423n zkkNrelUBG)Z$81d^{L}PrS&ylBj+Lb2|of)MEsh^@++UCg^#AhC5p`4&$xZQx43X? zNc42+`LWN%#h5p8lbR+Uppv~5)!ovG{?q%AJLTG#FAVye?r}KsM zA(<;K(3OGF8o#(?zZd`7sMSDT`u+=#?Q@;|UcxWE+q=Q{4W*2=?BcVsk=1%si!7#W zi>Cq$0mY1mHUl)19{26s9?(ekzh^<|585Kuyu`V;k?5~izme1UfsE&DHLdoCHtUww z2igK!u^niBsbBx41z!+Z>QBwWg3~C4!F6(bKP#>LHV}i=r%wR(+2>$qYx*J#JKOcf z<0}Ut8;V6=TaC=pB6m`bF47suo;m2N9)s0>+>LIeXP%oy{s&h4JLV%;pR6^h&!j^( zFk#}t19d_K#ME2_)!2!ojXXhkn)GympMKf!^H`G$?~SJ$(zYB$N>3ucYTF-nft-r`| zn4z8Ci!H}3FH$a$?NL@UldSU^qV=#$o)ZLxi~=18FO@Lu{y`|>-s|%UW3ScacjRj} zXy*b6*%=e4VgpT$*cRlgVJmaVD5I`_* z0u7XcT27!5D|}Lf%Iq@bbtjAUxc1ml`H(Ix&9_Hb?MW$$0v#{rr*+=z;6npXtu75x z&WE%k`KYI6a6^={-Bc?)j{oB|FO>8%V^Nsn#S70%9!dr7e$#w4nQ6qY>}ak6i3L9k$@2=xzR57uFaPTD%awNcHQ62BY7i5ei*XtLJI5$U3(^XG-P7CmQ4ujO%uYE{aUs5 zat3{0tc`7z464Ngt@A!EWeZ)RA`bxfP%UYKBk3b8M$52d9#=jXc@q=!9m1~qVGdW- zSE4WzzVKF$;kak7-h}g~lk&Bn-ZIJF#zQ<@k>zR2$0+Ae?u4!u^vJ3~kJiB{4ln6I~tN=MW?2dX_R500a3bzL(O{gZVGLGJwfB*{YT{@zeCQ#NTAMk20i5^i4)#4%G z-cIsTkL-$PZmQ`|EAY_b@6{7O;1ZLc@G`7wMiAoIA<4@yMsGuxRkt8mQN~35WmSsd z0>2fK3w1tW=TXa*q!@4>tJ{)jyfD(B)yIFu^MuXR!3DO}m$!&iSXE|GLIpR&Bnm+WPjnYQakUu=wI!F}OKzif z;pcr>qz6{6)FuW!EM~+URd2)Y+w-ATFy06Az?9jRa*|w48YGxBYl=DIp-m+m0W*zC zU%lIc@*PZg)l4U1q!lHUjd?ES+};xy@x|-DZ#I;fTmbY|@wBRYC&<1;-6^YRIv<(4 z%{P0x!&1jv8p4!i|Eyat?pWmo1!3ZXUUTl0Y?bVT~iyi zXz@v*^1zCd*TIMb7KTDZ?1KZxle2|*UG~BZP=2&~b#lZE(#@CTJ-+;9UL3>2r?TCX z+h(fex`c(o)vxZM1#+)5CW7bppecQNU&-ez_PCZ4^Ee9=@%Hm(!R>cWrMo* zoOn-6hgXGO$Bs}&L*Zkhr84_N;mNdgFR<*!9ZwEaVn)195lC^*#>e$WEYqGSy{f2Y z@h^%m*(+6r_>vnUc7So#wq%$>dQMBCtB?ypL@UEa0-95RNNb-+?JE1YUE4L1$?M_s z%*o1YLZW&-E2zA!YP5J4-_4gEiHRv zb;O5MkJ0a4(K`IRUWAZ^l`s)2%sIpx|8C_u*VsF_zu(;*M$cvMa@-@0lDUW=n{V+7 z4Ur}XPI*87TF*-4-5gn(yjB&lbXWCb))eW3Lkh7i zq|~o9*2{Sb0(+feIhjme1BN!xFNE!c%TRZc#fr&@>9#1n=)#GB7jx!OqsvvBw#8Y! zy{f?kg9RnRyTfM#zJ?!Q{4h()0`21HVP?RQs`I4CxC$c@>UPcSrhf{Y@)*4zZhTNp z6Ou6j=LwS4I3{iDkuxM5Dlr(cGh`4~m6lKkny=W8O%;nw4K{hPkLPW5FL`lK&zwU| zg&ldnBzCvV%kNBV^Kf+V%irUPZ)JE)y^L8_fw7 zV$WLl-VXIEesS4?6lW1S7k{jCz${)Ws0V&^Y3ebfzsHo}pbe3`U`rxVFEQ{OP0-f1z;5%^Ip89E z;@LmmWdFC{32noOqu*P?=xxR@noMjc;v9fDph-A<4t5Pn2I1L>6|g>zyW=|%V;(=m zY>zafnL;pY8b8msWyI&O=98quiO{)mjjoAl82&5>bP z!CxrV-e_Um=S1=%z2yzSmJOejm`DQ=rBoQz8>D(kJ&|-RZL8TJ#8MDYPHiDf(i=>H zcs{Zo>l|23<*lZlNt~lM)u8FO`?DH4-RalioWEi@jEeN#`QSsfm%c+TW$FSvoR79Q zc=+2HNJt21l34s93iOwbysXCxkCtT+!f>&&kqY1#~0wovzw4f#6fEO41L$u zyRAnvw5eN)ompRz{DH@RrlvEeQvsds9uS-bqF&fDU<|nj7g4)`wkIepgDv&z5qO#m zo$(uui$n`9ScL7Qs)Dzt(sB?>7t85Zd|TCbG^YSN5Mlp!a|*z4{3#hsDFT4|!~j@=yb}aUJAaSdniki#vX0xgf~<=cq|tKbk`*K$11X zogM>P{C9}Qt-+_$8s4x0TM6WrYJwyEZMr`_k?*xrh8LL>M{A*5P-hflu&nLwCx4j= zSI~{B=+9$-XI*;l+;_-`FHH;@3AqwASf*0T|+kb(hx5z+M_*wUyZt=%W6M^6`6mBW_Kb3be^e8_|!xQ6^4-5$-^c zCAVNs5#J&H(U0M{lJj2?@E;!5wL}E{JxAdW;oh|n@N|+Q|Kw)T_QTD>t0ekQZWbL{ zM}VBcCLT%QDZyoShLkiQSNITZfMb^XRmltg0eDV~rEJQ)2)RXnhgg#!=hEB6jJHl9 zlf3`StmxmgAveAMRWogL3K$~LGeM@l?L|9yi{=xC_-w04#J!j4136Yh2<|a_qNGCG z9FL7cuHArDz-aM)lN4`cIyr&}raN#9`m@~|B{gaL-o%a=JZXVuD!4}ezVQ6p&JK_g}@H=|wn-@qe(~{c^$~Abq?TFnDlZw%%C>;SEA67<%aMU}dY#h{%b< zL~I)(F-zv=Q z11_gT=ZE|`xr>%%))S+Ky#|ji#(!geC3#mza;GVC7QGWuPDPZvgyxaG^T=~nMDfJ5 zGW)NK1^U8oV%tA7M?!WPJ8eTBJ^)ITHot2qWBf8nKG5~HVL55NekdgTO1nMQ?iPh$jv^bQ@jB!-zrgi^J!j;(lp38fT z9!Ur^?)5Y1hwz#*zy2To%3tsP`9qQI4@I^=6xsezWcz=w$d+H^W@)o>FX*xRk+h2b zLr*F*En-s>PsuiLy~Ag@GJe64eK(haff}wJ$mJszadd?rI>rCVm$cRJE=y?y6Ozv- zUu};mhp?dhaS#5l9)5$u@q_zvEyp0-M;NoFL$wos*Khkzk+gnI`v3N+=wJNmKMqv~ z{y0>zLjE{Z{gjn4{BfxI5l`0sUmvPe$0hZW^;u)tl#et(e5JaXOsMtbt2^xPBNwt93z8u)Fx0ujYv?{A}ZMFAY;XaYu zzbwPc;P*!~|D#0qM~Uo@64}p10>(c|WWSIa{-HlNfaNvt_k94o*BXG;+r2lC5G^V@ znSoHe^+T#+z^OMjUVa}}=|V_6md3gXBOd{k)#=3^dP4*HCso!h z1Ua_O96$qX-yslC5te`v0LOGEaFMVA?`%u}rA+~S5)yw~lCv-S^Lv9R>Zw2p-%cU?f{3_pjr?3UF!HFZVpx0zTn~OSC?s0$&DU1AO3d3Pj zS(WLMUG65e4^q zbG2o*XAND8stjl%CF;R%e)4H)XPd@_e!dGK)&BTkK6er7?bc1UmgSL4q~nXz1f|cLSl?3 z_i4%CHxqRp?uLu@zf^;6k$^XLsv^Z0nm>x6-%lXm+M=IRyWDBOabu_+)XIlgVY42a zO-o%z0XzZ-VeO@WB{t-50`j2rdqYBbzkYkkoVe8x*_3h;wmyQO%%6KtfdoH>ZY{%T z8wb7whyFU$yf^9_{>Cs10UFO4#FpwPl(Gl7`U;lJudiU9yE3lNk(^Rhu+t9$`An$t zVpiea<8o_>T4!dyj&A+1dYn`K+3K;1_RDX|O>H3^-~Drg)KRB04`jUBn${8p(Ql>E z-E)5<#kP$~bCYoChTT#8t{N|I-G@I6ynaUqdH9I_~mT7+mmO1=-9aeK%C8bl* zzg^~ko#}r*l_0-<3Os3s&mkJgYBX^FE}eK@2@{+DOysx@7GBOru3g* z$6uXsjv#ES9V4qvuOa4x0qk?85`FR#It+Sh3+82vcZggiA5^5Kx0V8w^bA`7Qz}`}~7xySnpko8h_t@mOpkN}m^w-AYuU?1q&*&a_O249eA^_dH@t4rO zw{&I`7P2b-y0eac^pX-AbPJzC-tR9!%e&LJ`~%_xw6&igKHP85tFGQ^Xzrhz^ABke zEM8(Llrah2OU~a~*$zsVDgiChL&;tj*hz4YI9Q=7D z{xDr_`(e8J>+1h$CAR-_SJhD`|6?T%h8d{?NRP#LQ#Wh#hd28EmwbvKW>egML7q z@e99R?EfHCiF*@1f&IX1Sv{8tMksmuSrjy_e*nleYx)N#^c*nx{F7%?;}4V12EfLU zNkb~o7T5sElx#Vw;^5E>H2j{rDc zI1&=__FsjGm=%d8bVCd{Krb;qLx0umX8yAJ!<1A?tqEb=MUzX%>AgdLMw-k4++TT% ztr;*2WQl0f96cj!!D11+;}_h?jot|ClKgSPh^1Xm(DSc6){7`~Jw7;c+Xzvjr2rgc z%|W*2joA3y)onxU!U6tH-;CO_i$B4Olt}g`m|GF#&0W!)JGpeH8K3bUwrkha{jvqO zW{*0|?Vl)30);;6{FFF z+ltYnXUguG_0Fly`qft(rXC%8s=79`ThjCOwPM;>H*+mDm|X2hki&1r!sRB1iWdSx zI?nJFp83KSGd`gHcrv2;W@6kUQJ0)-R&+VKCDU;R+lb|$S`JY6XNhDq?^YpxUG%%= zccNU&yM8Sub0MrvOwg41DR>qz8G0)MRxg8cB$p5u-`2wivg`Zm*rpHMq;ab+P2X;0 zv|C`Yc*rZT0*P;Shd z`ttP0%NJno1+OeV&WsN7GDHKyYIQ3D`pwt?;p#Q4+*;4r`(l>!X@0%o`Rd?870#ju zqIb)9;)y!2s{tO(=>qX6?fihkCgDfIiT4U^^#h%J*_`%A_A9;Ij_7+f@5*@gzZFEb z5TF69US@uhU^>YV(zP8@xg$Wvp0F_z`7NnD=1H<|Wxv#({c^L4on8WvnJ@6Cq41ZW zv3*9r1;>VBC%e3$ni6jpsnGbx-t9Y~^`YRwaxBX__1c&7enRaHJG+^nOFcx#R)i>= z#|_Ivb;Jvz_WHKEZH)5bJf5buF@DHwPdK`&85=M9w6&`767$x?HBRgaL*2gY(Gk@y;OJ8{d#z&VU!|?fGB|zK# zM^=rBG!uP!7L>dN@Iwze)!FP(BCT%<6+((USWmM-s& zcjiY5Uz(jWY>;u=EBd2BpN*D+bEkNXh$Hy|teUf+Xr_R<#cEjwc= zR(@jkf(&bS>!wme;bH$wVvi8l3ItZdz|l+$Xh3DjLTD$!*J!QJm^64rBViSqP!6ho zX>Cne-!=N^wW`50jJy4muB0p+Zi|HwSIK@|OXUD~S$PeBmxs7kP;L89y#fyd#6l=L zkl=Gj!^2n_RW$T4BBwk=dmq|cWIp(Dc%}~YNZ)<8C^*k1 zZa7yAxq`AIpTd_UOhD0$0o%Q^bB4*&a!-weq9Up9+GYI})jtz;#9@G@HB3uI&)W|o z2tI1$Zcyj^_rJjWiYiSu#N)z4IN}=WcT~Kt?!A;{n_?Un^>FjujW!>jkx)$y$X31` zI5Py$JLnPv-=LiCx0uQ{(=+?4)ArQVJ%4H~TFc3NJXOnd7wkVa&YI9jM5%uRHbPvb zlDHXmGuUj>syz0><>X)^!@Kf#&XpB6;?p6UMvyJcIdG{y^bU=uIX(sBjG{@^BZd^uV0q?m)z!SHyfJ9y#a|%r z+N);9dz~53ftl%IaMQ(W-p)s#n1|Hc&F$FY-g{Xxgm9t#0NhbW;`D3A-)e+eH=aq^m&w z&HpnjPqKtS$ufBKZT&DRgPV_M9@|3&lAWDO*Q{XzSXG$&-Z~-fZZ7y-`Q)F`5?i^f(nN9S+`2S# zNo^K=bP>1Bs+WsOEvntGYXd*1TnfimpB3z8&g*9G6b9lZO(bU`Q#rZ>blwkSt(w)O zu1%#`r5sPzmv3h1zLIrdM;cYrg=7tiV*Vg%GCDyZ`IEen#{h-NyBjN0*1w}qJXhl_MzsaF@V@Ionw)9mP zmirmj+Y-?iyM{-nW#YbhfdiNBr!Sv;3sY-r11hZ@^^$7t=7wrNV`Td`toHvFF*u^9 zK>992(MBl~Q&VQTSY2;bXjCPrjT{oUH?s!qnGWX(0YGB>c=r!f^nXph`NQOJ9NoYx zDkKfNZKWELa2IfO>#=}@ZRD~h5Ce`27T}Bwc)n_k{e^?@oxkuS`e+fLP`etUf*xWs z0H>VmgLBrK)OcAGZsR`MA;2CSO`ZzSr_o>AvDuX~*brU@0uWcnG*{L;|xM6V*D|*s@FaRz0;+8NGKUD(k?GEWmby?|n9?A>9@yLb^*x zsjuAQ>CGIyv{CE0djW-2E%r-YQoAO!K;C-B_z8VUam2hCR)K45r|Qlai+FhPxC%c? zhd=eqU7LC3bTVl_SF#^>*{(}{H;zDdKp;dcS%#UKh6PS_t&3>dp-<0tlr()P^e`dI zVOow|dm#}Cjm%+qay=Jush8ZI4gInAMx!jqr=Em#OdKYp#b}+%byyVsQfBL>VK%0J z=)?GK({~92mPIYIhBVNRVO_SjV=3-ubb^^mEJk`2v--?nV&Vy zH#+E_ObmT<;{+ISUUJ|Ja&(c@T3ScWCk(c2GrvXdF!Ytu5PMX0WZWe1%Mr6yy~yDP zC)v&vv&PU9yCDA9vg&13ViH|>ak4CJ`M6hp`%JO;c!|om`&;S6LuQ=MKl7bBFV*_? zl+#HOi`!3RAO1$0|-|zHYRP zk4R+cgt*ZMag|oh2*vff{m&H!=S$mh;d?Kw?f2IauYD)lII1-Up>#JW9{Q4fGQ3FuOX2 z>A=iFI$>) zAV%+zG6-Wh))3)^8pnXDiEWQ%$JUz6l!~L<*B-PBkG?H!*==&5j&YgkbkVO#U1_yd;gm0}(G{zFAi>+D@}+Fl*7crNWCMh- z8hH=hXozrAceP7o2Dzvj^H|&G@+lu%O?^g$Q8A8?- zqr{Jr%V(hEgR4!Qh?%G`$2{`j+ZqdcL_uZTEL(57bB>C?fyRO4V?8g`S0g{RKxih} z&`=yc&uS#P85fJ#iFTsF7hIQ77R7n==SiZlLr;wZo^(Byv@c!k7Vl>uMJJ%Fr=a2J zJ-!J3h4-oAVuZx|PHQ=W!+Q?!t2>J4v&L#B?}DI?6INS+W+bS)u${q9M>kE1kzJ-t z!mi~fYwsU0cAZPb>$U8*a+OMIaB_0luPV>c-v(j)iVZhM`H@Ws)h!wzl=%|{P7KU- zKYdk?0ZIZV6SWlt`GzO2oV#{425$fGZK5HNRzrru_tVEGtL~@gV?sCsaL37K2n%fa z<$WQ2ZepFEW@OA?KTdiOAKLL$h53tV%Y|-ceZmJ)8trjCGZo!~b1@lZ3Kp@6Zuw%> zRIkuCZ6KMfCoQSZa;HdGr}VC)zP^*pjl=Gcc{Dy}vZAE9Yq$U|NRF7IKP$CCE?P0$ zyz~eVb+>%ErQq3-thdZ^@nh{w6Z`Pe0mg@)P@rvNmktpTB~32Pr&v^k3yt-z{xRP;=m+6Lb>$hWj857C+6bt$gx+( z;ZD(V&BKw0rd-GL?9V+&DDtV@6UyEok|jnSq}YUX;41RN=g|dnBiWF+X+_`I<(d(Ml#|kg$=I6| zr`txIuk=Ok^zfymD~q!-TS-c%L_k25t|F~%bp(o}rtG$oalFY>Zn2(<%EvpDjBxd<)K?6FBu$KoQUMM>i#eKUf_71Z+* z`)jsJgo{`^PF!o?E>WKmg9wl+X^-J*Unj*-JQP`aSDMYyh-oju3urgvX?7dcWX*sN zXLsnmc#$1hc|ZAFyL@c!v250(lPq9Q%$wjIs2JBhOUpySrfY>1a_vv_ne|HCkw4v_ z`HXW%<8c}H4?^kI6reQPtJv+`wu}{wW(f??}|W zhPwm<=5AP-*Rtl5kY2>jzG=^Il~tH$ZZ|rerrDf)KHH3)3>)srz#huIa`#GRZEUhp zvJTvt5Z&4}sz@3krp5Rw4$rh|=r@@})x+wJa=Hsxo*Of{ga?~2S9;HM~f zQHOc46XFdMu9pw+$NRrs<)oSs^&HxcS17aaRlVuxIH9{Et7A=RuQtcxOu zL8Cs-9R^iwndCs72aghH$Su4W6y`|2IX@eCX=kM0p`DnQGpm}Yv6t}&?Lz%wd`^*uj*RPU7UF-Qxo67A6y zizr7gbK$imo0#manrK0Z zf~mIgG%T7fLA_3{o~niJ`M751p2hL98q*QJKa%>2QGVw&>(ha>by3FkQhaDIXru>j zl!!{qvH)R_)QrKmy)Cpwavd*mJz7Q zX>>FW{}j8sj;{daR71Pw1eRr z^U@TN)HLI@M9HCE9k>W~%tg)UPkny-D$|F@6-2oWxn__0wo6Iwc?4QrtO#kXB|v0% z2nheEVu$CNM--c&Q8kAazYc7tEI;uVuv&TLH4`&ra4zZnvjkI(7jws?7|ok-75<>} zUXkif7<}tBLOI`~Wj=)#Ien-CBXHqFD*fa`w)NYIhu@b-8SlNhOE;RMfm7!gNaI2> zh_RE6=?<@bXA)FU>aPZtmOM9D4zZ4_5-VTb>q@^2_Z2@MdSEZ=y@AvfNH8A-LDc{f zF`Cl}Y8S}uloPISc|sZ~=wR<56%h|>RBL?Tnb{{SlLr^qeL!C}4>4SkYziZomqgbi zdev+|v!JI$udfBNR=cB^bmEC0!@yhf#G>8WYYf|5KQ|Hny(XDoV1!@({Na=HTf^3W zC3gG;qpSa|gVrppVy>x#Y>O!0#Gn|&US-Xh_b9L@b3K|kskMlvCWzi-*m9Yg6db4b z!gk}FF~c^Rt*;N03r(Z#PNB0jFF2X=5Z?{EdHZ0l@G@*hQ+YF{>>LoZ`>G!jG}_Nh zhUY~z^Sj0+Bt7SN;2q_;He$gFp;8{UKom@Z5n+x=Wu zDBXp64gT&G{653C>g7g?nFq2Q5vF2}FJ>LIB%sqFmxw2)YS%`nqP}Zx>Y&HbpEObi|0lc^q2EIeKZ3|7AAChbF zGGPeUF7m<6=2ZwvSL1H)RL>z*KFY3!D3hbwh_eAeQFC?4rJD>KLnDkDFJ$5*rP?Hj6+rRXoU~e#hNQ*8_bxc_ zyos6>>D>QuP2`R(%Evn6nN-sKL>5>7ySD()(TOO_Ujk_CnSez+Gmw)3fw8NWFc-bz zdBxUKoSjR?mdQHvN;EoMNjK`F4%j)12_+-}a^aN82m;u`TQy@x z*w~wGq;tnp;t_~qX6;HAHUZ*|+?gNMMrUmzhL1KY>nW~KHQMu6_g?F`b@k43sC50u zC&Xc!P=pTPiEMvFFRha0RrAx1|W#u0k=g^*KTs`Bye!8#1 zgQjbfdZ2>h;pSW$Xu$IFIro0QXz00h3lelJG30He~uH2;W{ zhjgh;Uoyjzm*fEXfDdI->87yBufMArQ-(;8s``!5*TpG!!-=VW~kIiiQ zrlZ*9bwMcgj$<|yP0L!T$mj~iaa_B(UpK!xQYFdjWjNoiCl?jPS`P;U1Bk>r4|t{` zl^@!0WT@t04q>A`fMI9RP(Nxbo=F`V2iAN zt}QSXCqm&MHw!;$dmPpTy;I&MTv#2aRi;Ddt3HuS7@N$5Xwq3#Ahc|>lcgAXN+APF z3GRKiBcIB@!#JQ`n0hRAjYod>w6*Ys#>SicbCU2G6^706yw1umMQVQL`l@xrq9jA1 zUG{=b7r6R4frlGGph51AstH-Px-|pfCuc<52mv7%6or+9PQSZ3^u;bwZFkR_w#Bm@ z=x{&FK_tNv5r40X)4!Zhn#GL{d+%ld3QM82(AH3RGkK1wz?Wb$&0ZBX>MMjgR4Dy$ z=3y-UB<-);b|Zz$~A>IVnzNMAq5E`&ulousu*}aC#)0>SX>VH%0C6$97w{Z0KXO zyVuZSBdlV{=4_>Ji94L17#)rN%r(ufIh)K?d+SL|%Ea^U5RK(4zCg$$btmI&r^}4V zat39$yUhtI`b{Y)%CwS*@;vsDrD&$(`%d4M+p4v|UVW=WF^bVQWzeUy-mFP2C1OZ^ zZ~9Oc;t`I|9Nx*RK8@G-+Nb&{NzLg(=n1(KWGNRHV6lXx3+a-;ZRdXL2Z+rhBwbsF zgM5Y}Nyd;YX%Fct0cL}3#DL`WW*h!o-^)a|W%omK_V5dB@2>mDEQ$UZ?LPC6`p& zzJp<3o87e|rp)uw5EQ5~?m8E3o<=eRvJ6MU#eB{LjqM48J~;2v9at2M_-~@-A+K|!}(4XIeLk)-aRgMMXlx~4y)1M{5UQ`6z&9 z({uhFK=w4!zC$E`0I^V3S`vExxgoegF=N2)od;Fk=*WpJ2)R`f*$008S6ES_1ce z%FKkyxqzf%0C@7pMg1{Sf6S>rR^J~X;g8_?|D$aNmCWocE(5A30=k*~X!A z8o}qzue^0N%d>8vdQFdVfyu6rF`Pl@Goucytl}^6@P1zq`kVhTe4v^Gf~g}wlPM6; zm2h^0VJnDo1l`U~=;~buaPw?QPZ?}Wpl=I^Rh_1v0NP1Lbx0bZx;@3;O1U-H5c#v8 zoG|(&nwq)VilH#L!L}8Isf$2sDJjw8I>|(=xz7I;HVQN8iAM5NjM`&MdABhb*FGGlb8vPUX;wur+pXddk$4E zqz&A@G_%vTHCyR3&+FY?`@;5y-tu2E0sgUZBpovGgbk4q>DuL?l6<<# zC__R($B;eVDUSVJ0Ju|V7P1rq^QnV8LrD&CBHr#i0?UfLt z$o;YnM;-AX$^jIVTu2zjTMRa=jsU*c+X-lzLc17uS?{r5#?vGNAl%Mj( zCWR9?uP`=kpob<&Z)h2n31)*NL-OQfLwu)^&kYp>??IPv-8R_kB({r?E9P7mAx@g7 zAVJJaTZB`Dpsvmdon5{rN02oVH&en1{$ zmpqP`9}R8BCW!B-B-E!pw;5Yl%(GP-@s+!fXPL|uXVLZ`yP!#b6R@n8&>qO4?-05G zjUGW1X@D!DnNY@rRHl?-J&`M=g=q$P2iIQ=%D-RCtR3UKI0gxX#h_if$OkrC?HTC8 zR7>(Q-d^Tyb++xopoR)Nsdmw)=>_+Mc0>FZwbmjIj^p46R85v6px+|E+vg|uXz}sp zp6Qjm=!|9Zc$gVysB#;^R?ZzD$OyRzw5Qr|fFW}aXs1p#VK@U2Z6oy&&m7!qX538; zbGuYZ<@H!{n}jaMM|F5M>~s|E1~f)fviDR;91dE!8dZO|I`Afz)A~(bTUwaZm9^xY zy`l3{_J%u|Os@mGW|JNCP9O%04jq6JCh>!9upJBHyA$0w9&9gh3oSW1OHHxYWhkBX zldHLs3~7JIgaM=@pfuaGIs!(%!Xzd{_5!(Pn$30*BVg~)F6HkpoX}F-&qgvBkjOew z^zs15f{xxkosm$+6%7d4n9dr99}hZNO7;Y0HM7CaH)OXL&4ZyFNm@O|JL7|7yI4$$ z8?vDK+9^G*uf^nT^4bAl#Ff1}FutS@qDq4mw%>g>hGs6)| z-#wsUvr1uZVeRmg+6y_qWhTc7VLEjrl{F`&DndsL-Don@G<^jpyqA`Xc3+LDa!@6v z-p^67W!F$rq3mH|ZMx}=xm(M7DnSSRDAn;U!zcPMtQ?VubM>^EirftU7R>Y>vCE_L zxjW?LlgYX&!rX>^b!%wbf6ndozNgBgIr zE)kfQaQTSDrH)cPT_+O=SG?KC^Bj*sTS|_+ZZHQ>XqJ@{zvkJxYw0b_M zobh(r%gHi*y|ZfItYCJ#L!cBS5K85%c|U1JPz#qD=!+WO3gYX;_K~R%=x)Ez8?A<77Ez~jRK`L zwM42I#S?Xk_M}qV_IO$oT%j9RE?>Mi%MmS#n-_hbz!|T&E%m4@LW-dZB8<^7s?Z4^y2XNweclbDK1F&M@0jvNjpJcsVK9KGgouBtZxflGwv~f9hMy_w0Lry4K78d)GSa-o5wP z`#azFoyG2p(OPMbeEQZrN>bmQG4?djhSsrg{^})L=p5_KI}Z3X%HmVo@`U|dCzOM5 z=e(ppZ;ZKNz#x;Y9@MUS(}WTrAfb-WW^*=LQ{RLxc>&izbic%9Nk_6@Fip;AaJR0z zCwNK9Z@8bl#m%iblbvsqanx5d;VlUx9v@dHaEixkUbffOkV(brvTfqzVBz!ZTe0O4 zt80hsvU5*g-IE?(_k9s%7I!MFZ((yN;`VXURltH{S}DZf5cu#=(GwkP%@2PKB zFk)hH(PypjhXn7K$hE!UY37abE=|3alZ83j?zRZ+i)JfTSFPr(&u%9=QAKv*Rb2Am z495IMG6^($VtQVGhH=p#^Zm2Cp;|==Q|s z8Lc;hEZX0xhMBG`h^qidpA#`<#G7ygBQ|{;SejS+x)rBTQD9mH9aX=0wC2aSoiJ(D zVddY3@`j0so-W9r&8W;qj+{)YX!3d@ZP=$c;yVdB{hI6%ZGU^saLl3w*WI|O?7P=0 zqXuJ2Z2s}#u`B07AW7Nu8~lA*u}LbMa$G%b9VZ9xONcpY;}luCi4wlzVdbVm+M|2b zwXlSF-R!=^(X0Kilg6gn2FIQsfOnKHR5Vv2sij`YBCYs|T%N-Fs0mU>FpuKs7_8XT zmpE;`8#mSliLckYjp<3UHKE^DJX2Z9&w5fe#zc)bti7}MS<*FTWi#FF6SyJDd-)5d!l7~mg)P5rLBaX zJ|PfeD-{qVnE=|`u~8HTn9{en_A)r&`oCWX0Ki1_n}@)@#{WUQL>thkXN>_$I7JxD zlbxZG#>KYP07>)!NMaAzEH+EwxU~#HbR@tVe+GD?e0B0~CyzUMR>&)gyc)~*iod+i zCZiPqakLN^zROlfzJ7`_PqD`dDc!W|=T66O_>~Z6eE&obU^XA1yP}OMT(yp8H=4ls zM?PwiK268^L=&P4dA@Ak|Ik`UkP?#+(#rnn zx#*$7{neiJv?+$*UXwb3?JAg&tv(EtWb^160pXnpeG+9Q_2O<^TyYC99XuuJGEL1^ z@}kL@9%yFH<9K^TR}r$*j+tGuQXxVL{RX)38OwHkfUG( zN0rcpk`IvmDQ9Iwz!KoB`vB1gSQ}IDt1QbNi0wl)#cXLAI8866E%gw}8+LzyIDbPB ztR!3@v~tBjI^v?!#t>VdlSIjSTYze2KstqfHf8`!l*|}n1xDvvG^bd*JCBPPs^%f7 z*Z6+@qWx2W{`m><`nmQ)G}AUUH5EfX9FI)$)1dVh*(Xl_Hn(?P`E0-zS6o{_>^4ia z^}!40^Dj5L+a*ehSZwWCR_*8iE*mGzMA54DRJ~^no4Kd9>g{#!n+~gPy!*?tJpPuc z1mWAs-(I5T&!M*9l<+WthXf^n@k3p*_(3-f7>zM`F!wZmMiYO7Vr{m)4e|xiap*S$@fJl9jLx~!Ull*2$&?ZV2Oat`ni>RkVyUIhZ3=h7I zjDBlrtPjDuP7;-2C6AC5?vS~xkCyGUbg{)}rCV1O?2fw{VeAeZTfJLftEYj0PpyDd-gWD9- z`=4zy{mMHb`t%RnGD%wO2gqw4xX8fH7Nu`1IdLK(aw7BcXqWBzoC0M9Qzy`T^Y*)A z1BonUvA0Ham9#w5tP z0alMi@Z>9|2{9%8Yr23;s<|T7Ni;uh;7_t1ayCO?AK1Uzqf@@&hVu1(htIM*`?qHg z|Kric-|2Oz;&gr~B|M=FH%tM@CuTL>o(^jfOCvk%I@jGg>72_{T9>*3mFPLHdg)UhG_jqp_6DU?nfBlA$* zMemw`z^Il^Kf{~KmmzPX{*@Q|$wK_+QTOu;!+(5I?MR4{L_B7JKN0clD<8@DmyRm0 z=IzRMOg@Ju6)DLUn1K=1$^~)L&Ih5fUrqv=ZlHTt+>0Bjn)w@YqNzlZcj< zFek$JCX&00Z8Ju9N>ira%Sp$~E-)=;^=LV14&1DLKFQx66sD4sE2rWPg@vx0+55g* zNzv?>Z~lPx$*^@n(X|CniAa;RI>>9zb{3-}Y932%SOa#_rFpXILmdixlV`I2F>ef3 zg=JW3{*6P|wqF# Date: Mon, 14 Mar 2022 17:55:30 +0300 Subject: [PATCH 31/42] Fix finding end of WAL on safekeepers after f86cf93435133ee11. That commit dropped wal_start_lsn, now we're looking since commit_lsn, which is the real end of WAL if no records follow it. ref #1351 --- postgres_ffi/src/xlog_utils.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/postgres_ffi/src/xlog_utils.rs b/postgres_ffi/src/xlog_utils.rs index caf1940a9c..d2b2b5c122 100644 --- a/postgres_ffi/src/xlog_utils.rs +++ b/postgres_ffi/src/xlog_utils.rs @@ -132,6 +132,8 @@ pub fn get_current_timestamp() -> TimestampTz { } } +/// Return offset of the last valid record in the segment segno, starting +/// looking at start_offset. Returns start_offset if no records found. fn find_end_of_wal_segment( data_dir: &Path, segno: XLogSegNo, @@ -147,7 +149,7 @@ fn find_end_of_wal_segment( let mut rec_offs: usize = 0; let mut buf = [0u8; XLOG_BLCKSZ]; let file_name = XLogFileName(tli, segno, wal_seg_size); - let mut last_valid_rec_pos: usize = 0; + let mut last_valid_rec_pos: usize = start_offset; // assume at given start_offset begins new record let mut file = File::open(data_dir.join(file_name.clone() + ".partial")).unwrap(); file.seek(SeekFrom::Start(offs as u64))?; let mut rec_hdr = [0u8; XLOG_RECORD_CRC_OFFS]; From 9c1a9a1d9f315adac161e5490b314dde63e3e292 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 14 Mar 2022 20:06:25 +0200 Subject: [PATCH 32/42] Update Cargo.lock for new dependencies (#1354) Commit b2ad8342d2 added dependency on 'criterion', which pulled along some other crates. --- Cargo.lock | 183 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index ad38a41d91..b1ebe6c07a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -260,6 +260,18 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5988cb1d626264ac94100be357308f29ff7cbdd3b36bda27f450a4ee3f713426" +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.9.1" @@ -281,6 +293,15 @@ dependencies = [ "serde", ] +[[package]] +name = "cast" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c24dab4283a142afa2fdca129b80ad2c6284e073930f964c3a1293c225ee39a" +dependencies = [ + "rustc_version", +] + [[package]] name = "cc" version = "1.0.72" @@ -447,6 +468,76 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "criterion" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1604dafd25fba2fe2d5895a9da139f8dc9b319a5fe5354ca137cbbce4e178d10" +dependencies = [ + "atty", + "cast", + "clap 2.34.0", + "criterion-plot", + "csv", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d00996de9f2f7559f7f4dc286073197f83e92256a59ed395f9aac01fe717da57" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c00d6d2ea26e8b151d99093005cb442fb9a37aeaca582a03ec70946f49ab5ed9" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-utils" version = "0.8.7" @@ -477,6 +568,28 @@ dependencies = [ "subtle", ] +[[package]] +name = "csv" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +dependencies = [ + "bstr", + "csv-core", + "itoa 0.4.8", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + [[package]] name = "daemonize" version = "0.4.1" @@ -1260,6 +1373,12 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "opaque-debug" version = "0.3.0" @@ -1444,6 +1563,34 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "plotters" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a3fd9ec30b9749ce28cd91f255d569591cdf937fe280c312143e3c4bad6f2a" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d88417318da0eaf0fdcdb51a0ee6c3bed624333bff8f946733049380be67ac1c" + +[[package]] +name = "plotters-svg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521fa9638fa597e1dc53e9412a4f9cefb01187ee1f7413076f9e6749e2885ba9" +dependencies = [ + "plotters-backend", +] + [[package]] name = "postgres" version = "0.19.1" @@ -1664,6 +1811,31 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rayon" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + [[package]] name = "rcgen" version = "0.8.14" @@ -2233,6 +2405,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.5.1" @@ -2855,6 +3037,7 @@ dependencies = [ "bincode", "byteorder", "bytes", + "criterion", "git-version", "hex", "hex-literal", From 705f51db2777228e3e61db77573625cc9929585c Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 16 Mar 2022 21:20:04 +0300 Subject: [PATCH 33/42] [proxy] Propagate some errors to user (#1329) * [proxy] Propagate most errors to user This change enables propagation of most errors to the user (e.g. auth and connectivity errors). Some of them will be stripped of sensitive information. As a side effect, most occurrences of `anyhow::Error` were replaced with concrete error types. * [proxy] Box weighty errors --- Cargo.lock | 2 + proxy/Cargo.toml | 2 + proxy/src/auth.rs | 124 ++++++++++++++++++++------- proxy/src/cancellation.rs | 2 +- proxy/src/compute.rs | 61 +++++++++++-- proxy/src/config.rs | 6 +- proxy/src/cplane_api.rs | 111 ++++++++++++++++++------ proxy/src/error.rs | 17 ++++ proxy/src/http.rs | 2 +- proxy/src/main.rs | 3 +- proxy/src/mgmt.rs | 20 +++-- proxy/src/proxy.rs | 176 +++++++++++++++++++++++--------------- proxy/src/stream.rs | 84 +++++++++++++++--- proxy/src/waiters.rs | 37 ++++++-- 14 files changed, 481 insertions(+), 166 deletions(-) create mode 100644 proxy/src/error.rs diff --git a/Cargo.lock b/Cargo.lock index b1ebe6c07a..750ac0edc2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1739,6 +1739,7 @@ dependencies = [ "anyhow", "bytes", "clap 3.0.14", + "fail", "futures", "hashbrown 0.11.2", "hex", @@ -1754,6 +1755,7 @@ dependencies = [ "scopeguard", "serde", "serde_json", + "thiserror", "tokio", "tokio-postgres 0.7.1 (git+https://github.com/zenithdb/rust-postgres.git?rev=2949d98df52587d562986aad155dd4e889e408b7)", "tokio-postgres-rustls", diff --git a/proxy/Cargo.toml b/proxy/Cargo.toml index d8d5cbe5bf..dda018a1d8 100644 --- a/proxy/Cargo.toml +++ b/proxy/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" anyhow = "1.0" bytes = { version = "1.0.1", features = ['serde'] } clap = "3.0" +fail = "0.5.0" futures = "0.3.13" hashbrown = "0.11.2" hex = "0.4.3" @@ -21,6 +22,7 @@ rustls = "0.19.1" scopeguard = "1.1.0" serde = "1" serde_json = "1" +thiserror = "1.0" tokio = { version = "1.11", features = ["macros"] } tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" } tokio-rustls = "0.22.0" diff --git a/proxy/src/auth.rs b/proxy/src/auth.rs index a5bdaeaeca..5e6357fe80 100644 --- a/proxy/src/auth.rs +++ b/proxy/src/auth.rs @@ -1,11 +1,79 @@ use crate::compute::DatabaseInfo; use crate::config::ProxyConfig; use crate::cplane_api::{self, CPlaneApi}; +use crate::error::UserFacingError; use crate::stream::PqStream; -use anyhow::{anyhow, bail, Context}; +use crate::waiters; use std::collections::HashMap; +use thiserror::Error; use tokio::io::{AsyncRead, AsyncWrite}; -use zenith_utils::pq_proto::{BeMessage as Be, BeParameterStatusMessage, FeMessage as Fe}; +use zenith_utils::pq_proto::{BeMessage as Be, BeParameterStatusMessage}; + +/// Common authentication error. +#[derive(Debug, Error)] +pub enum AuthErrorImpl { + /// Authentication error reported by the console. + #[error(transparent)] + Console(#[from] cplane_api::AuthError), + + /// For passwords that couldn't be processed by [`parse_password`]. + #[error("Malformed password message")] + MalformedPassword, + + /// Errors produced by [`PqStream`]. + #[error(transparent)] + Io(#[from] std::io::Error), +} + +impl AuthErrorImpl { + pub fn auth_failed(msg: impl Into) -> Self { + AuthErrorImpl::Console(cplane_api::AuthError::auth_failed(msg)) + } +} + +impl From for AuthErrorImpl { + fn from(e: waiters::RegisterError) -> Self { + AuthErrorImpl::Console(cplane_api::AuthError::from(e)) + } +} + +impl From for AuthErrorImpl { + fn from(e: waiters::WaitError) -> Self { + AuthErrorImpl::Console(cplane_api::AuthError::from(e)) + } +} + +#[derive(Debug, Error)] +#[error(transparent)] +pub struct AuthError(Box); + +impl From for AuthError +where + AuthErrorImpl: From, +{ + fn from(e: T) -> Self { + AuthError(Box::new(e.into())) + } +} + +impl UserFacingError for AuthError { + fn to_string_client(&self) -> String { + use AuthErrorImpl::*; + match self.0.as_ref() { + Console(e) => e.to_string_client(), + MalformedPassword => self.to_string(), + _ => "Internal error".to_string(), + } + } +} + +#[derive(Debug, Error)] +pub enum ClientCredsParseError { + #[error("Parameter `{0}` is missing in startup packet")] + MissingKey(&'static str), +} + +impl UserFacingError for ClientCredsParseError {} /// Various client credentials which we use for authentication. #[derive(Debug, PartialEq, Eq)] @@ -15,13 +83,13 @@ pub struct ClientCredentials { } impl TryFrom> for ClientCredentials { - type Error = anyhow::Error; + type Error = ClientCredsParseError; fn try_from(mut value: HashMap) -> Result { let mut get_param = |key| { value .remove(key) - .with_context(|| format!("{} is missing in startup packet", key)) + .ok_or(ClientCredsParseError::MissingKey(key)) }; let user = get_param("user")?; @@ -37,10 +105,14 @@ impl ClientCredentials { self, config: &ProxyConfig, client: &mut PqStream, - ) -> anyhow::Result { + ) -> Result { + fail::fail_point!("proxy-authenticate", |_| { + Err(AuthError::auth_failed("failpoint triggered")) + }); + use crate::config::ClientAuthMethod::*; use crate::config::RouterConfig::*; - let db_info = match &config.router_config { + match &config.router_config { Static { host, port } => handle_static(host.clone(), *port, client, self).await, Dynamic(Mixed) => { if self.user.ends_with("@zenith") { @@ -51,9 +123,7 @@ impl ClientCredentials { } Dynamic(Password) => handle_existing_user(config, client, self).await, Dynamic(Link) => handle_new_user(config, client).await, - }; - - db_info.context("failed to authenticate client") + } } } @@ -66,18 +136,14 @@ async fn handle_static( port: u16, client: &mut PqStream, creds: ClientCredentials, -) -> anyhow::Result { +) -> Result { client .write_message(&Be::AuthenticationCleartextPassword) .await?; // Read client's password bytes - let msg = match client.read_message().await? { - Fe::PasswordMessage(msg) => msg, - bad => bail!("unexpected message type: {:?}", bad), - }; - - let cleartext_password = std::str::from_utf8(&msg)?.split('\0').next().unwrap(); + let msg = client.read_password_message().await?; + let cleartext_password = parse_password(&msg).ok_or(AuthErrorImpl::MalformedPassword)?; let db_info = DatabaseInfo { host, @@ -98,7 +164,7 @@ async fn handle_existing_user( config: &ProxyConfig, client: &mut PqStream, creds: ClientCredentials, -) -> anyhow::Result { +) -> Result { let psql_session_id = new_psql_session_id(); let md5_salt = rand::random(); @@ -107,18 +173,12 @@ async fn handle_existing_user( .await?; // Read client's password hash - let msg = match client.read_message().await? { - Fe::PasswordMessage(msg) => msg, - bad => bail!("unexpected message type: {:?}", bad), - }; + let msg = client.read_password_message().await?; + let md5_response = parse_password(&msg).ok_or(AuthErrorImpl::MalformedPassword)?; - let (_trailing_null, md5_response) = msg - .split_last() - .ok_or_else(|| anyhow!("unexpected password message"))?; - - let cplane = CPlaneApi::new(&config.auth_endpoint); + let cplane = CPlaneApi::new(config.auth_endpoint.clone()); let db_info = cplane - .authenticate_proxy_request(creds, md5_response, &md5_salt, &psql_session_id) + .authenticate_proxy_client(creds, md5_response, &md5_salt, &psql_session_id) .await?; client @@ -131,7 +191,7 @@ async fn handle_existing_user( async fn handle_new_user( config: &ProxyConfig, client: &mut PqStream, -) -> anyhow::Result { +) -> Result { let psql_session_id = new_psql_session_id(); let greeting = hello_message(&config.redirect_uri, &psql_session_id); @@ -143,8 +203,8 @@ async fn handle_new_user( .write_message(&Be::NoticeResponse(greeting)) .await?; - // Wait for web console response - waiter.await?.map_err(|e| anyhow!(e)) + // Wait for web console response (see `mgmt`) + waiter.await?.map_err(AuthErrorImpl::auth_failed) }) .await?; @@ -153,6 +213,10 @@ async fn handle_new_user( Ok(db_info) } +fn parse_password(bytes: &[u8]) -> Option<&str> { + std::str::from_utf8(bytes).ok()?.strip_suffix('\0') +} + fn hello_message(redirect_uri: &str, session_id: &str) -> String { format!( concat![ diff --git a/proxy/src/cancellation.rs b/proxy/src/cancellation.rs index c1a7e81be9..07d3bcc71a 100644 --- a/proxy/src/cancellation.rs +++ b/proxy/src/cancellation.rs @@ -6,7 +6,7 @@ use tokio::net::TcpStream; use tokio_postgres::{CancelToken, NoTls}; use zenith_utils::pq_proto::CancelKeyData; -/// Enables serving CancelRequests. +/// Enables serving `CancelRequest`s. #[derive(Default)] pub struct CancelMap(Mutex>>); diff --git a/proxy/src/compute.rs b/proxy/src/compute.rs index 7c294bd488..64ce5d0a5a 100644 --- a/proxy/src/compute.rs +++ b/proxy/src/compute.rs @@ -1,6 +1,27 @@ -use anyhow::Context; +use crate::cancellation::CancelClosure; +use crate::error::UserFacingError; use serde::{Deserialize, Serialize}; -use std::net::{SocketAddr, ToSocketAddrs}; +use std::io; +use std::net::SocketAddr; +use thiserror::Error; +use tokio::net::TcpStream; +use tokio_postgres::NoTls; + +#[derive(Debug, Error)] +pub enum ConnectionError { + /// This error doesn't seem to reveal any secrets; for instance, + /// [`tokio_postgres::error::Kind`] doesn't contain ip addresses and such. + #[error("Failed to connect to the compute node: {0}")] + Postgres(#[from] tokio_postgres::Error), + + #[error("Failed to connect to the compute node")] + FailedToConnectToCompute, + + #[error("Failed to fetch compute node version")] + FailedToFetchPgVersion, +} + +impl UserFacingError for ConnectionError {} /// Compute node connection params. #[derive(Serialize, Deserialize, Debug, Default)] @@ -12,14 +33,38 @@ pub struct DatabaseInfo { pub password: Option, } +/// PostgreSQL version as [`String`]. +pub type Version = String; + impl DatabaseInfo { - pub fn socket_addr(&self) -> anyhow::Result { + async fn connect_raw(&self) -> io::Result<(SocketAddr, TcpStream)> { let host_port = format!("{}:{}", self.host, self.port); - host_port - .to_socket_addrs() - .with_context(|| format!("cannot resolve {} to SocketAddr", host_port))? - .next() - .context("cannot resolve at least one SocketAddr") + let socket = TcpStream::connect(host_port).await?; + let socket_addr = socket.peer_addr()?; + + Ok((socket_addr, socket)) + } + + /// Connect to a corresponding compute node. + pub async fn connect(self) -> Result<(TcpStream, Version, CancelClosure), ConnectionError> { + let (socket_addr, mut socket) = self + .connect_raw() + .await + .map_err(|_| ConnectionError::FailedToConnectToCompute)?; + + // TODO: establish a secure connection to the DB + let (client, conn) = tokio_postgres::Config::from(self) + .connect_raw(&mut socket, NoTls) + .await?; + + let version = conn + .parameter("server_version") + .ok_or(ConnectionError::FailedToFetchPgVersion)? + .into(); + + let cancel_closure = CancelClosure::new(socket_addr, client.cancel_token()); + + Ok((socket, version, cancel_closure)) } } diff --git a/proxy/src/config.rs b/proxy/src/config.rs index 9ab64db795..077ff02898 100644 --- a/proxy/src/config.rs +++ b/proxy/src/config.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, ensure, Context}; +use anyhow::{anyhow, bail, ensure, Context}; use rustls::{internal::pemfile, NoClientAuth, ProtocolVersion, ServerConfig}; use std::net::SocketAddr; use std::str::FromStr; @@ -29,7 +29,7 @@ impl FromStr for ClientAuthMethod { "password" => Ok(Password), "link" => Ok(Link), "mixed" => Ok(Mixed), - _ => Err(anyhow::anyhow!("Invlid option for router")), + _ => bail!("Invalid option for router: `{}`", s), } } } @@ -53,7 +53,7 @@ pub struct ProxyConfig { pub redirect_uri: String, /// control plane address where we would check auth. - pub auth_endpoint: String, + pub auth_endpoint: reqwest::Url, pub tls_config: Option, } diff --git a/proxy/src/cplane_api.rs b/proxy/src/cplane_api.rs index 187809717f..21fce79df3 100644 --- a/proxy/src/cplane_api.rs +++ b/proxy/src/cplane_api.rs @@ -1,52 +1,113 @@ use crate::auth::ClientCredentials; use crate::compute::DatabaseInfo; -use crate::waiters::{Waiter, Waiters}; -use anyhow::{anyhow, bail}; +use crate::error::UserFacingError; +use crate::mgmt; +use crate::waiters::{self, Waiter, Waiters}; use lazy_static::lazy_static; use serde::{Deserialize, Serialize}; +use thiserror::Error; lazy_static! { - static ref CPLANE_WAITERS: Waiters> = Default::default(); + static ref CPLANE_WAITERS: Waiters = Default::default(); } /// Give caller an opportunity to wait for cplane's reply. -pub async fn with_waiter(psql_session_id: impl Into, f: F) -> anyhow::Result +pub async fn with_waiter( + psql_session_id: impl Into, + action: impl FnOnce(Waiter<'static, mgmt::ComputeReady>) -> R, +) -> Result where - F: FnOnce(Waiter<'static, Result>) -> R, - R: std::future::Future>, + R: std::future::Future>, + E: From, { let waiter = CPLANE_WAITERS.register(psql_session_id.into())?; - f(waiter).await + action(waiter).await } -pub fn notify(psql_session_id: &str, msg: Result) -> anyhow::Result<()> { +pub fn notify( + psql_session_id: &str, + msg: Result, +) -> Result<(), waiters::NotifyError> { CPLANE_WAITERS.notify(psql_session_id, msg) } /// Zenith console API wrapper. -pub struct CPlaneApi<'a> { - auth_endpoint: &'a str, +pub struct CPlaneApi { + auth_endpoint: reqwest::Url, } -impl<'a> CPlaneApi<'a> { - pub fn new(auth_endpoint: &'a str) -> Self { +impl CPlaneApi { + pub fn new(auth_endpoint: reqwest::Url) -> Self { Self { auth_endpoint } } } -impl CPlaneApi<'_> { - pub async fn authenticate_proxy_request( +#[derive(Debug, Error)] +pub enum AuthErrorImpl { + /// Authentication error reported by the console. + #[error("Authentication failed: {0}")] + AuthFailed(String), + + /// HTTP status (other than 200) returned by the console. + #[error("Console responded with an HTTP status: {0}")] + HttpStatus(reqwest::StatusCode), + + #[error("Console responded with a malformed JSON: {0}")] + MalformedResponse(#[from] serde_json::Error), + + #[error(transparent)] + Transport(#[from] reqwest::Error), + + #[error(transparent)] + WaiterRegister(#[from] waiters::RegisterError), + + #[error(transparent)] + WaiterWait(#[from] waiters::WaitError), +} + +#[derive(Debug, Error)] +#[error(transparent)] +pub struct AuthError(Box); + +impl AuthError { + /// Smart constructor for authentication error reported by `mgmt`. + pub fn auth_failed(msg: impl Into) -> Self { + AuthError(Box::new(AuthErrorImpl::AuthFailed(msg.into()))) + } +} + +impl From for AuthError +where + AuthErrorImpl: From, +{ + fn from(e: T) -> Self { + AuthError(Box::new(e.into())) + } +} + +impl UserFacingError for AuthError { + fn to_string_client(&self) -> String { + use AuthErrorImpl::*; + match self.0.as_ref() { + AuthFailed(_) | HttpStatus(_) => self.to_string(), + _ => "Internal error".to_string(), + } + } +} + +impl CPlaneApi { + pub async fn authenticate_proxy_client( &self, creds: ClientCredentials, - md5_response: &[u8], + md5_response: &str, salt: &[u8; 4], psql_session_id: &str, - ) -> anyhow::Result { - let mut url = reqwest::Url::parse(self.auth_endpoint)?; + ) -> Result { + let mut url = self.auth_endpoint.clone(); url.query_pairs_mut() .append_pair("login", &creds.user) .append_pair("database", &creds.dbname) - .append_pair("md5response", std::str::from_utf8(md5_response)?) + .append_pair("md5response", md5_response) .append_pair("salt", &hex::encode(salt)) .append_pair("psql_session_id", psql_session_id); @@ -55,18 +116,20 @@ impl CPlaneApi<'_> { // TODO: leverage `reqwest::Client` to reuse connections let resp = reqwest::get(url).await?; if !resp.status().is_success() { - bail!("Auth failed: {}", resp.status()) + return Err(AuthErrorImpl::HttpStatus(resp.status()).into()); } let auth_info: ProxyAuthResponse = serde_json::from_str(resp.text().await?.as_str())?; println!("got auth info: #{:?}", auth_info); use ProxyAuthResponse::*; - match auth_info { - Ready { conn_info } => Ok(conn_info), - Error { error } => bail!(error), - NotReady { .. } => waiter.await?.map_err(|e| anyhow!(e)), - } + let db_info = match auth_info { + Ready { conn_info } => conn_info, + Error { error } => return Err(AuthErrorImpl::AuthFailed(error).into()), + NotReady { .. } => waiter.await?.map_err(AuthErrorImpl::AuthFailed)?, + }; + + Ok(db_info) }) .await } diff --git a/proxy/src/error.rs b/proxy/src/error.rs new file mode 100644 index 0000000000..e98e553f83 --- /dev/null +++ b/proxy/src/error.rs @@ -0,0 +1,17 @@ +/// Marks errors that may be safely shown to a client. +/// This trait can be seen as a specialized version of [`ToString`]. +/// +/// NOTE: This trait should not be implemented for [`anyhow::Error`], since it +/// is way too convenient and tends to proliferate all across the codebase, +/// ultimately leading to accidental leaks of sensitive data. +pub trait UserFacingError: ToString { + /// Format the error for client, stripping all sensitive info. + /// + /// Although this might be a no-op for many types, it's highly + /// recommended to override the default impl in case error type + /// contains anything sensitive: various IDs, IP addresses etc. + #[inline(always)] + fn to_string_client(&self) -> String { + self.to_string() + } +} diff --git a/proxy/src/http.rs b/proxy/src/http.rs index 0b693d88dd..33d134678f 100644 --- a/proxy/src/http.rs +++ b/proxy/src/http.rs @@ -7,7 +7,7 @@ use zenith_utils::http::json::json_response; use zenith_utils::http::{RouterBuilder, RouterService}; async fn status_handler(_: Request) -> Result, ApiError> { - Ok(json_response(StatusCode::OK, "")?) + json_response(StatusCode::OK, "") } fn make_router() -> RouterBuilder { diff --git a/proxy/src/main.rs b/proxy/src/main.rs index de618ccde9..bd99d0a639 100644 --- a/proxy/src/main.rs +++ b/proxy/src/main.rs @@ -20,13 +20,14 @@ mod cancellation; mod compute; mod config; mod cplane_api; +mod error; mod http; mod mgmt; mod proxy; mod stream; mod waiters; -/// Flattens Result> into Result. +/// Flattens `Result>` into `Result`. async fn flatten_err( f: impl Future, JoinError>>, ) -> anyhow::Result<()> { diff --git a/proxy/src/mgmt.rs b/proxy/src/mgmt.rs index 55b49b441f..e53542dfd2 100644 --- a/proxy/src/mgmt.rs +++ b/proxy/src/mgmt.rs @@ -79,6 +79,18 @@ enum PsqlSessionResult { Failure(String), } +/// A message received by `mgmt` when a compute node is ready. +pub type ComputeReady = Result; + +impl PsqlSessionResult { + fn into_compute_ready(self) -> ComputeReady { + match self { + Self::Success(db_info) => Ok(db_info), + Self::Failure(message) => Err(message), + } + } +} + impl postgres_backend::Handler for MgmtHandler { fn process_query( &mut self, @@ -99,13 +111,7 @@ fn try_process_query(pgb: &mut PostgresBackend, query_string: &str) -> anyhow::R let resp: PsqlSessionResponse = serde_json::from_str(query_string)?; - use PsqlSessionResult::*; - let msg = match resp.result { - Success(db_info) => Ok(db_info), - Failure(message) => Err(message), - }; - - match cplane_api::notify(&resp.session_id, msg) { + match cplane_api::notify(&resp.session_id, resp.result.into_compute_ready()) { Ok(()) => { pgb.write_message_noflush(&SINGLE_COL_ROWDESC)? .write_message_noflush(&BeMessage::DataRow(&[Some(b"ok")]))? diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index 1dc301b792..3c7f59bc26 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -1,17 +1,18 @@ use crate::auth; -use crate::cancellation::{self, CancelClosure, CancelMap}; -use crate::compute::DatabaseInfo; +use crate::cancellation::{self, CancelMap}; use crate::config::{ProxyConfig, TlsConfig}; use crate::stream::{MetricsStream, PqStream, Stream}; use anyhow::{bail, Context}; +use futures::TryFutureExt; use lazy_static::lazy_static; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; -use tokio::net::TcpStream; -use tokio_postgres::NoTls; use zenith_metrics::{new_common_metric_name, register_int_counter, IntCounter}; use zenith_utils::pq_proto::{BeMessage as Be, *}; +const ERR_INSECURE_CONNECTION: &str = "connection is insecure (try using `sslmode=require`)"; +const ERR_PROTO_VIOLATION: &str = "protocol violation"; + lazy_static! { static ref NUM_CONNECTIONS_ACCEPTED_COUNTER: IntCounter = register_int_counter!( new_common_metric_name("num_connections_accepted"), @@ -30,6 +31,7 @@ lazy_static! { .unwrap(); } +/// A small combinator for pluggable error logging. async fn log_error(future: F) -> F::Output where F: std::future::Future>, @@ -76,20 +78,21 @@ async fn handle_client( } let tls = config.tls_config.clone(); - if let Some((client, creds)) = handshake(stream, tls, cancel_map).await? { - cancel_map - .with_session(|session| async { - connect_client_to_db(config, session, client, creds).await - }) - .await?; - } + let (stream, creds) = match handshake(stream, tls, cancel_map).await? { + Some(x) => x, + None => return Ok(()), // it's a cancellation request + }; - Ok(()) + let client = Client::new(stream, creds); + cancel_map + .with_session(|session| client.connect_to_db(config, session)) + .await } -/// Handle a connection from one client. -/// For better testing experience, `stream` can be -/// any object satisfying the traits. +/// Establish a (most probably, secure) connection with the client. +/// For better testing experience, `stream` can be any object satisfying the traits. +/// It's easier to work with owned `stream` here as we need to updgrade it to TLS; +/// we also take an extra care of propagating only the select handshake errors to client. async fn handshake( stream: S, mut tls: Option, @@ -119,7 +122,7 @@ async fn handshake( stream = PqStream::new(stream.into_inner().upgrade(tls).await?); } } - _ => bail!("protocol violation"), + _ => bail!(ERR_PROTO_VIOLATION), }, GssEncRequest => match stream.get_ref() { Stream::Raw { .. } if !tried_gss => { @@ -128,18 +131,21 @@ async fn handshake( // Currently, we don't support GSSAPI stream.write_message(&Be::EncryptionResponse(false)).await?; } - _ => bail!("protocol violation"), + _ => bail!(ERR_PROTO_VIOLATION), }, StartupMessage { params, .. } => { // Check that the config has been consumed during upgrade // OR we didn't provide it at all (for dev purposes). if tls.is_some() { - let msg = "connection is insecure (try using `sslmode=require`)"; - stream.write_message(&Be::ErrorResponse(msg)).await?; - bail!(msg); + stream.throw_error_str(ERR_INSECURE_CONNECTION).await?; } - break Ok(Some((stream, params.try_into()?))); + // Here and forth: `or_else` demands that we use a future here + let creds = async { params.try_into() } + .or_else(|e| stream.throw_error(e)) + .await?; + + break Ok(Some((stream, creds))); } CancelRequest(cancel_key_data) => { cancel_map.cancel_session(cancel_key_data).await?; @@ -150,58 +156,60 @@ async fn handshake( } } -async fn connect_client_to_db( - config: &ProxyConfig, - session: cancellation::Session<'_>, - mut client: PqStream, +/// Thin connection context. +struct Client { + /// The underlying libpq protocol stream. + stream: PqStream, + /// Client credentials that we care about. creds: auth::ClientCredentials, -) -> anyhow::Result<()> { - let db_info = creds.authenticate(config, &mut client).await?; - let (db, version, cancel_closure) = connect_to_db(db_info).await?; - let cancel_key_data = session.enable_cancellation(cancel_closure); - - client - .write_message_noflush(&BeMessage::ParameterStatus( - BeParameterStatusMessage::ServerVersion(&version), - ))? - .write_message_noflush(&Be::BackendKeyData(cancel_key_data))? - .write_message(&BeMessage::ReadyForQuery) - .await?; - - // This function will be called for writes to either direction. - fn inc_proxied(cnt: usize) { - // Consider inventing something more sophisticated - // if this ever becomes a bottleneck (cacheline bouncing). - NUM_BYTES_PROXIED_COUNTER.inc_by(cnt as u64); - } - - let mut db = MetricsStream::new(db, inc_proxied); - let mut client = MetricsStream::new(client.into_inner(), inc_proxied); - let _ = tokio::io::copy_bidirectional(&mut client, &mut db).await?; - - Ok(()) } -/// Connect to a corresponding compute node. -async fn connect_to_db( - db_info: DatabaseInfo, -) -> anyhow::Result<(TcpStream, String, CancelClosure)> { - // TODO: establish a secure connection to the DB - let socket_addr = db_info.socket_addr()?; - let mut socket = TcpStream::connect(socket_addr).await?; +impl Client { + /// Construct a new connection context. + fn new(stream: PqStream, creds: auth::ClientCredentials) -> Self { + Self { stream, creds } + } +} - let (client, conn) = tokio_postgres::Config::from(db_info) - .connect_raw(&mut socket, NoTls) - .await?; +impl Client { + /// Let the client authenticate and connect to the designated compute node. + async fn connect_to_db( + self, + config: &ProxyConfig, + session: cancellation::Session<'_>, + ) -> anyhow::Result<()> { + let Self { mut stream, creds } = self; - let version = conn - .parameter("server_version") - .context("failed to fetch postgres server version")? - .into(); + // Authenticate and connect to a compute node. + let auth = creds.authenticate(config, &mut stream).await; + let db_info = async { auth }.or_else(|e| stream.throw_error(e)).await?; - let cancel_closure = CancelClosure::new(socket_addr, client.cancel_token()); + let (db, version, cancel_closure) = + db_info.connect().or_else(|e| stream.throw_error(e)).await?; + let cancel_key_data = session.enable_cancellation(cancel_closure); - Ok((socket, version, cancel_closure)) + stream + .write_message_noflush(&BeMessage::ParameterStatus( + BeParameterStatusMessage::ServerVersion(&version), + ))? + .write_message_noflush(&Be::BackendKeyData(cancel_key_data))? + .write_message(&BeMessage::ReadyForQuery) + .await?; + + /// This function will be called for writes to either direction. + fn inc_proxied(cnt: usize) { + // Consider inventing something more sophisticated + // if this ever becomes a bottleneck (cacheline bouncing). + NUM_BYTES_PROXIED_COUNTER.inc_by(cnt as u64); + } + + // Starting from here we only proxy the client's traffic. + let mut db = MetricsStream::new(db, inc_proxied); + let mut client = MetricsStream::new(stream.into_inner(), inc_proxied); + let _ = tokio::io::copy_bidirectional(&mut client, &mut db).await?; + + Ok(()) + } } #[cfg(test)] @@ -210,7 +218,7 @@ mod tests { use tokio::io::DuplexStream; use tokio_postgres::config::SslMode; - use tokio_postgres::tls::MakeTlsConnect; + use tokio_postgres::tls::{MakeTlsConnect, NoTls}; use tokio_postgres_rustls::MakeRustlsConnect; async fn dummy_proxy( @@ -264,7 +272,7 @@ mod tests { let proxy = tokio::spawn(dummy_proxy(client, Some(server_config.into()))); - tokio_postgres::Config::new() + let client_err = tokio_postgres::Config::new() .user("john_doe") .dbname("earth") .ssl_mode(SslMode::Disable) @@ -273,11 +281,15 @@ mod tests { .err() // -> Option .context("client shouldn't be able to connect")?; - proxy + assert!(client_err.to_string().contains(ERR_INSECURE_CONNECTION)); + + let server_err = proxy .await? .err() // -> Option .context("server shouldn't accept client")?; + assert!(client_err.to_string().contains(&server_err.to_string())); + Ok(()) } @@ -329,4 +341,30 @@ mod tests { proxy.await? } + + #[tokio::test] + async fn give_user_an_error_for_bad_creds() -> anyhow::Result<()> { + let (client, server) = tokio::io::duplex(1024); + + let proxy = tokio::spawn(dummy_proxy(client, None)); + + let client_err = tokio_postgres::Config::new() + .ssl_mode(SslMode::Disable) + .connect_raw(server, NoTls) + .await + .err() // -> Option + .context("client shouldn't be able to connect")?; + + // TODO: this is ugly, but `format!` won't allow us to extract fmt string + assert!(client_err.to_string().contains("missing in startup packet")); + + let server_err = proxy + .await? + .err() // -> Option + .context("server shouldn't accept client")?; + + assert!(client_err.to_string().contains(&server_err.to_string())); + + Ok(()) + } } diff --git a/proxy/src/stream.rs b/proxy/src/stream.rs index 8fd5bef388..fb0be84584 100644 --- a/proxy/src/stream.rs +++ b/proxy/src/stream.rs @@ -1,10 +1,12 @@ -use anyhow::Context; +use crate::error::UserFacingError; +use anyhow::bail; use bytes::BytesMut; use pin_project_lite::pin_project; use rustls::ServerConfig; use std::pin::Pin; use std::sync::Arc; use std::{io, task}; +use thiserror::Error; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf}; use tokio_rustls::server::TlsStream; use zenith_utils::pq_proto::{BeMessage, FeMessage, FeStartupPacket}; @@ -35,38 +37,63 @@ impl PqStream { self.stream } - /// Get a reference to the underlying stream. + /// Get a shared reference to the underlying stream. pub fn get_ref(&self) -> &S { &self.stream } } +fn err_connection() -> io::Error { + io::Error::new(io::ErrorKind::ConnectionAborted, "connection is lost") +} + +// TODO: change error type of `FeMessage::read_fut` +fn from_anyhow(e: anyhow::Error) -> io::Error { + io::Error::new(io::ErrorKind::Other, e.to_string()) +} + impl PqStream { /// Receive [`FeStartupPacket`], which is a first packet sent by a client. - pub async fn read_startup_packet(&mut self) -> anyhow::Result { - match FeStartupPacket::read_fut(&mut self.stream).await? { - Some(FeMessage::StartupPacket(packet)) => Ok(packet), - None => anyhow::bail!("connection is lost"), - other => anyhow::bail!("bad message type: {:?}", other), + pub async fn read_startup_packet(&mut self) -> io::Result { + // TODO: `FeStartupPacket::read_fut` should return `FeStartupPacket` + let msg = FeStartupPacket::read_fut(&mut self.stream) + .await + .map_err(from_anyhow)? + .ok_or_else(err_connection)?; + + match msg { + FeMessage::StartupPacket(packet) => Ok(packet), + _ => panic!("unreachable state"), } } - pub async fn read_message(&mut self) -> anyhow::Result { + pub async fn read_password_message(&mut self) -> io::Result { + match self.read_message().await? { + FeMessage::PasswordMessage(msg) => Ok(msg), + bad => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("unexpected message type: {:?}", bad), + )), + } + } + + async fn read_message(&mut self) -> io::Result { FeMessage::read_fut(&mut self.stream) - .await? - .context("connection is lost") + .await + .map_err(from_anyhow)? + .ok_or_else(err_connection) } } impl PqStream { /// Write the message into an internal buffer, but don't flush the underlying stream. - pub fn write_message_noflush<'a>(&mut self, message: &BeMessage<'a>) -> io::Result<&mut Self> { + pub fn write_message_noflush(&mut self, message: &BeMessage<'_>) -> io::Result<&mut Self> { BeMessage::write(&mut self.buffer, message)?; Ok(self) } /// Write the message into an internal buffer and flush it. - pub async fn write_message<'a>(&mut self, message: &BeMessage<'a>) -> io::Result<&mut Self> { + pub async fn write_message(&mut self, message: &BeMessage<'_>) -> io::Result<&mut Self> { self.write_message_noflush(message)?; self.flush().await?; Ok(self) @@ -79,6 +106,25 @@ impl PqStream { self.stream.flush().await?; Ok(self) } + + /// Write the error message using [`Self::write_message`], then re-throw it. + /// Allowing string literals is safe under the assumption they might not contain any runtime info. + pub async fn throw_error_str(&mut self, error: &'static str) -> anyhow::Result { + // This method exists due to `&str` not implementing `Into` + self.write_message(&BeMessage::ErrorResponse(error)).await?; + bail!(error) + } + + /// Write the error message using [`Self::write_message`], then re-throw it. + /// Trait [`UserFacingError`] acts as an allowlist for error types. + pub async fn throw_error(&mut self, error: E) -> anyhow::Result + where + E: UserFacingError + Into, + { + let msg = error.to_string_client(); + self.write_message(&BeMessage::ErrorResponse(&msg)).await?; + bail!(error) + } } pin_project! { @@ -101,15 +147,25 @@ impl Stream { } } +#[derive(Debug, Error)] +#[error("Can't upgrade TLS stream")] +pub enum StreamUpgradeError { + #[error("Bad state reached: can't upgrade TLS stream")] + AlreadyTls, + + #[error("Can't upgrade stream: IO error: {0}")] + Io(#[from] io::Error), +} + impl Stream { /// If possible, upgrade raw stream into a secure TLS-based stream. - pub async fn upgrade(self, cfg: Arc) -> anyhow::Result { + pub async fn upgrade(self, cfg: Arc) -> Result { match self { Stream::Raw { raw } => { let tls = Box::new(tokio_rustls::TlsAcceptor::from(cfg).accept(raw).await?); Ok(Stream::Tls { tls }) } - Stream::Tls { .. } => anyhow::bail!("can't upgrade TLS stream"), + Stream::Tls { .. } => Err(StreamUpgradeError::AlreadyTls), } } } diff --git a/proxy/src/waiters.rs b/proxy/src/waiters.rs index 9fda3ed94f..799d45a165 100644 --- a/proxy/src/waiters.rs +++ b/proxy/src/waiters.rs @@ -1,11 +1,32 @@ -use anyhow::{anyhow, Context}; use hashbrown::HashMap; use parking_lot::Mutex; use pin_project_lite::pin_project; use std::pin::Pin; use std::task; +use thiserror::Error; use tokio::sync::oneshot; +#[derive(Debug, Error)] +pub enum RegisterError { + #[error("Waiter `{0}` already registered")] + Occupied(String), +} + +#[derive(Debug, Error)] +pub enum NotifyError { + #[error("Notify failed: waiter `{0}` not registered")] + NotFound(String), + + #[error("Notify failed: channel hangup")] + Hangup, +} + +#[derive(Debug, Error)] +pub enum WaitError { + #[error("Wait failed: channel hangup")] + Hangup, +} + pub struct Waiters(pub(self) Mutex>>); impl Default for Waiters { @@ -15,13 +36,13 @@ impl Default for Waiters { } impl Waiters { - pub fn register(&self, key: String) -> anyhow::Result> { + pub fn register(&self, key: String) -> Result, RegisterError> { let (tx, rx) = oneshot::channel(); self.0 .lock() .try_insert(key.clone(), tx) - .map_err(|_| anyhow!("waiter already registered"))?; + .map_err(|e| RegisterError::Occupied(e.entry.key().clone()))?; Ok(Waiter { receiver: rx, @@ -32,7 +53,7 @@ impl Waiters { }) } - pub fn notify(&self, key: &str, value: T) -> anyhow::Result<()> + pub fn notify(&self, key: &str, value: T) -> Result<(), NotifyError> where T: Send + Sync, { @@ -40,9 +61,9 @@ impl Waiters { .0 .lock() .remove(key) - .with_context(|| format!("key {} not found", key))?; + .ok_or_else(|| NotifyError::NotFound(key.to_string()))?; - tx.send(value).map_err(|_| anyhow!("waiter channel hangup")) + tx.send(value).map_err(|_| NotifyError::Hangup) } } @@ -66,13 +87,13 @@ pin_project! { } impl std::future::Future for Waiter<'_, T> { - type Output = anyhow::Result; + type Output = Result; fn poll(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> task::Poll { self.project() .receiver .poll(cx) - .map_err(|_| anyhow!("channel hangup")) + .map_err(|_| WaitError::Hangup) } } From 1fddb0556f9e3dea86857f62fdc42b2d0db3d6d0 Mon Sep 17 00:00:00 2001 From: Andrey Taranik Date: Thu, 17 Mar 2022 00:01:17 +0300 Subject: [PATCH 34/42] deploy playbook fix - interaction with console (#1374) --- .circleci/ansible/deploy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/ansible/deploy.yaml b/.circleci/ansible/deploy.yaml index c95524a8a5..2dd109f99a 100644 --- a/.circleci/ansible/deploy.yaml +++ b/.circleci/ansible/deploy.yaml @@ -119,7 +119,7 @@ shell: cmd: | INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) - curl -sfS -d '{"version": {{ current_version }} }' -X POST {{ console_mgmt_base_url }}/api/v1/pageservers/$INSTANCE_ID + curl -sfS -d '{"version": {{ current_version }} }' -X PATCH {{ console_mgmt_base_url }}/api/v1/pageservers/$INSTANCE_ID tags: - pageserver @@ -169,6 +169,6 @@ shell: cmd: | INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) - curl -sfS -d '{"version": {{ current_version }} }' -X POST {{ console_mgmt_base_url }}/api/v1/safekeepers/$INSTANCE_ID + curl -sfS -d '{"version": {{ current_version }} }' -X PATCH {{ console_mgmt_base_url }}/api/v1/safekeepers/$INSTANCE_ID tags: - safekeeper From a7544eead59b4039ce18fcfc226b9e175f6521ed Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 17 Mar 2022 16:46:58 +0300 Subject: [PATCH 35/42] Remove the last non-borrowed string from `BeMessage` (#1376) --- proxy/src/auth.rs | 4 ++-- zenith_utils/src/pq_proto.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/proxy/src/auth.rs b/proxy/src/auth.rs index 5e6357fe80..e8fe65c081 100644 --- a/proxy/src/auth.rs +++ b/proxy/src/auth.rs @@ -200,7 +200,7 @@ async fn handle_new_user( client .write_message_noflush(&Be::AuthenticationOk)? .write_message_noflush(&BeParameterStatusMessage::encoding())? - .write_message(&Be::NoticeResponse(greeting)) + .write_message(&Be::NoticeResponse(&greeting)) .await?; // Wait for web console response (see `mgmt`) @@ -208,7 +208,7 @@ async fn handle_new_user( }) .await?; - client.write_message_noflush(&Be::NoticeResponse("Connecting to database.".into()))?; + client.write_message_noflush(&Be::NoticeResponse("Connecting to database."))?; Ok(db_info) } diff --git a/zenith_utils/src/pq_proto.rs b/zenith_utils/src/pq_proto.rs index 355b38fc95..cb69418c07 100644 --- a/zenith_utils/src/pq_proto.rs +++ b/zenith_utils/src/pq_proto.rs @@ -425,7 +425,7 @@ pub enum BeMessage<'a> { ReadyForQuery, RowDescription(&'a [RowDescriptor<'a>]), XLogData(XLogDataBody<'a>), - NoticeResponse(String), + NoticeResponse(&'a str), KeepAlive(WalSndKeepAlive), } From 7738254f83c86e46795b34db834d18af97197d8d Mon Sep 17 00:00:00 2001 From: Dmitry Rodionov Date: Thu, 17 Mar 2022 13:21:00 +0400 Subject: [PATCH 36/42] refactor timeline memory state management --- control_plane/src/storage.rs | 16 +- pageserver/src/bin/pageserver.rs | 49 +- pageserver/src/http/models.rs | 96 +++- pageserver/src/http/routes.rs | 188 +++++-- pageserver/src/layered_repository.rs | 471 ++++++++---------- pageserver/src/page_service.rs | 16 +- pageserver/src/remote_storage.rs | 37 +- pageserver/src/remote_storage/storage_sync.rs | 274 +++++----- .../remote_storage/storage_sync/download.rs | 73 +-- .../src/remote_storage/storage_sync/index.rs | 126 ++++- .../src/remote_storage/storage_sync/upload.rs | 110 ++-- pageserver/src/repository.rs | 257 ++++++---- pageserver/src/tenant_mgr.rs | 169 +++---- pageserver/src/timelines.rs | 348 +++++++------ pageserver/src/walreceiver.rs | 56 ++- .../batch_others/test_remote_storage.py | 39 +- .../batch_others/test_tenant_relocation.py | 81 ++- test_runner/fixtures/zenith_fixtures.py | 89 ++++ zenith/src/main.rs | 105 ++-- zenith_utils/src/http/error.rs | 6 + 20 files changed, 1484 insertions(+), 1122 deletions(-) diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index f6b7173067..ef43ba3c1e 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -1,4 +1,3 @@ -use std::convert::TryFrom; use std::io::Write; use std::net::TcpStream; use std::path::PathBuf; @@ -10,7 +9,7 @@ use anyhow::{bail, Context}; use nix::errno::Errno; use nix::sys::signal::{kill, Signal}; use nix::unistd::Pid; -use pageserver::http::models::{TenantCreateRequest, TimelineCreateRequest, TimelineInfoResponse}; +use pageserver::http::models::{TenantCreateRequest, TimelineCreateRequest}; use pageserver::timelines::TimelineInfo; use postgres::{Config, NoTls}; use reqwest::blocking::{Client, RequestBuilder, Response}; @@ -358,7 +357,7 @@ impl PageServerNode { } pub fn timeline_list(&self, tenant_id: &ZTenantId) -> anyhow::Result> { - let timeline_infos: Vec = self + let timeline_infos: Vec = self .http_request( Method::GET, format!("{}/tenant/{}/timeline", self.http_base_url, tenant_id), @@ -367,10 +366,7 @@ impl PageServerNode { .error_from_body()? .json()?; - timeline_infos - .into_iter() - .map(TimelineInfo::try_from) - .collect() + Ok(timeline_infos) } pub fn timeline_create( @@ -392,10 +388,8 @@ impl PageServerNode { }) .send()? .error_from_body()? - .json::>()?; + .json::>()?; - timeline_info_response - .map(TimelineInfo::try_from) - .transpose() + Ok(timeline_info_response) } } diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index d37ba0cece..05fb14daca 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -18,7 +18,10 @@ use daemonize::Daemonize; use pageserver::{ config::{defaults::*, PageServerConf}, - http, page_cache, page_service, remote_storage, tenant_mgr, thread_mgr, + http, page_cache, page_service, + remote_storage::{self, SyncStartupData}, + repository::TimelineSyncStatusUpdate, + tenant_mgr, thread_mgr, thread_mgr::ThreadKind, timelines, virtual_file, LOG_FILE_NAME, }; @@ -227,11 +230,47 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<() } let signals = signals::install_shutdown_handlers()?; - let sync_startup = remote_storage::start_local_timeline_sync(conf) + + // Initialize repositories with locally available timelines. + // Timelines that are only partially available locally (remote storage has more data than this pageserver) + // are scheduled for download and added to the repository once download is completed. + let SyncStartupData { + remote_index, + local_timeline_init_statuses, + } = remote_storage::start_local_timeline_sync(conf) .context("Failed to set up local files sync with external storage")?; - // Initialize tenant manager. - tenant_mgr::set_timeline_states(conf, sync_startup.initial_timeline_states); + for (tenant_id, local_timeline_init_statuses) in local_timeline_init_statuses { + // initialize local tenant + let repo = tenant_mgr::load_local_repo(conf, tenant_id, &remote_index); + for (timeline_id, init_status) in local_timeline_init_statuses { + match init_status { + remote_storage::LocalTimelineInitStatus::LocallyComplete => { + debug!("timeline {} for tenant {} is locally complete, registering it in repository", tenant_id, timeline_id); + // Lets fail here loudly to be on the safe side. + // XXX: It may be a better api to actually distinguish between repository startup + // and processing of newly downloaded timelines. + repo.apply_timeline_remote_sync_status_update( + timeline_id, + TimelineSyncStatusUpdate::Downloaded, + ) + .with_context(|| { + format!( + "Failed to bootstrap timeline {} for tenant {}", + timeline_id, tenant_id + ) + })? + } + remote_storage::LocalTimelineInitStatus::NeedsSync => { + debug!( + "timeline {} for tenant {} needs sync, \ + so skipped for adding into repository until sync is finished", + tenant_id, timeline_id + ); + } + } + } + } // initialize authentication for incoming connections let auth = match &conf.auth_type { @@ -253,7 +292,7 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<() None, "http_endpoint_thread", move || { - let router = http::make_router(conf, auth_cloned); + let router = http::make_router(conf, auth_cloned, remote_index); endpoint::serve_thread_main(router, http_listener, thread_mgr::shutdown_watcher()) }, )?; diff --git a/pageserver/src/http/models.rs b/pageserver/src/http/models.rs index 9844e7ea82..8827713f11 100644 --- a/pageserver/src/http/models.rs +++ b/pageserver/src/http/models.rs @@ -1,11 +1,12 @@ -use crate::timelines::TimelineInfo; -use anyhow::{anyhow, bail, Context}; +use anyhow::Context; use serde::{Deserialize, Serialize}; use zenith_utils::{ lsn::Lsn, zid::{HexZTenantId, HexZTimelineId, ZNodeId, ZTenantId, ZTimelineId}, }; +use crate::timelines::{LocalTimelineInfo, TimelineInfo}; + #[derive(Serialize, Deserialize)] pub struct TimelineCreateRequest { pub new_timeline_id: Option, @@ -18,8 +19,28 @@ pub struct TenantCreateRequest { pub new_tenant_id: Option, } +#[derive(Clone)] +pub enum TimelineInfoV1 { + Local { + timeline_id: ZTimelineId, + tenant_id: ZTenantId, + last_record_lsn: Lsn, + prev_record_lsn: Option, + ancestor_timeline_id: Option, + ancestor_lsn: Option, + disk_consistent_lsn: Lsn, + current_logical_size: Option, + current_logical_size_non_incremental: Option, + }, + Remote { + timeline_id: ZTimelineId, + tenant_id: ZTenantId, + disk_consistent_lsn: Lsn, + }, +} + #[derive(Serialize, Deserialize)] -pub struct TimelineInfoResponse { +pub struct TimelineInfoResponseV1 { pub kind: String, #[serde(with = "hex")] timeline_id: ZTimelineId, @@ -34,10 +55,10 @@ pub struct TimelineInfoResponse { current_logical_size_non_incremental: Option, } -impl From for TimelineInfoResponse { - fn from(other: TimelineInfo) -> Self { +impl From for TimelineInfoResponseV1 { + fn from(other: TimelineInfoV1) -> Self { match other { - TimelineInfo::Local { + TimelineInfoV1::Local { timeline_id, tenant_id, last_record_lsn, @@ -47,23 +68,23 @@ impl From for TimelineInfoResponse { disk_consistent_lsn, current_logical_size, current_logical_size_non_incremental, - } => TimelineInfoResponse { + } => TimelineInfoResponseV1 { kind: "Local".to_owned(), timeline_id, tenant_id, disk_consistent_lsn: disk_consistent_lsn.to_string(), last_record_lsn: Some(last_record_lsn.to_string()), - prev_record_lsn: Some(prev_record_lsn.to_string()), + prev_record_lsn: prev_record_lsn.map(|lsn| lsn.to_string()), ancestor_timeline_id: ancestor_timeline_id.map(HexZTimelineId::from), ancestor_lsn: ancestor_lsn.map(|lsn| lsn.to_string()), - current_logical_size: Some(current_logical_size), + current_logical_size, current_logical_size_non_incremental, }, - TimelineInfo::Remote { + TimelineInfoV1::Remote { timeline_id, tenant_id, disk_consistent_lsn, - } => TimelineInfoResponse { + } => TimelineInfoResponseV1 { kind: "Remote".to_owned(), timeline_id, tenant_id, @@ -79,10 +100,10 @@ impl From for TimelineInfoResponse { } } -impl TryFrom for TimelineInfo { +impl TryFrom for TimelineInfoV1 { type Error = anyhow::Error; - fn try_from(other: TimelineInfoResponse) -> anyhow::Result { + fn try_from(other: TimelineInfoResponseV1) -> anyhow::Result { let parse_lsn_hex_string = |lsn_string: String| { lsn_string .parse::() @@ -91,33 +112,68 @@ impl TryFrom for TimelineInfo { let disk_consistent_lsn = parse_lsn_hex_string(other.disk_consistent_lsn)?; Ok(match other.kind.as_str() { - "Local" => TimelineInfo::Local { + "Local" => TimelineInfoV1::Local { timeline_id: other.timeline_id, tenant_id: other.tenant_id, last_record_lsn: other .last_record_lsn - .ok_or(anyhow!("Local timeline should have last_record_lsn")) + .ok_or(anyhow::anyhow!( + "Local timeline should have last_record_lsn" + )) .and_then(parse_lsn_hex_string)?, prev_record_lsn: other .prev_record_lsn - .ok_or(anyhow!("Local timeline should have prev_record_lsn")) - .and_then(parse_lsn_hex_string)?, + .map(parse_lsn_hex_string) + .transpose()?, ancestor_timeline_id: other.ancestor_timeline_id.map(ZTimelineId::from), ancestor_lsn: other.ancestor_lsn.map(parse_lsn_hex_string).transpose()?, disk_consistent_lsn, - current_logical_size: other.current_logical_size.ok_or(anyhow!("No "))?, + current_logical_size: other.current_logical_size, current_logical_size_non_incremental: other.current_logical_size_non_incremental, }, - "Remote" => TimelineInfo::Remote { + "Remote" => TimelineInfoV1::Remote { timeline_id: other.timeline_id, tenant_id: other.tenant_id, disk_consistent_lsn, }, - unknown => bail!("Unknown timeline kind: {}", unknown), + unknown => anyhow::bail!("Unknown timeline kind: {}", unknown), }) } } +fn from_local( + tenant_id: ZTenantId, + timeline_id: ZTimelineId, + local: &LocalTimelineInfo, +) -> TimelineInfoV1 { + TimelineInfoV1::Local { + timeline_id, + tenant_id, + last_record_lsn: local.last_record_lsn, + prev_record_lsn: local.prev_record_lsn, + ancestor_timeline_id: local.ancestor_timeline_id.map(ZTimelineId::from), + ancestor_lsn: local.ancestor_lsn, + disk_consistent_lsn: local.disk_consistent_lsn, + current_logical_size: local.current_logical_size, + current_logical_size_non_incremental: local.current_logical_size_non_incremental, + } +} + +impl From for TimelineInfoV1 { + fn from(t: TimelineInfo) -> Self { + match (t.local.as_ref(), t.remote.as_ref()) { + (None, None) => unreachable!(), + (None, Some(remote)) => TimelineInfoV1::Remote { + timeline_id: t.timeline_id, + tenant_id: t.tenant_id, + disk_consistent_lsn: remote.remote_consistent_lsn.unwrap_or(Lsn(0)), + }, + (Some(local), None) => from_local(t.tenant_id, t.timeline_id, local), + (Some(local), Some(_)) => from_local(t.tenant_id, t.timeline_id, local), + } + } +} + #[derive(Serialize)] pub struct StatusResponse { pub id: ZNodeId, diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 8365601042..2d913afe4e 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use anyhow::Result; use hyper::StatusCode; use hyper::{Body, Request, Response, Uri}; +use tokio::sync::RwLock; use tracing::*; use zenith_utils::auth::JwtAuth; use zenith_utils::http::endpoint::attach_openapi_ui; @@ -16,24 +17,32 @@ use zenith_utils::http::{ request::parse_request_param, }; use zenith_utils::http::{RequestExt, RouterBuilder}; -use zenith_utils::zid::{HexZTenantId, ZTimelineId}; +use zenith_utils::zid::{HexZTenantId, ZTenantTimelineId, ZTimelineId}; use super::models::{ - StatusResponse, TenantCreateRequest, TimelineCreateRequest, TimelineInfoResponse, + StatusResponse, TenantCreateRequest, TimelineCreateRequest, TimelineInfoResponseV1, + TimelineInfoV1, +}; +use crate::remote_storage::{schedule_timeline_download, RemoteTimelineIndex}; +use crate::timelines::{ + extract_remote_timeline_info, LocalTimelineInfo, RemoteTimelineInfo, TimelineInfo, }; -use crate::repository::RepositoryTimeline; -use crate::timelines::TimelineInfo; use crate::{config::PageServerConf, tenant_mgr, timelines, ZTenantId}; #[derive(Debug)] struct State { conf: &'static PageServerConf, auth: Option>, + remote_index: Arc>, allowlist_routes: Vec, } impl State { - fn new(conf: &'static PageServerConf, auth: Option>) -> Self { + fn new( + conf: &'static PageServerConf, + auth: Option>, + remote_index: Arc>, + ) -> Self { let allowlist_routes = ["/v1/status", "/v1/doc", "/swagger.yml"] .iter() .map(|v| v.parse().unwrap()) @@ -42,6 +51,7 @@ impl State { conf, auth, allowlist_routes, + remote_index, } } } @@ -88,7 +98,7 @@ async fn timeline_create_handler(mut request: Request) -> Result json_response(StatusCode::CREATED, TimelineInfoResponse::from(info))?, + Some(info) => json_response(StatusCode::CREATED, info)?, None => json_response(StatusCode::CONFLICT, ())?, }) } @@ -97,15 +107,24 @@ async fn timeline_list_handler(request: Request) -> Result, let tenant_id: ZTenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; let include_non_incremental_logical_size = get_include_non_incremental_logical_size(&request); - let response_data: Vec = tokio::task::spawn_blocking(move || { + let local_timeline_infos = tokio::task::spawn_blocking(move || { let _enter = info_span!("timeline_list", tenant = %tenant_id).entered(); - crate::timelines::get_timelines(tenant_id, include_non_incremental_logical_size) + crate::timelines::get_local_timelines(tenant_id, include_non_incremental_logical_size) }) .await - .map_err(ApiError::from_err)?? - .into_iter() - .map(TimelineInfoResponse::from) - .collect(); + .map_err(ApiError::from_err)??; + + let remote_index = get_state(&request).remote_index.read().await; + let mut response_data = Vec::with_capacity(local_timeline_infos.len()); + for (timeline_id, local_timeline_info) in local_timeline_infos { + response_data.push(TimelineInfo { + tenant_id, + timeline_id, + local: Some(local_timeline_info), + remote: extract_remote_timeline_info(tenant_id, timeline_id, &remote_index), + }) + } + Ok(json_response(StatusCode::OK, response_data)?) } @@ -124,30 +143,76 @@ fn get_include_non_incremental_logical_size(request: &Request) -> bool { .unwrap_or(false) } -async fn timeline_detail_handler(request: Request) -> Result, ApiError> { +// common part for v1 and v2 handlers +async fn timeline_detail_common(request: Request) -> Result { let tenant_id: ZTenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; let timeline_id: ZTimelineId = parse_request_param(&request, "timeline_id")?; + let include_non_incremental_logical_size = get_include_non_incremental_logical_size(&request); - let response_data = tokio::task::spawn_blocking(move || { - let _enter = - info_span!("timeline_detail_handler", tenant = %tenant_id, timeline = %timeline_id) - .entered(); + let span = info_span!("timeline_detail_handler", tenant = %tenant_id, timeline = %timeline_id); + + let (local_timeline_info, span) = tokio::task::spawn_blocking(move || { + let entered = span.entered(); let repo = tenant_mgr::get_repository_for_tenant(tenant_id)?; - let include_non_incremental_logical_size = - get_include_non_incremental_logical_size(&request); - Ok::<_, anyhow::Error>(TimelineInfo::from_repo_timeline( - tenant_id, - repo.get_timeline(timeline_id)?, - include_non_incremental_logical_size, - )) + let local_timeline = { + repo.get_timeline(timeline_id) + .map(|timeline| { + LocalTimelineInfo::from_repo_timeline( + timeline, + include_non_incremental_logical_size, + ) + }) + .transpose()? + }; + Ok::<_, anyhow::Error>((local_timeline, entered.exit())) }) .await - .map_err(ApiError::from_err)? - .map(TimelineInfoResponse::from)?; + .map_err(ApiError::from_err)??; - Ok(json_response(StatusCode::OK, response_data)?) + let remote_timeline_info = { + let remote_index_read = get_state(&request).remote_index.read().await; + remote_index_read + .timeline_entry(&ZTenantTimelineId { + tenant_id, + timeline_id, + }) + .map(|remote_entry| RemoteTimelineInfo { + remote_consistent_lsn: remote_entry.disk_consistent_lsn(), + awaits_download: remote_entry.get_awaits_download(), + }) + }; + + let _enter = span.entered(); + + if local_timeline_info.is_none() && remote_timeline_info.is_none() { + return Err(ApiError::NotFound( + "Timeline is not found neither locally nor remotely".to_string(), + )); + } + + Ok(TimelineInfo { + tenant_id, + timeline_id, + local: local_timeline_info, + remote: remote_timeline_info, + }) +} + +// TODO remove when console adopts v2 +async fn timeline_detail_handler_v1(request: Request) -> Result, ApiError> { + let timeline_info = timeline_detail_common(request).await?; + Ok(json_response( + StatusCode::OK, + TimelineInfoResponseV1::from(TimelineInfoV1::from(timeline_info)), + )?) +} + +async fn timeline_detail_handler_v2(request: Request) -> Result, ApiError> { + let timeline_info = timeline_detail_common(request).await?; + + Ok(json_response(StatusCode::OK, timeline_info)?) } async fn timeline_attach_handler(request: Request) -> Result, ApiError> { @@ -155,31 +220,37 @@ async fn timeline_attach_handler(request: Request) -> Result { - anyhow::bail!("Timeline with id {} is already local", timeline_id) - } - RepositoryTimeline::Remote { - id: _, - disk_consistent_lsn: _, - } => { - // FIXME (rodionov) get timeline already schedules timeline for download, and duplicate tasks can cause errors - // first should be fixed in https://github.com/zenithdb/zenith/issues/997 - // TODO (rodionov) change timeline state to awaits download (incapsulate it somewhere in the repo) - // TODO (rodionov) can we safely request replication on the timeline before sync is completed? (can be implemented on top of the #997) - Ok(()) - } - } + let span = tokio::task::spawn_blocking(move || { + let entered = span.entered(); + if tenant_mgr::get_timeline_for_tenant_load(tenant_id, timeline_id).is_ok() { + anyhow::bail!("Timeline is already present locally") + }; + Ok(entered.exit()) }) .await .map_err(ApiError::from_err)??; + let mut remote_index_write = get_state(&request).remote_index.write().await; + + let _enter = span.entered(); // entered guard cannot live across awaits (non Send) + let index_entry = remote_index_write + .timeline_entry_mut(&ZTenantTimelineId { + tenant_id, + timeline_id, + }) + .ok_or_else(|| ApiError::BadRequest("Unknown remote timeline".to_string()))?; + + if index_entry.get_awaits_download() { + return Err(ApiError::NotFound( + "Timeline download is already in progress".to_string(), + )); + } + + index_entry.set_awaits_download(true); + schedule_timeline_download(tenant_id, timeline_id); + Ok(json_response(StatusCode::ACCEPTED, ())?) } @@ -221,13 +292,17 @@ async fn tenant_create_handler(mut request: Request) -> Result) -> Result, ApiError> { pub fn make_router( conf: &'static PageServerConf, auth: Option>, + remote_index: Arc>, ) -> RouterBuilder { let spec = include_bytes!("openapi_spec.yml"); let mut router = attach_openapi_ui(endpoint::make_router(), spec, "/swagger.yml", "/v1/doc"); @@ -263,7 +339,7 @@ pub fn make_router( } router - .data(Arc::new(State::new(conf, auth))) + .data(Arc::new(State::new(conf, auth, remote_index))) .get("/v1/status", status_handler) .get("/v1/tenant", tenant_list_handler) .post("/v1/tenant", tenant_create_handler) @@ -271,7 +347,11 @@ pub fn make_router( .post("/v1/tenant/:tenant_id/timeline", timeline_create_handler) .get( "/v1/tenant/:tenant_id/timeline/:timeline_id", - timeline_detail_handler, + timeline_detail_handler_v1, + ) + .get( + "/v2/tenant/:tenant_id/timeline/:timeline_id", + timeline_detail_handler_v2, ) .post( "/v1/tenant/:tenant_id/timeline/:timeline_id/attach", diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index 9e0df5dab2..c17df84689 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -35,9 +35,9 @@ use self::metadata::{metadata_path, TimelineMetadata, METADATA_FILE_NAME}; use crate::config::PageServerConf; use crate::page_cache; use crate::relish::*; -use crate::remote_storage::{schedule_timeline_checkpoint_upload, schedule_timeline_download}; +use crate::remote_storage::{schedule_timeline_checkpoint_upload, RemoteTimelineIndex}; use crate::repository::{ - BlockNumber, GcResult, Repository, RepositoryTimeline, Timeline, TimelineSyncState, + BlockNumber, GcResult, Repository, RepositoryTimeline, Timeline, TimelineSyncStatusUpdate, TimelineWriter, ZenithWalRecord, }; use crate::thread_mgr; @@ -129,27 +129,46 @@ pub struct LayeredRepository { // timeout... gc_cs: Mutex<()>, walredo_mgr: Arc, + + // provides access to timeline data sitting in the remote storage + // supposed to be used for retrieval of remote consistent lsn in walreceiver + remote_index: Arc>, + /// Makes every timeline to backup their files to remote storage. upload_relishes: bool, } /// Public interface impl Repository for LayeredRepository { - fn get_timeline(&self, timelineid: ZTimelineId) -> Result { - Ok(RepositoryTimeline::from(self.get_or_init_timeline( - timelineid, - &mut self.timelines.lock().unwrap(), - )?)) + fn get_timeline(&self, timelineid: ZTimelineId) -> Option { + let timelines = self.timelines.lock().unwrap(); + self.get_timeline_internal(timelineid, &timelines) + .map(RepositoryTimeline::from) } - fn list_timelines(&self) -> Result> { - Ok(self - .timelines + fn get_timeline_load(&self, timelineid: ZTimelineId) -> Result> { + let mut timelines = self.timelines.lock().unwrap(); + match self.get_timeline_load_internal(timelineid, &mut timelines)? { + Some(local_loaded_timeline) => Ok(local_loaded_timeline as _), + None => anyhow::bail!( + "cannot get local timeline: unknown timeline id: {}", + timelineid + ), + } + } + + fn list_timelines(&self) -> Vec<(ZTimelineId, RepositoryTimeline)> { + self.timelines .lock() .unwrap() - .values() - .map(|timeline_entry| RepositoryTimeline::from(timeline_entry.clone())) - .collect()) + .iter() + .map(|(timeline_id, timeline_entry)| { + ( + *timeline_id, + RepositoryTimeline::from(timeline_entry.clone()), + ) + }) + .collect() } fn create_empty_timeline( @@ -176,10 +195,16 @@ impl Repository for LayeredRepository { self.upload_relishes, ); - let timeline_rc = Arc::new(timeline); - let r = timelines.insert(timelineid, LayeredTimelineEntry::Local(timeline_rc.clone())); - assert!(r.is_none()); - Ok(timeline_rc) + let timeline = Arc::new(timeline); + let r = timelines.insert( + timelineid, + LayeredTimelineEntry::Loaded(Arc::clone(&timeline)), + ); + ensure!( + r.is_none(), + "assertion failure, inserted duplicate timeline" + ); + Ok(timeline) } /// Branch a timeline @@ -190,14 +215,12 @@ impl Repository for LayeredRepository { let _gc_cs = self.gc_cs.lock().unwrap(); let mut timelines = self.timelines.lock().unwrap(); - let src_timeline = match self.get_or_init_timeline(src, &mut timelines)? { - LayeredTimelineEntry::Local(timeline) => timeline, - LayeredTimelineEntry::Remote { .. } => { - bail!("Cannot branch off the timeline {} that's not local", src) - } - }; + let src_timeline = self + .get_timeline_load_internal(src, &mut timelines) + // message about timeline being remote is one .context up in the stack + .context("failed to load timeline for branching")? + .ok_or_else(|| anyhow::anyhow!("unknown timeline id: {}", &src))?; let latest_gc_cutoff_lsn = src_timeline.get_latest_gc_cutoff_lsn(); - src_timeline .check_lsn_is_in_scope(start_lsn, &latest_gc_cutoff_lsn) .context("invalid branch start lsn")?; @@ -232,6 +255,7 @@ impl Repository for LayeredRepository { ); crashsafe_dir::create_dir_all(self.conf.timeline_path(&dst, &self.tenantid))?; Self::save_metadata(self.conf, dst, self.tenantid, &metadata, true)?; + timelines.insert(dst, LayeredTimelineEntry::Unloaded { id: dst, metadata }); info!("branched timeline {} from {} at {}", dst, src, start_lsn); @@ -261,11 +285,19 @@ impl Repository for LayeredRepository { fn checkpoint_iteration(&self, cconf: CheckpointConfig) -> Result<()> { // Scan through the hashmap and collect a list of all the timelines, // while holding the lock. Then drop the lock and actually perform the - // checkpoints. We don't want to block everything else while the + // checkpoints. We don't want to block everything else while the // checkpoint runs. let timelines = self.timelines.lock().unwrap(); let timelines_to_checkpoint = timelines .iter() + // filter to get only loaded timelines + .filter_map(|(timelineid, entry)| match entry { + LayeredTimelineEntry::Loaded(timeline) => Some((timelineid, timeline)), + LayeredTimelineEntry::Unloaded { .. } => { + debug!("Skipping checkpoint for unloaded timeline {}", timelineid); + None + } + }) .map(|(timelineid, timeline)| (*timelineid, timeline.clone())) .collect::>(); drop(timelines); @@ -273,13 +305,7 @@ impl Repository for LayeredRepository { for (timelineid, timeline) in &timelines_to_checkpoint { let _entered = info_span!("checkpoint", timeline = %timelineid, tenant = %self.tenantid).entered(); - match timeline { - LayeredTimelineEntry::Local(timeline) => timeline.checkpoint(cconf)?, - LayeredTimelineEntry::Remote { .. } => debug!( - "Cannot run the checkpoint for remote timeline {}", - timelineid - ), - } + timeline.checkpoint(cconf)?; } Ok(()) @@ -288,32 +314,10 @@ impl Repository for LayeredRepository { // Detaches the timeline from the repository. fn detach_timeline(&self, timeline_id: ZTimelineId) -> Result<()> { let mut timelines = self.timelines.lock().unwrap(); - match timelines.entry(timeline_id) { - Entry::Vacant(_) => { - bail!("cannot detach non existing timeline"); - } - Entry::Occupied(mut entry) => { - let timeline_entry = entry.get_mut(); + if timelines.remove(&timeline_id).is_none() { + bail!("cannot detach timeline that is not available locally"); + } - let timeline = match timeline_entry { - LayeredTimelineEntry::Remote { .. } => { - bail!("cannot detach remote timeline {}", timeline_id); - } - LayeredTimelineEntry::Local(timeline) => timeline, - }; - - // TODO (rodionov) keep local state in timeline itself (refactoring related to https://github.com/zenithdb/zenith/issues/997 and #1104) - - // FIXME this is local disk consistent lsn, need to keep the latest succesfully uploaded checkpoint lsn in timeline (metadata?) - // https://github.com/zenithdb/zenith/issues/1104 - let remote_disk_consistent_lsn = timeline.disk_consistent_lsn.load(); - // reference to timeline is dropped here - entry.insert(LayeredTimelineEntry::Remote { - id: timeline_id, - disk_consistent_lsn: remote_disk_consistent_lsn, - }); - } - }; // Release the lock to shutdown and remove the files without holding it drop(timelines); // shutdown the timeline (this shuts down the walreceiver) @@ -324,158 +328,142 @@ impl Repository for LayeredRepository { Ok(()) } - // TODO this method currentlly does not do anything to prevent (or react to) state updates between a sync task schedule and a sync task end (that causes this update). - // Sync task is enqueued and can error and be rescheduled, so some significant time may pass between the events. - // - /// Reacts on the timeline sync state change, changing pageserver's memory state for this timeline (unload or load of the timeline files). - fn set_timeline_state( + fn apply_timeline_remote_sync_status_update( &self, timeline_id: ZTimelineId, - new_state: TimelineSyncState, + timeline_sync_status_update: TimelineSyncStatusUpdate, ) -> Result<()> { debug!( - "set_timeline_state: timeline_id: {}, new_state: {:?}", - timeline_id, new_state + "apply_timeline_remote_sync_status_update timeline_id: {} update: {:?}", + timeline_id, timeline_sync_status_update ); - let mut timelines_accessor = self.timelines.lock().unwrap(); - - match new_state { - TimelineSyncState::Ready(_) => { - let reloaded_timeline = - self.init_local_timeline(timeline_id, &mut timelines_accessor)?; - timelines_accessor - .insert(timeline_id, LayeredTimelineEntry::Local(reloaded_timeline)); - None + match timeline_sync_status_update { + TimelineSyncStatusUpdate::Uploaded => { /* nothing to do, remote consistent lsn is managed by the remote storage */ } - TimelineSyncState::Evicted(_) => timelines_accessor.remove(&timeline_id), - TimelineSyncState::AwaitsDownload(disk_consistent_lsn) - | TimelineSyncState::CloudOnly(disk_consistent_lsn) => timelines_accessor.insert( - timeline_id, - LayeredTimelineEntry::Remote { - id: timeline_id, - disk_consistent_lsn, - }, - ), - }; - // NOTE we do not delete local data in case timeline became cloud only, this is performed in detach_timeline - drop(timelines_accessor); - + TimelineSyncStatusUpdate::Downloaded => { + match self.timelines.lock().unwrap().entry(timeline_id) { + Entry::Occupied(_) => bail!("We completed a download for a timeline that already exists in repository. This is a bug."), + Entry::Vacant(entry) => { + // we need to get metadata of a timeline, another option is to pass it along with Downloaded status + let metadata = Self::load_metadata(self.conf, timeline_id, self.tenantid).context("failed to load local metadata")?; + // finally we make newly downloaded timeline visible to repository + entry.insert(LayeredTimelineEntry::Unloaded { id: timeline_id, metadata, }) + }, + }; + } + } Ok(()) } - /// Layered repo does not store anything but - /// * local, fully loaded timelines, ready for usage - /// * remote timelines, that need a download task scheduled first before they can be used - /// - /// [`TimelineSyncState::Evicted`] and other non-local and non-remote states are not stored in the layered repo at all, - /// hence their statuses cannot be returned by the repo. - fn get_timeline_state(&self, timeline_id: ZTimelineId) -> Option { - let timelines_accessor = self.timelines.lock().unwrap(); - let timeline_entry = timelines_accessor.get(&timeline_id)?; - Some( - if timeline_entry - .local_or_schedule_download(self.tenantid) - .is_some() - { - TimelineSyncState::Ready(timeline_entry.disk_consistent_lsn()) - } else { - TimelineSyncState::CloudOnly(timeline_entry.disk_consistent_lsn()) - }, - ) + fn get_remote_index(&self) -> &tokio::sync::RwLock { + self.remote_index.as_ref() } } #[derive(Clone)] enum LayeredTimelineEntry { - Local(Arc), - Remote { + Loaded(Arc), + Unloaded { id: ZTimelineId, - /// metadata contents of the latest successfully uploaded checkpoint - disk_consistent_lsn: Lsn, + metadata: TimelineMetadata, }, } impl LayeredTimelineEntry { fn timeline_id(&self) -> ZTimelineId { match self { - LayeredTimelineEntry::Local(timeline) => timeline.timelineid, - LayeredTimelineEntry::Remote { id, .. } => *id, + LayeredTimelineEntry::Loaded(timeline) => timeline.timelineid, + LayeredTimelineEntry::Unloaded { id, .. } => *id, } } - /// Gets local timeline data, if it's present. Otherwise schedules a download fot the remote timeline and returns `None`. - fn local_or_schedule_download(&self, tenant_id: ZTenantId) -> Option<&LayeredTimeline> { + fn ancestor_timeline_id(&self) -> Option { match self { - Self::Local(local) => Some(local.as_ref()), - Self::Remote { - id: timeline_id, .. - } => { - debug!( - "Accessed a remote timeline {} for tenant {}, scheduling a timeline download", - timeline_id, tenant_id - ); - schedule_timeline_download(tenant_id, *timeline_id); - None + LayeredTimelineEntry::Loaded(timeline) => { + timeline.ancestor_timeline.as_ref().map(|t| t.timeline_id()) } + LayeredTimelineEntry::Unloaded { metadata, .. } => metadata.ancestor_timeline(), } } - /// Gets a current (latest for the remote case) disk consistent Lsn for the timeline. - fn disk_consistent_lsn(&self) -> Lsn { + fn ancestor_lsn(&self) -> Lsn { match self { - Self::Local(local) => local.disk_consistent_lsn.load(), - Self::Remote { - disk_consistent_lsn, - .. - } => *disk_consistent_lsn, + LayeredTimelineEntry::Loaded(timeline) => timeline.ancestor_lsn, + LayeredTimelineEntry::Unloaded { metadata, .. } => metadata.ancestor_lsn(), + } + } + + fn ensure_loaded(&self) -> anyhow::Result<&Arc> { + match self { + LayeredTimelineEntry::Loaded(timeline) => Ok(timeline), + LayeredTimelineEntry::Unloaded { .. } => { + anyhow::bail!("timeline is unloaded") + } } } } impl From for RepositoryTimeline { - fn from(layered_timeline: LayeredTimelineEntry) -> Self { - match layered_timeline { - LayeredTimelineEntry::Local(timeline) => RepositoryTimeline::Local { - id: timeline.timelineid, - timeline, - }, - LayeredTimelineEntry::Remote { - id, - disk_consistent_lsn, - } => RepositoryTimeline::Remote { - id, - disk_consistent_lsn, - }, + fn from(entry: LayeredTimelineEntry) -> Self { + match entry { + LayeredTimelineEntry::Loaded(timeline) => RepositoryTimeline::Loaded(timeline as _), + LayeredTimelineEntry::Unloaded { metadata, .. } => { + RepositoryTimeline::Unloaded { metadata } + } } } } /// Private functions impl LayeredRepository { - // Implementation of the public `get_timeline` function. This differs from the public - // interface in that the caller must already hold the mutex on the 'timelines' hashmap. - fn get_or_init_timeline( + // Implementation of the public `get_timeline` function. + // Differences from the public: + // * interface in that the caller must already hold the mutex on the 'timelines' hashmap. + fn get_timeline_internal( + &self, + timelineid: ZTimelineId, + timelines: &HashMap, + ) -> Option { + timelines.get(&timelineid).cloned() + } + + // Implementation of the public `get_timeline_load` function. + // Differences from the public: + // * interface in that the caller must already hold the mutex on the 'timelines' hashmap. + fn get_timeline_load_internal( &self, timelineid: ZTimelineId, timelines: &mut HashMap, - ) -> Result { + ) -> anyhow::Result>> { match timelines.get(&timelineid) { - Some(timeline_entry) => { - let _ = timeline_entry.local_or_schedule_download(self.tenantid); - Ok(timeline_entry.clone()) - } + Some(entry) => match entry { + LayeredTimelineEntry::Loaded(local_timeline) => { + trace!("timeline {} found loaded", &timelineid); + return Ok(Some(Arc::clone(local_timeline))); + } + LayeredTimelineEntry::Unloaded { .. } => { + trace!("timeline {} found unloaded", &timelineid) + } + }, None => { - let timeline = self.init_local_timeline(timelineid, timelines)?; - timelines.insert( - timelineid, - LayeredTimelineEntry::Local(Arc::clone(&timeline)), - ); - Ok(LayeredTimelineEntry::Local(timeline)) + trace!("timeline {} not found", &timelineid); + return Ok(None); } - } + }; + let timeline = self.load_local_timeline(timelineid, timelines)?; + let was_loaded = timelines.insert( + timelineid, + LayeredTimelineEntry::Loaded(Arc::clone(&timeline)), + ); + ensure!( + was_loaded.is_none() + || matches!(was_loaded, Some(LayeredTimelineEntry::Unloaded { .. })), + "assertion failure, inserted wrong timeline in an incorrect state" + ); + Ok(Some(timeline)) } - fn init_local_timeline( + fn load_local_timeline( &self, timelineid: ZTimelineId, timelines: &mut HashMap, @@ -486,8 +474,18 @@ impl LayeredRepository { let ancestor = metadata .ancestor_timeline() - .map(|ancestor_timelineid| self.get_or_init_timeline(ancestor_timelineid, timelines)) - .transpose()?; + .map(|ancestor_timeline_id| { + trace!( + "loading {}'s ancestor {}", + timelineid, + &ancestor_timeline_id + ); + self.get_timeline_load_internal(ancestor_timeline_id, timelines) + }) + .transpose() + .context("cannot load ancestor timeline")? + .flatten() + .map(LayeredTimelineEntry::Loaded); let _enter = info_span!("loading timeline", timeline = %timelineid, tenant = %self.tenantid) .entered(); @@ -513,6 +511,7 @@ impl LayeredRepository { conf: &'static PageServerConf, walredo_mgr: Arc, tenantid: ZTenantId, + remote_index: Arc>, upload_relishes: bool, ) -> LayeredRepository { LayeredRepository { @@ -521,6 +520,7 @@ impl LayeredRepository { timelines: Mutex::new(HashMap::new()), gc_cs: Mutex::new(()), walredo_mgr, + remote_index, upload_relishes, } } @@ -608,86 +608,46 @@ impl LayeredRepository { // grab mutex to prevent new timelines from being created here. let _gc_cs = self.gc_cs.lock().unwrap(); - let mut timelines = self.timelines.lock().unwrap(); - // Scan all timelines. For each timeline, remember the timeline ID and // the branch point where it was created. - // - let mut timelineids: Vec = Vec::new(); - - // We scan the directory, not the in-memory hash table, because the hash - // table only contains entries for timelines that have been accessed. We - // need to take all timelines into account, not only the active ones. - let timelines_path = self.conf.timelines_path(&self.tenantid); - - for direntry in fs::read_dir(timelines_path)? { - let direntry = direntry?; - if let Some(fname) = direntry.file_name().to_str() { - if let Ok(timelineid) = fname.parse::() { - timelineids.push(timelineid); - } - } - } - - // Now collect info about branchpoints let mut all_branchpoints: BTreeSet<(ZTimelineId, Lsn)> = BTreeSet::new(); - for &timelineid in &timelineids { - let timeline = match self.get_or_init_timeline(timelineid, &mut timelines)? { - LayeredTimelineEntry::Local(timeline) => timeline, - LayeredTimelineEntry::Remote { .. } => { - warn!( - "Timeline {} is not local, cannot proceed with gc", - timelineid - ); - return Ok(totals); - } - }; + let mut timeline_ids = Vec::new(); + let mut timelines = self.timelines.lock().unwrap(); - if let Some(ancestor_timeline) = &timeline.ancestor_timeline { - let ancestor_timeline = - match ancestor_timeline.local_or_schedule_download(self.tenantid) { - Some(timeline) => timeline, - None => { - warn!( - "Timeline {} has ancestor {} is not local, cannot proceed with gc", - timelineid, - ancestor_timeline.timeline_id() - ); - return Ok(totals); - } - }; + for (timeline_id, timeline_entry) in timelines.iter() { + timeline_ids.push(*timeline_id); + + // This is unresolved question for now, how to do gc in presense of remote timelines + // especially when this is combined with branching. + // Somewhat related: https://github.com/zenithdb/zenith/issues/999 + if let Some(ancestor_timeline_id) = &timeline_entry.ancestor_timeline_id() { // If target_timeline is specified, we only need to know branchpoints of its children if let Some(timelineid) = target_timelineid { - if ancestor_timeline.timelineid == timelineid { + if ancestor_timeline_id == &timelineid { all_branchpoints - .insert((ancestor_timeline.timelineid, timeline.ancestor_lsn)); + .insert((*ancestor_timeline_id, timeline_entry.ancestor_lsn())); } } // Collect branchpoints for all timelines else { - all_branchpoints.insert((ancestor_timeline.timelineid, timeline.ancestor_lsn)); + all_branchpoints.insert((*ancestor_timeline_id, timeline_entry.ancestor_lsn())); } } } // Ok, we now know all the branch points. // Perform GC for each timeline. - for timelineid in timelineids { + for timelineid in timeline_ids.into_iter() { if thread_mgr::is_shutdown_requested() { // We were requested to shut down. Stop and return with the progress we // made. break; } - // We have already loaded all timelines above - // so this operation is just a quick map lookup. - let timeline = match self.get_or_init_timeline(timelineid, &mut *timelines)? { - LayeredTimelineEntry::Local(timeline) => timeline, - LayeredTimelineEntry::Remote { .. } => { - debug!("Skipping GC for non-local timeline {}", timelineid); - continue; - } - }; + // Timeline is known to be local and loaded. + let timeline = self + .get_timeline_load_internal(timelineid, &mut *timelines)? + .expect("checked above that timeline is local and loaded"); // If target_timeline is specified, only GC it if let Some(target_timelineid) = target_timelineid { @@ -989,13 +949,13 @@ impl Timeline for LayeredTimeline { match &timeline.ancestor_timeline { None => break, Some(ancestor_entry) => { - match ancestor_entry.local_or_schedule_download(self.tenantid) { - Some(ancestor) => { - timeline = ancestor; - continue; - } - None => bail!("Cannot list relishes for timeline {} tenant {} due to its ancestor being remote only", self.timelineid, self.tenantid), - } + timeline = ancestor_entry.ensure_loaded().with_context( + || format!( + "cannot list relishes for timeline {} tenant {} due to its ancestor {} being either unloaded", + self.timelineid, self.tenantid, ancestor_entry.timeline_id(), + ) + )?; + continue; } } } @@ -1313,19 +1273,15 @@ impl LayeredTimeline { while lsn < timeline.ancestor_lsn { trace!("going into ancestor {} ", timeline.ancestor_lsn); - timeline = match timeline - .ancestor_timeline - .as_ref() - .and_then(|ancestor_entry| ancestor_entry.local_or_schedule_download(self.tenantid)) - { - Some(timeline) => timeline, - None => { - bail!( - "Cannot get the whole layer for read locked: timeline {} is not present locally", - self.timelineid - ) - } - }; + timeline = timeline + .ancestor_timeline + .as_ref() + .expect("there should be an ancestor") + .ensure_loaded() + .with_context(|| format!( + "Cannot get the whole layer for read locked: timeline {} is not present locally", + self.get_ancestor_timeline_id().unwrap()) + )?; } // Now we have the right starting timeline for our search. @@ -1366,18 +1322,13 @@ impl LayeredTimeline { // If not, check if there's a layer on the ancestor timeline match &timeline.ancestor_timeline { Some(ancestor_entry) => { - match ancestor_entry.local_or_schedule_download(self.tenantid) { - Some(ancestor) => { - lsn = timeline.ancestor_lsn; - timeline = ancestor; - trace!("recursing into ancestor at {}/{}", timeline.timelineid, lsn); - continue; - } - None => bail!( - "Cannot get a layer for read from remote ancestor timeline {}", - self.timelineid - ), - } + let ancestor = ancestor_entry + .ensure_loaded() + .context("cannot get a layer for read from ancestor because it is either remote or unloaded")?; + lsn = timeline.ancestor_lsn; + timeline = ancestor; + trace!("recursing into ancestor at {}/{}", timeline.timelineid, lsn); + continue; } None => return Ok(None), } @@ -1501,7 +1452,6 @@ impl LayeredTimeline { fn checkpoint_internal(&self, checkpoint_distance: u64, reconstruct_pages: bool) -> Result<()> { // Prevent concurrent checkpoints let _checkpoint_cs = self.checkpoint_cs.lock().unwrap(); - let write_guard = self.write_lock.lock().unwrap(); let mut layers = self.layers.lock().unwrap(); @@ -1862,10 +1812,10 @@ impl LayeredTimeline { ); } // Now check ancestor timelines, if any are present locally - else if let Some(ancestor) = - self.ancestor_timeline.as_ref().and_then(|timeline_entry| { - timeline_entry.local_or_schedule_download(self.tenantid) - }) + else if let Some(ancestor) = self + .ancestor_timeline + .as_ref() + .and_then(|timeline_entry| timeline_entry.ensure_loaded().ok()) { let prior_lsn = ancestor.get_last_record_lsn(); if seg.rel.is_blocky() { @@ -2435,9 +2385,8 @@ mod tests { metadata_bytes[512 - 4 - 2] ^= 1; std::fs::write(metadata_path, metadata_bytes)?; - let new_repo = harness.load(); - let err = new_repo.get_timeline(TIMELINE_ID).err().unwrap(); - assert_eq!(err.to_string(), "failed to load metadata"); + let err = harness.try_load().err().expect("should fail"); + assert_eq!(err.to_string(), "failed to load local metadata"); assert_eq!( err.source().unwrap().to_string(), "metadata checksum mismatch" @@ -2527,7 +2476,7 @@ mod tests { // Load the timeline. This will cause the files in the "future" to be renamed // away. let new_repo = harness.load(); - new_repo.get_timeline(TIMELINE_ID).unwrap(); + new_repo.get_timeline_load(TIMELINE_ID).unwrap(); drop(new_repo); for filename in future_filenames.iter() { @@ -2544,7 +2493,7 @@ mod tests { } let new_repo = harness.load(); - new_repo.get_timeline(TIMELINE_ID).unwrap(); + new_repo.get_timeline_load(TIMELINE_ID).unwrap(); drop(new_repo); for filename in future_filenames.iter() { diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 42a099cca5..6e6b6415f3 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -322,8 +322,8 @@ impl PageServerHandler { let _enter = info_span!("pagestream", timeline = %timelineid, tenant = %tenantid).entered(); // Check that the timeline exists - let timeline = tenant_mgr::get_timeline_for_tenant(tenantid, timelineid) - .context("Cannot handle pagerequests for a remote timeline")?; + let timeline = tenant_mgr::get_timeline_for_tenant_load(tenantid, timelineid) + .context("Cannot load local timeline")?; /* switch client to COPYBOTH */ pgb.write_message(&BeMessage::CopyBothResponse)?; @@ -520,8 +520,8 @@ impl PageServerHandler { let _enter = span.enter(); // check that the timeline exists - let timeline = tenant_mgr::get_timeline_for_tenant(tenantid, timelineid) - .context("Cannot handle basebackup request for a remote timeline")?; + let timeline = tenant_mgr::get_timeline_for_tenant_load(tenantid, timelineid) + .context("Cannot load local timeline")?; let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn(); if let Some(lsn) = lsn { timeline @@ -655,8 +655,8 @@ impl postgres_backend::Handler for PageServerHandler { info_span!("callmemaybe", timeline = %timelineid, tenant = %tenantid).entered(); // Check that the timeline exists - tenant_mgr::get_timeline_for_tenant(tenantid, timelineid) - .context("Failed to fetch local timeline for callmemaybe requests")?; + tenant_mgr::get_timeline_for_tenant_load(tenantid, timelineid) + .context("Cannot load local timeline")?; walreceiver::launch_wal_receiver(self.conf, tenantid, timelineid, &connstr)?; @@ -778,8 +778,8 @@ impl postgres_backend::Handler for PageServerHandler { let tenantid = ZTenantId::from_str(caps.get(1).unwrap().as_str())?; let timelineid = ZTimelineId::from_str(caps.get(2).unwrap().as_str())?; - let timeline = tenant_mgr::get_timeline_for_tenant(tenantid, timelineid) - .context("Failed to fetch local timeline for checkpoint request")?; + let timeline = tenant_mgr::get_timeline_for_tenant_load(tenantid, timelineid) + .context("Cannot load local timeline")?; timeline.checkpoint(CheckpointConfig::Forced)?; pgb.write_message_noflush(&SINGLE_COL_ROWDESC)? diff --git a/pageserver/src/remote_storage.rs b/pageserver/src/remote_storage.rs index 4af1f8ed56..08fb16a679 100644 --- a/pageserver/src/remote_storage.rs +++ b/pageserver/src/remote_storage.rs @@ -89,32 +89,38 @@ use std::{ collections::HashMap, ffi, fs, path::{Path, PathBuf}, + sync::Arc, }; use anyhow::{bail, Context}; -use tokio::io; +use tokio::{io, sync::RwLock}; use tracing::{error, info}; use zenith_utils::zid::{ZTenantId, ZTenantTimelineId, ZTimelineId}; +pub use self::storage_sync::index::{RemoteTimelineIndex, TimelineIndexEntry}; pub use self::storage_sync::{schedule_timeline_checkpoint_upload, schedule_timeline_download}; use self::{local_fs::LocalFs, rust_s3::S3}; use crate::{ config::{PageServerConf, RemoteStorageKind}, layered_repository::metadata::{TimelineMetadata, METADATA_FILE_NAME}, - repository::TimelineSyncState, }; pub use storage_sync::compression; +#[derive(Clone, Copy, Debug)] +pub enum LocalTimelineInitStatus { + LocallyComplete, + NeedsSync, +} + +type LocalTimelineInitStatuses = HashMap>; + /// A structure to combine all synchronization data to share with pageserver after a successful sync loop initialization. /// Successful initialization includes a case when sync loop is not started, in which case the startup data is returned still, /// to simplify the received code. pub struct SyncStartupData { - /// A sync state, derived from initial comparison of local timeline files and the remote archives, - /// before any sync tasks are executed. - /// To reuse the local file scan logic, the timeline states are returned even if no sync loop get started during init: - /// in this case, no remote files exist and all local timelines with correct metadata files are considered ready. - pub initial_timeline_states: HashMap>, + pub remote_index: Arc>, + pub local_timeline_init_statuses: LocalTimelineInitStatuses, } /// Based on the config, initiates the remote storage connection and starts a separate thread @@ -154,23 +160,18 @@ pub fn start_local_timeline_sync( .context("Failed to spawn the storage sync thread"), None => { info!("No remote storage configured, skipping storage sync, considering all local timelines with correct metadata files enabled"); - let mut initial_timeline_states: HashMap< - ZTenantId, - HashMap, - > = HashMap::new(); - for (ZTenantTimelineId{tenant_id, timeline_id}, (timeline_metadata, _)) in + let mut local_timeline_init_statuses = LocalTimelineInitStatuses::new(); + for (ZTenantTimelineId { tenant_id, timeline_id }, _) in local_timeline_files { - initial_timeline_states + local_timeline_init_statuses .entry(tenant_id) .or_default() - .insert( - timeline_id, - TimelineSyncState::Ready(timeline_metadata.disk_consistent_lsn()), - ); + .insert(timeline_id, LocalTimelineInitStatus::LocallyComplete); } Ok(SyncStartupData { - initial_timeline_states, + local_timeline_init_statuses, + remote_index: Arc::new(RwLock::new(RemoteTimelineIndex::empty())), }) } } diff --git a/pageserver/src/remote_storage/storage_sync.rs b/pageserver/src/remote_storage/storage_sync.rs index d14f849e15..f1483375cb 100644 --- a/pageserver/src/remote_storage/storage_sync.rs +++ b/pageserver/src/remote_storage/storage_sync.rs @@ -58,7 +58,7 @@ //! Synchronization never removes any local from pageserver workdir or remote files from the remote storage, yet there could be overwrites of the same files (metadata file updates; future checksum mismatch fixes). //! NOTE: No real contents or checksum check happens right now and is a subject to improve later. //! -//! After the whole timeline is downloaded, [`crate::tenant_mgr::set_timeline_states`] function is used to update pageserver memory stage for the timeline processed. +//! After the whole timeline is downloaded, [`crate::tenant_mgr::apply_timeline_sync_status_updates`] function is used to update pageserver memory stage for the timeline processed. //! //! When pageserver signals shutdown, current sync task gets finished and the loop exists. @@ -93,17 +93,25 @@ use self::{ download::{download_timeline, DownloadedTimeline}, index::{ ArchiveDescription, ArchiveId, RemoteTimeline, RemoteTimelineIndex, TimelineIndexEntry, + TimelineIndexEntryInner, }, upload::upload_timeline_checkpoint, }; -use super::{RemoteStorage, SyncStartupData, ZTenantTimelineId}; +use super::{ + LocalTimelineInitStatus, LocalTimelineInitStatuses, RemoteStorage, SyncStartupData, + ZTenantTimelineId, +}; use crate::{ config::PageServerConf, layered_repository::metadata::TimelineMetadata, - remote_storage::storage_sync::compression::read_archive_header, repository::TimelineSyncState, - tenant_mgr::set_timeline_states, thread_mgr, thread_mgr::ThreadKind, + remote_storage::storage_sync::compression::read_archive_header, + repository::TimelineSyncStatusUpdate, tenant_mgr::apply_timeline_sync_status_updates, + thread_mgr, thread_mgr::ThreadKind, }; -use zenith_metrics::{register_histogram_vec, register_int_gauge, HistogramVec, IntGauge}; +use zenith_metrics::{ + register_histogram_vec, register_int_counter, register_int_gauge, HistogramVec, IntCounter, + IntGauge, +}; use zenith_utils::zid::{ZTenantId, ZTimelineId}; lazy_static! { @@ -112,6 +120,11 @@ lazy_static! { "Number of storage sync items left in the queue" ) .expect("failed to register pageserver remote storage remaining sync items int gauge"); + static ref FATAL_TASK_FAILURES: IntCounter = register_int_counter!( + "pageserver_remote_storage_fatal_task_failures", + "Number of critically failed tasks" + ) + .expect("failed to register pageserver remote storage remaining sync items int gauge"); static ref IMAGE_SYNC_TIME: HistogramVec = register_histogram_vec!( "pageserver_remote_storage_image_sync_time", "Time took to synchronize (download or upload) a whole pageserver image. \ @@ -379,10 +392,13 @@ pub(super) fn spawn_storage_sync_thread< None } }); - let remote_index = RemoteTimelineIndex::try_parse_descriptions_from_paths(conf, download_paths); - - let initial_timeline_states = schedule_first_sync_tasks(&remote_index, local_timeline_files); + let mut remote_index = + RemoteTimelineIndex::try_parse_descriptions_from_paths(conf, download_paths); + let local_timeline_init_statuses = + schedule_first_sync_tasks(&mut remote_index, local_timeline_files); + let remote_index = Arc::new(RwLock::new(remote_index)); + let remote_index_cloned = Arc::clone(&remote_index); thread_mgr::spawn( ThreadKind::StorageSync, None, @@ -393,7 +409,7 @@ pub(super) fn spawn_storage_sync_thread< runtime, conf, receiver, - remote_index, + remote_index_cloned, storage, max_concurrent_sync, max_sync_errors, @@ -402,12 +418,13 @@ pub(super) fn spawn_storage_sync_thread< ) .context("Failed to spawn remote storage sync thread")?; Ok(SyncStartupData { - initial_timeline_states, + remote_index, + local_timeline_init_statuses, }) } enum LoopStep { - NewStates(HashMap>), + SyncStatusUpdates(HashMap>), Shutdown, } @@ -419,13 +436,14 @@ fn storage_sync_loop< runtime: Runtime, conf: &'static PageServerConf, mut receiver: UnboundedReceiver, - index: RemoteTimelineIndex, + index: Arc>, storage: S, max_concurrent_sync: NonZeroUsize, max_sync_errors: NonZeroU32, ) -> anyhow::Result<()> { - let remote_assets = Arc::new((storage, RwLock::new(index))); + let remote_assets = Arc::new((storage, Arc::clone(&index))); loop { + let index = Arc::clone(&index); let loop_step = runtime.block_on(async { tokio::select! { new_timeline_states = loop_step( @@ -435,15 +453,15 @@ fn storage_sync_loop< max_concurrent_sync, max_sync_errors, ) - .instrument(debug_span!("storage_sync_loop_step")) => LoopStep::NewStates(new_timeline_states), + .instrument(debug_span!("storage_sync_loop_step")) => LoopStep::SyncStatusUpdates(new_timeline_states), _ = thread_mgr::shutdown_watcher() => LoopStep::Shutdown, } }); match loop_step { - LoopStep::NewStates(new_timeline_states) => { + LoopStep::SyncStatusUpdates(new_timeline_states) => { // Batch timeline download registration to ensure that the external registration code won't block any running tasks before. - set_timeline_states(conf, new_timeline_states); + apply_timeline_sync_status_updates(conf, index, new_timeline_states); debug!("Sync loop step completed"); } LoopStep::Shutdown => { @@ -462,10 +480,10 @@ async fn loop_step< >( conf: &'static PageServerConf, receiver: &mut UnboundedReceiver, - remote_assets: Arc<(S, RwLock)>, + remote_assets: Arc<(S, Arc>)>, max_concurrent_sync: NonZeroUsize, max_sync_errors: NonZeroU32, -) -> HashMap> { +) -> HashMap> { let max_concurrent_sync = max_concurrent_sync.get(); let mut next_tasks = BTreeSet::new(); @@ -516,8 +534,10 @@ async fn loop_step< }) .collect::>(); - let mut new_timeline_states: HashMap> = - HashMap::with_capacity(max_concurrent_sync); + let mut new_timeline_states: HashMap< + ZTenantId, + HashMap, + > = HashMap::with_capacity(max_concurrent_sync); while let Some((sync_id, state_update)) = task_batch.next().await { debug!("Finished storage sync task for sync id {}", sync_id); if let Some(state_update) = state_update { @@ -540,24 +560,19 @@ async fn process_task< S: RemoteStorage + Send + Sync + 'static, >( conf: &'static PageServerConf, - remote_assets: Arc<(S, RwLock)>, + remote_assets: Arc<(S, Arc>)>, task: SyncTask, max_sync_errors: NonZeroU32, -) -> Option { +) -> Option { if task.retries > max_sync_errors.get() { error!( "Evicting task {:?} that failed {} times, exceeding the error threshold", task.kind, task.retries ); - return Some(TimelineSyncState::Evicted( - remote_assets - .as_ref() - .1 - .read() - .await - .timeline_entry(&task.sync_id) - .and_then(TimelineIndexEntry::disk_consistent_lsn), - )); + FATAL_TASK_FAILURES.inc(); + // FIXME (rodionov) this can potentially leave holes in timeline uploads + // planneed to be fixed as part of https://github.com/zenithdb/zenith/issues/977 + return None; } if task.retries > 0 { @@ -569,6 +584,8 @@ async fn process_task< tokio::time::sleep(Duration::from_secs_f64(seconds_to_wait)).await; } + let remote_index = Arc::clone(&remote_assets.1); + let sync_start = Instant::now(); let sync_name = task.kind.sync_name(); match task.kind { @@ -585,19 +602,25 @@ async fn process_task< match download_result { DownloadedTimeline::Abort => { register_sync_status(sync_start, sync_name, None); + remote_index + .write() + .await + .set_awaits_download(&task.sync_id, false) + .expect("timeline should be present in remote index"); None } - DownloadedTimeline::FailedAndRescheduled { - disk_consistent_lsn, - } => { + DownloadedTimeline::FailedAndRescheduled => { register_sync_status(sync_start, sync_name, Some(false)); - Some(TimelineSyncState::AwaitsDownload(disk_consistent_lsn)) + None } - DownloadedTimeline::Successful { - disk_consistent_lsn, - } => { + DownloadedTimeline::Successful => { register_sync_status(sync_start, sync_name, Some(true)); - Some(TimelineSyncState::Ready(disk_consistent_lsn)) + remote_index + .write() + .await + .set_awaits_download(&task.sync_id, false) + .expect("timeline should be present in remote index"); + Some(TimelineSyncStatusUpdate::Downloaded) } } } @@ -617,45 +640,45 @@ async fn process_task< } fn schedule_first_sync_tasks( - index: &RemoteTimelineIndex, + index: &mut RemoteTimelineIndex, local_timeline_files: HashMap)>, -) -> HashMap> { - let mut initial_timeline_statuses: HashMap> = - HashMap::new(); +) -> LocalTimelineInitStatuses { + let mut local_timeline_init_statuses = LocalTimelineInitStatuses::new(); let mut new_sync_tasks = VecDeque::with_capacity(local_timeline_files.len().max(local_timeline_files.len())); for (sync_id, (local_metadata, local_files)) in local_timeline_files { - let local_disk_consistent_lsn = local_metadata.disk_consistent_lsn(); - let ZTenantTimelineId { tenant_id, timeline_id, } = sync_id; - match index.timeline_entry(&sync_id) { + match index.timeline_entry_mut(&sync_id) { Some(index_entry) => { - let timeline_status = compare_local_and_remote_timeline( + let (timeline_status, awaits_download) = compare_local_and_remote_timeline( &mut new_sync_tasks, sync_id, local_metadata, local_files, index_entry, ); - match timeline_status { - Some(timeline_status) => { - initial_timeline_statuses - .entry(tenant_id) - .or_default() - .insert(timeline_id, timeline_status); - } - None => error!( - "Failed to compare local and remote timeline for task {}", - sync_id - ), + let was_there = local_timeline_init_statuses + .entry(tenant_id) + .or_default() + .insert(timeline_id, timeline_status); + + if was_there.is_some() { + // defensive check + warn!( + "Overwriting timeline init sync status. Status {:?} Timeline {}", + timeline_status, timeline_id + ); } + index_entry.set_awaits_download(awaits_download); } None => { + // TODO (rodionov) does this mean that we've crashed during tenant creation? + // is it safe to upload this checkpoint? could it be half broken? new_sync_tasks.push_back(SyncTask::new( sync_id, 0, @@ -664,56 +687,18 @@ fn schedule_first_sync_tasks( metadata: local_metadata, }), )); - initial_timeline_statuses + local_timeline_init_statuses .entry(tenant_id) .or_default() - .insert( - timeline_id, - TimelineSyncState::Ready(local_disk_consistent_lsn), - ); + .insert(timeline_id, LocalTimelineInitStatus::LocallyComplete); } } } - let unprocessed_remote_ids = |remote_id: &ZTenantTimelineId| { - initial_timeline_statuses - .get(&remote_id.tenant_id) - .and_then(|timelines| timelines.get(&remote_id.timeline_id)) - .is_none() - }; - for unprocessed_remote_id in index - .all_sync_ids() - .filter(unprocessed_remote_ids) - .collect::>() - { - let ZTenantTimelineId { - tenant_id: cloud_only_tenant_id, - timeline_id: cloud_only_timeline_id, - } = unprocessed_remote_id; - match index - .timeline_entry(&unprocessed_remote_id) - .and_then(TimelineIndexEntry::disk_consistent_lsn) - { - Some(remote_disk_consistent_lsn) => { - initial_timeline_statuses - .entry(cloud_only_tenant_id) - .or_default() - .insert( - cloud_only_timeline_id, - TimelineSyncState::CloudOnly(remote_disk_consistent_lsn), - ); - } - None => error!( - "Failed to find disk consistent LSN for remote timeline {}", - unprocessed_remote_id - ), - } - } - new_sync_tasks.into_iter().for_each(|task| { sync_queue::push(task); }); - initial_timeline_statuses + local_timeline_init_statuses } fn compare_local_and_remote_timeline( @@ -722,10 +707,21 @@ fn compare_local_and_remote_timeline( local_metadata: TimelineMetadata, local_files: Vec, remote_entry: &TimelineIndexEntry, -) -> Option { +) -> (LocalTimelineInitStatus, bool) { let local_lsn = local_metadata.disk_consistent_lsn(); let uploads = remote_entry.uploaded_checkpoints(); + let mut initial_timeline_status = LocalTimelineInitStatus::LocallyComplete; + + let mut awaits_download = false; + // TODO probably here we need more sophisticated logic, + // if more data is available remotely can we just download whats there? + // without trying to upload something. It may be tricky, needs further investigation. + // For now looks strange that we can request upload + // and dowload for the same timeline simultaneously. + // (upload needs to be only for previously unsynced files, not whole timeline dir). + // If one of the tasks fails they will be reordered in the queue which can lead + // to timeline being stuck in evicted state if !uploads.contains(&local_lsn) { new_sync_tasks.push_back(SyncTask::new( sync_id, @@ -735,6 +731,7 @@ fn compare_local_and_remote_timeline( metadata: local_metadata, }), )); + // Note that status here doesnt change. } let uploads_count = uploads.len(); @@ -743,7 +740,7 @@ fn compare_local_and_remote_timeline( .filter(|upload_lsn| upload_lsn <= &local_lsn) .map(ArchiveId) .collect(); - Some(if archives_to_skip.len() != uploads_count { + if archives_to_skip.len() != uploads_count { new_sync_tasks.push_back(SyncTask::new( sync_id, 0, @@ -752,10 +749,12 @@ fn compare_local_and_remote_timeline( archives_to_skip, }), )); - TimelineSyncState::AwaitsDownload(remote_entry.disk_consistent_lsn()?) - } else { - TimelineSyncState::Ready(remote_entry.disk_consistent_lsn().unwrap_or(local_lsn)) - }) + initial_timeline_status = LocalTimelineInitStatus::NeedsSync; + awaits_download = true; + // we do not need to manupulate with remote consistent lsn here + // because it will be updated when sync will be completed + } + (initial_timeline_status, awaits_download) } fn register_sync_status(sync_start: Instant, sync_name: &str, sync_status: Option) { @@ -769,21 +768,23 @@ fn register_sync_status(sync_start: Instant, sync_name: &str, sync_status: Optio .observe(secs_elapsed) } -async fn update_index_description< +async fn fetch_full_index< P: Send + Sync + 'static, S: RemoteStorage + Send + Sync + 'static, >( - (storage, index): &(S, RwLock), + (storage, index): &(S, Arc>), timeline_dir: &Path, id: ZTenantTimelineId, ) -> anyhow::Result { - let mut index_write = index.write().await; - let full_index = match index_write.timeline_entry(&id) { + let index_read = index.read().await; + let full_index = match index_read.timeline_entry(&id).map(|e| e.inner()) { None => bail!("Timeline not found for sync id {}", id), - Some(TimelineIndexEntry::Full(_)) => bail!("Index is already populated for sync id {}", id), - Some(TimelineIndexEntry::Description(description)) => { + Some(TimelineIndexEntryInner::Full(_)) => { + bail!("Index is already populated for sync id {}", id) + } + Some(TimelineIndexEntryInner::Description(description)) => { let mut archive_header_downloads = FuturesUnordered::new(); - for (&archive_id, description) in description { + for (archive_id, description) in description { archive_header_downloads.push(async move { let header = download_archive_header(storage, timeline_dir, description) .await @@ -795,18 +796,22 @@ async fn update_index_description< let mut full_index = RemoteTimeline::empty(); while let Some(header_data) = archive_header_downloads.next().await { match header_data { - Ok((archive_id, header_size, header)) => full_index.update_archive_contents(archive_id.0, header, header_size), - Err((e, archive_id)) => bail!( - "Failed to download archive header for tenant {}, timeline {}, archive for Lsn {}: {}", - id.tenant_id, id.timeline_id, archive_id.0, - e - ), - } + Ok((archive_id, header_size, header)) => full_index.update_archive_contents(archive_id.0, header, header_size), + Err((e, archive_id)) => bail!( + "Failed to download archive header for tenant {}, timeline {}, archive for Lsn {}: {}", + id.tenant_id, id.timeline_id, archive_id.0, + e + ), + } } full_index } }; - index_write.add_timeline_entry(id, TimelineIndexEntry::Full(full_index.clone())); + drop(index_read); // tokio rw lock is not upgradeable + let mut index_write = index.write().await; + index_write + .upgrade_timeline_entry(&id, full_index.clone()) + .context("cannot upgrade timeline entry in remote index")?; Ok(full_index) } @@ -850,7 +855,7 @@ mod test_utils { #[track_caller] pub async fn ensure_correct_timeline_upload( harness: &RepoHarness, - remote_assets: Arc<(LocalFs, RwLock)>, + remote_assets: Arc<(LocalFs, Arc>)>, timeline_id: ZTimelineId, new_upload: NewCheckpoint, ) { @@ -909,11 +914,14 @@ mod test_utils { } pub async fn expect_timeline( - index: &RwLock, + index: &Arc>, sync_id: ZTenantTimelineId, ) -> RemoteTimeline { - if let Some(TimelineIndexEntry::Full(remote_timeline)) = - index.read().await.timeline_entry(&sync_id) + if let Some(TimelineIndexEntryInner::Full(remote_timeline)) = index + .read() + .await + .timeline_entry(&sync_id) + .map(|e| e.inner()) { remote_timeline.clone() } else { @@ -926,7 +934,7 @@ mod test_utils { #[track_caller] pub async fn assert_index_descriptions( - index: &RwLock, + index: &Arc>, expected_index_with_descriptions: RemoteTimelineIndex, ) { let index_read = index.read().await; @@ -965,26 +973,26 @@ mod test_utils { sync_id ) }); - let expected_timeline_description = match expected_timeline_description { - TimelineIndexEntry::Description(description) => description, - TimelineIndexEntry::Full(_) => panic!("Expected index entry for sync id {} is a full entry, while a description was expected", sync_id), + let expected_timeline_description = match expected_timeline_description.inner() { + TimelineIndexEntryInner::Description(description) => description, + TimelineIndexEntryInner::Full(_) => panic!("Expected index entry for sync id {} is a full entry, while a description was expected", sync_id), }; - match actual_timeline_entry { - TimelineIndexEntry::Description(actual_descriptions) => { + match actual_timeline_entry.inner() { + TimelineIndexEntryInner::Description(description) => { assert_eq!( - actual_descriptions, expected_timeline_description, + description, expected_timeline_description, "Index contains unexpected descriptions entry for sync id {}", sync_id ) } - TimelineIndexEntry::Full(actual_full_entry) => { + TimelineIndexEntryInner::Full(remote_timeline) => { let expected_lsns = expected_timeline_description .values() .map(|description| description.disk_consistent_lsn) .collect::>(); assert_eq!( - actual_full_entry.checkpoints().collect::>(), + remote_timeline.checkpoints().collect::>(), expected_lsns, "Timeline {} should have the same checkpoints uploaded", sync_id, diff --git a/pageserver/src/remote_storage/storage_sync/download.rs b/pageserver/src/remote_storage/storage_sync/download.rs index 00115ba8d5..e5362b2973 100644 --- a/pageserver/src/remote_storage/storage_sync/download.rs +++ b/pageserver/src/remote_storage/storage_sync/download.rs @@ -5,14 +5,14 @@ use std::{borrow::Cow, collections::BTreeSet, path::PathBuf, sync::Arc}; use anyhow::{ensure, Context}; use tokio::{fs, sync::RwLock}; use tracing::{debug, error, trace, warn}; -use zenith_utils::{lsn::Lsn, zid::ZTenantId}; +use zenith_utils::zid::ZTenantId; use crate::{ config::PageServerConf, layered_repository::metadata::{metadata_path, TimelineMetadata}, remote_storage::{ storage_sync::{ - compression, index::TimelineIndexEntry, sync_queue, update_index_description, SyncKind, + compression, fetch_full_index, index::TimelineIndexEntryInner, sync_queue, SyncKind, SyncTask, }, RemoteStorage, ZTenantTimelineId, @@ -30,10 +30,10 @@ pub(super) enum DownloadedTimeline { Abort, /// Remote timeline data is found, its latest checkpoint's metadata contents (disk_consistent_lsn) is known. /// Initial download failed due to some error, the download task is rescheduled for another retry. - FailedAndRescheduled { disk_consistent_lsn: Lsn }, + FailedAndRescheduled, /// Remote timeline data is found, its latest checkpoint's metadata contents (disk_consistent_lsn) is known. /// Initial download successful. - Successful { disk_consistent_lsn: Lsn }, + Successful, } /// Attempts to download and uncompress files from all remote archives for the timeline given. @@ -47,7 +47,7 @@ pub(super) async fn download_timeline< S: RemoteStorage + Send + Sync + 'static, >( conf: &'static PageServerConf, - remote_assets: Arc<(S, RwLock)>, + remote_assets: Arc<(S, Arc>)>, sync_id: ZTenantTimelineId, mut download: TimelineDownload, retries: u32, @@ -58,19 +58,26 @@ pub(super) async fn download_timeline< tenant_id, timeline_id, } = sync_id; - let index_read = remote_assets.1.read().await; + let index = &remote_assets.1; + + let index_read = index.read().await; let remote_timeline = match index_read.timeline_entry(&sync_id) { None => { - error!("Cannot download: no timeline is present in the index for given ids"); + error!("Cannot download: no timeline is present in the index for given id"); return DownloadedTimeline::Abort; } - Some(index_entry) => match index_entry { - TimelineIndexEntry::Full(remote_timeline) => Cow::Borrowed(remote_timeline), - TimelineIndexEntry::Description(_) => { + + Some(index_entry) => match index_entry.inner() { + TimelineIndexEntryInner::Full(remote_timeline) => Cow::Borrowed(remote_timeline), + TimelineIndexEntryInner::Description(_) => { + // we do not check here for awaits_download because it is ok + // to call this function while the download is in progress + // so it is not a concurrent download, it is the same one + let remote_disk_consistent_lsn = index_entry.disk_consistent_lsn(); drop(index_read); debug!("Found timeline description for the given ids, downloading the full index"); - match update_index_description( + match fetch_full_index( remote_assets.as_ref(), &conf.timeline_path(&timeline_id, &tenant_id), sync_id, @@ -80,16 +87,15 @@ pub(super) async fn download_timeline< Ok(remote_timeline) => Cow::Owned(remote_timeline), Err(e) => { error!("Failed to download full timeline index: {:?}", e); + return match remote_disk_consistent_lsn { - Some(disk_consistent_lsn) => { + Some(_) => { sync_queue::push(SyncTask::new( sync_id, retries, SyncKind::Download(download), )); - DownloadedTimeline::FailedAndRescheduled { - disk_consistent_lsn, - } + DownloadedTimeline::FailedAndRescheduled } None => { error!("Cannot download: no disk consistent Lsn is present for the index entry"); @@ -101,12 +107,9 @@ pub(super) async fn download_timeline< } }, }; - let disk_consistent_lsn = match remote_timeline.checkpoints().max() { - Some(lsn) => lsn, - None => { - debug!("Cannot download: no disk consistent Lsn is present for the remote timeline"); - return DownloadedTimeline::Abort; - } + if remote_timeline.checkpoints().max().is_none() { + debug!("Cannot download: no disk consistent Lsn is present for the remote timeline"); + return DownloadedTimeline::Abort; }; debug!("Downloading timeline archives"); @@ -125,7 +128,7 @@ pub(super) async fn download_timeline< conf, sync_id, Arc::clone(&remote_assets), - remote_timeline.as_ref(), + &remote_timeline, archive_id, Arc::clone(&download.files_to_skip), ) @@ -142,9 +145,7 @@ pub(super) async fn download_timeline< retries, SyncKind::Download(download), )); - return DownloadedTimeline::FailedAndRescheduled { - disk_consistent_lsn, - }; + return DownloadedTimeline::FailedAndRescheduled; } Ok(()) => { debug!("Successfully downloaded archive {:?}", archive_id); @@ -154,9 +155,7 @@ pub(super) async fn download_timeline< } debug!("Finished downloading all timeline's archives"); - DownloadedTimeline::Successful { - disk_consistent_lsn, - } + DownloadedTimeline::Successful } async fn try_download_archive< @@ -168,7 +167,7 @@ async fn try_download_archive< tenant_id, timeline_id, }: ZTenantTimelineId, - remote_assets: Arc<(S, RwLock)>, + remote_assets: Arc<(S, Arc>)>, remote_timeline: &RemoteTimeline, archive_id: ArchiveId, files_to_skip: Arc>, @@ -256,13 +255,15 @@ mod tests { let repo_harness = RepoHarness::create("test_download_timeline")?; let sync_id = ZTenantTimelineId::new(repo_harness.tenant_id, TIMELINE_ID); let storage = LocalFs::new(tempdir()?.path().to_owned(), &repo_harness.conf.workdir)?; - let index = RwLock::new(RemoteTimelineIndex::try_parse_descriptions_from_paths( - repo_harness.conf, - storage - .list() - .await? - .into_iter() - .map(|storage_path| storage.local_path(&storage_path).unwrap()), + let index = Arc::new(RwLock::new( + RemoteTimelineIndex::try_parse_descriptions_from_paths( + repo_harness.conf, + storage + .list() + .await? + .into_iter() + .map(|storage_path| storage.local_path(&storage_path).unwrap()), + ), )); let remote_assets = Arc::new((storage, index)); let storage = &remote_assets.0; diff --git a/pageserver/src/remote_storage/storage_sync/index.rs b/pageserver/src/remote_storage/storage_sync/index.rs index 81c99754c9..7d6b4881f7 100644 --- a/pageserver/src/remote_storage/storage_sync/index.rs +++ b/pageserver/src/remote_storage/storage_sync/index.rs @@ -11,7 +11,7 @@ use std::{ use anyhow::{bail, ensure, Context}; use serde::{Deserialize, Serialize}; -use tracing::debug; +use tracing::*; use zenith_utils::{ lsn::Lsn, zid::{ZTenantId, ZTimelineId}, @@ -52,10 +52,16 @@ impl RelativePath { /// Currently, timeline archive files are tracked only. #[derive(Debug, Clone)] pub struct RemoteTimelineIndex { - timeline_files: HashMap, + timeline_entries: HashMap, } impl RemoteTimelineIndex { + pub fn empty() -> Self { + Self { + timeline_entries: HashMap::new(), + } + } + /// Attempts to parse file paths (not checking the file contents) and find files /// that can be tracked wiht the index. /// On parse falures, logs the error and continues, so empty index can be created from not suitable paths. @@ -63,9 +69,7 @@ impl RemoteTimelineIndex { conf: &'static PageServerConf, paths: impl Iterator, ) -> Self { - let mut index = Self { - timeline_files: HashMap::new(), - }; + let mut index = Self::empty(); for path in paths { if let Err(e) = try_parse_index_entry(&mut index, conf, path.as_ref()) { debug!( @@ -79,40 +83,100 @@ impl RemoteTimelineIndex { } pub fn timeline_entry(&self, id: &ZTenantTimelineId) -> Option<&TimelineIndexEntry> { - self.timeline_files.get(id) + self.timeline_entries.get(id) } pub fn timeline_entry_mut( &mut self, id: &ZTenantTimelineId, ) -> Option<&mut TimelineIndexEntry> { - self.timeline_files.get_mut(id) + self.timeline_entries.get_mut(id) } pub fn add_timeline_entry(&mut self, id: ZTenantTimelineId, entry: TimelineIndexEntry) { - self.timeline_files.insert(id, entry); + self.timeline_entries.insert(id, entry); + } + + pub fn upgrade_timeline_entry( + &mut self, + id: &ZTenantTimelineId, + remote_timeline: RemoteTimeline, + ) -> anyhow::Result<()> { + let mut entry = self.timeline_entries.get_mut(id).ok_or(anyhow::anyhow!( + "timeline is unexpectedly missing from remote index" + ))?; + + if !matches!(entry.inner, TimelineIndexEntryInner::Description(_)) { + anyhow::bail!("timeline entry is not a description entry") + }; + + entry.inner = TimelineIndexEntryInner::Full(remote_timeline); + + Ok(()) } pub fn all_sync_ids(&self) -> impl Iterator + '_ { - self.timeline_files.keys().copied() + self.timeline_entries.keys().copied() + } + + pub fn set_awaits_download( + &mut self, + id: &ZTenantTimelineId, + awaits_download: bool, + ) -> anyhow::Result<()> { + self.timeline_entry_mut(id) + .ok_or_else(|| anyhow::anyhow!("unknown timeline sync {}", id))? + .set_awaits_download(awaits_download); + Ok(()) } } +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct DescriptionTimelineIndexEntry { + pub description: BTreeMap, + pub awaits_download: bool, +} + #[derive(Debug, Clone, PartialEq, Eq)] -pub enum TimelineIndexEntry { - /// An archive found on the remote storage, but not yet downloaded, only a metadata from its storage path is available, without archive contents. +pub struct FullTimelineIndexEntry { + pub remote_timeline: RemoteTimeline, + pub awaits_download: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TimelineIndexEntryInner { Description(BTreeMap), - /// Full archive metadata, including the file list, parsed from the archive header. Full(RemoteTimeline), } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TimelineIndexEntry { + inner: TimelineIndexEntryInner, + awaits_download: bool, +} + impl TimelineIndexEntry { + pub fn new(inner: TimelineIndexEntryInner, awaits_download: bool) -> Self { + Self { + inner, + awaits_download, + } + } + + pub fn inner(&self) -> &TimelineIndexEntryInner { + &self.inner + } + + pub fn inner_mut(&mut self) -> &mut TimelineIndexEntryInner { + &mut self.inner + } + pub fn uploaded_checkpoints(&self) -> BTreeSet { - match self { - Self::Description(description) => { + match &self.inner { + TimelineIndexEntryInner::Description(description) => { description.keys().map(|archive_id| archive_id.0).collect() } - Self::Full(remote_timeline) => remote_timeline + TimelineIndexEntryInner::Full(remote_timeline) => remote_timeline .checkpoint_archives .keys() .map(|archive_id| archive_id.0) @@ -122,17 +186,25 @@ impl TimelineIndexEntry { /// Gets latest uploaded checkpoint's disk consisten Lsn for the corresponding timeline. pub fn disk_consistent_lsn(&self) -> Option { - match self { - Self::Description(description) => { + match &self.inner { + TimelineIndexEntryInner::Description(description) => { description.keys().map(|archive_id| archive_id.0).max() } - Self::Full(remote_timeline) => remote_timeline + TimelineIndexEntryInner::Full(remote_timeline) => remote_timeline .checkpoint_archives .keys() .map(|archive_id| archive_id.0) .max(), } } + + pub fn get_awaits_download(&self) -> bool { + self.awaits_download + } + + pub fn set_awaits_download(&mut self, awaits_download: bool) { + self.awaits_download = awaits_download; + } } /// Checkpoint archive's id, corresponding to the `disk_consistent_lsn` from the timeline's metadata file during checkpointing. @@ -331,13 +403,15 @@ fn try_parse_index_entry( tenant_id, timeline_id, }; - let timeline_index_entry = index - .timeline_files - .entry(sync_id) - .or_insert_with(|| TimelineIndexEntry::Description(BTreeMap::new())); - match timeline_index_entry { - TimelineIndexEntry::Description(descriptions) => { - descriptions.insert( + let timeline_index_entry = index.timeline_entries.entry(sync_id).or_insert_with(|| { + TimelineIndexEntry::new( + TimelineIndexEntryInner::Description(BTreeMap::default()), + false, + ) + }); + match timeline_index_entry.inner_mut() { + TimelineIndexEntryInner::Description(description) => { + description.insert( ArchiveId(disk_consistent_lsn), ArchiveDescription { header_size, @@ -346,7 +420,7 @@ fn try_parse_index_entry( }, ); } - TimelineIndexEntry::Full(_) => { + TimelineIndexEntryInner::Full(_) => { bail!("Cannot add parsed archive description to its full context in index with sync id {}", sync_id) } } diff --git a/pageserver/src/remote_storage/storage_sync/upload.rs b/pageserver/src/remote_storage/storage_sync/upload.rs index d064039ecc..8fdd91dd18 100644 --- a/pageserver/src/remote_storage/storage_sync/upload.rs +++ b/pageserver/src/remote_storage/storage_sync/upload.rs @@ -10,9 +10,9 @@ use crate::{ config::PageServerConf, remote_storage::{ storage_sync::{ - compression, - index::{RemoteTimeline, TimelineIndexEntry}, - sync_queue, update_index_description, SyncKind, SyncTask, + compression, fetch_full_index, + index::{RemoteTimeline, TimelineIndexEntry, TimelineIndexEntryInner}, + sync_queue, SyncKind, SyncTask, }, RemoteStorage, ZTenantTimelineId, }, @@ -30,7 +30,7 @@ pub(super) async fn upload_timeline_checkpoint< S: RemoteStorage + Send + Sync + 'static, >( config: &'static PageServerConf, - remote_assets: Arc<(S, RwLock)>, + remote_assets: Arc<(S, Arc>)>, sync_id: ZTenantTimelineId, new_checkpoint: NewCheckpoint, retries: u32, @@ -49,22 +49,24 @@ pub(super) async fn upload_timeline_checkpoint< let index_read = index.read().await; let remote_timeline = match index_read.timeline_entry(&sync_id) { None => None, - Some(TimelineIndexEntry::Full(remote_timeline)) => Some(Cow::Borrowed(remote_timeline)), - Some(TimelineIndexEntry::Description(_)) => { - debug!("Found timeline description for the given ids, downloading the full index"); - match update_index_description(remote_assets.as_ref(), &timeline_dir, sync_id).await { - Ok(remote_timeline) => Some(Cow::Owned(remote_timeline)), - Err(e) => { - error!("Failed to download full timeline index: {:?}", e); - sync_queue::push(SyncTask::new( - sync_id, - retries, - SyncKind::Upload(new_checkpoint), - )); - return Some(false); + Some(entry) => match entry.inner() { + TimelineIndexEntryInner::Full(remote_timeline) => Some(Cow::Borrowed(remote_timeline)), + TimelineIndexEntryInner::Description(_) => { + debug!("Found timeline description for the given ids, downloading the full index"); + match fetch_full_index(remote_assets.as_ref(), &timeline_dir, sync_id).await { + Ok(remote_timeline) => Some(Cow::Owned(remote_timeline)), + Err(e) => { + error!("Failed to download full timeline index: {:?}", e); + sync_queue::push(SyncTask::new( + sync_id, + retries, + SyncKind::Upload(new_checkpoint), + )); + return Some(false); + } } } - } + }, }; let already_contains_upload_lsn = remote_timeline @@ -95,22 +97,40 @@ pub(super) async fn upload_timeline_checkpoint< { Ok((archive_header, header_size)) => { let mut index_write = index.write().await; - match index_write.timeline_entry_mut(&sync_id) { - Some(TimelineIndexEntry::Full(remote_timeline)) => { - remote_timeline.update_archive_contents( - new_checkpoint.metadata.disk_consistent_lsn(), - archive_header, - header_size, - ); - } - None | Some(TimelineIndexEntry::Description(_)) => { + match index_write + .timeline_entry_mut(&sync_id) + .map(|e| e.inner_mut()) + { + None => { let mut new_timeline = RemoteTimeline::empty(); new_timeline.update_archive_contents( new_checkpoint.metadata.disk_consistent_lsn(), archive_header, header_size, ); - index_write.add_timeline_entry(sync_id, TimelineIndexEntry::Full(new_timeline)); + index_write.add_timeline_entry( + sync_id, + TimelineIndexEntry::new(TimelineIndexEntryInner::Full(new_timeline), false), + ) + } + Some(TimelineIndexEntryInner::Full(remote_timeline)) => { + remote_timeline.update_archive_contents( + new_checkpoint.metadata.disk_consistent_lsn(), + archive_header, + header_size, + ); + } + Some(TimelineIndexEntryInner::Description(_)) => { + let mut new_timeline = RemoteTimeline::empty(); + new_timeline.update_archive_contents( + new_checkpoint.metadata.disk_consistent_lsn(), + archive_header, + header_size, + ); + index_write.add_timeline_entry( + sync_id, + TimelineIndexEntry::new(TimelineIndexEntryInner::Full(new_timeline), false), + ) } } debug!("Checkpoint uploaded successfully"); @@ -136,7 +156,7 @@ async fn try_upload_checkpoint< S: RemoteStorage + Send + Sync + 'static, >( config: &'static PageServerConf, - remote_assets: Arc<(S, RwLock)>, + remote_assets: Arc<(S, Arc>)>, sync_id: ZTenantTimelineId, new_checkpoint: &NewCheckpoint, files_to_skip: BTreeSet, @@ -209,13 +229,15 @@ mod tests { let repo_harness = RepoHarness::create("reupload_timeline")?; let sync_id = ZTenantTimelineId::new(repo_harness.tenant_id, TIMELINE_ID); let storage = LocalFs::new(tempdir()?.path().to_owned(), &repo_harness.conf.workdir)?; - let index = RwLock::new(RemoteTimelineIndex::try_parse_descriptions_from_paths( - repo_harness.conf, - storage - .list() - .await? - .into_iter() - .map(|storage_path| storage.local_path(&storage_path).unwrap()), + let index = Arc::new(RwLock::new( + RemoteTimelineIndex::try_parse_descriptions_from_paths( + repo_harness.conf, + storage + .list() + .await? + .into_iter() + .map(|storage_path| storage.local_path(&storage_path).unwrap()), + ), )); let remote_assets = Arc::new((storage, index)); let index = &remote_assets.1; @@ -405,13 +427,15 @@ mod tests { let repo_harness = RepoHarness::create("reupload_timeline_rejected")?; let sync_id = ZTenantTimelineId::new(repo_harness.tenant_id, TIMELINE_ID); let storage = LocalFs::new(tempdir()?.path().to_owned(), &repo_harness.conf.workdir)?; - let index = RwLock::new(RemoteTimelineIndex::try_parse_descriptions_from_paths( - repo_harness.conf, - storage - .list() - .await? - .into_iter() - .map(|storage_path| storage.local_path(&storage_path).unwrap()), + let index = Arc::new(RwLock::new( + RemoteTimelineIndex::try_parse_descriptions_from_paths( + repo_harness.conf, + storage + .list() + .await? + .into_iter() + .map(|storage_path| storage.local_path(&storage_path).unwrap()), + ), )); let remote_assets = Arc::new((storage, index)); let storage = &remote_assets.0; diff --git a/pageserver/src/repository.rs b/pageserver/src/repository.rs index be937b8d26..e335f42519 100644 --- a/pageserver/src/repository.rs +++ b/pageserver/src/repository.rs @@ -1,4 +1,6 @@ +use crate::layered_repository::metadata::TimelineMetadata; use crate::relish::*; +use crate::remote_storage::RemoteTimelineIndex; use crate::walrecord::MultiXactMember; use crate::CheckpointConfig; use anyhow::Result; @@ -6,6 +8,7 @@ use bytes::Bytes; use postgres_ffi::{MultiXactId, MultiXactOffset, TransactionId}; use serde::{Deserialize, Serialize}; use std::collections::HashSet; +use std::fmt::Display; use std::ops::{AddAssign, Deref}; use std::sync::{Arc, RwLockReadGuard}; use std::time::Duration; @@ -15,30 +18,43 @@ use zenith_utils::zid::ZTimelineId; /// Block number within a relish. This matches PostgreSQL's BlockNumber type. pub type BlockNumber = u32; +#[derive(Clone, Copy, Debug)] +pub enum TimelineSyncStatusUpdate { + Uploaded, + Downloaded, +} + +impl Display for TimelineSyncStatusUpdate { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + TimelineSyncStatusUpdate::Uploaded => "Uploaded", + TimelineSyncStatusUpdate::Downloaded => "Downloaded", + }; + f.write_str(s) + } +} /// /// A repository corresponds to one .zenith directory. One repository holds multiple /// timelines, forked off from the same initial call to 'initdb'. pub trait Repository: Send + Sync { - fn detach_timeline(&self, timeline_id: ZTimelineId) -> Result<()>; - - /// Updates timeline based on the new sync state, received from the remote storage synchronization. + /// Updates timeline based on the `TimelineSyncStatusUpdate`, received from the remote storage synchronization. /// See [`crate::remote_storage`] for more details about the synchronization. - fn set_timeline_state( + fn apply_timeline_remote_sync_status_update( &self, timeline_id: ZTimelineId, - new_state: TimelineSyncState, + timeline_sync_status_update: TimelineSyncStatusUpdate, ) -> Result<()>; - /// Gets current synchronization state of the timeline. - /// See [`crate::remote_storage`] for more details about the synchronization. - fn get_timeline_state(&self, timeline_id: ZTimelineId) -> Option; - /// Get Timeline handle for given zenith timeline ID. - fn get_timeline(&self, timelineid: ZTimelineId) -> Result; + /// This function is idempotent. It doesnt change internal state in any way. + fn get_timeline(&self, timelineid: ZTimelineId) -> Option; + + /// Get Timeline handle for locally available timeline. Load it into memory if it is not loaded. + fn get_timeline_load(&self, timelineid: ZTimelineId) -> Result>; /// Lists timelines the repository contains. /// Up to repository's implementation to omit certain timelines that ar not considered ready for use. - fn list_timelines(&self) -> Result>; + fn list_timelines(&self) -> Vec<(ZTimelineId, RepositoryTimeline)>; /// Create a new, empty timeline. The caller is responsible for loading data into it /// Initdb lsn is provided for timeline impl to be able to perform checks for some operations against it. @@ -70,72 +86,44 @@ pub trait Repository: Send + Sync { /// perform one checkpoint iteration, flushing in-memory data on disk. /// this function is periodically called by checkponter thread. fn checkpoint_iteration(&self, cconf: CheckpointConfig) -> Result<()>; + + /// detaches locally available timeline by stopping all threads and removing all the data. + fn detach_timeline(&self, timeline_id: ZTimelineId) -> Result<()>; + + // Allows to retrieve remote timeline index from the repo. Used in walreceiver to grab remote consistent lsn. + fn get_remote_index(&self) -> &tokio::sync::RwLock; } /// A timeline, that belongs to the current repository. pub enum RepositoryTimeline { /// Timeline, with its files present locally in pageserver's working directory. /// Loaded into pageserver's memory and ready to be used. - Local { - id: ZTimelineId, - timeline: Arc, - }, - /// Timeline, found on the pageserver's remote storage, but not yet downloaded locally. - Remote { - id: ZTimelineId, - /// metadata contents of the latest successfully uploaded checkpoint - disk_consistent_lsn: Lsn, + Loaded(Arc), + + /// All the data is available locally, but not loaded into memory, so loading have to be done before actually using the timeline + Unloaded { + // It is ok to keep metadata here, because it is not changed when timeline is unloaded. + // FIXME can s3 sync actually change it? It can change it when timeline is in awaiting download state. + // but we currently do not download something for the timeline once it is local (even if there are new checkpoints) is it correct? + // also it is not that good to keep TimelineMetadata here, because it is layered repo implementation detail + metadata: TimelineMetadata, }, } -impl RepositoryTimeline { - pub fn local_timeline(&self) -> Option> { - if let Self::Local { timeline, .. } = self { - Some(Arc::clone(timeline)) - } else { - None - } - } - - pub fn id(&self) -> ZTimelineId { - match self { - Self::Local { id, .. } => *id, - Self::Remote { id, .. } => *id, - } - } -} - -/// A state of the timeline synchronization with the remote storage. -/// Contains `disk_consistent_lsn` of the corresponding remote timeline (latest checkpoint's disk_consistent_lsn). #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum TimelineSyncState { - /// No further downloads from the remote storage are needed. - /// The timeline state is up-to-date or ahead of the remote storage one, - /// ready to be used in any pageserver operation. - Ready(Lsn), - /// Timeline is scheduled for downloading, but its current local state is not up to date with the remote storage. - /// The timeline is not ready to be used in any pageserver operations, otherwise it might diverge its local state from the remote version, - /// making it impossible to sync it further. - AwaitsDownload(Lsn), - /// Timeline was not in the pageserver's local working directory, but was found on the remote storage, ready to be downloaded. - /// Cannot be used in any pageserver operations due to complete absence locally. - CloudOnly(Lsn), - /// Timeline was evicted from the pageserver's local working directory due to conflicting remote and local states or too many errors during the synchronization. - /// Such timelines cannot have their state synchronized further and may not have the data about remote timeline's disk_consistent_lsn, since eviction may happen - /// due to errors before the remote timeline contents is known. - Evicted(Option), +pub enum LocalTimelineState { + // timeline is loaded into memory (with layer map and all the bits), + Loaded, + // timeline is on disk locally and ready to be loaded into memory. + Unloaded, } -impl TimelineSyncState { - pub fn remote_disk_consistent_lsn(&self) -> Option { - Some(match self { - TimelineSyncState::Evicted(None) => return None, - TimelineSyncState::Ready(lsn) => lsn, - TimelineSyncState::AwaitsDownload(lsn) => lsn, - TimelineSyncState::CloudOnly(lsn) => lsn, - TimelineSyncState::Evicted(Some(lsn)) => lsn, - }) - .copied() +impl<'a> From<&'a RepositoryTimeline> for LocalTimelineState { + fn from(local_timeline_entry: &'a RepositoryTimeline) -> Self { + match local_timeline_entry { + RepositoryTimeline::Loaded(_) => LocalTimelineState::Loaded, + RepositoryTimeline::Unloaded { .. } => LocalTimelineState::Unloaded, + } } } @@ -362,7 +350,7 @@ pub mod repo_harness { use crate::{ config::PageServerConf, - layered_repository::{LayeredRepository, TIMELINES_SEGMENT_NAME}, + layered_repository::LayeredRepository, walredo::{WalRedoError, WalRedoManager}, }; @@ -395,7 +383,6 @@ pub mod repo_harness { let repo_dir = PageServerConf::test_repo_dir(test_name); let _ = fs::remove_dir_all(&repo_dir); fs::create_dir_all(&repo_dir)?; - fs::create_dir_all(&repo_dir.join(TIMELINES_SEGMENT_NAME))?; let conf = PageServerConf::dummy_conf(repo_dir); // Make a static copy of the config. This can never be free'd, but that's @@ -404,19 +391,45 @@ pub mod repo_harness { let tenant_id = ZTenantId::generate(); fs::create_dir_all(conf.tenant_path(&tenant_id))?; + fs::create_dir_all(conf.timelines_path(&tenant_id))?; Ok(Self { conf, tenant_id }) } pub fn load(&self) -> Box { + self.try_load().expect("failed to load test repo") + } + + pub fn try_load(&self) -> Result> { let walredo_mgr = Arc::new(TestRedoManager); - Box::new(LayeredRepository::new( + let repo = Box::new(LayeredRepository::new( self.conf, walredo_mgr, self.tenant_id, + Arc::new(tokio::sync::RwLock::new(RemoteTimelineIndex::empty())), false, - )) + )); + // populate repo with locally available timelines + for timeline_dir_entry in fs::read_dir(self.conf.timelines_path(&self.tenant_id)) + .expect("should be able to read timelines dir") + { + let timeline_dir_entry = timeline_dir_entry.unwrap(); + let timeline_id: ZTimelineId = timeline_dir_entry + .path() + .file_name() + .unwrap() + .to_string_lossy() + .parse() + .unwrap(); + + repo.apply_timeline_remote_sync_status_update( + timeline_id, + TimelineSyncStatusUpdate::Downloaded, + )?; + } + + Ok(repo) } pub fn timeline_path(&self, timeline_id: &ZTimelineId) -> PathBuf { @@ -835,10 +848,9 @@ mod tests { // Create a branch, check that the relation is visible there repo.branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Lsn(0x30))?; - let newtline = match repo.get_timeline(NEW_TIMELINE_ID)?.local_timeline() { - Some(timeline) => timeline, - None => panic!("Should have a local timeline"), - }; + let newtline = repo + .get_timeline_load(NEW_TIMELINE_ID) + .expect("Should have a local timeline"); let new_writer = newtline.writer(); assert!(newtline @@ -896,10 +908,9 @@ mod tests { // Branch the history, modify relation differently on the new timeline repo.branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Lsn(0x30))?; - let newtline = match repo.get_timeline(NEW_TIMELINE_ID)?.local_timeline() { - Some(timeline) => timeline, - None => panic!("Should have a local timeline"), - }; + let newtline = repo + .get_timeline_load(NEW_TIMELINE_ID) + .expect("Should have a local timeline"); let new_writer = newtline.writer(); new_writer.put_page_image(TESTREL_A, 0, Lsn(0x40), TEST_IMG("bar blk 0 at 4"))?; @@ -1046,11 +1057,9 @@ mod tests { make_some_layers(&tline, Lsn(0x20))?; repo.branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Lsn(0x40))?; - let newtline = match repo.get_timeline(NEW_TIMELINE_ID)?.local_timeline() { - Some(timeline) => timeline, - None => panic!("Should have a local timeline"), - }; - + let newtline = repo + .get_timeline_load(NEW_TIMELINE_ID) + .expect("Should have a local timeline"); // this removes layers before lsn 40 (50 minus 10), so there are two remaining layers, image and delta for 31-50 repo.gc_iteration(Some(TIMELINE_ID), 0x10, false)?; assert!(newtline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x25)).is_ok()); @@ -1067,10 +1076,9 @@ mod tests { make_some_layers(&tline, Lsn(0x20))?; repo.branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Lsn(0x40))?; - let newtline = match repo.get_timeline(NEW_TIMELINE_ID)?.local_timeline() { - Some(timeline) => timeline, - None => panic!("Should have a local timeline"), - }; + let newtline = repo + .get_timeline_load(NEW_TIMELINE_ID) + .expect("Should have a local timeline"); make_some_layers(&newtline, Lsn(0x60))?; @@ -1143,4 +1151,81 @@ mod tests { Ok(()) } + + #[test] + fn timeline_load() -> Result<()> { + const TEST_NAME: &str = "timeline_load"; + let harness = RepoHarness::create(TEST_NAME)?; + { + let repo = harness.load(); + let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0x8000))?; + make_some_layers(&tline, Lsn(0x8000))?; + tline.checkpoint(CheckpointConfig::Forced)?; + } + + let repo = harness.load(); + let tline = repo + .get_timeline(TIMELINE_ID) + .expect("cannot load timeline"); + assert!(matches!(tline, RepositoryTimeline::Unloaded { .. })); + + assert!(repo.get_timeline_load(TIMELINE_ID).is_ok()); + + let tline = repo + .get_timeline(TIMELINE_ID) + .expect("cannot load timeline"); + assert!(matches!(tline, RepositoryTimeline::Loaded(_))); + + Ok(()) + } + + #[test] + fn timeline_load_with_ancestor() -> Result<()> { + const TEST_NAME: &str = "timeline_load"; + let harness = RepoHarness::create(TEST_NAME)?; + // create two timelines + { + let repo = harness.load(); + let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0))?; + + make_some_layers(&tline, Lsn(0x20))?; + tline.checkpoint(CheckpointConfig::Forced)?; + + repo.branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Lsn(0x40))?; + + let newtline = repo + .get_timeline_load(NEW_TIMELINE_ID) + .expect("Should have a local timeline"); + + make_some_layers(&newtline, Lsn(0x60))?; + tline.checkpoint(CheckpointConfig::Forced)?; + } + + // check that both of them are initially unloaded + let repo = harness.load(); + { + let tline = repo.get_timeline(TIMELINE_ID).expect("cannot get timeline"); + assert!(matches!(tline, RepositoryTimeline::Unloaded { .. })); + + let tline = repo + .get_timeline(NEW_TIMELINE_ID) + .expect("cannot get timeline"); + assert!(matches!(tline, RepositoryTimeline::Unloaded { .. })); + } + // load only child timeline + let _ = repo + .get_timeline_load(NEW_TIMELINE_ID) + .expect("cannot load timeline"); + + // check that both, child and ancestor are loaded + let tline = repo + .get_timeline(NEW_TIMELINE_ID) + .expect("cannot get timeline"); + assert!(matches!(tline, RepositoryTimeline::Loaded(_))); + + let tline = repo.get_timeline(TIMELINE_ID).expect("cannot get timeline"); + assert!(matches!(tline, RepositoryTimeline::Loaded(_))); + + Ok(()) + } } diff --git a/pageserver/src/tenant_mgr.rs b/pageserver/src/tenant_mgr.rs index 568088fc1d..8584bdd424 100644 --- a/pageserver/src/tenant_mgr.rs +++ b/pageserver/src/tenant_mgr.rs @@ -3,16 +3,19 @@ use crate::config::PageServerConf; use crate::layered_repository::LayeredRepository; -use crate::repository::{Repository, Timeline, TimelineSyncState}; +use crate::remote_storage::RemoteTimelineIndex; +use crate::repository::{Repository, Timeline, TimelineSyncStatusUpdate}; use crate::thread_mgr; use crate::thread_mgr::ThreadKind; use crate::timelines; +use crate::timelines::CreateRepo; use crate::walredo::PostgresRedoManager; use crate::CheckpointConfig; use anyhow::{Context, Result}; use lazy_static::lazy_static; use log::*; use serde::{Deserialize, Serialize}; +use std::collections::hash_map::Entry; use std::collections::HashMap; use std::fmt; use std::sync::{Arc, Mutex, MutexGuard}; @@ -57,79 +60,67 @@ fn access_tenants() -> MutexGuard<'static, HashMap> { TENANTS.lock().unwrap() } -/// Updates tenants' repositories, changing their timelines state in memory. -pub fn set_timeline_states( +// Sets up wal redo manager and repository for tenant. Reduces code duplocation. +// Used during pageserver startup, or when new tenant is attached to pageserver. +pub fn load_local_repo( conf: &'static PageServerConf, - timeline_states: HashMap>, -) { - if timeline_states.is_empty() { - debug!("no timeline state updates to perform"); - return; - } - - info!("Updating states for {} timelines", timeline_states.len()); - trace!("States: {:?}", timeline_states); - + tenant_id: ZTenantId, + remote_index: &Arc>, +) -> Arc { let mut m = access_tenants(); - for (tenant_id, timeline_states) in timeline_states { - let tenant = m.entry(tenant_id).or_insert_with(|| { - // TODO (rodionov) reuse one of the initialisation routines - // Set up a WAL redo manager, for applying WAL records. - let walredo_mgr = PostgresRedoManager::new(conf, tenant_id); + let tenant = m.entry(tenant_id).or_insert_with(|| { + // Set up a WAL redo manager, for applying WAL records. + let walredo_mgr = PostgresRedoManager::new(conf, tenant_id); - // Set up an object repository, for actual data storage. - let repo: Arc = Arc::new(LayeredRepository::new( - conf, - Arc::new(walredo_mgr), - tenant_id, - conf.remote_storage_config.is_some(), - )); - Tenant { - state: TenantState::Idle, - repo, - } - }); - if let Err(e) = put_timelines_into_tenant(tenant, tenant_id, timeline_states) { - error!( - "Failed to update timeline states for tenant {}: {:?}", - tenant_id, e - ); + // Set up an object repository, for actual data storage. + let repo: Arc = Arc::new(LayeredRepository::new( + conf, + Arc::new(walredo_mgr), + tenant_id, + Arc::clone(remote_index), + conf.remote_storage_config.is_some(), + )); + Tenant { + state: TenantState::Idle, + repo, } - } + }); + Arc::clone(&tenant.repo) } -fn put_timelines_into_tenant( - tenant: &mut Tenant, - tenant_id: ZTenantId, - timeline_states: HashMap, -) -> anyhow::Result<()> { - for (timeline_id, timeline_state) in timeline_states { - // If the timeline is being put into any other state than Ready, - // stop any threads operating on it. - // - // FIXME: This is racy. A page service thread could just get - // handle on the Timeline, before we call set_timeline_state() - if !matches!(timeline_state, TimelineSyncState::Ready(_)) { - thread_mgr::shutdown_threads(None, Some(tenant_id), Some(timeline_id)); - - // Should we run a final checkpoint to flush all the data to - // disk? Doesn't seem necessary; all of the states other than - // Ready imply that the data on local disk is corrupt or incomplete, - // and we don't want to flush that to disk. - } - - tenant - .repo - .set_timeline_state(timeline_id, timeline_state) - .with_context(|| { - format!( - "Failed to update timeline {} state to {:?}", - timeline_id, timeline_state - ) - })?; +/// Updates tenants' repositories, changing their timelines state in memory. +pub fn apply_timeline_sync_status_updates( + conf: &'static PageServerConf, + remote_index: Arc>, + sync_status_updates: HashMap>, +) { + if sync_status_updates.is_empty() { + debug!("no sync status updates to apply"); + return; } + info!( + "Applying sync status updates for {} timelines", + sync_status_updates.len() + ); + trace!("Sync status updates: {:?}", sync_status_updates); - Ok(()) + for (tenant_id, tenant_timelines_sync_status_updates) in sync_status_updates { + let repo = load_local_repo(conf, tenant_id, &remote_index); + + for (timeline_id, timeline_sync_status_update) in tenant_timelines_sync_status_updates { + match repo.apply_timeline_remote_sync_status_update(timeline_id, timeline_sync_status_update) + { + Ok(_) => debug!( + "successfully applied timeline sync status update: {} -> {}", + timeline_id, timeline_sync_status_update + ), + Err(e) => error!( + "Failed to apply timeline sync status update for tenant {}. timeline {} update {} Error: {:#}", + tenant_id, timeline_id, timeline_sync_status_update, e + ), + } + } + } } /// @@ -179,24 +170,30 @@ pub fn shutdown_all_tenants() { pub fn create_tenant_repository( conf: &'static PageServerConf, - new_tenant_id: Option, + tenantid: ZTenantId, + remote_index: Arc>, ) -> Result> { - let new_tenant_id = new_tenant_id.unwrap_or_else(ZTenantId::generate); - let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, new_tenant_id)); - match timelines::create_repo(conf, new_tenant_id, wal_redo_manager)? { - Some(repo) => { - access_tenants() - .entry(new_tenant_id) - .or_insert_with(|| Tenant { - state: TenantState::Idle, - repo, - }); - Ok(Some(new_tenant_id)) - } - None => { - debug!("repository already exists for tenant {}", new_tenant_id); + match access_tenants().entry(tenantid) { + Entry::Occupied(_) => { + debug!("tenant {} already exists", tenantid); Ok(None) } + Entry::Vacant(v) => { + let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenantid)); + let repo = timelines::create_repo( + conf, + tenantid, + CreateRepo::Real { + wal_redo_manager, + remote_index, + }, + )?; + v.insert(Tenant { + state: TenantState::Idle, + repo, + }); + Ok(Some(tenantid)) + } } } @@ -255,19 +252,19 @@ pub fn get_repository_for_tenant(tenantid: ZTenantId) -> Result Result> { get_repository_for_tenant(tenantid)? - .get_timeline(timelineid)? - .local_timeline() - .with_context(|| format!("cannot fetch timeline {}", timelineid)) + .get_timeline_load(timelineid) + .with_context(|| format!("Timeline {} not found for tenant {}", timelineid, tenantid)) } #[derive(Serialize, Deserialize, Clone)] diff --git a/pageserver/src/timelines.rs b/pageserver/src/timelines.rs index 4de131ef70..9cfc21b413 100644 --- a/pageserver/src/timelines.rs +++ b/pageserver/src/timelines.rs @@ -2,8 +2,9 @@ //! Timeline management code // -use anyhow::{anyhow, bail, Context, Result}; +use anyhow::{bail, Context, Result}; use postgres_ffi::ControlFileData; +use serde::{Deserialize, Serialize}; use std::{ fs, path::Path, @@ -12,135 +13,126 @@ use std::{ }; use tracing::*; -use zenith_utils::lsn::Lsn; -use zenith_utils::zid::{ZTenantId, ZTimelineId}; +use zenith_utils::zid::{ZTenantId, ZTenantTimelineId, ZTimelineId}; use zenith_utils::{crashsafe_dir, logging}; +use zenith_utils::{lsn::Lsn, zid::HexZTimelineId}; -use crate::{config::PageServerConf, repository::Repository}; +use crate::{ + config::PageServerConf, + layered_repository::metadata::TimelineMetadata, + remote_storage::RemoteTimelineIndex, + repository::{LocalTimelineState, Repository}, +}; use crate::{import_datadir, LOG_FILE_NAME}; use crate::{layered_repository::LayeredRepository, walredo::WalRedoManager}; use crate::{repository::RepositoryTimeline, tenant_mgr}; use crate::{repository::Timeline, CheckpointConfig}; -#[derive(Clone)] -pub enum TimelineInfo { - Local { - timeline_id: ZTimelineId, - tenant_id: ZTenantId, - last_record_lsn: Lsn, - prev_record_lsn: Lsn, - ancestor_timeline_id: Option, - ancestor_lsn: Option, - disk_consistent_lsn: Lsn, - current_logical_size: usize, - current_logical_size_non_incremental: Option, - }, - Remote { - timeline_id: ZTimelineId, - tenant_id: ZTenantId, - disk_consistent_lsn: Lsn, - }, +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct LocalTimelineInfo { + pub ancestor_timeline_id: Option, + pub ancestor_lsn: Option, + pub last_record_lsn: Lsn, + pub prev_record_lsn: Option, + pub disk_consistent_lsn: Lsn, + pub current_logical_size: Option, // is None when timeline is Unloaded + pub current_logical_size_non_incremental: Option, + pub timeline_state: LocalTimelineState, } -impl TimelineInfo { - pub fn from_repo_timeline( - tenant_id: ZTenantId, - repo_timeline: RepositoryTimeline, - include_non_incremental_logical_size: bool, - ) -> Self { - match repo_timeline { - RepositoryTimeline::Local { id, timeline } => { - let ancestor_timeline_id = timeline.get_ancestor_timeline_id(); - let ancestor_lsn = if ancestor_timeline_id.is_some() { - Some(timeline.get_ancestor_lsn()) - } else { - None - }; - - Self::Local { - timeline_id: id, - tenant_id, - last_record_lsn: timeline.get_last_record_lsn(), - prev_record_lsn: timeline.get_prev_record_lsn(), - ancestor_timeline_id, - ancestor_lsn, - disk_consistent_lsn: timeline.get_disk_consistent_lsn(), - current_logical_size: timeline.get_current_logical_size(), - current_logical_size_non_incremental: get_current_logical_size_non_incremental( - include_non_incremental_logical_size, - timeline.as_ref(), - ), - } - } - RepositoryTimeline::Remote { - id, - disk_consistent_lsn, - } => Self::Remote { - timeline_id: id, - tenant_id, - disk_consistent_lsn, - }, - } - } - - pub fn from_dyn_timeline( - tenant_id: ZTenantId, - timeline_id: ZTimelineId, +impl LocalTimelineInfo { + pub fn from_loaded_timeline( timeline: &dyn Timeline, include_non_incremental_logical_size: bool, - ) -> Self { - let ancestor_timeline_id = timeline.get_ancestor_timeline_id(); - let ancestor_lsn = if ancestor_timeline_id.is_some() { - Some(timeline.get_ancestor_lsn()) - } else { - None - }; - - Self::Local { - timeline_id, - tenant_id, - last_record_lsn: timeline.get_last_record_lsn(), - prev_record_lsn: timeline.get_prev_record_lsn(), - ancestor_timeline_id, - ancestor_lsn, + ) -> anyhow::Result { + let last_record_lsn = timeline.get_last_record_lsn(); + let info = LocalTimelineInfo { + ancestor_timeline_id: timeline + .get_ancestor_timeline_id() + .map(HexZTimelineId::from), + ancestor_lsn: { + match timeline.get_ancestor_lsn() { + Lsn(0) => None, + lsn @ Lsn(_) => Some(lsn), + } + }, disk_consistent_lsn: timeline.get_disk_consistent_lsn(), - current_logical_size: timeline.get_current_logical_size(), - current_logical_size_non_incremental: get_current_logical_size_non_incremental( - include_non_incremental_logical_size, - timeline, - ), + last_record_lsn, + prev_record_lsn: Some(timeline.get_prev_record_lsn()), + timeline_state: LocalTimelineState::Loaded, + current_logical_size: Some(timeline.get_current_logical_size()), + current_logical_size_non_incremental: if include_non_incremental_logical_size { + Some(timeline.get_current_logical_size_non_incremental(last_record_lsn)?) + } else { + None + }, + }; + Ok(info) + } + + pub fn from_unloaded_timeline(metadata: &TimelineMetadata) -> Self { + LocalTimelineInfo { + ancestor_timeline_id: metadata.ancestor_timeline().map(HexZTimelineId::from), + ancestor_lsn: { + match metadata.ancestor_lsn() { + Lsn(0) => None, + lsn @ Lsn(_) => Some(lsn), + } + }, + disk_consistent_lsn: metadata.disk_consistent_lsn(), + last_record_lsn: metadata.disk_consistent_lsn(), + prev_record_lsn: metadata.prev_record_lsn(), + timeline_state: LocalTimelineState::Unloaded, + current_logical_size: None, + current_logical_size_non_incremental: None, } } - pub fn timeline_id(&self) -> ZTimelineId { - match *self { - TimelineInfo::Local { timeline_id, .. } => timeline_id, - TimelineInfo::Remote { timeline_id, .. } => timeline_id, - } - } - - pub fn tenant_id(&self) -> ZTenantId { - match *self { - TimelineInfo::Local { tenant_id, .. } => tenant_id, - TimelineInfo::Remote { tenant_id, .. } => tenant_id, + pub fn from_repo_timeline( + repo_timeline: RepositoryTimeline, + include_non_incremental_logical_size: bool, + ) -> anyhow::Result { + match repo_timeline { + RepositoryTimeline::Loaded(timeline) => { + Self::from_loaded_timeline(timeline.as_ref(), include_non_incremental_logical_size) + } + RepositoryTimeline::Unloaded { metadata } => { + Ok(Self::from_unloaded_timeline(&metadata)) + } } } } -fn get_current_logical_size_non_incremental( - include_non_incremental_logical_size: bool, - timeline: &dyn Timeline, -) -> Option { - if !include_non_incremental_logical_size { - return None; - } - match timeline.get_current_logical_size_non_incremental(timeline.get_last_record_lsn()) { - Ok(size) => Some(size), - Err(e) => { - error!("Failed to get non-incremental logical size: {:?}", e); - None - } - } +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct RemoteTimelineInfo { + pub remote_consistent_lsn: Option, + pub awaits_download: bool, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct TimelineInfo { + #[serde(with = "hex")] + pub tenant_id: ZTenantId, + #[serde(with = "hex")] + pub timeline_id: ZTimelineId, + pub local: Option, + pub remote: Option, +} + +pub fn extract_remote_timeline_info( + tenant_id: ZTenantId, + timeline_id: ZTimelineId, + remote_index: &RemoteTimelineIndex, +) -> Option { + remote_index + .timeline_entry(&ZTenantTimelineId { + tenant_id, + timeline_id, + }) + .map(|remote_entry| RemoteTimelineInfo { + remote_consistent_lsn: remote_entry.disk_consistent_lsn(), + awaits_download: remote_entry.get_awaits_download(), + }) } #[derive(Debug, Clone, Copy)] @@ -158,25 +150,12 @@ pub fn init_pageserver( // use true as daemonize parameter because otherwise we pollute zenith cli output with a few pages long output of info messages let _log_file = logging::init(LOG_FILE_NAME, true)?; - // We don't use the real WAL redo manager, because we don't want to spawn the WAL redo - // process during repository initialization. - // - // FIXME: That caused trouble, because the WAL redo manager spawned a thread that launched - // initdb in the background, and it kept running even after the "zenith init" had exited. - // In tests, we started the page server immediately after that, so that initdb was still - // running in the background, and we failed to run initdb again in the same directory. This - // has been solved for the rapid init+start case now, but the general race condition remains - // if you restart the server quickly. The WAL redo manager doesn't use a separate thread - // anymore, but I think that could still happen. - let dummy_redo_mgr = Arc::new(crate::walredo::DummyRedoManager {}); - crashsafe_dir::create_dir_all(conf.tenants_path())?; if let Some(tenant_id) = create_tenant { println!("initializing tenantid {}", tenant_id); - let repo = create_repo(conf, tenant_id, dummy_redo_mgr) - .context("failed to create repo")? - .ok_or_else(|| anyhow!("For newely created pageserver, found already existing repository for tenant {}", tenant_id))?; + let repo = + create_repo(conf, tenant_id, CreateRepo::Dummy).context("failed to create repo")?; let new_timeline_id = initial_timeline_id.unwrap_or_else(ZTimelineId::generate); bootstrap_timeline(conf, tenant_id, new_timeline_id, repo.as_ref()) .context("failed to create initial timeline")?; @@ -189,15 +168,45 @@ pub fn init_pageserver( Ok(()) } +pub enum CreateRepo { + Real { + wal_redo_manager: Arc, + remote_index: Arc>, + }, + Dummy, +} + pub fn create_repo( conf: &'static PageServerConf, tenant_id: ZTenantId, - wal_redo_manager: Arc, -) -> Result>> { + create_repo: CreateRepo, +) -> Result> { + let (wal_redo_manager, remote_index) = match create_repo { + CreateRepo::Real { + wal_redo_manager, + remote_index, + } => (wal_redo_manager, remote_index), + CreateRepo::Dummy => { + // We don't use the real WAL redo manager, because we don't want to spawn the WAL redo + // process during repository initialization. + // + // FIXME: That caused trouble, because the WAL redo manager spawned a thread that launched + // initdb in the background, and it kept running even after the "zenith init" had exited. + // In tests, we started the page server immediately after that, so that initdb was still + // running in the background, and we failed to run initdb again in the same directory. This + // has been solved for the rapid init+start case now, but the general race condition remains + // if you restart the server quickly. The WAL redo manager doesn't use a separate thread + // anymore, but I think that could still happen. + let wal_redo_manager = Arc::new(crate::walredo::DummyRedoManager {}); + + let remote_index = Arc::new(tokio::sync::RwLock::new(RemoteTimelineIndex::empty())); + (wal_redo_manager as _, remote_index) + } + }; + let repo_dir = conf.tenant_path(&tenant_id); if repo_dir.exists() { - debug!("repo for {} already exists", tenant_id); - return Ok(None); + bail!("tenant {} directory already exists", tenant_id); } // top-level dir may exist if we are creating it through CLI @@ -206,12 +215,13 @@ pub fn create_repo( crashsafe_dir::create_dir(conf.timelines_path(&tenant_id))?; info!("created directory structure in {}", repo_dir.display()); - Ok(Some(Arc::new(LayeredRepository::new( + Ok(Arc::new(LayeredRepository::new( conf, wal_redo_manager, tenant_id, + remote_index, conf.remote_storage_config.is_some(), - )))) + ))) } // Returns checkpoint LSN from controlfile @@ -299,30 +309,25 @@ fn bootstrap_timeline( Ok(timeline) } -pub(crate) fn get_timelines( +pub(crate) fn get_local_timelines( tenant_id: ZTenantId, include_non_incremental_logical_size: bool, -) -> Result> { +) -> Result> { let repo = tenant_mgr::get_repository_for_tenant(tenant_id) .with_context(|| format!("Failed to get repo for tenant {}", tenant_id))?; + let repo_timelines = repo.list_timelines(); - Ok(repo - .list_timelines() - .with_context(|| format!("Failed to list timelines for tenant {}", tenant_id))? - .into_iter() - .filter_map(|timeline| match timeline { - RepositoryTimeline::Local { timeline, id } => Some((id, timeline)), - RepositoryTimeline::Remote { .. } => None, - }) - .map(|(timeline_id, timeline)| { - TimelineInfo::from_dyn_timeline( - tenant_id, - timeline_id, - timeline.as_ref(), + let mut local_timeline_info = Vec::with_capacity(repo_timelines.len()); + for (timeline_id, repository_timeline) in repo_timelines { + local_timeline_info.push(( + timeline_id, + LocalTimelineInfo::from_repo_timeline( + repository_timeline, include_non_incremental_logical_size, - ) - }) - .collect()) + )?, + )) + } + Ok(local_timeline_info) } pub(crate) fn create_timeline( @@ -336,16 +341,8 @@ pub(crate) fn create_timeline( let repo = tenant_mgr::get_repository_for_tenant(tenant_id)?; if conf.timeline_path(&new_timeline_id, &tenant_id).exists() { - match repo.get_timeline(new_timeline_id)? { - RepositoryTimeline::Local { id, .. } => { - debug!("timeline {} already exists", id); - return Ok(None); - } - RepositoryTimeline::Remote { id, .. } => bail!( - "timeline {} already exists in pageserver's remote storage", - id - ), - } + debug!("timeline {} already exists", new_timeline_id); + return Ok(None); } let mut start_lsn = ancestor_start_lsn.unwrap_or(Lsn(0)); @@ -353,15 +350,8 @@ pub(crate) fn create_timeline( let new_timeline_info = match ancestor_timeline_id { Some(ancestor_timeline_id) => { let ancestor_timeline = repo - .get_timeline(ancestor_timeline_id) - .with_context(|| format!("Cannot get ancestor timeline {}", ancestor_timeline_id))? - .local_timeline() - .with_context(|| { - format!( - "Cannot branch off the timeline {} that's not present locally", - ancestor_timeline_id - ) - })?; + .get_timeline_load(ancestor_timeline_id) + .context("Cannot branch off the timeline that's not present locally")?; if start_lsn == Lsn(0) { // Find end of WAL on the old timeline @@ -391,18 +381,20 @@ pub(crate) fn create_timeline( } repo.branch_timeline(ancestor_timeline_id, new_timeline_id, start_lsn)?; // load the timeline into memory - let loaded_timeline = repo.get_timeline(new_timeline_id)?; - TimelineInfo::from_repo_timeline(tenant_id, loaded_timeline, false) + let loaded_timeline = repo.get_timeline_load(new_timeline_id)?; + LocalTimelineInfo::from_loaded_timeline(loaded_timeline.as_ref(), false) + .context("cannot fill timeline info")? } None => { let new_timeline = bootstrap_timeline(conf, tenant_id, new_timeline_id, repo.as_ref())?; - TimelineInfo::from_dyn_timeline( - tenant_id, - new_timeline_id, - new_timeline.as_ref(), - false, - ) + LocalTimelineInfo::from_loaded_timeline(new_timeline.as_ref(), false) + .context("cannot fill timeline info")? } }; - Ok(Some(new_timeline_info)) + Ok(Some(TimelineInfo { + tenant_id, + timeline_id: new_timeline_id, + local: Some(new_timeline_info), + remote: None, + })) } diff --git a/pageserver/src/walreceiver.rs b/pageserver/src/walreceiver.rs index 6fff1d062d..305dd4b3a2 100644 --- a/pageserver/src/walreceiver.rs +++ b/pageserver/src/walreceiver.rs @@ -31,6 +31,7 @@ use tracing::*; use zenith_utils::lsn::Lsn; use zenith_utils::pq_proto::ZenithFeedback; use zenith_utils::zid::ZTenantId; +use zenith_utils::zid::ZTenantTimelineId; use zenith_utils::zid::ZTimelineId; // @@ -111,18 +112,18 @@ fn get_wal_producer_connstr(tenantid: ZTenantId, timelineid: ZTimelineId) -> Str // fn thread_main( conf: &'static PageServerConf, - tenantid: ZTenantId, - timelineid: ZTimelineId, + tenant_id: ZTenantId, + timeline_id: ZTimelineId, ) -> Result<()> { - let _enter = info_span!("WAL receiver", timeline = %timelineid, tenant = %tenantid).entered(); + let _enter = info_span!("WAL receiver", timeline = %timeline_id, tenant = %tenant_id).entered(); info!("WAL receiver thread started"); // Look up the current WAL producer address - let wal_producer_connstr = get_wal_producer_connstr(tenantid, timelineid); + let wal_producer_connstr = get_wal_producer_connstr(tenant_id, timeline_id); // Make a connection to the WAL safekeeper, or directly to the primary PostgreSQL server, // and start streaming WAL from it. - let res = walreceiver_main(conf, tenantid, timelineid, &wal_producer_connstr); + let res = walreceiver_main(conf, tenant_id, timeline_id, &wal_producer_connstr); // TODO cleanup info messages if let Err(e) = res { @@ -130,20 +131,20 @@ fn thread_main( } else { info!( "walreceiver disconnected tenant {}, timelineid {}", - tenantid, timelineid + tenant_id, timeline_id ); } // Drop it from list of active WAL_RECEIVERS // so that next callmemaybe request launched a new thread - drop_wal_receiver(tenantid, timelineid); + drop_wal_receiver(tenant_id, timeline_id); Ok(()) } fn walreceiver_main( _conf: &PageServerConf, - tenantid: ZTenantId, - timelineid: ZTimelineId, + tenant_id: ZTenantId, + timeline_id: ZTimelineId, wal_producer_connstr: &str, ) -> Result<(), Error> { // Connect to the database in replication mode. @@ -182,13 +183,16 @@ fn walreceiver_main( let end_of_wal = Lsn::from(u64::from(identify.xlogpos)); let mut caught_up = false; - let timeline = - tenant_mgr::get_timeline_for_tenant(tenantid, timelineid).with_context(|| { - format!( - "Can not start the walrecever for a remote tenant {}, timeline {}", - tenantid, timelineid, - ) - })?; + let repo = tenant_mgr::get_repository_for_tenant(tenant_id) + .with_context(|| format!("no repository found for tenant {}", tenant_id))?; + let timeline = repo.get_timeline_load(timeline_id).with_context(|| { + format!( + "local timeline {} not found for tenant {}", + timeline_id, tenant_id + ) + })?; + + let remote_index = repo.get_remote_index(); // // Start streaming the WAL, from where we left off previously. @@ -292,11 +296,19 @@ fn walreceiver_main( }; if let Some(last_lsn) = status_update { - let timeline_synced_disk_consistent_lsn = - tenant_mgr::get_repository_for_tenant(tenantid)? - .get_timeline_state(timelineid) - .and_then(|state| state.remote_disk_consistent_lsn()) - .unwrap_or(Lsn(0)); + let timeline_remote_consistent_lsn = runtime.block_on(async { + remote_index + .read() + .await + // here we either do not have this timeline in remote index + // or there were no checkpoints for it yet + .timeline_entry(&ZTenantTimelineId { + tenant_id, + timeline_id, + }) + .and_then(|e| e.disk_consistent_lsn()) + .unwrap_or(Lsn(0)) // no checkpoint was uploaded + }); // The last LSN we processed. It is not guaranteed to survive pageserver crash. let write_lsn = u64::from(last_lsn); @@ -304,7 +316,7 @@ fn walreceiver_main( let flush_lsn = u64::from(timeline.get_disk_consistent_lsn()); // The last LSN that is synced to remote storage and is guaranteed to survive pageserver crash // Used by safekeepers to remove WAL preceding `remote_consistent_lsn`. - let apply_lsn = u64::from(timeline_synced_disk_consistent_lsn); + let apply_lsn = u64::from(timeline_remote_consistent_lsn); let ts = SystemTime::now(); // Send zenith feedback message. diff --git a/test_runner/batch_others/test_remote_storage.py b/test_runner/batch_others/test_remote_storage.py index edcc768819..8689838089 100644 --- a/test_runner/batch_others/test_remote_storage.py +++ b/test_runner/batch_others/test_remote_storage.py @@ -5,7 +5,7 @@ import time, shutil, os from contextlib import closing from pathlib import Path from uuid import UUID -from fixtures.zenith_fixtures import ZenithEnvBuilder +from fixtures.zenith_fixtures import ZenithEnvBuilder, assert_local, wait_for, wait_for_last_record_lsn, wait_for_upload from fixtures.log_helper import log import pytest @@ -26,7 +26,6 @@ import pytest # * queries the specific data, ensuring that it matches the one stored before # # The tests are done for all types of remote storage pageserver supports. -@pytest.mark.skip(reason="will be fixed with https://github.com/zenithdb/zenith/issues/1193") @pytest.mark.parametrize('storage_type', ['local_fs', 'mock_s3']) def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder, storage_type: str): zenith_env_builder.rust_log_override = 'debug' @@ -45,6 +44,8 @@ def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder, env = zenith_env_builder.init_start() pg = env.postgres.create_start('main') + client = env.pageserver.http_client() + tenant_id = pg.safe_psql("show zenith.zenith_tenant")[0][0] timeline_id = pg.safe_psql("show zenith.zenith_timeline")[0][0] @@ -54,13 +55,21 @@ def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder, CREATE TABLE t1(id int primary key, secret text); INSERT INTO t1 VALUES ({data_id}, '{data_secret}'); ''') + cur.execute("SELECT pg_current_wal_flush_lsn()") + current_lsn = int(cur.fetchone()[0].split('/')[1], base=16) + + # wait until pageserver receives that data + wait_for_last_record_lsn(client, UUID(tenant_id), UUID(timeline_id), current_lsn) # run checkpoint manually to be sure that data landed in remote storage with closing(env.pageserver.connect()) as psconn: with psconn.cursor() as pscur: - pscur.execute(f"do_gc {tenant_id} {timeline_id}") - log.info("waiting for upload") # TODO api to check if upload is done - time.sleep(2) + pscur.execute(f"checkpoint {tenant_id} {timeline_id}") + + log.info("waiting for upload") + # wait until pageserver successfully uploaded a checkpoint to remote storage + wait_for_upload(client, UUID(tenant_id), UUID(timeline_id), current_lsn) + log.info("upload is done") ##### Stop the first pageserver instance, erase all its data env.postgres.stop_all() @@ -73,26 +82,12 @@ def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder, ##### Second start, restore the data and ensure it's the same env.pageserver.start() - client = env.pageserver.http_client() client.timeline_attach(UUID(tenant_id), UUID(timeline_id)) - # FIXME cannot handle duplicate download requests (which might be caused by repeated timeline detail calls) - # subject to fix in https://github.com/zenithdb/zenith/issues/997 - time.sleep(5) log.info("waiting for timeline redownload") - attempts = 0 - while True: - timeline_details = client.timeline_detail(UUID(tenant_id), UUID(timeline_id)) - assert timeline_details['timeline_id'] == timeline_id - assert timeline_details['tenant_id'] == tenant_id - if timeline_details['kind'] == 'Local': - log.info("timeline downloaded, checking its data") - break - attempts += 1 - if attempts > 10: - raise Exception("timeline redownload failed") - log.debug("still waiting") - time.sleep(1) + wait_for(number_of_iterations=10, + interval=1, + func=lambda: assert_local(client, UUID(tenant_id), UUID(timeline_id))) pg = env.postgres.create_start('main') with closing(pg.connect()) as conn: diff --git a/test_runner/batch_others/test_tenant_relocation.py b/test_runner/batch_others/test_tenant_relocation.py index 7a9d478f16..e4492e5393 100644 --- a/test_runner/batch_others/test_tenant_relocation.py +++ b/test_runner/batch_others/test_tenant_relocation.py @@ -3,17 +3,19 @@ import os import pathlib import subprocess import threading +from typing import Dict from uuid import UUID from fixtures.log_helper import log import time import signal import pytest -from fixtures.zenith_fixtures import PgProtocol, PortDistributor, Postgres, ZenithEnvBuilder, ZenithPageserverHttpClient, zenith_binpath, pg_distrib_dir +from fixtures.zenith_fixtures import PgProtocol, PortDistributor, Postgres, ZenithEnvBuilder, ZenithPageserverHttpClient, assert_local, wait_for, wait_for_last_record_lsn, wait_for_upload, zenith_binpath, pg_distrib_dir def assert_abs_margin_ratio(a: float, b: float, margin_ratio: float): - assert abs(a - b) / a < margin_ratio, (a, b, margin_ratio) + print("!" * 100, abs(a - b) / a) + assert abs(a - b) / a < margin_ratio, abs(a - b) / a @contextmanager @@ -34,6 +36,7 @@ def new_pageserver_helper(new_pageserver_dir: pathlib.Path, f"-c listen_pg_addr='localhost:{pg_port}'", f"-c listen_http_addr='localhost:{http_port}'", f"-c pg_distrib_dir='{pg_distrib_dir}'", + f"-c id=2", f"-c remote_storage={{local_path='{remote_storage_mock_path}'}}", ] @@ -57,20 +60,6 @@ def new_pageserver_helper(new_pageserver_dir: pathlib.Path, os.kill(pid, signal.SIGQUIT) -def wait_for(number_of_iterations: int, interval: int, func): - last_exception = None - for i in range(number_of_iterations): - try: - res = func() - except Exception as e: - log.info("waiting for %s iteration %s failed", func, i + 1) - last_exception = e - time.sleep(interval) - continue - return res - raise Exception("timed out while waiting for %s" % func) from last_exception - - @contextmanager def pg_cur(pg): with closing(pg.connect()) as conn: @@ -108,13 +97,6 @@ def load(pg: Postgres, stop_event: threading.Event, load_ok_event: threading.Eve log.info('load thread stopped') -def assert_local(pageserver_http_client: ZenithPageserverHttpClient, tenant: UUID, timeline: str): - timeline_detail = pageserver_http_client.timeline_detail(tenant, UUID(timeline)) - assert timeline_detail.get('type') == "Local", timeline_detail - return timeline_detail - - -@pytest.mark.skip(reason="will be fixed with https://github.com/zenithdb/zenith/issues/1193") @pytest.mark.parametrize('with_load', ['with_load', 'without_load']) def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder, port_distributor: PortDistributor, @@ -129,7 +111,7 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder, tenant = env.zenith_cli.create_tenant(UUID("74ee8b079a0e437eb0afea7d26a07209")) log.info("tenant to relocate %s", tenant) - + env.zenith_cli.create_root_branch('main', tenant_id=tenant) env.zenith_cli.create_branch('test_tenant_relocation', tenant_id=tenant) tenant_pg = env.postgres.create_start(branch_name='main', @@ -141,8 +123,8 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder, with conn.cursor() as cur: # save timeline for later gc call cur.execute("SHOW zenith.zenith_timeline") - timeline = cur.fetchone()[0] - log.info("timeline to relocate %s", timeline) + timeline = UUID(cur.fetchone()[0]) + log.info("timeline to relocate %s", timeline.hex) # we rely upon autocommit after each statement # as waiting for acceptors happens there @@ -150,6 +132,15 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder, cur.execute("INSERT INTO t SELECT generate_series(1,1000), 'some payload'") cur.execute("SELECT sum(key) FROM t") assert cur.fetchone() == (500500, ) + cur.execute("SELECT pg_current_wal_flush_lsn()") + + current_lsn = int(cur.fetchone()[0].split('/')[1], base=16) + + pageserver_http = env.pageserver.http_client() + + # wait until pageserver receives that data + wait_for_last_record_lsn(pageserver_http, tenant, timeline, current_lsn) + timeline_detail = pageserver_http.timeline_detail_v2(tenant, timeline) if with_load == 'with_load': # create load table @@ -165,12 +156,10 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder, # run checkpoint manually to be sure that data landed in remote storage with closing(env.pageserver.connect()) as psconn: with psconn.cursor() as pscur: - pscur.execute(f"do_gc {tenant.hex} {timeline}") + pscur.execute(f"checkpoint {tenant.hex} {timeline.hex}") - # ensure upload is completed - pageserver_http_client = env.pageserver.http_client() - timeline_detail = pageserver_http_client.timeline_detail(tenant, UUID(timeline)) - assert timeline_detail['disk_consistent_lsn'] == timeline_detail['timeline_state']['Ready'] + # wait until pageserver successfully uploaded a checkpoint to remote storage + wait_for_upload(pageserver_http, tenant, timeline, current_lsn) log.info("inititalizing new pageserver") # bootstrap second pageserver @@ -182,8 +171,7 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder, log.info("new pageserver ports pg %s http %s", new_pageserver_pg_port, new_pageserver_http_port) pageserver_bin = pathlib.Path(zenith_binpath) / 'pageserver' - new_pageserver_http_client = ZenithPageserverHttpClient(port=new_pageserver_http_port, - auth_token=None) + new_pageserver_http = ZenithPageserverHttpClient(port=new_pageserver_http_port, auth_token=None) with new_pageserver_helper(new_pageserver_dir, pageserver_bin, @@ -192,25 +180,18 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder, new_pageserver_http_port): # call to attach timeline to new pageserver - new_pageserver_http_client.timeline_attach(tenant, UUID(timeline)) - # FIXME cannot handle duplicate download requests, subject to fix in https://github.com/zenithdb/zenith/issues/997 - time.sleep(5) - # new pageserver should in sync (modulo wal tail or vacuum activity) with the old one because there was no new writes since checkpoint + new_pageserver_http.timeline_attach(tenant, timeline) + # new pageserver should be in sync (modulo wal tail or vacuum activity) with the old one because there was no new writes since checkpoint new_timeline_detail = wait_for( number_of_iterations=5, interval=1, - func=lambda: assert_local(new_pageserver_http_client, tenant, timeline)) - assert new_timeline_detail['timeline_state'].get('Ready'), new_timeline_detail + func=lambda: assert_local(new_pageserver_http, tenant, timeline)) + # when load is active these checks can break because lsns are not static # so lets check with some margin - if with_load == 'without_load': - # TODO revisit this once https://github.com/zenithdb/zenith/issues/1049 is fixed - assert_abs_margin_ratio(new_timeline_detail['disk_consistent_lsn'], - timeline_detail['disk_consistent_lsn'], - 0.01) - assert_abs_margin_ratio(new_timeline_detail['timeline_state']['Ready'], - timeline_detail['timeline_state']['Ready'], - 0.01) + assert_abs_margin_ratio(new_timeline_detail['local']['disk_consistent_lsn'], + timeline_detail['local']['disk_consistent_lsn'], + 0.03) # callmemaybe to start replication from safekeeper to the new pageserver # when there is no load there is a clean checkpoint and no wal delta @@ -219,7 +200,9 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder, with pg_cur(PgProtocol(host='localhost', port=new_pageserver_pg_port)) as cur: # "callmemaybe {} {} host={} port={} options='-c ztimelineid={} ztenantid={}'" safekeeper_connstring = f"host=localhost port={env.safekeepers[0].port.pg} options='-c ztimelineid={timeline} ztenantid={tenant} pageserver_connstr=postgresql://no_user:@localhost:{new_pageserver_pg_port}'" - cur.execute("callmemaybe {} {} {}".format(tenant, timeline, safekeeper_connstring)) + cur.execute("callmemaybe {} {} {}".format(tenant.hex, + timeline.hex, + safekeeper_connstring)) tenant_pg.stop() @@ -239,7 +222,7 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder, # detach tenant from old pageserver before we check # that all the data is there to be sure that old pageserver # is no longer involved, and if it is, we will see the errors - pageserver_http_client.timeline_detach(tenant, UUID(timeline)) + pageserver_http.timeline_detach(tenant, timeline) with pg_cur(tenant_pg) as cur: # check that data is still there diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index ec570a7dac..c44a6e431f 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -783,6 +783,15 @@ class ZenithPageserverHttpClient(requests.Session): assert isinstance(res_json, dict) return res_json + def timeline_detail_v2(self, tenant_id: uuid.UUID, timeline_id: uuid.UUID) -> Dict[Any, Any]: + res = self.get( + f"http://localhost:{self.port}/v2/tenant/{tenant_id.hex}/timeline/{timeline_id.hex}?include-non-incremental-logical-size=1" + ) + self.verbose_error(res) + res_json = res.json() + assert isinstance(res_json, dict) + return res_json + def get_metrics(self) -> str: res = self.get(f"http://localhost:{self.port}/metrics") self.verbose_error(res) @@ -866,6 +875,30 @@ class ZenithCli: return uuid.UUID(created_timeline_id) + def create_root_branch(self, branch_name: str, tenant_id: Optional[uuid.UUID] = None): + cmd = [ + 'timeline', + 'create', + '--branch-name', + branch_name, + '--tenant-id', + (tenant_id or self.env.initial_tenant).hex, + ] + + res = self.raw_cli(cmd) + res.check_returncode() + + matches = CREATE_TIMELINE_ID_EXTRACTOR.search(res.stdout) + + created_timeline_id = None + if matches is not None: + created_timeline_id = matches.group('timeline_id') + + if created_timeline_id is None: + raise Exception('could not find timeline id after `zenith timeline create` invocation') + else: + return uuid.UUID(created_timeline_id) + def create_branch(self, new_branch_name: str = DEFAULT_BRANCH_NAME, ancestor_branch_name: Optional[str] = None, @@ -1839,3 +1872,59 @@ def check_restored_datadir_content(test_output_dir: str, env: ZenithEnv, pg: Pos subprocess.run([cmd], stdout=stdout_f, shell=True) assert (mismatch, error) == ([], []) + + +def wait_for(number_of_iterations: int, interval: int, func): + last_exception = None + for i in range(number_of_iterations): + try: + res = func() + except Exception as e: + log.info("waiting for %s iteration %s failed", func, i + 1) + last_exception = e + time.sleep(interval) + continue + return res + raise Exception("timed out while waiting for %s" % func) from last_exception + + +def assert_local(pageserver_http_client: ZenithPageserverHttpClient, + tenant: uuid.UUID, + timeline: uuid.UUID): + timeline_detail = pageserver_http_client.timeline_detail_v2(tenant, timeline) + assert timeline_detail.get('local', {}).get("disk_consistent_lsn"), timeline_detail + return timeline_detail + + +def remote_consistent_lsn(pageserver_http_client: ZenithPageserverHttpClient, + tenant: uuid.UUID, + timeline: uuid.UUID) -> int: + detail = pageserver_http_client.timeline_detail_v2(tenant, timeline) + assert isinstance(detail['remote']['remote_consistent_lsn'], int) + return detail['remote']['remote_consistent_lsn'] + + +def wait_for_upload(pageserver_http_client: ZenithPageserverHttpClient, + tenant: uuid.UUID, + timeline: uuid.UUID, + lsn: int): + """waits for local timeline upload up to specified lsn""" + + wait_for(10, 1, lambda: remote_consistent_lsn(pageserver_http_client, tenant, timeline) >= lsn) + + +def last_record_lsn(pageserver_http_client: ZenithPageserverHttpClient, + tenant: uuid.UUID, + timeline: uuid.UUID) -> int: + detail = pageserver_http_client.timeline_detail_v2(tenant, timeline) + assert isinstance(detail['local']['last_record_lsn'], int) + return detail['local']['last_record_lsn'] + + +def wait_for_last_record_lsn(pageserver_http_client: ZenithPageserverHttpClient, + tenant: uuid.UUID, + timeline: uuid.UUID, + lsn: int): + """waits for pageserver to catch up to a certain lsn""" + + wait_for(10, 1, lambda: last_record_lsn(pageserver_http_client, tenant, timeline) >= lsn) diff --git a/zenith/src/main.rs b/zenith/src/main.rs index dd35427d5d..389c394103 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -299,42 +299,40 @@ fn print_timelines_tree( .iter() .map(|t| { ( - t.timeline_id(), + t.timeline_id, TimelineTreeEl { info: t.clone(), children: BTreeSet::new(), name: timeline_name_mappings - .remove(&ZTenantTimelineId::new(t.tenant_id(), t.timeline_id())), + .remove(&ZTenantTimelineId::new(t.tenant_id, t.timeline_id)), }, ) }) .collect::>(); // Memorize all direct children of each timeline. - for timeline in &timelines { - if let TimelineInfo::Local { - ancestor_timeline_id: Some(tid), - .. - } = timeline + for timeline in timelines.iter() { + if let Some(ancestor_timeline_id) = + timeline.local.as_ref().and_then(|l| l.ancestor_timeline_id) { timelines_hash - .get_mut(tid) + .get_mut(&ZTimelineId::from(ancestor_timeline_id)) .context("missing timeline info in the HashMap")? .children - .insert(timeline.timeline_id()); + .insert(timeline.timeline_id); } } for timeline in timelines_hash.values() { // Start with root local timelines (no ancestors) first. - if let TimelineInfo::Local { - ancestor_timeline_id, - .. - } = &timeline.info + if timeline + .info + .local + .as_ref() + .and_then(|l| l.ancestor_timeline_id) + .is_none() { - if ancestor_timeline_id.is_none() { - print_timeline(0, &Vec::from([true]), timeline, &timelines_hash)?; - } + print_timeline(0, &Vec::from([true]), timeline, &timelines_hash)?; } } @@ -350,20 +348,21 @@ fn print_timeline( timeline: &TimelineTreeEl, timelines: &HashMap, ) -> Result<()> { - let local_or_remote = match timeline.info { - TimelineInfo::Local { .. } => "(L)", - TimelineInfo::Remote { .. } => "(R)", + let local_remote = match (timeline.info.local.as_ref(), timeline.info.remote.as_ref()) { + (None, None) => unreachable!("in this case no info for a timeline is found"), + (None, Some(_)) => "(R)", + (Some(_), None) => "(L)", + (Some(_), Some(_)) => "(L+R)", }; // Draw main padding - print!("{} ", local_or_remote); + print!("{} ", local_remote); if nesting_level > 0 { - let lsn_string = match &timeline.info { - TimelineInfo::Local { ancestor_lsn, .. } => ancestor_lsn - .map(|lsn| lsn.to_string()) - .unwrap_or_else(|| "Unknown local Lsn".to_string()), - TimelineInfo::Remote { .. } => "unknown Lsn (remote)".to_string(), + let ancestor_lsn = match timeline.info.local.as_ref().and_then(|i| i.ancestor_lsn) { + Some(lsn) => lsn.to_string(), + None => "Unknown Lsn".to_string(), }; + let mut br_sym = "┣━"; // Draw each nesting padding with proper style @@ -383,14 +382,14 @@ fn print_timeline( br_sym = "┗━"; } - print!("{} @{}: ", br_sym, lsn_string); + print!("{} @{}: ", br_sym, ancestor_lsn); } // Finally print a timeline id and name with new line println!( "{} [{}]", timeline.name.as_deref().unwrap_or("_no_name_"), - timeline.info.timeline_id() + timeline.info.timeline_id ); let len = timeline.children.len(); @@ -430,7 +429,7 @@ fn get_timeline_infos( Ok(PageServerNode::from_env(env) .timeline_list(tenant_id)? .into_iter() - .map(|timeline_info| (timeline_info.timeline_id(), timeline_info)) + .map(|timeline_info| (timeline_info.timeline_id, timeline_info)) .collect()) } @@ -555,26 +554,17 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - let timeline = pageserver .timeline_create(tenant_id, None, None, None)? .ok_or_else(|| anyhow!("Failed to create new timeline for tenant {}", tenant_id))?; - let new_timeline_id = timeline.timeline_id(); + let new_timeline_id = timeline.timeline_id; - let last_record_lsn = match timeline { - TimelineInfo::Local { - last_record_lsn, .. - } => last_record_lsn, - TimelineInfo::Remote { .. } => { - bail!( - "Timeline {} was created as remote, not local", - new_timeline_id - ) - } - }; + let last_record_lsn = timeline + .local + .expect("no local timeline info") + .last_record_lsn; env.register_branch_mapping(new_branch_name.to_string(), tenant_id, new_timeline_id)?; println!( "Created timeline '{}' at Lsn {} for tenant: {}", - timeline.timeline_id(), - last_record_lsn, - tenant_id, + timeline.timeline_id, last_record_lsn, tenant_id, ); } Some(("branch", branch_match)) => { @@ -602,26 +592,18 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - let timeline = pageserver .timeline_create(tenant_id, None, start_lsn, Some(ancestor_timeline_id))? .ok_or_else(|| anyhow!("Failed to create new timeline for tenant {}", tenant_id))?; - let new_timeline_id = timeline.timeline_id(); + let new_timeline_id = timeline.timeline_id; - let last_record_lsn = match timeline { - TimelineInfo::Local { - last_record_lsn, .. - } => last_record_lsn, - TimelineInfo::Remote { .. } => bail!( - "Timeline {} was created as remote, not local", - new_timeline_id - ), - }; + let last_record_lsn = timeline + .local + .expect("no local timeline info") + .last_record_lsn; env.register_branch_mapping(new_branch_name.to_string(), tenant_id, new_timeline_id)?; println!( "Created timeline '{}' at Lsn {} for tenant: {}. Ancestor timeline: '{}'", - timeline.timeline_id(), - last_record_lsn, - tenant_id, - ancestor_branch_name, + timeline.timeline_id, last_record_lsn, tenant_id, ancestor_branch_name, ); } Some((sub_name, _)) => bail!("Unexpected tenant subcommand '{}'", sub_name), @@ -662,13 +644,8 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { // older point in time, or following but lagging behind the primary. let lsn_str = timeline_infos .get(&node.timeline_id) - .map(|bi| match bi { - TimelineInfo::Local { - last_record_lsn, .. - } => last_record_lsn.to_string(), - TimelineInfo::Remote { .. } => "? (remote)".to_string(), - }) - .unwrap_or_else(|| '?'.to_string()); + .and_then(|bi| bi.local.as_ref().map(|l| l.last_record_lsn.to_string())) + .unwrap_or_else(|| "?".to_string()); let branch_name = timeline_name_mappings .get(&ZTenantTimelineId::new(tenant_id, node.timeline_id)) diff --git a/zenith_utils/src/http/error.rs b/zenith_utils/src/http/error.rs index 3262c33a51..b23fa029d4 100644 --- a/zenith_utils/src/http/error.rs +++ b/zenith_utils/src/http/error.rs @@ -14,6 +14,9 @@ pub enum ApiError { #[error("Unauthorized: {0}")] Unauthorized(String), + #[error("NotFound: {0}")] + NotFound(String), + #[error(transparent)] InternalServerError(#[from] anyhow::Error), } @@ -36,6 +39,9 @@ impl ApiError { self.to_string(), StatusCode::UNAUTHORIZED, ), + ApiError::NotFound(_) => { + HttpErrorBody::response_from_msg_and_status(self.to_string(), StatusCode::NOT_FOUND) + } ApiError::InternalServerError(err) => HttpErrorBody::response_from_msg_and_status( err.to_string(), StatusCode::INTERNAL_SERVER_ERROR, From b19870cd88ed125101f928ddf533f393a7236f2f Mon Sep 17 00:00:00 2001 From: Dmitry Rodionov Date: Thu, 17 Mar 2022 21:36:17 +0400 Subject: [PATCH 37/42] guard against partial uploads to local storage --- pageserver/src/remote_storage/local_fs.rs | 37 ++++++++++++++++++----- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/pageserver/src/remote_storage/local_fs.rs b/pageserver/src/remote_storage/local_fs.rs index 01f6028d17..6cce127a7c 100644 --- a/pageserver/src/remote_storage/local_fs.rs +++ b/pageserver/src/remote_storage/local_fs.rs @@ -5,6 +5,7 @@ //! volume is mounted to the local FS. use std::{ + ffi::OsString, future::Future, path::{Path, PathBuf}, pin::Pin, @@ -83,11 +84,21 @@ impl RemoteStorage for LocalFs { ) -> anyhow::Result<()> { let target_file_path = self.resolve_in_storage(to)?; create_target_directory(&target_file_path).await?; + // We need this dance with sort of durable rename (without fsyncs) + // to prevent partial uploads. This was really hit when pageserver shutdown + // cancelled the upload and partial file was left on the fs + let mut temp_extension = target_file_path + .extension() + .unwrap_or_default() + .to_os_string(); + + temp_extension.push(OsString::from(".temp")); + let temp_file_path = target_file_path.with_extension(temp_extension); let mut destination = io::BufWriter::new( fs::OpenOptions::new() .write(true) .create(true) - .open(&target_file_path) + .open(&temp_file_path) .await .with_context(|| { format!( @@ -101,16 +112,26 @@ impl RemoteStorage for LocalFs { .await .with_context(|| { format!( - "Failed to upload file to the local storage at '{}'", + "Failed to upload file (write temp) to the local storage at '{}'", + temp_file_path.display() + ) + })?; + + destination.flush().await.with_context(|| { + format!( + "Failed to upload (flush temp) file to the local storage at '{}'", + temp_file_path.display() + ) + })?; + + fs::rename(temp_file_path, &target_file_path) + .await + .with_context(|| { + format!( + "Failed to upload (rename) file to the local storage at '{}'", target_file_path.display() ) })?; - destination.flush().await.with_context(|| { - format!( - "Failed to upload file to the local storage at '{}'", - target_file_path.display() - ) - })?; Ok(()) } From 3b069f5aef3fbcfc370814f825767f17d6997f67 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 18 Mar 2022 21:27:48 +0200 Subject: [PATCH 38/42] Fix name of directory used in unit test. There's another test called 'timeline_load'. If the two tests run in parallel, they would conflict and fail. --- pageserver/src/repository.rs | 2 +- vendor/postgres | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pageserver/src/repository.rs b/pageserver/src/repository.rs index e335f42519..074bdf4d01 100644 --- a/pageserver/src/repository.rs +++ b/pageserver/src/repository.rs @@ -1181,7 +1181,7 @@ mod tests { #[test] fn timeline_load_with_ancestor() -> Result<()> { - const TEST_NAME: &str = "timeline_load"; + const TEST_NAME: &str = "timeline_load_with_ancestor"; let harness = RepoHarness::create(TEST_NAME)?; // create two timelines { diff --git a/vendor/postgres b/vendor/postgres index 093aa160e5..5e9bc37322 160000 --- a/vendor/postgres +++ b/vendor/postgres @@ -1 +1 @@ -Subproject commit 093aa160e5df19814ff19b995d36dd5ee03c7f8b +Subproject commit 5e9bc3732266c072151df20d6772b47ca51e233f From 063f9ba81dfaa8f6c9b0b8797d41532715a40669 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Sat, 19 Mar 2022 02:38:29 +0200 Subject: [PATCH 39/42] Use serde_with to (de)serialize ZId and Lsn to hex --- Cargo.lock | 74 ++++++- control_plane/Cargo.toml | 1 + control_plane/src/local_env.rs | 26 +-- control_plane/src/storage.rs | 10 +- pageserver/Cargo.toml | 3 +- pageserver/src/http/models.rs | 84 ++++---- pageserver/src/http/routes.rs | 8 +- pageserver/src/tenant_mgr.rs | 4 +- pageserver/src/timelines.rs | 24 ++- .../batch_others/test_remote_storage.py | 3 +- .../batch_others/test_tenant_relocation.py | 7 +- test_runner/fixtures/zenith_fixtures.py | 14 +- zenith/src/main.rs | 8 +- zenith_utils/Cargo.toml | 1 + zenith_utils/src/auth.rs | 15 +- zenith_utils/src/zid.rs | 199 +----------------- 16 files changed, 192 insertions(+), 289 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 750ac0edc2..a9de71420b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -441,6 +441,7 @@ dependencies = [ "regex", "reqwest", "serde", + "serde_with", "tar", "thiserror", "toml", @@ -600,6 +601,41 @@ dependencies = [ "libc", ] +[[package]] +name = "darling" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d720b8683f8dd83c65155f0530560cba68cd2bf395f6513a483caee57ff7f4" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a340f241d2ceed1deb47ae36c4144b2707ec7dd0b649f894cb39bb595986324" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72c41b3b7352feb3211a0d743dc5700a4e3b60f51bd2b368892d1e0f9a95f44b" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "digest" version = "0.9.0" @@ -1038,6 +1074,12 @@ dependencies = [ "tokio-rustls 0.23.2", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.2.3" @@ -1422,7 +1464,6 @@ dependencies = [ "daemonize", "fail", "futures", - "hex", "hex-literal", "humantime", "hyper", @@ -1440,6 +1481,7 @@ dependencies = [ "scopeguard", "serde", "serde_json", + "serde_with", "signal-hook", "tar", "tempfile", @@ -2075,6 +2117,12 @@ dependencies = [ "rustls 0.19.1", ] +[[package]] +name = "rustversion" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" + [[package]] name = "ryu" version = "1.0.9" @@ -2187,6 +2235,29 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec1e6ec4d8950e5b1e894eac0d360742f3b1407a6078a604a731c4b3f49cefbc" +dependencies = [ + "rustversion", + "serde", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12e47be9471c72889ebafb5e14d5ff930d89ae7a67bbdb5f8abb564f845a927e" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "sha2" version = "0.9.9" @@ -3056,6 +3127,7 @@ dependencies = [ "rustls-split", "serde", "serde_json", + "serde_with", "signal-hook", "tempfile", "thiserror", diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index eff6b3ef2d..b52c7ad5a9 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" tar = "0.4.33" postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" } serde = { version = "1.0", features = ["derive"] } +serde_with = "1.12.0" toml = "0.5" lazy_static = "1.4" regex = "1" diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 2a1d51fe08..00ace431e6 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -5,6 +5,7 @@ use anyhow::{bail, ensure, Context}; use serde::{Deserialize, Serialize}; +use serde_with::{serde_as, DisplayFromStr}; use std::collections::HashMap; use std::env; use std::fs; @@ -12,9 +13,7 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use zenith_utils::auth::{encode_from_key_file, Claims, Scope}; use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{ - HexZTenantId, HexZTimelineId, ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId, -}; +use zenith_utils::zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId}; use crate::safekeeper::SafekeeperNode; @@ -25,6 +24,7 @@ use crate::safekeeper::SafekeeperNode; // to 'zenith init --config=' option. See control_plane/simple.conf for // an example. // +#[serde_as] #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] pub struct LocalEnv { // Base directory for all the nodes (the pageserver, safekeepers and @@ -50,7 +50,8 @@ pub struct LocalEnv { // Default tenant ID to use with the 'zenith' command line utility, when // --tenantid is not explicitly specified. #[serde(default)] - pub default_tenant_id: Option, + #[serde_as(as = "Option")] + pub default_tenant_id: Option, // used to issue tokens during e.g pg start #[serde(default)] @@ -66,7 +67,8 @@ pub struct LocalEnv { // A `HashMap>` would be more appropriate here, // but deserialization into a generic toml object as `toml::Value::try_from` fails with an error. // https://toml.io/en/v1.0.0 does not contain a concept of "a table inside another table". - branch_name_mappings: HashMap>, + #[serde_as(as = "HashMap<_, Vec<(DisplayFromStr, DisplayFromStr)>>")] + branch_name_mappings: HashMap>, } #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] @@ -164,9 +166,6 @@ impl LocalEnv { .entry(branch_name.clone()) .or_default(); - let tenant_id = HexZTenantId::from(tenant_id); - let timeline_id = HexZTimelineId::from(timeline_id); - let existing_ids = existing_values .iter() .find(|(existing_tenant_id, _)| existing_tenant_id == &tenant_id); @@ -193,7 +192,6 @@ impl LocalEnv { branch_name: &str, tenant_id: ZTenantId, ) -> Option { - let tenant_id = HexZTenantId::from(tenant_id); self.branch_name_mappings .get(branch_name)? .iter() @@ -207,13 +205,7 @@ impl LocalEnv { .iter() .flat_map(|(name, tenant_timelines)| { tenant_timelines.iter().map(|&(tenant_id, timeline_id)| { - ( - ZTenantTimelineId::new( - ZTenantId::from(tenant_id), - ZTimelineId::from(timeline_id), - ), - name.clone(), - ) + (ZTenantTimelineId::new(tenant_id, timeline_id), name.clone()) }) }) .collect() @@ -259,7 +251,7 @@ impl LocalEnv { // If no initial tenant ID was given, generate it. if env.default_tenant_id.is_none() { - env.default_tenant_id = Some(HexZTenantId::from(ZTenantId::generate())); + env.default_tenant_id = Some(ZTenantId::generate()); } env.base_data_dir = base_path(); diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index ef43ba3c1e..835c93bf1d 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -18,7 +18,7 @@ use thiserror::Error; use zenith_utils::http::error::HttpErrorBody; use zenith_utils::lsn::Lsn; use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{HexZTenantId, HexZTimelineId, ZTenantId, ZTimelineId}; +use zenith_utils::zid::{ZTenantId, ZTimelineId}; use crate::local_env::LocalEnv; use crate::{fill_rust_env_vars, read_pidfile}; @@ -337,9 +337,7 @@ impl PageServerNode { ) -> anyhow::Result> { let tenant_id_string = self .http_request(Method::POST, format!("{}/tenant", self.http_base_url)) - .json(&TenantCreateRequest { - new_tenant_id: new_tenant_id.map(HexZTenantId::from), - }) + .json(&TenantCreateRequest { new_tenant_id }) .send()? .error_from_body()? .json::>()?; @@ -382,9 +380,9 @@ impl PageServerNode { format!("{}/tenant/{}/timeline", self.http_base_url, tenant_id), ) .json(&TimelineCreateRequest { - new_timeline_id: new_timeline_id.map(HexZTimelineId::from), + new_timeline_id, ancestor_start_lsn, - ancestor_timeline_id: ancestor_timeline_id.map(HexZTimelineId::from), + ancestor_timeline_id, }) .send()? .error_from_body()? diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index cfcb453732..efd2fa4a38 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -25,11 +25,12 @@ tokio-stream = "0.1.8" anyhow = { version = "1.0", features = ["backtrace"] } crc32c = "0.6.0" thiserror = "1.0" -hex = { version = "0.4.3", features = ["serde"] } tar = "0.4.33" humantime = "2.1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1" +serde_with = "1.12.0" + toml_edit = { version = "0.13", features = ["easy"] } scopeguard = "1.1.0" async-trait = "0.1" diff --git a/pageserver/src/http/models.rs b/pageserver/src/http/models.rs index 8827713f11..c28cd0def7 100644 --- a/pageserver/src/http/models.rs +++ b/pageserver/src/http/models.rs @@ -1,24 +1,39 @@ -use anyhow::Context; use serde::{Deserialize, Serialize}; +use serde_with::{serde_as, DisplayFromStr}; use zenith_utils::{ lsn::Lsn, - zid::{HexZTenantId, HexZTimelineId, ZNodeId, ZTenantId, ZTimelineId}, + zid::{ZNodeId, ZTenantId, ZTimelineId}, }; use crate::timelines::{LocalTimelineInfo, TimelineInfo}; +#[serde_as] #[derive(Serialize, Deserialize)] pub struct TimelineCreateRequest { - pub new_timeline_id: Option, - pub ancestor_timeline_id: Option, + #[serde(default)] + #[serde_as(as = "Option")] + pub new_timeline_id: Option, + #[serde(default)] + #[serde_as(as = "Option")] + pub ancestor_timeline_id: Option, + #[serde(default)] + #[serde_as(as = "Option")] pub ancestor_start_lsn: Option, } +#[serde_as] #[derive(Serialize, Deserialize)] pub struct TenantCreateRequest { - pub new_tenant_id: Option, + #[serde(default)] + #[serde_as(as = "Option")] + pub new_tenant_id: Option, } +#[serde_as] +#[derive(Serialize, Deserialize)] +#[serde(transparent)] +pub struct TenantCreateResponse(#[serde_as(as = "DisplayFromStr")] pub ZTenantId); + #[derive(Clone)] pub enum TimelineInfoV1 { Local { @@ -39,18 +54,24 @@ pub enum TimelineInfoV1 { }, } +#[serde_as] #[derive(Serialize, Deserialize)] pub struct TimelineInfoResponseV1 { pub kind: String, - #[serde(with = "hex")] + #[serde_as(as = "DisplayFromStr")] timeline_id: ZTimelineId, - #[serde(with = "hex")] + #[serde_as(as = "DisplayFromStr")] tenant_id: ZTenantId, - disk_consistent_lsn: String, - last_record_lsn: Option, - prev_record_lsn: Option, - ancestor_timeline_id: Option, - ancestor_lsn: Option, + #[serde_as(as = "DisplayFromStr")] + disk_consistent_lsn: Lsn, + #[serde_as(as = "Option")] + last_record_lsn: Option, + #[serde_as(as = "Option")] + prev_record_lsn: Option, + #[serde_as(as = "Option")] + ancestor_timeline_id: Option, + #[serde_as(as = "Option")] + ancestor_lsn: Option, current_logical_size: Option, current_logical_size_non_incremental: Option, } @@ -72,11 +93,11 @@ impl From for TimelineInfoResponseV1 { kind: "Local".to_owned(), timeline_id, tenant_id, - disk_consistent_lsn: disk_consistent_lsn.to_string(), - last_record_lsn: Some(last_record_lsn.to_string()), - prev_record_lsn: prev_record_lsn.map(|lsn| lsn.to_string()), - ancestor_timeline_id: ancestor_timeline_id.map(HexZTimelineId::from), - ancestor_lsn: ancestor_lsn.map(|lsn| lsn.to_string()), + disk_consistent_lsn, + last_record_lsn: Some(last_record_lsn), + prev_record_lsn, + ancestor_timeline_id, + ancestor_lsn, current_logical_size, current_logical_size_non_incremental, }, @@ -88,7 +109,7 @@ impl From for TimelineInfoResponseV1 { kind: "Remote".to_owned(), timeline_id, tenant_id, - disk_consistent_lsn: disk_consistent_lsn.to_string(), + disk_consistent_lsn, last_record_lsn: None, prev_record_lsn: None, ancestor_timeline_id: None, @@ -104,37 +125,24 @@ impl TryFrom for TimelineInfoV1 { type Error = anyhow::Error; fn try_from(other: TimelineInfoResponseV1) -> anyhow::Result { - let parse_lsn_hex_string = |lsn_string: String| { - lsn_string - .parse::() - .with_context(|| format!("Failed to parse Lsn as hex string from '{}'", lsn_string)) - }; - - let disk_consistent_lsn = parse_lsn_hex_string(other.disk_consistent_lsn)?; Ok(match other.kind.as_str() { "Local" => TimelineInfoV1::Local { timeline_id: other.timeline_id, tenant_id: other.tenant_id, - last_record_lsn: other - .last_record_lsn - .ok_or(anyhow::anyhow!( - "Local timeline should have last_record_lsn" - )) - .and_then(parse_lsn_hex_string)?, - prev_record_lsn: other - .prev_record_lsn - .map(parse_lsn_hex_string) - .transpose()?, + last_record_lsn: other.last_record_lsn.ok_or(anyhow::anyhow!( + "Local timeline should have last_record_lsn" + ))?, + prev_record_lsn: other.prev_record_lsn, ancestor_timeline_id: other.ancestor_timeline_id.map(ZTimelineId::from), - ancestor_lsn: other.ancestor_lsn.map(parse_lsn_hex_string).transpose()?, - disk_consistent_lsn, + ancestor_lsn: other.ancestor_lsn, + disk_consistent_lsn: other.disk_consistent_lsn, current_logical_size: other.current_logical_size, current_logical_size_non_incremental: other.current_logical_size_non_incremental, }, "Remote" => TimelineInfoV1::Remote { timeline_id: other.timeline_id, tenant_id: other.tenant_id, - disk_consistent_lsn, + disk_consistent_lsn: other.disk_consistent_lsn, }, unknown => anyhow::bail!("Unknown timeline kind: {}", unknown), }) diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 2d913afe4e..a1249f463a 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -17,11 +17,11 @@ use zenith_utils::http::{ request::parse_request_param, }; use zenith_utils::http::{RequestExt, RouterBuilder}; -use zenith_utils::zid::{HexZTenantId, ZTenantTimelineId, ZTimelineId}; +use zenith_utils::zid::{ZTenantTimelineId, ZTimelineId}; use super::models::{ - StatusResponse, TenantCreateRequest, TimelineCreateRequest, TimelineInfoResponseV1, - TimelineInfoV1, + StatusResponse, TenantCreateRequest, TenantCreateResponse, TimelineCreateRequest, + TimelineInfoResponseV1, TimelineInfoV1, }; use crate::remote_storage::{schedule_timeline_download, RemoteTimelineIndex}; use crate::timelines::{ @@ -308,7 +308,7 @@ async fn tenant_create_handler(mut request: Request) -> Result json_response(StatusCode::CREATED, HexZTenantId::from(id))?, + Some(id) => json_response(StatusCode::CREATED, TenantCreateResponse(id))?, None => json_response(StatusCode::CONFLICT, ())?, }) } diff --git a/pageserver/src/tenant_mgr.rs b/pageserver/src/tenant_mgr.rs index 8584bdd424..4d6dfd7488 100644 --- a/pageserver/src/tenant_mgr.rs +++ b/pageserver/src/tenant_mgr.rs @@ -15,6 +15,7 @@ use anyhow::{Context, Result}; use lazy_static::lazy_static; use log::*; use serde::{Deserialize, Serialize}; +use serde_with::{serde_as, DisplayFromStr}; use std::collections::hash_map::Entry; use std::collections::HashMap; use std::fmt; @@ -267,9 +268,10 @@ pub fn get_timeline_for_tenant_load( .with_context(|| format!("Timeline {} not found for tenant {}", timelineid, tenantid)) } +#[serde_as] #[derive(Serialize, Deserialize, Clone)] pub struct TenantInfo { - #[serde(with = "hex")] + #[serde_as(as = "DisplayFromStr")] pub id: ZTenantId, pub state: TenantState, } diff --git a/pageserver/src/timelines.rs b/pageserver/src/timelines.rs index 9cfc21b413..00dd0f8f9c 100644 --- a/pageserver/src/timelines.rs +++ b/pageserver/src/timelines.rs @@ -5,6 +5,7 @@ use anyhow::{bail, Context, Result}; use postgres_ffi::ControlFileData; use serde::{Deserialize, Serialize}; +use serde_with::{serde_as, DisplayFromStr}; use std::{ fs, path::Path, @@ -13,9 +14,9 @@ use std::{ }; use tracing::*; +use zenith_utils::lsn::Lsn; use zenith_utils::zid::{ZTenantId, ZTenantTimelineId, ZTimelineId}; use zenith_utils::{crashsafe_dir, logging}; -use zenith_utils::{lsn::Lsn, zid::HexZTimelineId}; use crate::{ config::PageServerConf, @@ -28,12 +29,18 @@ use crate::{layered_repository::LayeredRepository, walredo::WalRedoManager}; use crate::{repository::RepositoryTimeline, tenant_mgr}; use crate::{repository::Timeline, CheckpointConfig}; +#[serde_as] #[derive(Debug, Serialize, Deserialize, Clone)] pub struct LocalTimelineInfo { - pub ancestor_timeline_id: Option, + #[serde_as(as = "Option")] + pub ancestor_timeline_id: Option, + #[serde_as(as = "Option")] pub ancestor_lsn: Option, + #[serde_as(as = "DisplayFromStr")] pub last_record_lsn: Lsn, + #[serde_as(as = "Option")] pub prev_record_lsn: Option, + #[serde_as(as = "DisplayFromStr")] pub disk_consistent_lsn: Lsn, pub current_logical_size: Option, // is None when timeline is Unloaded pub current_logical_size_non_incremental: Option, @@ -47,9 +54,7 @@ impl LocalTimelineInfo { ) -> anyhow::Result { let last_record_lsn = timeline.get_last_record_lsn(); let info = LocalTimelineInfo { - ancestor_timeline_id: timeline - .get_ancestor_timeline_id() - .map(HexZTimelineId::from), + ancestor_timeline_id: timeline.get_ancestor_timeline_id(), ancestor_lsn: { match timeline.get_ancestor_lsn() { Lsn(0) => None, @@ -72,7 +77,7 @@ impl LocalTimelineInfo { pub fn from_unloaded_timeline(metadata: &TimelineMetadata) -> Self { LocalTimelineInfo { - ancestor_timeline_id: metadata.ancestor_timeline().map(HexZTimelineId::from), + ancestor_timeline_id: metadata.ancestor_timeline(), ancestor_lsn: { match metadata.ancestor_lsn() { Lsn(0) => None, @@ -103,17 +108,20 @@ impl LocalTimelineInfo { } } +#[serde_as] #[derive(Debug, Serialize, Deserialize, Clone)] pub struct RemoteTimelineInfo { + #[serde_as(as = "Option")] pub remote_consistent_lsn: Option, pub awaits_download: bool, } +#[serde_as] #[derive(Debug, Serialize, Deserialize, Clone)] pub struct TimelineInfo { - #[serde(with = "hex")] + #[serde_as(as = "DisplayFromStr")] pub tenant_id: ZTenantId, - #[serde(with = "hex")] + #[serde_as(as = "DisplayFromStr")] pub timeline_id: ZTimelineId, pub local: Option, pub remote: Option, diff --git a/test_runner/batch_others/test_remote_storage.py b/test_runner/batch_others/test_remote_storage.py index 8689838089..07a122ede9 100644 --- a/test_runner/batch_others/test_remote_storage.py +++ b/test_runner/batch_others/test_remote_storage.py @@ -7,6 +7,7 @@ from pathlib import Path from uuid import UUID from fixtures.zenith_fixtures import ZenithEnvBuilder, assert_local, wait_for, wait_for_last_record_lsn, wait_for_upload from fixtures.log_helper import log +from fixtures.utils import lsn_from_hex import pytest @@ -56,7 +57,7 @@ def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder, INSERT INTO t1 VALUES ({data_id}, '{data_secret}'); ''') cur.execute("SELECT pg_current_wal_flush_lsn()") - current_lsn = int(cur.fetchone()[0].split('/')[1], base=16) + current_lsn = lsn_from_hex(cur.fetchone()[0]) # wait until pageserver receives that data wait_for_last_record_lsn(client, UUID(tenant_id), UUID(timeline_id), current_lsn) diff --git a/test_runner/batch_others/test_tenant_relocation.py b/test_runner/batch_others/test_tenant_relocation.py index e4492e5393..12ce3eb760 100644 --- a/test_runner/batch_others/test_tenant_relocation.py +++ b/test_runner/batch_others/test_tenant_relocation.py @@ -11,6 +11,7 @@ import signal import pytest from fixtures.zenith_fixtures import PgProtocol, PortDistributor, Postgres, ZenithEnvBuilder, ZenithPageserverHttpClient, assert_local, wait_for, wait_for_last_record_lsn, wait_for_upload, zenith_binpath, pg_distrib_dir +from fixtures.utils import lsn_from_hex def assert_abs_margin_ratio(a: float, b: float, margin_ratio: float): @@ -134,7 +135,7 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder, assert cur.fetchone() == (500500, ) cur.execute("SELECT pg_current_wal_flush_lsn()") - current_lsn = int(cur.fetchone()[0].split('/')[1], base=16) + current_lsn = lsn_from_hex(cur.fetchone()[0]) pageserver_http = env.pageserver.http_client() @@ -189,8 +190,8 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder, # when load is active these checks can break because lsns are not static # so lets check with some margin - assert_abs_margin_ratio(new_timeline_detail['local']['disk_consistent_lsn'], - timeline_detail['local']['disk_consistent_lsn'], + assert_abs_margin_ratio(lsn_from_hex(new_timeline_detail['local']['disk_consistent_lsn']), + lsn_from_hex(timeline_detail['local']['disk_consistent_lsn']), 0.03) # callmemaybe to start replication from safekeeper to the new pageserver diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index c44a6e431f..fa68c4f476 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -33,7 +33,7 @@ from typing_extensions import Literal import requests import backoff # type: ignore -from .utils import (get_self_dir, mkdir_if_needed, subprocess_capture) +from .utils import (get_self_dir, lsn_from_hex, mkdir_if_needed, subprocess_capture) from fixtures.log_helper import log """ This file contains pytest fixtures. A fixture is a test resource that can be @@ -1900,8 +1900,10 @@ def remote_consistent_lsn(pageserver_http_client: ZenithPageserverHttpClient, tenant: uuid.UUID, timeline: uuid.UUID) -> int: detail = pageserver_http_client.timeline_detail_v2(tenant, timeline) - assert isinstance(detail['remote']['remote_consistent_lsn'], int) - return detail['remote']['remote_consistent_lsn'] + + lsn_str = detail['remote']['remote_consistent_lsn'] + assert isinstance(lsn_str, str) + return lsn_from_hex(lsn_str) def wait_for_upload(pageserver_http_client: ZenithPageserverHttpClient, @@ -1917,8 +1919,10 @@ def last_record_lsn(pageserver_http_client: ZenithPageserverHttpClient, tenant: uuid.UUID, timeline: uuid.UUID) -> int: detail = pageserver_http_client.timeline_detail_v2(tenant, timeline) - assert isinstance(detail['local']['last_record_lsn'], int) - return detail['local']['last_record_lsn'] + + lsn_str = detail['local']['last_record_lsn'] + assert isinstance(lsn_str, str) + return lsn_from_hex(lsn_str) def wait_for_last_record_lsn(pageserver_http_client: ZenithPageserverHttpClient, diff --git a/zenith/src/main.rs b/zenith/src/main.rs index 389c394103..f5d4184e63 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -316,7 +316,7 @@ fn print_timelines_tree( timeline.local.as_ref().and_then(|l| l.ancestor_timeline_id) { timelines_hash - .get_mut(&ZTimelineId::from(ancestor_timeline_id)) + .get_mut(&ancestor_timeline_id) .context("missing timeline info in the HashMap")? .children .insert(timeline.timeline_id); @@ -437,8 +437,8 @@ fn get_timeline_infos( fn get_tenant_id(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow::Result { if let Some(tenant_id_from_arguments) = parse_tenant_id(sub_match).transpose() { tenant_id_from_arguments - } else if let Some(tenantid_conf) = env.default_tenant_id { - Ok(ZTenantId::from(tenantid_conf)) + } else if let Some(default_id) = env.default_tenant_id { + Ok(default_id) } else { bail!("No tenant id. Use --tenant-id, or set 'default_tenant_id' in the config file"); } @@ -479,7 +479,7 @@ fn handle_init(init_match: &ArgMatches) -> Result { .context("Failed to initialize zenith repository")?; // default_tenantid was generated by the `env.init()` call above - let initial_tenant_id = ZTenantId::from(env.default_tenant_id.unwrap()); + let initial_tenant_id = env.default_tenant_id.unwrap(); // Call 'pageserver init'. let pageserver = PageServerNode::from_env(&env); diff --git a/zenith_utils/Cargo.toml b/zenith_utils/Cargo.toml index daaf345f8f..8e7f5f233c 100644 --- a/zenith_utils/Cargo.toml +++ b/zenith_utils/Cargo.toml @@ -27,6 +27,7 @@ hex = { version = "0.4.3", features = ["serde"] } rustls = "0.19.1" rustls-split = "0.2.1" git-version = "0.3.5" +serde_with = "1.12.0" zenith_metrics = { path = "../zenith_metrics" } workspace_hack = { path = "../workspace_hack" } diff --git a/zenith_utils/src/auth.rs b/zenith_utils/src/auth.rs index cbc4fcee61..8271121c63 100644 --- a/zenith_utils/src/auth.rs +++ b/zenith_utils/src/auth.rs @@ -14,8 +14,9 @@ use jsonwebtoken::{ decode, encode, Algorithm, DecodingKey, EncodingKey, Header, TokenData, Validation, }; use serde::{Deserialize, Serialize}; +use serde_with::{serde_as, DisplayFromStr}; -use crate::zid::{HexZTenantId, ZTenantId}; +use crate::zid::ZTenantId; const JWT_ALGORITHM: Algorithm = Algorithm::RS256; @@ -26,18 +27,18 @@ pub enum Scope { PageServerApi, } +#[serde_as] #[derive(Debug, Serialize, Deserialize, Clone)] pub struct Claims { - pub tenant_id: Option, + #[serde(default)] + #[serde_as(as = "Option")] + pub tenant_id: Option, pub scope: Scope, } impl Claims { pub fn new(tenant_id: Option, scope: Scope) -> Self { - Self { - tenant_id: tenant_id.map(HexZTenantId::from), - scope, - } + Self { tenant_id, scope } } } @@ -47,7 +48,7 @@ pub fn check_permission(claims: &Claims, tenantid: Option) -> Result< bail!("Attempt to access management api with tenant scope. Permission denied") } (Scope::Tenant, Some(tenantid)) => { - if ZTenantId::from(claims.tenant_id.unwrap()) != tenantid { + if claims.tenant_id.unwrap() != tenantid { bail!("Tenant id mismatch. Permission denied") } Ok(()) diff --git a/zenith_utils/src/zid.rs b/zenith_utils/src/zid.rs index e047e38da7..fce5ed97c1 100644 --- a/zenith_utils/src/zid.rs +++ b/zenith_utils/src/zid.rs @@ -2,100 +2,19 @@ use std::{fmt, str::FromStr}; use hex::FromHex; use rand::Rng; -use serde::{ - de::{self, Visitor}, - Deserialize, Serialize, -}; - -macro_rules! mutual_from { - ($id1:ident, $id2:ident) => { - impl From<$id1> for $id2 { - fn from(id1: $id1) -> Self { - Self(id1.0.into()) - } - } - - impl From<$id2> for $id1 { - fn from(id2: $id2) -> Self { - Self(id2.0.into()) - } - } - }; -} +use serde::{Deserialize, Serialize}; /// Zenith ID is a 128-bit random ID. /// Used to represent various identifiers. Provides handy utility methods and impls. /// /// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look /// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`. -/// Use [`HexZId`] to serialize it as hex string instead: `ad50847381e248feaac9876cc71ae418`. +/// +/// Use `#[serde_as(as = "DisplayFromStr")]` to (de)serialize it as hex string instead: `ad50847381e248feaac9876cc71ae418`. +/// Check the `serde_with::serde_as` documentation for options for more complex types. #[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)] struct ZId([u8; 16]); -/// [`ZId`] version that serializes and deserializes as a hex string. -/// Useful for various json serializations, where hex byte array from original id is not convenient. -/// -/// Plain `ZId` could be (de)serialized into hex string with `#[serde(with = "hex")]` attribute. -/// This however won't work on nested types like `Option` or `Vec`, see https://github.com/serde-rs/serde/issues/723 for the details. -/// Every separate type currently needs a new (de)serializing method for every type separately. -/// -/// To provide a generic way to serialize the ZId as a hex string where `#[serde(with = "hex")]` is not enough, this wrapper is created. -/// The default wrapper serialization is left unchanged due to -/// * byte array (de)serialization being faster and simpler -/// * byte deserialization being used in Safekeeper already, with those bytes coming from compute (see `ProposerGreeting` in safekeeper) -/// * current `HexZId`'s deserialization impl breaks on compute byte array deserialization, having it by default is dangerous -#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -struct HexZId([u8; 16]); - -impl Serialize for HexZId { - fn serialize(&self, ser: S) -> Result - where - S: serde::Serializer, - { - hex::encode(self.0).serialize(ser) - } -} - -impl<'de> Deserialize<'de> for HexZId { - fn deserialize(de: D) -> Result - where - D: serde::Deserializer<'de>, - { - de.deserialize_bytes(HexVisitor) - } -} - -struct HexVisitor; - -impl<'de> Visitor<'de> for HexVisitor { - type Value = HexZId; - - fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "A hexadecimal representation of a 128-bit random Zenith ID" - ) - } - - fn visit_bytes(self, hex_bytes: &[u8]) -> Result - where - E: de::Error, - { - ZId::from_hex(hex_bytes) - .map(HexZId::from) - .map_err(de::Error::custom) - } - - fn visit_str(self, hex_bytes_str: &str) -> Result - where - E: de::Error, - { - Self::visit_bytes(self, hex_bytes_str.as_bytes()) - } -} - -mutual_from!(ZId, HexZId); - impl ZId { pub fn get_from_buf(buf: &mut dyn bytes::Buf) -> ZId { let mut arr = [0u8; 16]; @@ -256,76 +175,22 @@ macro_rules! zid_newtype { /// /// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look /// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`. -/// Use [`HexZTimelineId`] to serialize it as hex string instead: `ad50847381e248feaac9876cc71ae418`. +/// See [`ZId`] for alternative ways to serialize it. #[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize)] pub struct ZTimelineId(ZId); -/// A [`ZTimelineId`] version that gets (de)serialized as a hex string. -/// Use in complex types, where `#[serde(with = "hex")]` does not work. -/// See [`HexZId`] for more details. -#[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize)] -pub struct HexZTimelineId(HexZId); - -impl std::fmt::Debug for HexZTimelineId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - ZTimelineId::from(*self).fmt(f) - } -} - -impl std::fmt::Display for HexZTimelineId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - ZTimelineId::from(*self).fmt(f) - } -} - -impl FromStr for HexZTimelineId { - type Err = ::Err; - - fn from_str(s: &str) -> Result { - Ok(HexZTimelineId::from(ZTimelineId::from_str(s)?)) - } -} - zid_newtype!(ZTimelineId); -mutual_from!(ZTimelineId, HexZTimelineId); /// Zenith Tenant Id represents identifiar of a particular tenant. /// Is used for distinguishing requests and data belonging to different users. /// /// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look /// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`. -/// Use [`HexZTenantId`] to serialize it as hex string instead: `ad50847381e248feaac9876cc71ae418`. +/// See [`ZId`] for alternative ways to serialize it. #[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)] pub struct ZTenantId(ZId); -/// A [`ZTenantId`] version that gets (de)serialized as a hex string. -/// Use in complex types, where `#[serde(with = "hex")]` does not work. -/// See [`HexZId`] for more details. -#[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize)] -pub struct HexZTenantId(HexZId); - -impl std::fmt::Debug for HexZTenantId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - ZTenantId::from(*self).fmt(f) - } -} - -impl std::fmt::Display for HexZTenantId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - ZTenantId::from(*self).fmt(f) - } -} - -impl FromStr for HexZTenantId { - type Err = ::Err; - - fn from_str(s: &str) -> Result { - Ok(HexZTenantId::from(ZTenantId::from_str(s)?)) - } -} - zid_newtype!(ZTenantId); -mutual_from!(ZTenantId, HexZTenantId); // A pair uniquely identifying Zenith instance. #[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash)] @@ -368,55 +233,3 @@ impl fmt::Display for ZNodeId { write!(f, "{}", self.0) } } - -#[cfg(test)] -mod tests { - use std::fmt::Display; - - use super::*; - use hex::FromHexError; - use hex_literal::hex; - - #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] - struct TestStruct + Display> { - field: Option, - } - - #[test] - fn test_hex_serializations_tenant_id() { - let original_struct = TestStruct { - field: Some(HexZTenantId::from(ZTenantId::from_array(hex!( - "11223344556677881122334455667788" - )))), - }; - - let serialized_string = serde_json::to_string(&original_struct).unwrap(); - assert_eq!( - serialized_string, - r#"{"field":"11223344556677881122334455667788"}"# - ); - - let deserialized_struct: TestStruct = - serde_json::from_str(&serialized_string).unwrap(); - assert_eq!(original_struct, deserialized_struct); - } - - #[test] - fn test_hex_serializations_timeline_id() { - let original_struct = TestStruct { - field: Some(HexZTimelineId::from(ZTimelineId::from_array(hex!( - "AA223344556677881122334455667788" - )))), - }; - - let serialized_string = serde_json::to_string(&original_struct).unwrap(); - assert_eq!( - serialized_string, - r#"{"field":"aa223344556677881122334455667788"}"# - ); - - let deserialized_struct: TestStruct = - serde_json::from_str(&serialized_string).unwrap(); - assert_eq!(original_struct, deserialized_struct); - } -} From 37ebbb598d625341db904e276d5ff5185ad311b2 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Tue, 15 Mar 2022 10:46:27 +0200 Subject: [PATCH 40/42] Add a macOs build --- .github/workflows/testing.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 218783387b..27e2962712 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -13,7 +13,7 @@ jobs: # If we want to duplicate this job for different # Rust toolchains (e.g. nightly or 1.37.0), add them here. rust_toolchain: [stable] - os: [ubuntu-latest] + os: [ubuntu-latest, macos-latest] timeout-minutes: 30 name: run regression test suite runs-on: ${{ matrix.os }} @@ -32,11 +32,17 @@ jobs: toolchain: ${{ matrix.rust_toolchain }} override: true - - name: Install postgres dependencies + - name: Install Ubuntu postgres dependencies + if: matrix.os == 'ubuntu-latest' run: | sudo apt update sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev + - name: Install macOs postgres dependencies + if: matrix.os == 'macos-latest' + run: | + brew install flex bison + - name: Set pg revision for caching id: pg_ver run: echo ::set-output name=pg_rev::$(git rev-parse HEAD:vendor/postgres) From 77ed2a0fa039fcb20e2617a597b4db39ee20155a Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Thu, 17 Mar 2022 10:06:42 +0200 Subject: [PATCH 41/42] Run GitHub testing workflow on every push --- .github/workflows/testing.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 27e2962712..83e46ce6be 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -1,10 +1,6 @@ name: Build and Test -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] +on: push jobs: regression-check: From bd6bef468c2a619ac8c39c04355c517334847b24 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Sun, 20 Mar 2022 21:13:23 +0200 Subject: [PATCH 42/42] Provide single list timelines HTTP API handle --- pageserver/src/http/models.rs | 150 ------------------ pageserver/src/http/openapi_spec.yml | 33 +++- pageserver/src/http/routes.rs | 27 +--- .../batch_others/test_pageserver_api.py | 6 +- .../batch_others/test_tenant_relocation.py | 2 +- .../batch_others/test_timeline_size.py | 24 +-- test_runner/batch_others/test_wal_acceptor.py | 46 +++--- test_runner/fixtures/zenith_fixtures.py | 15 +- 8 files changed, 83 insertions(+), 220 deletions(-) diff --git a/pageserver/src/http/models.rs b/pageserver/src/http/models.rs index c28cd0def7..d1dfb911ba 100644 --- a/pageserver/src/http/models.rs +++ b/pageserver/src/http/models.rs @@ -5,8 +5,6 @@ use zenith_utils::{ zid::{ZNodeId, ZTenantId, ZTimelineId}, }; -use crate::timelines::{LocalTimelineInfo, TimelineInfo}; - #[serde_as] #[derive(Serialize, Deserialize)] pub struct TimelineCreateRequest { @@ -34,154 +32,6 @@ pub struct TenantCreateRequest { #[serde(transparent)] pub struct TenantCreateResponse(#[serde_as(as = "DisplayFromStr")] pub ZTenantId); -#[derive(Clone)] -pub enum TimelineInfoV1 { - Local { - timeline_id: ZTimelineId, - tenant_id: ZTenantId, - last_record_lsn: Lsn, - prev_record_lsn: Option, - ancestor_timeline_id: Option, - ancestor_lsn: Option, - disk_consistent_lsn: Lsn, - current_logical_size: Option, - current_logical_size_non_incremental: Option, - }, - Remote { - timeline_id: ZTimelineId, - tenant_id: ZTenantId, - disk_consistent_lsn: Lsn, - }, -} - -#[serde_as] -#[derive(Serialize, Deserialize)] -pub struct TimelineInfoResponseV1 { - pub kind: String, - #[serde_as(as = "DisplayFromStr")] - timeline_id: ZTimelineId, - #[serde_as(as = "DisplayFromStr")] - tenant_id: ZTenantId, - #[serde_as(as = "DisplayFromStr")] - disk_consistent_lsn: Lsn, - #[serde_as(as = "Option")] - last_record_lsn: Option, - #[serde_as(as = "Option")] - prev_record_lsn: Option, - #[serde_as(as = "Option")] - ancestor_timeline_id: Option, - #[serde_as(as = "Option")] - ancestor_lsn: Option, - current_logical_size: Option, - current_logical_size_non_incremental: Option, -} - -impl From for TimelineInfoResponseV1 { - fn from(other: TimelineInfoV1) -> Self { - match other { - TimelineInfoV1::Local { - timeline_id, - tenant_id, - last_record_lsn, - prev_record_lsn, - ancestor_timeline_id, - ancestor_lsn, - disk_consistent_lsn, - current_logical_size, - current_logical_size_non_incremental, - } => TimelineInfoResponseV1 { - kind: "Local".to_owned(), - timeline_id, - tenant_id, - disk_consistent_lsn, - last_record_lsn: Some(last_record_lsn), - prev_record_lsn, - ancestor_timeline_id, - ancestor_lsn, - current_logical_size, - current_logical_size_non_incremental, - }, - TimelineInfoV1::Remote { - timeline_id, - tenant_id, - disk_consistent_lsn, - } => TimelineInfoResponseV1 { - kind: "Remote".to_owned(), - timeline_id, - tenant_id, - disk_consistent_lsn, - last_record_lsn: None, - prev_record_lsn: None, - ancestor_timeline_id: None, - ancestor_lsn: None, - current_logical_size: None, - current_logical_size_non_incremental: None, - }, - } - } -} - -impl TryFrom for TimelineInfoV1 { - type Error = anyhow::Error; - - fn try_from(other: TimelineInfoResponseV1) -> anyhow::Result { - Ok(match other.kind.as_str() { - "Local" => TimelineInfoV1::Local { - timeline_id: other.timeline_id, - tenant_id: other.tenant_id, - last_record_lsn: other.last_record_lsn.ok_or(anyhow::anyhow!( - "Local timeline should have last_record_lsn" - ))?, - prev_record_lsn: other.prev_record_lsn, - ancestor_timeline_id: other.ancestor_timeline_id.map(ZTimelineId::from), - ancestor_lsn: other.ancestor_lsn, - disk_consistent_lsn: other.disk_consistent_lsn, - current_logical_size: other.current_logical_size, - current_logical_size_non_incremental: other.current_logical_size_non_incremental, - }, - "Remote" => TimelineInfoV1::Remote { - timeline_id: other.timeline_id, - tenant_id: other.tenant_id, - disk_consistent_lsn: other.disk_consistent_lsn, - }, - unknown => anyhow::bail!("Unknown timeline kind: {}", unknown), - }) - } -} - -fn from_local( - tenant_id: ZTenantId, - timeline_id: ZTimelineId, - local: &LocalTimelineInfo, -) -> TimelineInfoV1 { - TimelineInfoV1::Local { - timeline_id, - tenant_id, - last_record_lsn: local.last_record_lsn, - prev_record_lsn: local.prev_record_lsn, - ancestor_timeline_id: local.ancestor_timeline_id.map(ZTimelineId::from), - ancestor_lsn: local.ancestor_lsn, - disk_consistent_lsn: local.disk_consistent_lsn, - current_logical_size: local.current_logical_size, - current_logical_size_non_incremental: local.current_logical_size_non_incremental, - } -} - -impl From for TimelineInfoV1 { - fn from(t: TimelineInfo) -> Self { - match (t.local.as_ref(), t.remote.as_ref()) { - (None, None) => unreachable!(), - (None, Some(remote)) => TimelineInfoV1::Remote { - timeline_id: t.timeline_id, - tenant_id: t.tenant_id, - disk_consistent_lsn: remote.remote_consistent_lsn.unwrap_or(Lsn(0)), - }, - (Some(local), None) => from_local(t.tenant_id, t.timeline_id, local), - (Some(local), Some(_)) => from_local(t.tenant_id, t.timeline_id, local), - } - } -} - #[derive(Serialize)] pub struct StatusResponse { pub id: ZNodeId, diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index d322b051a6..a9101d4bd6 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -148,6 +148,7 @@ paths: format: hex ancestor_start_lsn: type: string + format: hex responses: "201": description: TimelineInfo @@ -289,7 +290,6 @@ components: required: - timeline_id - tenant_id - - disk_consistent_lsn properties: timeline_id: type: string @@ -297,17 +297,44 @@ components: tenant_id: type: string format: hex + local: + $ref: "#/components/schemas/LocalTimelineInfo" + remote: + $ref: "#/components/schemas/RemoteTimelineInfo" + RemoteTimelineInfo: + type: object + required: + - awaits_download + properties: + awaits_download: + type: boolean + remote_consistent_lsn: + type: string + format: hex + LocalTimelineInfo: + type: object + required: + - last_record_lsn + - disk_consistent_lsn + - timeline_state + properties: last_record_lsn: type: string - prev_record_lsn: + format: hex + disk_consistent_lsn: + type: string + format: hex + timeline_state: type: string ancestor_timeline_id: type: string format: hex ancestor_lsn: type: string - disk_consistent_lsn: + format: hex + prev_record_lsn: type: string + format: hex current_logical_size: type: integer current_logical_size_non_incremental: diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index a1249f463a..3ca8b6334a 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -21,7 +21,6 @@ use zenith_utils::zid::{ZTenantTimelineId, ZTimelineId}; use super::models::{ StatusResponse, TenantCreateRequest, TenantCreateResponse, TimelineCreateRequest, - TimelineInfoResponseV1, TimelineInfoV1, }; use crate::remote_storage::{schedule_timeline_download, RemoteTimelineIndex}; use crate::timelines::{ @@ -143,8 +142,7 @@ fn get_include_non_incremental_logical_size(request: &Request) -> bool { .unwrap_or(false) } -// common part for v1 and v2 handlers -async fn timeline_detail_common(request: Request) -> Result { +async fn timeline_detail_handler(request: Request) -> Result, ApiError> { let tenant_id: ZTenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; @@ -192,25 +190,12 @@ async fn timeline_detail_common(request: Request) -> Result) -> Result, ApiError> { - let timeline_info = timeline_detail_common(request).await?; - Ok(json_response( - StatusCode::OK, - TimelineInfoResponseV1::from(TimelineInfoV1::from(timeline_info)), - )?) -} - -async fn timeline_detail_handler_v2(request: Request) -> Result, ApiError> { - let timeline_info = timeline_detail_common(request).await?; + }; Ok(json_response(StatusCode::OK, timeline_info)?) } @@ -347,11 +332,7 @@ pub fn make_router( .post("/v1/tenant/:tenant_id/timeline", timeline_create_handler) .get( "/v1/tenant/:tenant_id/timeline/:timeline_id", - timeline_detail_handler_v1, - ) - .get( - "/v2/tenant/:tenant_id/timeline/:timeline_id", - timeline_detail_handler_v2, + timeline_detail_handler, ) .post( "/v1/tenant/:tenant_id/timeline/:timeline_id/attach", diff --git a/test_runner/batch_others/test_pageserver_api.py b/test_runner/batch_others/test_pageserver_api.py index 2aa3686904..965ba9bcc3 100644 --- a/test_runner/batch_others/test_pageserver_api.py +++ b/test_runner/batch_others/test_pageserver_api.py @@ -39,10 +39,14 @@ def check_client(client: ZenithPageserverHttpClient, initial_tenant: UUID): timeline_id_str = str(timeline['timeline_id']) timeline_details = client.timeline_detail(tenant_id=tenant_id, timeline_id=UUID(timeline_id_str)) - assert timeline_details['kind'] == 'Local' + assert timeline_details['tenant_id'] == tenant_id.hex assert timeline_details['timeline_id'] == timeline_id_str + local_timeline_details = timeline_details.get('local') + assert local_timeline_details is not None + assert local_timeline_details['timeline_state'] == 'Loaded' + def test_pageserver_http_api_client(zenith_simple_env: ZenithEnv): env = zenith_simple_env diff --git a/test_runner/batch_others/test_tenant_relocation.py b/test_runner/batch_others/test_tenant_relocation.py index 12ce3eb760..32fbc8f872 100644 --- a/test_runner/batch_others/test_tenant_relocation.py +++ b/test_runner/batch_others/test_tenant_relocation.py @@ -141,7 +141,7 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder, # wait until pageserver receives that data wait_for_last_record_lsn(pageserver_http, tenant, timeline, current_lsn) - timeline_detail = pageserver_http.timeline_detail_v2(tenant, timeline) + timeline_detail = assert_local(pageserver_http, tenant, timeline) if with_load == 'with_load': # create load table diff --git a/test_runner/batch_others/test_timeline_size.py b/test_runner/batch_others/test_timeline_size.py index 7d8ab551b0..0b341746ee 100644 --- a/test_runner/batch_others/test_timeline_size.py +++ b/test_runner/batch_others/test_timeline_size.py @@ -2,7 +2,7 @@ from contextlib import closing from uuid import UUID import psycopg2.extras import psycopg2.errors -from fixtures.zenith_fixtures import ZenithEnv, ZenithEnvBuilder, Postgres +from fixtures.zenith_fixtures import ZenithEnv, ZenithEnvBuilder, Postgres, assert_local from fixtures.log_helper import log import time @@ -13,8 +13,9 @@ def test_timeline_size(zenith_simple_env: ZenithEnv): new_timeline_id = env.zenith_cli.create_branch('test_timeline_size', 'empty') client = env.pageserver.http_client() - res = client.timeline_detail(tenant_id=env.initial_tenant, timeline_id=new_timeline_id) - assert res["current_logical_size"] == res["current_logical_size_non_incremental"] + timeline_details = assert_local(client, env.initial_tenant, new_timeline_id) + assert timeline_details['local']['current_logical_size'] == timeline_details['local'][ + 'current_logical_size_non_incremental'] pgmain = env.postgres.create_start("test_timeline_size") log.info("postgres is running on 'test_timeline_size' branch") @@ -31,12 +32,16 @@ def test_timeline_size(zenith_simple_env: ZenithEnv): FROM generate_series(1, 10) g """) - res = client.timeline_detail(tenant_id=env.initial_tenant, timeline_id=new_timeline_id) - assert res["current_logical_size"] == res["current_logical_size_non_incremental"] + res = assert_local(client, env.initial_tenant, new_timeline_id) + local_details = res['local'] + assert local_details["current_logical_size"] == local_details[ + "current_logical_size_non_incremental"] cur.execute("TRUNCATE foo") - res = client.timeline_detail(tenant_id=env.initial_tenant, timeline_id=new_timeline_id) - assert res["current_logical_size"] == res["current_logical_size_non_incremental"] + res = assert_local(client, env.initial_tenant, new_timeline_id) + local_details = res['local'] + assert local_details["current_logical_size"] == local_details[ + "current_logical_size_non_incremental"] # wait until received_lsn_lag is 0 @@ -71,8 +76,9 @@ def test_timeline_size_quota(zenith_env_builder: ZenithEnvBuilder): new_timeline_id = env.zenith_cli.create_branch('test_timeline_size_quota') client = env.pageserver.http_client() - res = client.timeline_detail(tenant_id=env.initial_tenant, timeline_id=new_timeline_id) - assert res["current_logical_size"] == res["current_logical_size_non_incremental"] + res = assert_local(client, env.initial_tenant, new_timeline_id) + assert res['local']["current_logical_size"] == res['local'][ + "current_logical_size_non_incremental"] pgmain = env.postgres.create_start( "test_timeline_size_quota", diff --git a/test_runner/batch_others/test_wal_acceptor.py b/test_runner/batch_others/test_wal_acceptor.py index bdc4c4f63c..37ce1a8bca 100644 --- a/test_runner/batch_others/test_wal_acceptor.py +++ b/test_runner/batch_others/test_wal_acceptor.py @@ -89,29 +89,33 @@ def test_many_timelines(zenith_env_builder: ZenithEnvBuilder): sk_metrics = [sk.http_client().get_metrics() for sk in env.safekeepers] timeline_metrics = [] - with env.pageserver.http_client() as pageserver_http: - for timeline_detail in timeline_details: - timeline_id: str = timeline_detail["timeline_id"] + for timeline_detail in timeline_details: + timeline_id: str = timeline_detail["timeline_id"] - m = TimelineMetrics( - timeline_id=timeline_id, - last_record_lsn=lsn_from_hex(timeline_detail["last_record_lsn"]), - ) - for sk_m in sk_metrics: - m.flush_lsns.append(sk_m.flush_lsn_inexact[(tenant_id.hex, timeline_id)]) - m.commit_lsns.append(sk_m.commit_lsn_inexact[(tenant_id.hex, timeline_id)]) + local_timeline_detail = timeline_detail.get('local') + if local_timeline_detail is None: + log.debug(f"Timeline {timeline_id} is not present locally, skipping") + continue - for flush_lsn, commit_lsn in zip(m.flush_lsns, m.commit_lsns): - # Invariant. May be < when transaction is in progress. - assert commit_lsn <= flush_lsn - # We only call collect_metrics() after a transaction is confirmed by - # the compute node, which only happens after a consensus of safekeepers - # has confirmed the transaction. We assume majority consensus here. - assert (2 * sum(m.last_record_lsn <= lsn - for lsn in m.flush_lsns) > zenith_env_builder.num_safekeepers) - assert (2 * sum(m.last_record_lsn <= lsn - for lsn in m.commit_lsns) > zenith_env_builder.num_safekeepers) - timeline_metrics.append(m) + m = TimelineMetrics( + timeline_id=timeline_id, + last_record_lsn=lsn_from_hex(local_timeline_detail['last_record_lsn']), + ) + for sk_m in sk_metrics: + m.flush_lsns.append(sk_m.flush_lsn_inexact[(tenant_id.hex, timeline_id)]) + m.commit_lsns.append(sk_m.commit_lsn_inexact[(tenant_id.hex, timeline_id)]) + + for flush_lsn, commit_lsn in zip(m.flush_lsns, m.commit_lsns): + # Invariant. May be < when transaction is in progress. + assert commit_lsn <= flush_lsn + # We only call collect_metrics() after a transaction is confirmed by + # the compute node, which only happens after a consensus of safekeepers + # has confirmed the transaction. We assume majority consensus here. + assert (2 * sum(m.last_record_lsn <= lsn + for lsn in m.flush_lsns) > zenith_env_builder.num_safekeepers) + assert (2 * sum(m.last_record_lsn <= lsn + for lsn in m.commit_lsns) > zenith_env_builder.num_safekeepers) + timeline_metrics.append(m) log.info(f"{message}: {timeline_metrics}") return timeline_metrics diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index fa68c4f476..08ac09ee4c 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -783,15 +783,6 @@ class ZenithPageserverHttpClient(requests.Session): assert isinstance(res_json, dict) return res_json - def timeline_detail_v2(self, tenant_id: uuid.UUID, timeline_id: uuid.UUID) -> Dict[Any, Any]: - res = self.get( - f"http://localhost:{self.port}/v2/tenant/{tenant_id.hex}/timeline/{timeline_id.hex}?include-non-incremental-logical-size=1" - ) - self.verbose_error(res) - res_json = res.json() - assert isinstance(res_json, dict) - return res_json - def get_metrics(self) -> str: res = self.get(f"http://localhost:{self.port}/metrics") self.verbose_error(res) @@ -1891,7 +1882,7 @@ def wait_for(number_of_iterations: int, interval: int, func): def assert_local(pageserver_http_client: ZenithPageserverHttpClient, tenant: uuid.UUID, timeline: uuid.UUID): - timeline_detail = pageserver_http_client.timeline_detail_v2(tenant, timeline) + timeline_detail = pageserver_http_client.timeline_detail(tenant, timeline) assert timeline_detail.get('local', {}).get("disk_consistent_lsn"), timeline_detail return timeline_detail @@ -1899,7 +1890,7 @@ def assert_local(pageserver_http_client: ZenithPageserverHttpClient, def remote_consistent_lsn(pageserver_http_client: ZenithPageserverHttpClient, tenant: uuid.UUID, timeline: uuid.UUID) -> int: - detail = pageserver_http_client.timeline_detail_v2(tenant, timeline) + detail = pageserver_http_client.timeline_detail(tenant, timeline) lsn_str = detail['remote']['remote_consistent_lsn'] assert isinstance(lsn_str, str) @@ -1918,7 +1909,7 @@ def wait_for_upload(pageserver_http_client: ZenithPageserverHttpClient, def last_record_lsn(pageserver_http_client: ZenithPageserverHttpClient, tenant: uuid.UUID, timeline: uuid.UUID) -> int: - detail = pageserver_http_client.timeline_detail_v2(tenant, timeline) + detail = pageserver_http_client.timeline_detail(tenant, timeline) lsn_str = detail['local']['last_record_lsn'] assert isinstance(lsn_str, str)