Support parallel test running for python tests

Support is done via pytest-xdist plugin.
To use the feature add -n<concurrency> to pytest invocation
e.g. pytest -n8 to run 8 tests in parallel.

Changes in code are mostly about ports assigning. Previously port for
pageserver was hardcoded without the ability to override through zenith
cli and ports for started compute nodes were calculated twice, in zenith
cli and in test code. Now zenith cli supports port arguments for
pageserver and compute nodes to be passed explicitly.

Tests are modified in such a way that each worker gets a non overlapping
port range which can be configured and now contains 100 ports. These
ports are distributed to test services (pageserver, wal acceptors,
compute nodes) so they can work independently.

control_plane/src/compute.rs

@@ -88,15 +88,17 @@ impl ComputeControlPlane {
         &mut self,
         tenantid: ZTenantId,
         branch_name: &str,
+        port: Option<u16>,
     ) -> Result<Arc<PostgresNode>> {
         let timeline_id = self
             .pageserver
             .branch_get_by_name(&tenantid, branch_name)?
             .timeline_id;
 
+        let port = port.unwrap_or(self.get_port());
         let node = Arc::new(PostgresNode {
             name: branch_name.to_owned(),
-            address: SocketAddr::new("127.0.0.1".parse().unwrap(), self.get_port()),
+            address: SocketAddr::new("127.0.0.1".parse().unwrap(), port),
             env: self.env.clone(),
             pageserver: Arc::clone(&self.pageserver),
             is_test: false,

control_plane/src/local_env.rs

@@ -20,8 +20,9 @@ use zenith_utils::zid::ZTenantId;
 //
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct LocalEnv {
-    // Pageserver connection strings
-    pub pageserver_connstring: String,
+    // Pageserver connection settings
+    pub pageserver_pg_port: u16,
+    pub pageserver_http_port: u16,
 
     // Base directory for both pageserver and compute nodes
     pub base_data_dir: PathBuf,
@@ -88,7 +89,12 @@ fn base_path() -> PathBuf {
 //
 // Initialize a new Zenith repository
 //
-pub fn init(tenantid: ZTenantId, auth_type: AuthType) -> Result<()> {
+pub fn init(
+    pageserver_pg_port: u16,
+    pageserver_http_port: u16,
+    tenantid: ZTenantId,
+    auth_type: AuthType,
+) -> Result<()> {
     // check if config already exists
     let base_path = base_path();
     if base_path.exists() {
@@ -159,7 +165,8 @@ pub fn init(tenantid: ZTenantId, auth_type: AuthType) -> Result<()> {
     }
 
     let conf = LocalEnv {
-        pageserver_connstring: "postgresql://127.0.0.1:6400".to_string(),
+        pageserver_pg_port,
+        pageserver_http_port,
         pg_distrib_dir,
         zenith_distrib_dir,
         base_data_dir: base_path,

control_plane/src/storage.rs

@@ -22,7 +22,6 @@ use crate::read_pidfile;
 use pageserver::branches::BranchInfo;
 use zenith_utils::connstring::connection_address;
 
-const HTTP_BASE_URL: &str = "http://127.0.0.1:9898/v1";
 
 #[derive(Error, Debug)]
 pub enum PageserverHttpError {
@@ -81,27 +80,36 @@ impl PageServerNode {
 
         PageServerNode {
             kill_on_exit: false,
-            pg_connection_config: Self::default_config(password), // default
+            pg_connection_config: Self::pageserver_connection_config(
+                password,
+                env.pageserver_pg_port,
+            ),
             env: env.clone(),
             http_client: Client::new(),
-            http_base_url: HTTP_BASE_URL.to_owned(),
+            http_base_url: format!("http://localhost:{}/v1", env.pageserver_http_port),
         }
     }
 
-    fn default_config(password: &str) -> Config {
-        format!("postgresql://no_user:{}@localhost:64000/no_db", password)
+    fn pageserver_connection_config(password: &str, port: u16) -> Config {
+        format!("postgresql://no_user:{}@localhost:{}/no_db", password, port)
             .parse()
             .unwrap()
     }
 
     pub fn init(&self, create_tenant: Option<&str>, enable_auth: bool) -> anyhow::Result<()> {
         let mut cmd = Command::new(self.env.pageserver_bin()?);
+        let listen_pg = format!("localhost:{}", self.env.pageserver_pg_port);
+        let listen_http = format!("localhost:{}", self.env.pageserver_http_port);
         let mut args = vec![
             "--init",
             "-D",
             self.env.base_data_dir.to_str().unwrap(),
             "--postgres-distrib",
             self.env.pg_distrib_dir.to_str().unwrap(),
+            "--listen-pg",
+            &listen_pg,
+            "--listen-http",
+            &listen_http,
         ];
 
         if enable_auth {