test_runner: fix mypy errors and force it on CI (#774)

* Fix bugs found by mypy
* Add some missing types and runtime checks, remove unused code
* Make ZenithPageserver start right away for better type safety
* Add `types-*` packages to Pipfile
* Pin mypy version and run it on CircleCI

test_runner/batch_others/test_createdropdb.py

@@ -87,12 +87,14 @@ def test_dropdb(
     pg_before.connect(dbname='foodb').close()
 
     # Test that database subdir exists on the branch before drop
+    assert pg_before.pgdata_dir
     dbpath = pathlib.Path(pg_before.pgdata_dir) / 'base' / str(dboid)
     log.info(dbpath)
 
     assert os.path.isdir(dbpath) == True
 
     # Test that database subdir doesn't exist on the branch after drop
+    assert pg_after.pgdata_dir
     dbpath = pathlib.Path(pg_after.pgdata_dir) / 'base' / str(dboid)
     log.info(dbpath)
 

test_runner/batch_others/test_pageserver_api.py

@@ -3,7 +3,8 @@ from uuid import uuid4
 import pytest
 import psycopg2
 import requests
-from fixtures.zenith_fixtures import ZenithPageserver, ZenithPageserverHttpClient
+from fixtures.zenith_fixtures import ZenithCli, ZenithPageserver, ZenithPageserverHttpClient
+from typing import cast
 
 pytest_plugins = ("fixtures.zenith_fixtures")
 
@@ -53,7 +54,7 @@ def test_branch_list_psql(pageserver: ZenithPageserver, zenith_cli):
     conn.close()
 
 
-def test_tenant_list_psql(pageserver: ZenithPageserver, zenith_cli):
+def test_tenant_list_psql(pageserver: ZenithPageserver, zenith_cli: ZenithCli):
     res = zenith_cli.run(["tenant", "list"])
     res.check_returncode()
     tenants = sorted(map(lambda t: t.split()[0], res.stdout.splitlines()))
@@ -74,7 +75,7 @@ def test_tenant_list_psql(pageserver: ZenithPageserver, zenith_cli):
     cur.execute('tenant_list')
 
     # compare tenants list
-    new_tenants = sorted(map(lambda t: t['id'], json.loads(cur.fetchone()[0])))
+    new_tenants = sorted(map(lambda t: cast(str, t['id']), json.loads(cur.fetchone()[0])))
     assert sorted([pageserver.initial_tenant, tenant1]) == new_tenants
 
 

test_runner/batch_others/test_readonly_node.py

@@ -84,6 +84,6 @@ def test_readonly_node(zenith_cli, pageserver: ZenithPageserver, postgres: Postg
     # Create node at pre-initdb lsn
     try:
         zenith_cli.run(["pg", "start", "test_branch_preinitdb", "test_readonly_node@0/42"])
-        assert false, "compute node startup with invalid LSN should have failed"
+        assert False, "compute node startup with invalid LSN should have failed"
     except Exception:
         print("Node creation with pre-initdb LSN failed (as expected)")

test_runner/batch_others/test_zenith_cli.py

@@ -1,19 +1,24 @@
 import json
 import uuid
 
+from psycopg2.extensions import cursor as PgCursor
 from fixtures.zenith_fixtures import ZenithCli, ZenithPageserver
+from typing import cast
 
 pytest_plugins = ("fixtures.zenith_fixtures")
 
 
-def helper_compare_branch_list(page_server_cur, zenith_cli, initial_tenant: str):
+def helper_compare_branch_list(page_server_cur: PgCursor,
+                               zenith_cli: ZenithCli,
+                               initial_tenant: str):
     """
     Compare branches list returned by CLI and directly via API.
     Filters out branches created by other tests.
     """
 
     page_server_cur.execute(f'branch_list {initial_tenant}')
-    branches_api = sorted(map(lambda b: b['name'], json.loads(page_server_cur.fetchone()[0])))
+    branches_api = sorted(
+        map(lambda b: cast(str, b['name']), json.loads(page_server_cur.fetchone()[0])))
     branches_api = [b for b in branches_api if b.startswith('test_cli_') or b in ('empty', 'main')]
 
     res = zenith_cli.run(["branch"])
@@ -32,7 +37,7 @@ def helper_compare_branch_list(page_server_cur, zenith_cli, initial_tenant: str)
     assert branches_api == branches_cli == branches_cli_with_tenant_arg
 
 
-def test_cli_branch_list(pageserver: ZenithPageserver, zenith_cli):
+def test_cli_branch_list(pageserver: ZenithPageserver, zenith_cli: ZenithCli):
     page_server_conn = pageserver.connect()
     page_server_cur = page_server_conn.cursor()
 
@@ -58,9 +63,10 @@ def test_cli_branch_list(pageserver: ZenithPageserver, zenith_cli):
     assert 'test_cli_branch_list_nested' in branches_cli
 
 
-def helper_compare_tenant_list(page_server_cur, zenith_cli: ZenithCli):
+def helper_compare_tenant_list(page_server_cur: PgCursor, zenith_cli: ZenithCli):
     page_server_cur.execute(f'tenant_list')
-    tenants_api = sorted(map(lambda t: t['id'], json.loads(page_server_cur.fetchone()[0])))
+    tenants_api = sorted(
+        map(lambda t: cast(str, t['id']), json.loads(page_server_cur.fetchone()[0])))
 
     res = zenith_cli.run(["tenant", "list"])
     assert res.stderr == ''