rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-04 20:12:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reformat all python files by black & isort

Browse Source
This commit is contained in:
Alexander Bayandin
2022-08-18 13:37:28 +01:00
committed by Alexander Bayandin
parent 6b2e1d9065
commit 4c2bb43775
84 changed files with 3282 additions and 2687 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
scripts/coverage
View File

@@ -9,13 +9,6 @@
# * https://github.com/taiki-e/cargo-llvm-cov
# * https://github.com/llvm/llvm-project/tree/main/llvm/test/tools/llvm-cov
from abc import ABC, abstractmethod
from dataclasses import dataclass
from pathlib import Path
from tempfile import TemporaryDirectory
from textwrap import dedent
from typing import Any, Dict, Iterator, Iterable, List, Optional
import argparse
import hashlib
import json
@@ -24,6 +17,12 @@ import shutil
import socket
import subprocess
import sys
from abc import ABC, abstractmethod
from dataclasses import dataclass
from pathlib import Path
from tempfile import TemporaryDirectory
from textwrap import dedent
from typing import Any, Dict, Iterable, Iterator, List, Optional
def file_mtime_or_zero(path: Path) -> int:

347
scripts/export_import_between_pageservers.py
View File

@@ -20,20 +20,21 @@
# For more context on how to use this, see:
# https://github.com/neondatabase/cloud/wiki/Storage-format-migration
import os
from os import path
import shutil
from pathlib import Path
import tempfile
from contextlib import closing
import psycopg2
import subprocess
import argparse
import os
import shutil
import subprocess
import tempfile
import time
import requests
import uuid
from contextlib import closing
from os import path
from pathlib import Path
from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, TypeVar, Union, cast
import psycopg2
import requests
from psycopg2.extensions import connection as PgConnection
from typing import Any, Callable, Dict, Iterator, List, Optional, TypeVar, cast, Union, Tuple
###############################################
### client-side utils copied from test fixtures
@@ -45,7 +46,7 @@ _global_counter = 0
def global_counter() -> int:
""" A really dumb global counter.
"""A really dumb global counter.
This is useful for giving output files a unique number, so if we run the
same command multiple times we can keep their output separate.
"""
@@ -55,7 +56,7 @@ def global_counter() -> int:
def subprocess_capture(capture_dir: str, cmd: List[str], **kwargs: Any) -> str:
""" Run a process and capture its output
"""Run a process and capture its output
Output will go to files named "cmd_NNN.stdout" and "cmd_NNN.stderr"
where "cmd" is the name of the program and NNN is an incrementing
counter.
@@ -63,13 +64,13 @@ def subprocess_capture(capture_dir: str, cmd: List[str], **kwargs: Any) -> str:
Returns basepath for files with captured output.
"""
assert type(cmd) is list
base = os.path.basename(cmd[0]) + '_{}'.format(global_counter())
base = os.path.basename(cmd[0]) + "_{}".format(global_counter())
basepath = os.path.join(capture_dir, base)
stdout_filename = basepath + '.stdout'
stderr_filename = basepath + '.stderr'
stdout_filename = basepath + ".stdout"
stderr_filename = basepath + ".stderr"
with open(stdout_filename, 'w') as stdout_f:
with open(stderr_filename, 'w') as stderr_f:
with open(stdout_filename, "w") as stdout_f:
with open(stderr_filename, "w") as stderr_f:
print('(capturing output to "{}.stdout")'.format(base))
subprocess.run(cmd, **kwargs, stdout=stdout_f, stderr=stderr_f)
@@ -77,15 +78,16 @@ def subprocess_capture(capture_dir: str, cmd: List[str], **kwargs: Any) -> str:
class PgBin:
""" A helper class for executing postgres binaries """
"""A helper class for executing postgres binaries"""
def __init__(self, log_dir: Path, pg_distrib_dir):
self.log_dir = log_dir
self.pg_bin_path = os.path.join(str(pg_distrib_dir), 'bin')
self.pg_bin_path = os.path.join(str(pg_distrib_dir), "bin")
self.env = os.environ.copy()
self.env['LD_LIBRARY_PATH'] = os.path.join(str(pg_distrib_dir), 'lib')
self.env["LD_LIBRARY_PATH"] = os.path.join(str(pg_distrib_dir), "lib")
def _fixpath(self, command: List[str]):
if '/' not in command[0]:
if "/" not in command[0]:
command[0] = os.path.join(self.pg_bin_path, command[0])
def _build_env(self, env_add: Optional[Env]) -> Env:
@@ -106,15 +108,17 @@ class PgBin:
"""
self._fixpath(command)
print('Running command "{}"'.format(' '.join(command)))
print('Running command "{}"'.format(" ".join(command)))
env = self._build_env(env)
subprocess.run(command, env=env, cwd=cwd, check=True)
def run_capture(self,
command: List[str],
env: Optional[Env] = None,
cwd: Optional[str] = None,
**kwargs: Any) -> str:
def run_capture(
self,
command: List[str],
env: Optional[Env] = None,
cwd: Optional[str] = None,
**kwargs: Any,
) -> str:
"""
Run one of the postgres binaries, with stderr and stdout redirected to a file.
This is just like `run`, but for chatty programs. Returns basepath for files
@@ -122,35 +126,33 @@ class PgBin:
"""
self._fixpath(command)
print('Running command "{}"'.format(' '.join(command)))
print('Running command "{}"'.format(" ".join(command)))
env = self._build_env(env)
return subprocess_capture(str(self.log_dir),
command,
env=env,
cwd=cwd,
check=True,
**kwargs)
return subprocess_capture(
str(self.log_dir), command, env=env, cwd=cwd, check=True, **kwargs
)
class PgProtocol:
""" Reusable connection logic """
"""Reusable connection logic"""
def __init__(self, **kwargs):
self.default_options = kwargs
def conn_options(self, **kwargs):
conn_options = self.default_options.copy()
if 'dsn' in kwargs:
conn_options.update(parse_dsn(kwargs['dsn']))
if "dsn" in kwargs:
conn_options.update(parse_dsn(kwargs["dsn"]))
conn_options.update(kwargs)
# Individual statement timeout in seconds. 2 minutes should be
# enough for our tests, but if you need a longer, you can
# change it by calling "SET statement_timeout" after
# connecting.
if 'options' in conn_options:
conn_options['options'] = f"-cstatement_timeout=120s " + conn_options['options']
if "options" in conn_options:
conn_options["options"] = f"-cstatement_timeout=120s " + conn_options["options"]
else:
conn_options['options'] = "-cstatement_timeout=120s"
conn_options["options"] = "-cstatement_timeout=120s"
return conn_options
# autocommit=True here by default because that's what we need most of the time
@@ -194,18 +196,18 @@ class PgProtocol:
class VanillaPostgres(PgProtocol):
def __init__(self, pgdatadir: Path, pg_bin: PgBin, port: int, init=True):
super().__init__(host='localhost', port=port, dbname='postgres')
super().__init__(host="localhost", port=port, dbname="postgres")
self.pgdatadir = pgdatadir
self.pg_bin = pg_bin
self.running = False
if init:
self.pg_bin.run_capture(['initdb', '-D', str(pgdatadir)])
self.pg_bin.run_capture(["initdb", "-D", str(pgdatadir)])
self.configure([f"port = {port}\n"])
def configure(self, options: List[str]):
"""Append lines into postgresql.conf file."""
assert not self.running
with open(os.path.join(self.pgdatadir, 'postgresql.conf'), 'a') as conf_file:
with open(os.path.join(self.pgdatadir, "postgresql.conf"), "a") as conf_file:
conf_file.write("\n".join(options))
def start(self, log_path: Optional[str] = None):
@@ -216,12 +218,13 @@ class VanillaPostgres(PgProtocol):
log_path = os.path.join(self.pgdatadir, "pg.log")
self.pg_bin.run_capture(
['pg_ctl', '-w', '-D', str(self.pgdatadir), '-l', log_path, 'start'])
["pg_ctl", "-w", "-D", str(self.pgdatadir), "-l", log_path, "start"]
)
def stop(self):
assert self.running
self.running = False
self.pg_bin.run_capture(['pg_ctl', '-w', '-D', str(self.pgdatadir), 'stop'])
self.pg_bin.run_capture(["pg_ctl", "-w", "-D", str(self.pgdatadir), "stop"])
def __enter__(self):
return self
@@ -246,9 +249,9 @@ class NeonPageserverHttpClient(requests.Session):
res.raise_for_status()
except requests.RequestException as e:
try:
msg = res.json()['msg']
msg = res.json()["msg"]
except:
msg = ''
msg = ""
raise NeonPageserverApiException(msg) from e
def check_status(self):
@@ -265,17 +268,17 @@ class NeonPageserverHttpClient(requests.Session):
res = self.post(
f"http://{self.host}:{self.port}/v1/tenant",
json={
'new_tenant_id': new_tenant_id.hex,
"new_tenant_id": new_tenant_id.hex,
},
)
if res.status_code == 409:
if ok_if_exists:
print(f'could not create tenant: already exists for id {new_tenant_id}')
print(f"could not create tenant: already exists for id {new_tenant_id}")
else:
res.raise_for_status()
elif res.status_code == 201:
print(f'created tenant {new_tenant_id}')
print(f"created tenant {new_tenant_id}")
else:
self.verbose_error(res)
@@ -299,47 +302,55 @@ class NeonPageserverHttpClient(requests.Session):
def lsn_to_hex(num: int) -> str:
""" Convert lsn from int to standard hex notation. """
return "{:X}/{:X}".format(num >> 32, num & 0xffffffff)
"""Convert lsn from int to standard hex notation."""
return "{:X}/{:X}".format(num >> 32, num & 0xFFFFFFFF)
def lsn_from_hex(lsn_hex: str) -> int:
""" Convert lsn from hex notation to int. """
l, r = lsn_hex.split('/')
"""Convert lsn from hex notation to int."""
l, r = lsn_hex.split("/")
return (int(l, 16) << 32) + int(r, 16)
def remote_consistent_lsn(pageserver_http_client: NeonPageserverHttpClient,
tenant: uuid.UUID,
timeline: uuid.UUID) -> int:
def remote_consistent_lsn(
pageserver_http_client: NeonPageserverHttpClient, tenant: uuid.UUID, timeline: uuid.UUID
) -> int:
detail = pageserver_http_client.timeline_detail(tenant, timeline)
if detail['remote'] is None:
if detail["remote"] is None:
# No remote information at all. This happens right after creating
# a timeline, before any part of it has been uploaded to remote
# storage yet.
return 0
else:
lsn_str = detail['remote']['remote_consistent_lsn']
lsn_str = detail["remote"]["remote_consistent_lsn"]
assert isinstance(lsn_str, str)
return lsn_from_hex(lsn_str)
def wait_for_upload(pageserver_http_client: NeonPageserverHttpClient,
tenant: uuid.UUID,
timeline: uuid.UUID,
lsn: int):
def wait_for_upload(
pageserver_http_client: NeonPageserverHttpClient,
tenant: uuid.UUID,
timeline: uuid.UUID,
lsn: int,
):
"""waits for local timeline upload up to specified lsn"""
for i in range(10):
current_lsn = remote_consistent_lsn(pageserver_http_client, tenant, timeline)
if current_lsn >= lsn:
return
print("waiting for remote_consistent_lsn to reach {}, now {}, iteration {}".format(
lsn_to_hex(lsn), lsn_to_hex(current_lsn), i + 1))
print(
"waiting for remote_consistent_lsn to reach {}, now {}, iteration {}".format(
lsn_to_hex(lsn), lsn_to_hex(current_lsn), i + 1
)
)
time.sleep(1)
raise Exception("timed out while waiting for remote_consistent_lsn to reach {}, was {}".format(
lsn_to_hex(lsn), lsn_to_hex(current_lsn)))
raise Exception(
"timed out while waiting for remote_consistent_lsn to reach {}, was {}".format(
lsn_to_hex(lsn), lsn_to_hex(current_lsn)
)
)
##############
@@ -399,7 +410,7 @@ def reconstruct_paths(log_dir, pg_bin, base_tar):
# Add all template0copy paths to template0
prefix = f"base/{oid}/"
if filepath.startswith(prefix):
suffix = filepath[len(prefix):]
suffix = filepath[len(prefix) :]
yield f"base/{template0_oid}/{suffix}"
elif filepath.startswith("global"):
print(f"skipping {database} global file {filepath}")
@@ -451,15 +462,17 @@ def get_rlsn(pageserver_connstr, tenant_id, timeline_id):
return last_lsn, prev_lsn
def import_timeline(args,
psql_path,
pageserver_connstr,
pageserver_http,
tenant_id,
timeline_id,
last_lsn,
prev_lsn,
tar_filename):
def import_timeline(
args,
psql_path,
pageserver_connstr,
pageserver_http,
tenant_id,
timeline_id,
last_lsn,
prev_lsn,
tar_filename,
):
# Import timelines to new pageserver
import_cmd = f"import basebackup {tenant_id} {timeline_id} {last_lsn} {last_lsn}"
full_cmd = rf"""cat {tar_filename} | {psql_path} {pageserver_connstr} -c '{import_cmd}' """
@@ -469,34 +482,30 @@ def import_timeline(args,
print(f"Running: {full_cmd}")
with open(stdout_filename, 'w') as stdout_f:
with open(stderr_filename2, 'w') as stderr_f:
with open(stdout_filename, "w") as stdout_f:
with open(stderr_filename2, "w") as stderr_f:
print(f"(capturing output to {stdout_filename})")
pg_bin = PgBin(args.work_dir, args.pg_distrib_dir)
subprocess.run(full_cmd,
stdout=stdout_f,
stderr=stderr_f,
env=pg_bin._build_env(None),
shell=True,
check=True)
subprocess.run(
full_cmd,
stdout=stdout_f,
stderr=stderr_f,
env=pg_bin._build_env(None),
shell=True,
check=True,
)
print(f"Done import")
# Wait until pageserver persists the files
wait_for_upload(pageserver_http,
uuid.UUID(tenant_id),
uuid.UUID(timeline_id),
lsn_from_hex(last_lsn))
wait_for_upload(
pageserver_http, uuid.UUID(tenant_id), uuid.UUID(timeline_id), lsn_from_hex(last_lsn)
)
def export_timeline(args,
psql_path,
pageserver_connstr,
tenant_id,
timeline_id,
last_lsn,
prev_lsn,
tar_filename):
def export_timeline(
args, psql_path, pageserver_connstr, tenant_id, timeline_id, last_lsn, prev_lsn, tar_filename
):
# Choose filenames
incomplete_filename = tar_filename + ".incomplete"
stderr_filename = path.join(args.work_dir, f"{tenant_id}_{timeline_id}.stderr")
@@ -507,15 +516,13 @@ def export_timeline(args,
# Run export command
print(f"Running: {cmd}")
with open(incomplete_filename, 'w') as stdout_f:
with open(stderr_filename, 'w') as stderr_f:
with open(incomplete_filename, "w") as stdout_f:
with open(stderr_filename, "w") as stderr_f:
print(f"(capturing output to {incomplete_filename})")
pg_bin = PgBin(args.work_dir, args.pg_distrib_dir)
subprocess.run(cmd,
stdout=stdout_f,
stderr=stderr_f,
env=pg_bin._build_env(None),
check=True)
subprocess.run(
cmd, stdout=stdout_f, stderr=stderr_f, env=pg_bin._build_env(None), check=True
)
# Add missing rels
pg_bin = PgBin(args.work_dir, args.pg_distrib_dir)
@@ -551,27 +558,28 @@ def main(args: argparse.Namespace):
for timeline in timelines:
# Skip timelines we don't need to export
if args.timelines and timeline['timeline_id'] not in args.timelines:
if args.timelines and timeline["timeline_id"] not in args.timelines:
print(f"Skipping timeline {timeline['timeline_id']}")
continue
# Choose filenames
tar_filename = path.join(args.work_dir,
f"{timeline['tenant_id']}_{timeline['timeline_id']}.tar")
tar_filename = path.join(
args.work_dir, f"{timeline['tenant_id']}_{timeline['timeline_id']}.tar"
)
# Export timeline from old pageserver
if args.only_import is False:
last_lsn, prev_lsn = get_rlsn(
old_pageserver_connstr,
timeline['tenant_id'],
timeline['timeline_id'],
timeline["tenant_id"],
timeline["timeline_id"],
)
export_timeline(
args,
psql_path,
old_pageserver_connstr,
timeline['tenant_id'],
timeline['timeline_id'],
timeline["tenant_id"],
timeline["timeline_id"],
last_lsn,
prev_lsn,
tar_filename,
@@ -583,8 +591,8 @@ def main(args: argparse.Namespace):
psql_path,
new_pageserver_connstr,
new_http_client,
timeline['tenant_id'],
timeline['timeline_id'],
timeline["tenant_id"],
timeline["timeline_id"],
last_lsn,
prev_lsn,
tar_filename,
@@ -592,117 +600,118 @@ def main(args: argparse.Namespace):
# Re-export and compare
re_export_filename = tar_filename + ".reexport"
export_timeline(args,
psql_path,
new_pageserver_connstr,
timeline['tenant_id'],
timeline['timeline_id'],
last_lsn,
prev_lsn,
re_export_filename)
export_timeline(
args,
psql_path,
new_pageserver_connstr,
timeline["tenant_id"],
timeline["timeline_id"],
last_lsn,
prev_lsn,
re_export_filename,
)
# Check the size is the same
old_size = os.path.getsize(tar_filename),
new_size = os.path.getsize(re_export_filename),
old_size = (os.path.getsize(tar_filename),)
new_size = (os.path.getsize(re_export_filename),)
if old_size != new_size:
raise AssertionError(f"Sizes don't match old: {old_size} new: {new_size}")
if __name__ == '__main__':
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
'--tenant-id',
dest='tenants',
"--tenant-id",
dest="tenants",
required=True,
nargs='+',
help='Id of the tenant to migrate. You can pass multiple arguments',
nargs="+",
help="Id of the tenant to migrate. You can pass multiple arguments",
)
parser.add_argument(
'--timeline-id',
dest='timelines',
"--timeline-id",
dest="timelines",
required=False,
nargs='+',
help='Id of the timeline to migrate. You can pass multiple arguments',
nargs="+",
help="Id of the timeline to migrate. You can pass multiple arguments",
)
parser.add_argument(
'--from-host',
dest='old_pageserver_host',
"--from-host",
dest="old_pageserver_host",
required=True,
help='Host of the pageserver to migrate data from',
help="Host of the pageserver to migrate data from",
)
parser.add_argument(
'--from-http-port',
dest='old_pageserver_http_port',
"--from-http-port",
dest="old_pageserver_http_port",
required=False,
type=int,
default=9898,
help='HTTP port of the pageserver to migrate data from. Default: 9898',
help="HTTP port of the pageserver to migrate data from. Default: 9898",
)
parser.add_argument(
'--from-pg-port',
dest='old_pageserver_pg_port',
"--from-pg-port",
dest="old_pageserver_pg_port",
required=False,
type=int,
default=6400,
help='pg port of the pageserver to migrate data from. Default: 6400',
help="pg port of the pageserver to migrate data from. Default: 6400",
)
parser.add_argument(
'--to-host',
dest='new_pageserver_host',
"--to-host",
dest="new_pageserver_host",
required=True,
help='Host of the pageserver to migrate data to',
help="Host of the pageserver to migrate data to",
)
parser.add_argument(
'--to-http-port',
dest='new_pageserver_http_port',
"--to-http-port",
dest="new_pageserver_http_port",
required=False,
default=9898,
type=int,
help='HTTP port of the pageserver to migrate data to. Default: 9898',
help="HTTP port of the pageserver to migrate data to. Default: 9898",
)
parser.add_argument(
'--to-pg-port',
dest='new_pageserver_pg_port',
"--to-pg-port",
dest="new_pageserver_pg_port",
required=False,
default=6400,
type=int,
help='pg port of the pageserver to migrate data to. Default: 6400',
help="pg port of the pageserver to migrate data to. Default: 6400",
)
parser.add_argument(
'--ignore-tenant-exists',
dest='ok_if_exists',
"--ignore-tenant-exists",
dest="ok_if_exists",
required=False,
help=
'Ignore error if we are trying to create the tenant that already exists. It can be dangerous if existing tenant already contains some data.',
help="Ignore error if we are trying to create the tenant that already exists. It can be dangerous if existing tenant already contains some data.",
)
parser.add_argument(
'--pg-distrib-dir',
dest='pg_distrib_dir',
"--pg-distrib-dir",
dest="pg_distrib_dir",
required=False,
default='/usr/local/',
help='Path where postgres binaries are installed. Default: /usr/local/',
default="/usr/local/",
help="Path where postgres binaries are installed. Default: /usr/local/",
)
parser.add_argument(
'--psql-path',
dest='psql_path',
"--psql-path",
dest="psql_path",
required=False,
default='/usr/local/bin/psql',
help='Path to the psql binary. Default: /usr/local/bin/psql',
default="/usr/local/bin/psql",
help="Path to the psql binary. Default: /usr/local/bin/psql",
)
parser.add_argument(
'--only-import',
dest='only_import',
"--only-import",
dest="only_import",
required=False,
default=False,
action='store_true',
help='Skip export and tenant creation part',
action="store_true",
help="Skip export and tenant creation part",
)
parser.add_argument(
'--work-dir',
dest='work_dir',
"--work-dir",
dest="work_dir",
required=True,
default=False,
help='directory where temporary tar files are stored',
help="directory where temporary tar files are stored",
)
args = parser.parse_args()
main(args)

152
scripts/generate_perf_report_page.py
View File

@@ -1,31 +1,36 @@
#!/usr/bin/env python3
import argparse
import json
from dataclasses import dataclass
from pathlib import Path
import json
from typing import Any, Dict, List, Optional, Tuple, cast
from jinja2 import Template
# skip 'input' columns. They are included in the header and just blow the table
EXCLUDE_COLUMNS = frozenset({
'scale',
'duration',
'number_of_clients',
'number_of_threads',
'init_start_timestamp',
'init_end_timestamp',
'run_start_timestamp',
'run_end_timestamp',
})
EXCLUDE_COLUMNS = frozenset(
{
"scale",
"duration",
"number_of_clients",
"number_of_threads",
"init_start_timestamp",
"init_end_timestamp",
"run_start_timestamp",
"run_end_timestamp",
}
)
KEY_EXCLUDE_FIELDS = frozenset({
'init_start_timestamp',
'init_end_timestamp',
'run_start_timestamp',
'run_end_timestamp',
})
NEGATIVE_COLOR = 'negative'
POSITIVE_COLOR = 'positive'
KEY_EXCLUDE_FIELDS = frozenset(
{
"init_start_timestamp",
"init_end_timestamp",
"run_start_timestamp",
"run_end_timestamp",
}
)
NEGATIVE_COLOR = "negative"
POSITIVE_COLOR = "positive"
EPS = 1e-6
@@ -55,75 +60,76 @@ def get_columns(values: List[Dict[Any, Any]]) -> Tuple[List[Tuple[str, str]], Li
value_columns = []
common_columns = []
for item in values:
if item['name'] in KEY_EXCLUDE_FIELDS:
if item["name"] in KEY_EXCLUDE_FIELDS:
continue
if item['report'] != 'test_param':
value_columns.append(cast(str, item['name']))
if item["report"] != "test_param":
value_columns.append(cast(str, item["name"]))
else:
common_columns.append((cast(str, item['name']), cast(str, item['value'])))
common_columns.append((cast(str, item["name"]), cast(str, item["value"])))
value_columns.sort()
common_columns.sort(key=lambda x: x[0]) # sort by name
return common_columns, value_columns
def format_ratio(ratio: float, report: str) -> Tuple[str, str]:
color = ''
sign = '+' if ratio > 0 else ''
color = ""
sign = "+" if ratio > 0 else ""
if abs(ratio) < 0.05:
return f'&nbsp({sign}{ratio:.2f})', color
return f"&nbsp({sign}{ratio:.2f})", color
if report not in {'test_param', 'higher_is_better', 'lower_is_better'}:
raise ValueError(f'Unknown report type: {report}')
if report not in {"test_param", "higher_is_better", "lower_is_better"}:
raise ValueError(f"Unknown report type: {report}")
if report == 'test_param':
return f'{ratio:.2f}', color
if report == "test_param":
return f"{ratio:.2f}", color
if ratio > 0:
if report == 'higher_is_better':
if report == "higher_is_better":
color = POSITIVE_COLOR
elif report == 'lower_is_better':
elif report == "lower_is_better":
color = NEGATIVE_COLOR
elif ratio < 0:
if report == 'higher_is_better':
if report == "higher_is_better":
color = NEGATIVE_COLOR
elif report == 'lower_is_better':
elif report == "lower_is_better":
color = POSITIVE_COLOR
return f'&nbsp({sign}{ratio:.2f})', color
return f"&nbsp({sign}{ratio:.2f})", color
def extract_value(name: str, suit_run: SuitRun) -> Optional[Dict[str, Any]]:
for item in suit_run.values['data']:
if item['name'] == name:
for item in suit_run.values["data"]:
if item["name"] == name:
return cast(Dict[str, Any], item)
return None
def get_row_values(columns: List[str], run_result: SuitRun,
prev_result: Optional[SuitRun]) -> List[RowValue]:
def get_row_values(
columns: List[str], run_result: SuitRun, prev_result: Optional[SuitRun]
) -> List[RowValue]:
row_values = []
for column in columns:
current_value = extract_value(column, run_result)
if current_value is None:
# should never happen
raise ValueError(f'{column} not found in {run_result.values}')
raise ValueError(f"{column} not found in {run_result.values}")
value = current_value["value"]
if isinstance(value, float):
value = f'{value:.2f}'
value = f"{value:.2f}"
if prev_result is None:
row_values.append(RowValue(value, '', ''))
row_values.append(RowValue(value, "", ""))
continue
prev_value = extract_value(column, prev_result)
if prev_value is None:
# this might happen when new metric is added and there is no value for it in previous run
# let this be here, TODO add proper handling when this actually happens
raise ValueError(f'{column} not found in previous result')
raise ValueError(f"{column} not found in previous result")
# adding `EPS` to each term to avoid ZeroDivisionError when the denominator is zero
ratio = (float(value) + EPS) / (float(prev_value['value']) + EPS) - 1
ratio_display, color = format_ratio(ratio, current_value['report'])
ratio = (float(value) + EPS) / (float(prev_value["value"]) + EPS) - 1
ratio_display, color = format_ratio(ratio, current_value["report"])
row_values.append(RowValue(value, color, ratio_display))
return row_values
@@ -139,8 +145,10 @@ def prepare_rows_from_runs(value_columns: List[str], runs: List[SuitRun]) -> Lis
prev_run = None
for run in runs:
rows.append(
SuiteRunTableRow(revision=run.revision,
values=get_row_values(value_columns, run, prev_run)))
SuiteRunTableRow(
revision=run.revision, values=get_row_values(value_columns, run, prev_run)
)
)
prev_run = run
return rows
@@ -152,27 +160,29 @@ def main(args: argparse.Namespace) -> None:
# we have files in form: <ctr>_<rev>.json
# fill them in the hashmap so we have grouped items for the
# same run configuration (scale, duration etc.) ordered by counter.
for item in sorted(input_dir.iterdir(), key=lambda x: int(x.name.split('_')[0])):
for item in sorted(input_dir.iterdir(), key=lambda x: int(x.name.split("_")[0])):
run_data = json.loads(item.read_text())
revision = run_data['revision']
revision = run_data["revision"]
for suit_result in run_data['result']:
key = "{}{}".format(run_data['platform'], suit_result['suit'])
for suit_result in run_data["result"]:
key = "{}{}".format(run_data["platform"], suit_result["suit"])
# pack total duration as a synthetic value
total_duration = suit_result['total_duration']
suit_result['data'].append({
'name': 'total_duration',
'value': total_duration,
'unit': 's',
'report': 'lower_is_better',
})
common_columns, value_columns = get_columns(suit_result['data'])
total_duration = suit_result["total_duration"]
suit_result["data"].append(
{
"name": "total_duration",
"value": total_duration,
"unit": "s",
"report": "lower_is_better",
}
)
common_columns, value_columns = get_columns(suit_result["data"])
grouped_runs.setdefault(
key,
SuitRuns(
platform=run_data['platform'],
suit=suit_result['suit'],
platform=run_data["platform"],
suit=suit_result["suit"],
common_columns=common_columns,
value_columns=value_columns,
runs=[],
@@ -184,26 +194,26 @@ def main(args: argparse.Namespace) -> None:
for result in grouped_runs.values():
suit = result.suit
context[suit] = {
'common_columns': result.common_columns,
'value_columns': result.value_columns,
'platform': result.platform,
"common_columns": result.common_columns,
"value_columns": result.value_columns,
"platform": result.platform,
# reverse the order so newest results are on top of the table
'rows': reversed(prepare_rows_from_runs(result.value_columns, result.runs)),
"rows": reversed(prepare_rows_from_runs(result.value_columns, result.runs)),
}
template = Template((Path(__file__).parent / 'perf_report_template.html').read_text())
template = Template((Path(__file__).parent / "perf_report_template.html").read_text())
Path(args.out).write_text(template.render(context=context))
if __name__ == '__main__':
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
'--input-dir',
dest='input_dir',
"--input-dir",
dest="input_dir",
required=True,
help='Directory with jsons generated by the test suite',
help="Directory with jsons generated by the test suite",
)
parser.add_argument('--out', required=True, help='Output html file path')
parser.add_argument("--out", required=True, help="Output html file path")
args = parser.parse_args()
main(args)

11
scripts/git-upload
View File

@@ -1,17 +1,16 @@
#!/usr/bin/env python3
from contextlib import contextmanager
import shlex
from tempfile import TemporaryDirectory
from distutils.dir_util import copy_tree
from pathlib import Path
import argparse
import os
import shlex
import shutil
import subprocess
import sys
import textwrap
from contextlib import contextmanager
from distutils.dir_util import copy_tree
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Optional

82
scripts/ingest_perf_test_result.py
View File

@@ -1,12 +1,13 @@
#!/usr/bin/env python3
import argparse
from contextlib import contextmanager
import json
import os
from contextlib import contextmanager
from datetime import datetime
from pathlib import Path
import psycopg2
import psycopg2.extras
from pathlib import Path
from datetime import datetime
CREATE_TABLE = """
CREATE TABLE IF NOT EXISTS perf_test_results (
@@ -24,15 +25,15 @@ CREATE TABLE IF NOT EXISTS perf_test_results (
def err(msg):
print(f'error: {msg}')
print(f"error: {msg}")
exit(1)
@contextmanager
def get_connection_cursor():
connstr = os.getenv('DATABASE_URL')
connstr = os.getenv("DATABASE_URL")
if not connstr:
err('DATABASE_URL environment variable is not set')
err("DATABASE_URL environment variable is not set")
with psycopg2.connect(connstr) as conn:
with conn.cursor() as cur:
yield cur
@@ -44,33 +45,35 @@ def create_table(cur):
def ingest_perf_test_result(cursor, data_dile: Path, recorded_at_timestamp: int) -> int:
run_data = json.loads(data_dile.read_text())
revision = run_data['revision']
platform = run_data['platform']
revision = run_data["revision"]
platform = run_data["platform"]
run_result = run_data['result']
run_result = run_data["result"]
args_list = []
for suit_result in run_result:
suit = suit_result['suit']
total_duration = suit_result['total_duration']
suit = suit_result["suit"]
total_duration = suit_result["total_duration"]
suit_result['data'].append({
'name': 'total_duration',
'value': total_duration,
'unit': 's',
'report': 'lower_is_better',
})
suit_result["data"].append(
{
"name": "total_duration",
"value": total_duration,
"unit": "s",
"report": "lower_is_better",
}
)
for metric in suit_result['data']:
for metric in suit_result["data"]:
values = {
'suit': suit,
'revision': revision,
'platform': platform,
'metric_name': metric['name'],
'metric_value': metric['value'],
'metric_unit': metric['unit'],
'metric_report_type': metric['report'],
'recorded_at_timestamp': datetime.utcfromtimestamp(recorded_at_timestamp),
"suit": suit,
"revision": revision,
"platform": platform,
"metric_name": metric["name"],
"metric_value": metric["value"],
"metric_unit": metric["unit"],
"metric_report_type": metric["report"],
"recorded_at_timestamp": datetime.utcfromtimestamp(recorded_at_timestamp),
}
args_list.append(values)
@@ -104,13 +107,16 @@ def ingest_perf_test_result(cursor, data_dile: Path, recorded_at_timestamp: int)
def main():
parser = argparse.ArgumentParser(description='Perf test result uploader. \
Database connection string should be provided via DATABASE_URL environment variable', )
parser = argparse.ArgumentParser(
description="Perf test result uploader. \
Database connection string should be provided via DATABASE_URL environment variable",
)
parser.add_argument(
'--ingest',
"--ingest",
type=Path,
help='Path to perf test result file, or directory with perf test result files')
parser.add_argument('--initdb', action='store_true', help='Initialuze database')
help="Path to perf test result file, or directory with perf test result files",
)
parser.add_argument("--initdb", action="store_true", help="Initialuze database")
args = parser.parse_args()
with get_connection_cursor() as cur:
@@ -118,19 +124,19 @@ def main():
create_table(cur)
if not args.ingest.exists():
err(f'ingest path {args.ingest} does not exist')
err(f"ingest path {args.ingest} does not exist")
if args.ingest:
if args.ingest.is_dir():
for item in sorted(args.ingest.iterdir(), key=lambda x: int(x.name.split('_')[0])):
recorded_at_timestamp = int(item.name.split('_')[0])
for item in sorted(args.ingest.iterdir(), key=lambda x: int(x.name.split("_")[0])):
recorded_at_timestamp = int(item.name.split("_")[0])
ingested = ingest_perf_test_result(cur, item, recorded_at_timestamp)
print(f'Ingested {ingested} metric values from {item}')
print(f"Ingested {ingested} metric values from {item}")
else:
recorded_at_timestamp = int(args.ingest.name.split('_')[0])
recorded_at_timestamp = int(args.ingest.name.split("_")[0])
ingested = ingest_perf_test_result(cur, args.ingest, recorded_at_timestamp)
print(f'Ingested {ingested} metric values from {args.ingest}')
print(f"Ingested {ingested} metric values from {args.ingest}")
if __name__ == '__main__':
if __name__ == "__main__":
main()