diff --git a/.circleci/config.yml b/.circleci/config.yml index 38534393ac..9e4b9ea4ba 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,13 +1,13 @@ version: 2.1 -orbs: - python: circleci/python@1.4.0 - executors: zenith-build-executor: resource_class: xlarge docker: - image: cimg/rust:1.55.0 + zenith-python-executor: + docker: + - image: cimg/python:3.7.10 # Oldest available 3.7 with Ubuntu 20.04 (for GLIBC and Rust) at CirlceCI jobs: check-codestyle: @@ -183,15 +183,13 @@ jobs: - "*" check-python: - executor: python/default + executor: zenith-python-executor steps: - checkout - run: - name: Install pipenv & deps + name: Install deps working_directory: test_runner - command: | - pip install pipenv - pipenv install --dev + command: pipenv --python 3.7 install --dev - run: name: Run yapf to ensure code format when: always @@ -204,8 +202,7 @@ jobs: command: pipenv run mypy . run-pytest: - #description: "Run pytest" - executor: python/default + executor: zenith-python-executor parameters: # pytest args to specify the tests to run. # @@ -240,11 +237,9 @@ jobs: steps: - run: git submodule update --init --depth 1 - run: - name: Install pipenv & deps + name: Install deps working_directory: test_runner - command: | - pip install pipenv - pipenv install + command: pipenv --python 3.7 install - run: name: Run pytest working_directory: test_runner diff --git a/README.md b/README.md index 977d015bfc..3d4392fe9d 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ libssl-dev clang pkg-config libpq-dev To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `tmp_install/bin` and `tmp_install/lib`, respectively. To run the integration tests (not required to use the code), install -Python (3.6 or higher), and install python3 packages with `pipenv` using `pipenv install` in the project directory. +Python (3.7 or higher), and install python3 packages with `pipenv` using `pipenv install` in the project directory. 2. Build zenith and patched postgres ```sh diff --git a/test_runner/Pipfile b/test_runner/Pipfile index 246a3d419f..e179553dbd 100644 --- a/test_runner/Pipfile +++ b/test_runner/Pipfile @@ -19,10 +19,11 @@ cached-property = "*" yapf = "==0.31.0" mypy = "==0.910" # Non-pinned packages follow. +pipenv = "*" flake8 = "*" types-requests = "*" types-psycopg2 = "*" [requires] -# we need at least 3.6, but pipenv doesn't allow to say this directly +# we need at least 3.7, but pipenv doesn't allow to say this directly python_version = "3" diff --git a/test_runner/Pipfile.lock b/test_runner/Pipfile.lock index 289639a2ee..185b91189b 100644 --- a/test_runner/Pipfile.lock +++ b/test_runner/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "48eef51e8eb3c7f0d7284879dd68a83d9b0d9dc835f7a3ff0452b4ddfe9c560a" + "sha256": "63b72760ef37375186a638066ba0ad5804dbace99ddc503ea654e9749070ab24" }, "pipfile-spec": 6, "requires": { @@ -162,6 +162,14 @@ "markers": "python_version >= '3'", "version": "==3.3" }, + "importlib-metadata": { + "hashes": [ + "sha256:b618b6d2d5ffa2f16add5697cf57a46c76a56229b0ed1c438322e4e95645bd15", + "sha256:f284b3e11256ad1e5d03ab86bb2ccd6f5339688ff17a4d797a0fe7df326f23b1" + ], + "markers": "python_version < '3.8'", + "version": "==4.8.1" + }, "iniconfig": { "hashes": [ "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", @@ -291,9 +299,47 @@ ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==1.26.7" + }, + "zipp": { + "hashes": [ + "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832", + "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc" + ], + "markers": "python_version >= '3.6'", + "version": "==3.6.0" } }, "develop": { + "backports.entry-points-selectable": { + "hashes": [ + "sha256:988468260ec1c196dab6ae1149260e2f5472c9110334e5d51adcb77867361f6a", + "sha256:a6d9a871cde5e15b4c4a53e3d43ba890cc6861ec1332c9c2428c92f977192acc" + ], + "markers": "python_version >= '2.7'", + "version": "==1.1.0" + }, + "certifi": { + "hashes": [ + "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872", + "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569" + ], + "version": "==2021.10.8" + }, + "distlib": { + "hashes": [ + "sha256:c8b54e8454e5bf6237cc84c20e8264c3e991e824ef27e8f1e81049867d861e31", + "sha256:d982d0751ff6eaaab5e2ec8e691d949ee80eddf01a62eaa96ddb11531fe16b05" + ], + "version": "==0.3.3" + }, + "filelock": { + "hashes": [ + "sha256:2b5eb3589e7fdda14599e7eb1a50e09b4cc14f34ed98b8ba56d33bfaafcbef2f", + "sha256:34a9f35f95c441e7b38209775d6e0337f9a3759f3565f6c5798f19618527c76f" + ], + "markers": "python_version >= '3.6'", + "version": "==3.3.1" + }, "flake8": { "hashes": [ "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d", @@ -302,6 +348,14 @@ "index": "pypi", "version": "==4.0.1" }, + "importlib-metadata": { + "hashes": [ + "sha256:b618b6d2d5ffa2f16add5697cf57a46c76a56229b0ed1c438322e4e95645bd15", + "sha256:f284b3e11256ad1e5d03ab86bb2ccd6f5339688ff17a4d797a0fe7df326f23b1" + ], + "markers": "python_version < '3.8'", + "version": "==4.8.1" + }, "mccabe": { "hashes": [ "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", @@ -345,6 +399,22 @@ ], "version": "==0.4.3" }, + "pipenv": { + "hashes": [ + "sha256:05958fadcd70b2de6a27542fcd2bd72dd5c59c6d35307fdac3e06361fb06e30e", + "sha256:d180f5be4775c552fd5e69ae18a9d6099d9dafb462efe54f11c72cb5f4d5e977" + ], + "index": "pypi", + "version": "==2021.5.29" + }, + "platformdirs": { + "hashes": [ + "sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2", + "sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d" + ], + "markers": "python_version >= '3.6'", + "version": "==2.4.0" + }, "pycodestyle": { "hashes": [ "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20", @@ -361,6 +431,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.4.0" }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, "toml": { "hashes": [ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", @@ -369,6 +447,42 @@ "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" }, + "typed-ast": { + "hashes": [ + "sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace", + "sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff", + "sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266", + "sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528", + "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6", + "sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808", + "sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4", + "sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363", + "sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341", + "sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04", + "sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41", + "sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e", + "sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3", + "sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899", + "sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805", + "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c", + "sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c", + "sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39", + "sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a", + "sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3", + "sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7", + "sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f", + "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075", + "sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0", + "sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40", + "sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428", + "sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927", + "sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3", + "sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f", + "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65" + ], + "markers": "python_version < '3.8'", + "version": "==1.4.3" + }, "types-psycopg2": { "hashes": [ "sha256:77ed80f2668582654623e04fb3d741ecce93effcc39c929d7e02f4a917a538ce", @@ -394,6 +508,22 @@ "index": "pypi", "version": "==3.10.0.2" }, + "virtualenv": { + "hashes": [ + "sha256:10062e34c204b5e4ec5f62e6ef2473f8ba76513a9a617e873f1f8fb4a519d300", + "sha256:bcc17f0b3a29670dd777d6f0755a4c04f28815395bca279cdcb213b97199a6b8" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==20.8.1" + }, + "virtualenv-clone": { + "hashes": [ + "sha256:418ee935c36152f8f153c79824bb93eaf6f0f7984bae31d3f48f350b9183501a", + "sha256:44d5263bceed0bac3e1424d64f798095233b64def1c5689afa43dc3223caf5b0" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.5.7" + }, "yapf": { "hashes": [ "sha256:408fb9a2b254c302f49db83c59f9aa0b4b0fd0ec25be3a5c51181327922ff63d", @@ -401,6 +531,14 @@ ], "index": "pypi", "version": "==0.31.0" + }, + "zipp": { + "hashes": [ + "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832", + "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc" + ], + "markers": "python_version >= '3.6'", + "version": "==3.6.0" } } } diff --git a/test_runner/README.md b/test_runner/README.md index fe37fdbcb2..4d42bbd914 100644 --- a/test_runner/README.md +++ b/test_runner/README.md @@ -3,10 +3,13 @@ This directory contains integration tests. Prerequisites: -- Python 3.6 or later +- Python 3.7 or later + - Development headers may also be needed to build `psycopg2` from source. + - Python 3.7 is recommended if you want to update tests. - Dependencies: install them via `pipenv install`. Note that Debian/Ubuntu packages are stale, as it commonly happens, so manual installation is not recommended. + Exact version of `pipenv` is not important unless you change dependencies. Run `pipenv shell` to activate the venv or use `pipenv run` to run a single command in the venv, e.g. `pipenv run pytest`. - Zenith and Postgres binaries @@ -93,18 +96,48 @@ Cleanup will happen even if the test fails (raises an unhandled exception). Python destructors, e.g. `__del__()` aren't recommended for cleanup. -### Code quality +### Before submitting a patch +#### Obligatory checks +Install dev dependencies via `pipenv --python 3.7 install --dev` (better) +or `pipenv install --dev` (if you don't have Python 3.7 and don't need to change dependencies). -We force code formatting via yapf and type hints via mypy: +We force code formatting via yapf and type hints via mypy. +Run the following commands in the `test_runner/` directory: -1. Install `yapf` and other tools (`flake8`, `mypy`) with `pipenv install --dev`. -1. Reformat all your code by running `pipenv run yapf -ri .` in the `test_runner/` directory. -1. Ensure there are no type errors by running `pipenv run mypy .` in the `test_runner/` directory. - -Before submitting a patch, please consider: +```bash +pipenv run yapf -ri . # All code is reformatted +pipenv run mypy . # Ensure there are no typing errors +``` +#### Advisable actions * Writing a couple of docstrings to clarify the reasoning behind a new test. * Running `flake8` (or a linter of your choice, e.g. `pycodestyle`) and fixing possible defects, if any. * Adding more type hints to your code to avoid `Any`, especially: * For fixture parameters, they are not automatically deduced. * For function arguments and return values. + +#### Changing dependencies +You have to update `Pipfile.lock` if you have changed `Pipfile`: + +```bash +pipenv --python 3.7 install --dev # Re-create venv for Python 3.7 and install recent pipenv inside +pipenv run pipenv --version # Should be at least 2021.5.29 +pipenv run pipenv lock # Regenerate Pipfile.lock +``` + +As the minimal supported version is Python 3.7 and we use it in CI, +you have to use a Python 3.7 environment when updating `Pipfile.lock`. +Otherwise some back-compatibility packages will be missing. + +It is also important to run recent `pipenv`. +Older versions remove markers from `Pipfile.lock`. + +If you don't have Python 3.7, you should install it and its headers (for `psycopg2`) +separately, e.g.: + +```bash +# In Ubuntu +sudo add-apt-repository ppa:deadsnakes/ppa +sudo apt update +sudo apt install python3.7 python3.7-dev +```