diff --git a/.github/scripts/detect-updated-pytests.py b/.github/scripts/detect-updated-pytests.py new file mode 100755 index 0000000000..2d6e36d9ff --- /dev/null +++ b/.github/scripts/detect-updated-pytests.py @@ -0,0 +1,143 @@ +import os +import re +import shutil +import subprocess +import sys + +commit_sha = os.getenv("COMMIT_SHA") +base_sha = os.getenv("BASE_SHA") + +cmd = ["git", "merge-base", base_sha, commit_sha] +print(f"Running: {' '.join(cmd)}...") +result = subprocess.run(cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) +if result.returncode != 0 or not (baseline := result.stdout.strip()): + print("Baseline commit for PR is not found, detection skipped.") + sys.exit(0) +print(f"Baseline commit: {baseline}") + +cmd = ["git", "diff", "--name-only", f"{baseline}..{commit_sha}", "test_runner/regress/"] +print(f"Running: {' '.join(cmd)}...") +result = subprocess.run(cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) +if result.returncode != 0: + print(f"Git diff returned code {result.returncode}\n{result.stdout}\nDetection skipped.") + sys.exit(0) + + +def collect_tests(test_file_name): + cmd = ["./scripts/pytest", "--collect-only", "-q", test_file_name] + print(f"Running: {' '.join(cmd)}...") + result = subprocess.run(cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if result.returncode != 0: + print( + f"pytest --collect-only returned code {result.returncode}\n{result.stdout}\nDetection skipped." + ) + sys.exit(0) + + tests = [] + for test_item in result.stdout.split("\n"): + if not test_item.startswith(test_file_name): + break + test_name = re.sub(r"(.*::)([^\[]+)(\[.*)", r"\2", test_item) + if test_name not in tests: + tests.append(test_name) + return tests + + +all_new_tests = [] +all_updated_tests = [] +temp_test_file = "test_runner/regress/__temp__.py" +temp_file = None +for test_file in result.stdout.split("\n"): + if not test_file: + continue + print(f"Test file modified: {test_file}.") + + # Get and compare two lists of items collected by pytest to detect new tests in the PR + if temp_file: + temp_file.close() + temp_file = open(temp_test_file, "w") + cmd = ["git", "show", f"{baseline}:{test_file}"] + print(f"Running: {' '.join(cmd)}...") + result = subprocess.run(cmd, text=True, stdout=temp_file) + if result.returncode != 0: + tests0 = [] + else: + tests0 = collect_tests(temp_test_file) + + tests1 = collect_tests(test_file) + + new_tests = set(tests1).difference(tests0) + for test_name in new_tests: + all_new_tests.append(f"{test_file}::{test_name}") + + # Detect pre-existing test functions updated in the PR + cmd = ["git", "diff", f"{baseline}..{commit_sha}", test_file] + print(f"Running: {' '.join(cmd)}...") + result = subprocess.run(cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if result.returncode != 0: + print(f"Git diff returned code {result.returncode}\n{result.stdout}\nDetection skipped.") + sys.exit(0) + updated_funcs = [] + for diff_line in result.stdout.split("\n"): + print(diff_line) + # TODO: detect functions with added/modified parameters + if not diff_line.startswith("@@"): + continue + + # Extract names of functions with updated content relying on hunk header + m = re.match(r"^(@@[0-9, +-]+@@ def )([^(]+)(.*)", diff_line) + if not m: + continue + func_name = m.group(2) + print(func_name) ## + + # Ignore functions not collected by pytest + if func_name not in tests1: + continue + if func_name not in updated_funcs: + updated_funcs.append(func_name) + + for func_name in updated_funcs: + print(f"Function modified: {func_name}.") + # Extract changes within the function + + cmd = ["git", "log", f"{baseline}..{commit_sha}", "-L", f":{func_name}:{test_file}"] + print(f"Running: {' '.join(cmd)}...") + result = subprocess.run(cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if result.returncode != 0: + continue + + patch_contents = result.stdout + + # Revert changes to get the file with only this function updated + # (applying the patch might fail if it contains a change for the next function declaraion) + shutil.copy(test_file, temp_test_file) + + cmd = ["patch", "-R", "-p1", "--no-backup-if-mismatch", "-r", "/dev/null", temp_test_file] + print(f"Running: {' '.join(cmd)}...") + result = subprocess.run( + cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, input=patch_contents + ) + print(f"result: {result.returncode}; {result.stdout}") + if result.returncode != 0: + continue + + # Ignore whitespace-only changes + cmd = ["diff", "-w", test_file, temp_test_file] + print(f"Running: {' '.join(cmd)}...") + result = subprocess.run(cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if result.returncode == 0: + continue + all_updated_tests.append(f"{test_file}::{func_name}") + +if temp_file: + temp_file.close() +if os.path.exists(temp_test_file): + os.remove(temp_test_file) + +if github_output := os.getenv("GITHUB_OUTPUT"): + with open(github_output, "a") as f: + if all_new_tests or all_updated_tests: + f.write("tests=") + f.write(" ".join(all_new_tests + all_updated_tests)) + f.write("\n") diff --git a/.github/workflows/build_and_run_selected_test.yml b/.github/workflows/build_and_test_tests.yml similarity index 55% rename from .github/workflows/build_and_run_selected_test.yml rename to .github/workflows/build_and_test_tests.yml index 7f1eb991c4..fcde773c01 100644 --- a/.github/workflows/build_and_run_selected_test.yml +++ b/.github/workflows/build_and_test_tests.yml @@ -1,10 +1,10 @@ -name: Build and Run Selected Test +name: Build and Run Selected Tests on: workflow_dispatch: inputs: test-selection: - description: 'Specification of selected test(s), as accepted by pytest -k' + description: 'Specification of selected test(s), e. g.: test_runner/regress/test_pg_regress.py::test_pg_regress' required: true type: string run-count: @@ -26,6 +26,8 @@ on: default: '[{"pg_version":"v17"}]' required: true type: string + workflow_call: + pull_request: # TODO: remove before merge defaults: run: @@ -42,26 +44,71 @@ jobs: github-event-name: ${{ github.event_name }} github-event-json: ${{ toJSON(github.event) }} - build-and-test-locally: - needs: [ meta ] + + choose-test-parameters: + runs-on: [ self-hosted, small ] + container: + image: ghcr.io/neondatabase/build-tools:pinned-bookworm + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --init + + outputs: + tests: ${{ inputs.test-selection != '' && inputs.test-selection || steps.detect_tests_to_test.outputs.tests }} + archs: ${{ inputs.test-selection != '' && inputs.archs || '["x64", "arm64"]' }} + build-types: ${{ inputs.test-selection != '' && inputs.build-types || '["release"]' }} + pg-versions: ${{ inputs.test-selection != '' && inputs.pg-versions || '[{"pg_version":"v14"}, {"pg_version":"v17"}]' }} + run-count: ${{ inputs.test-selection != '' && inputs.run-count || 5 }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + if: inputs.test-selection == '' + with: + submodules: false + clean: false + fetch-depth: 1000 + + - name: Cache poetry deps + if: inputs.test-selection == '' + uses: actions/cache@v4 + with: + path: ~/.cache/pypoetry/virtualenvs + key: v2-${{ runner.os }}-${{ runner.arch }}-python-deps-bookworm-${{ hashFiles('poetry.lock') }} + + - name: Install Python deps + if: inputs.test-selection == '' + shell: bash -euxo pipefail {0} + run: ./scripts/pysync + + - name: Detect new and updated tests + id: detect_tests_to_test + if: github.event.pull_request.head.sha && inputs.test-selection == '' + env: + COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.sha }} + run: python3 .github/scripts/detect-updated-pytests.py + + build-and-test-tests: + needs: [ meta, choose-test-parameters ] + if: needs.choose-test-parameters.outputs.tests != '' strategy: fail-fast: false matrix: - arch: ${{ fromJson(inputs.archs) }} - build-type: ${{ fromJson(inputs.build-types) }} + arch: ${{ fromJson(needs.choose-test-parameters.outputs.archs) }} + build-type: ${{ fromJson(needs.choose-test-parameters.outputs.build-types) }} uses: ./.github/workflows/_build-and-test-locally.yml with: arch: ${{ matrix.arch }} build-tools-image: ghcr.io/neondatabase/build-tools:pinned-bookworm build-tag: ${{ needs.meta.outputs.build-tag }} build-type: ${{ matrix.build-type }} - test-cfg: ${{ inputs.pg-versions }} - test-selection: ${{ inputs.test-selection }} - test-run-count: ${{ fromJson(inputs.run-count) }} + test-cfg: ${{ needs.choose-test-parameters.outputs.pg-versions }} + test-selection: ${{ needs.choose-test-parameters.outputs.tests }} + test-run-count: ${{ fromJson(needs.choose-test-parameters.outputs.run-count) }} secrets: inherit create-test-report: - needs: [ build-and-test-locally ] + needs: [ build-and-test-tests ] if: ${{ !cancelled() }} permissions: id-token: write # aws-actions/configure-aws-credentials @@ -96,6 +143,7 @@ jobs: aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} env: REGRESS_TEST_RESULT_CONNSTR_NEW: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_DEV }} + REPORT_EXT: '-ext' - uses: actions/github-script@v7 if: ${{ !cancelled() }}