diff --git a/.github/actions/run-python-test-set/action.yml b/.github/actions/run-python-test-set/action.yml index c9987053ce..fcc8983e40 100644 --- a/.github/actions/run-python-test-set/action.yml +++ b/.github/actions/run-python-test-set/action.yml @@ -27,6 +27,26 @@ inputs: description: 'Whether to upload the performance report' required: false default: 'false' + run_with_real_s3: + description: 'Whether to pass real s3 credentials to the test suite' + required: false + default: 'false' + real_s3_bucket: + description: 'Bucket name for real s3 tests' + required: false + default: '' + real_s3_region: + description: 'Region name for real s3 tests' + required: false + default: '' + real_s3_access_key_id: + description: 'Access key id' + required: false + default: '' + real_s3_secret_access_key: + description: 'Secret access key' + required: false + default: '' runs: using: "composite" @@ -63,7 +83,9 @@ runs: # this variable will be embedded in perf test report # and is needed to distinguish different environments PLATFORM: github-actions-selfhosted - shell: bash -euxo pipefail {0} + AWS_ACCESS_KEY_ID: ${{ inputs.real_s3_access_key_id }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.real_s3_secret_access_key }} + shell: bash -euxo pipefail {0} {0} run: | PERF_REPORT_DIR="$(realpath test_runner/perf-report-local)" rm -rf $PERF_REPORT_DIR @@ -77,6 +99,14 @@ runs: if [[ "${{ inputs.run_in_parallel }}" == "true" ]]; then EXTRA_PARAMS="-n4 $EXTRA_PARAMS" fi + + if [[ "${{ inputs.run_with_real_s3 }}" == "true" ]]; then + echo "REAL S3 ENABLED" + export ENABLE_REAL_S3_REMOTE_STORAGE=nonempty + export REMOTE_STORAGE_S3_BUCKET=${{ inputs.real_s3_bucket }} + export REMOTE_STORAGE_S3_REGION=${{ inputs.real_s3_region }} + fi + if [[ "${{ inputs.save_perf_report }}" == "true" ]]; then if [[ "$GITHUB_REF" == "refs/heads/main" ]]; then mkdir -p "$PERF_REPORT_DIR" diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index ec7579a0d1..4e784f0920 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -209,7 +209,11 @@ jobs: build_type: ${{ matrix.build_type }} rust_toolchain: ${{ matrix.rust_toolchain }} test_selection: batch_others - + run_with_real_s3: true + real_s3_bucket: ci-tests-s3 + real_s3_region: us-west-2 + real_s3_access_key_id: "${{ secrets.AWS_ACCESS_KEY_ID_CI_TESTS_S3 }}" + real_s3_secret_access_key: "${{ secrets.AWS_SECRET_ACCESS_KEY_CI_TESTS_S3 }}" - name: Merge and upload coverage data if: matrix.build_type == 'debug' uses: ./.github/actions/save-coverage-data diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 87a598b387..9b39bf2b39 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -457,8 +457,11 @@ class RemoteStorageKind(enum.Enum): def available_remote_storages() -> List[RemoteStorageKind]: remote_storages = [RemoteStorageKind.LOCAL_FS, RemoteStorageKind.MOCK_S3] - if os.getenv("ENABLE_REAL_S3_REMOTE_STORAGE"): + if os.getenv("ENABLE_REAL_S3_REMOTE_STORAGE") is not None: remote_storages.append(RemoteStorageKind.REAL_S3) + log.info("Enabling real s3 storage for tests") + else: + log.info("Using mock implementations to test remote storage") return remote_storages @@ -609,7 +612,8 @@ class NeonEnvBuilder: region_name=mock_region, aws_access_key_id=self.mock_s3_server.access_key(), aws_secret_access_key=self.mock_s3_server.secret_key(), - ).create_bucket(Bucket=bucket_name) + ) + self.remote_storage_client.create_bucket(Bucket=bucket_name) self.remote_storage = S3Storage( bucket_name=bucket_name,