mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-08 05:52:55 +00:00
138 lines
4.6 KiB
Python
138 lines
4.6 KiB
Python
import os
|
|
import shutil
|
|
from contextlib import closing
|
|
from pathlib import Path
|
|
from typing import Any, Dict
|
|
|
|
import pytest
|
|
from fixtures.log_helper import log
|
|
from fixtures.neon_fixtures import (
|
|
NeonEnvBuilder,
|
|
)
|
|
from fixtures.pg_version import PgVersion
|
|
from pytest_httpserver import HTTPServer
|
|
from werkzeug.wrappers.request import Request
|
|
from werkzeug.wrappers.response import Response
|
|
|
|
|
|
# use neon_env_builder_local fixture to override the default neon_env_builder fixture
|
|
# and use a test-specific pg_install instead of shared one
|
|
@pytest.fixture(scope="function")
|
|
def neon_env_builder_local(
|
|
neon_env_builder: NeonEnvBuilder,
|
|
test_output_dir: Path,
|
|
pg_distrib_dir: Path,
|
|
pg_version: PgVersion,
|
|
) -> NeonEnvBuilder:
|
|
test_local_pginstall = test_output_dir / "pg_install"
|
|
log.info(f"copy {pg_distrib_dir} to {test_local_pginstall}")
|
|
shutil.copytree(
|
|
pg_distrib_dir / pg_version.v_prefixed, test_local_pginstall / pg_version.v_prefixed
|
|
)
|
|
|
|
neon_env_builder.pg_distrib_dir = test_local_pginstall
|
|
log.info(f"local neon_env_builder.pg_distrib_dir: {neon_env_builder.pg_distrib_dir}")
|
|
|
|
return neon_env_builder
|
|
|
|
|
|
def test_remote_extensions(
|
|
httpserver: HTTPServer,
|
|
neon_env_builder_local: NeonEnvBuilder,
|
|
httpserver_listen_address,
|
|
pg_version,
|
|
):
|
|
if pg_version == PgVersion.V16:
|
|
pytest.skip("TODO: PG16 extension building")
|
|
|
|
# setup mock http server
|
|
# that expects request for anon.tar.zst
|
|
# and returns the requested file
|
|
(host, port) = httpserver_listen_address
|
|
extensions_endpoint = f"http://{host}:{port}/pg-ext-s3-gateway"
|
|
|
|
build_tag = os.environ.get("BUILD_TAG", "latest")
|
|
archive_path = f"{build_tag}/v{pg_version}/extensions/anon.tar.zst"
|
|
|
|
def endpoint_handler_build_tag(request: Request) -> Response:
|
|
log.info(f"request: {request}")
|
|
|
|
file_name = "anon.tar.zst"
|
|
file_path = f"test_runner/regress/data/extension_test/5670669815/v{pg_version}/extensions/anon.tar.zst"
|
|
file_size = os.path.getsize(file_path)
|
|
fh = open(file_path, "rb")
|
|
|
|
return Response(
|
|
fh,
|
|
mimetype="application/octet-stream",
|
|
headers=[
|
|
("Content-Length", str(file_size)),
|
|
("Content-Disposition", 'attachment; filename="%s"' % file_name),
|
|
],
|
|
direct_passthrough=True,
|
|
)
|
|
|
|
httpserver.expect_request(
|
|
f"/pg-ext-s3-gateway/{archive_path}", method="GET"
|
|
).respond_with_handler(endpoint_handler_build_tag)
|
|
|
|
# Start a compute node with remote_extension spec
|
|
# and check that it can download the extensions and use them to CREATE EXTENSION.
|
|
env = neon_env_builder_local.init_start()
|
|
env.neon_cli.create_branch("test_remote_extensions")
|
|
endpoint = env.endpoints.create(
|
|
"test_remote_extensions",
|
|
config_lines=["log_min_messages=debug3"],
|
|
)
|
|
|
|
# mock remote_extensions spec
|
|
spec: Dict[str, Any] = {
|
|
"library_index": {
|
|
"anon": "anon",
|
|
},
|
|
"extension_data": {
|
|
"anon": {
|
|
"archive_path": "",
|
|
"control_data": {
|
|
"anon.control": "# PostgreSQL Anonymizer (anon) extension\ncomment = 'Data anonymization tools'\ndefault_version = '1.1.0'\ndirectory='extension/anon'\nrelocatable = false\nrequires = 'pgcrypto'\nsuperuser = false\nmodule_pathname = '$libdir/anon'\ntrusted = true\n"
|
|
},
|
|
},
|
|
},
|
|
}
|
|
spec["extension_data"]["anon"]["archive_path"] = archive_path
|
|
|
|
endpoint.create_remote_extension_spec(spec)
|
|
|
|
endpoint.start(
|
|
remote_ext_config=extensions_endpoint,
|
|
)
|
|
|
|
# this is expected to fail if there's no pgcrypto extension, that's ok
|
|
# we just want to check that the extension was downloaded
|
|
try:
|
|
with closing(endpoint.connect()) as conn:
|
|
with conn.cursor() as cur:
|
|
# Check that appropriate files were downloaded
|
|
cur.execute("CREATE EXTENSION anon")
|
|
res = [x[0] for x in cur.fetchall()]
|
|
log.info(res)
|
|
except Exception as err:
|
|
assert "pgcrypto" in str(err), f"unexpected error creating anon extension {err}"
|
|
|
|
httpserver.check()
|
|
|
|
|
|
# TODO
|
|
# 1. Test downloading remote library.
|
|
#
|
|
# 2. Test a complex extension, which has multiple extensions in one archive
|
|
# using postgis as an example
|
|
#
|
|
# 3.Test that extension is downloaded after endpoint restart,
|
|
# when the library is used in the query.
|
|
# Run the test with mutliple simultaneous connections to an endpoint.
|
|
# to ensure that the extension is downloaded only once.
|
|
#
|
|
# 4. Test that private extensions are only downloaded when they are present in the spec.
|
|
#
|