Add Versioning + move Access to control plane for remote ext (#4760)

test_runner/fixtures/neon_fixtures.py

@@ -721,7 +721,10 @@ class NeonEnvBuilder:
         self.remote_storage = LocalFsStorage(Path(self.repo_dir / "local_fs_remote_storage"))
 
     def enable_mock_s3_remote_storage(
-        self, bucket_name: str, force_enable: bool = True, enable_remote_extensions: bool = False
+        self,
+        bucket_name: str,
+        force_enable: bool = True,
+        enable_remote_extensions: bool = False,
     ):
         """
         Sets up the pageserver to use the S3 mock server, creates the bucket, if it's not present already.
@@ -763,7 +766,10 @@ class NeonEnvBuilder:
             )
 
     def enable_real_s3_remote_storage(
-        self, test_name: str, force_enable: bool = True, enable_remote_extensions: bool = False
+        self,
+        test_name: str,
+        force_enable: bool = True,
+        enable_remote_extensions: bool = False,
     ):
         """
         Sets up configuration to use real s3 endpoint without mock server
@@ -810,7 +816,7 @@ class NeonEnvBuilder:
                 bucket_region="eu-central-1",
                 access_key=access_key,
                 secret_key=secret_key,
-                prefix_in_bucket="5555",
+                prefix_in_bucket="",
             )
 
     def cleanup_local_storage(self):

test_runner/regress/data/extension_test/111/v14/ext_index.json

@@ -0,0 +1,10 @@
+{
+  "embedding": {
+    "control_file_content": "comment = 'hnsw index' \ndefault_version = '0.1.0' \nmodule_pathname = '$libdir/embedding' \nrelocatable = true \ntrusted = true",
+    "extension_archive": "111/v14/extensions/embedding.tar.zst"
+  },
+  "anon": {
+    "control_file_content": "# PostgreSQL Anonymizer (anon) extension \ncomment = 'Data anonymization tools' \ndefault_version = '1.1.0' \ndirectory='extension/anon' \nrelocatable = false \nrequires = 'pgcrypto' \nsuperuser = false \nmodule_pathname = '$libdir/anon' \ntrusted = true \n",
+    "extension_archive": "111/v14/extensions/anon.tar.zst"
+  }
+}

test_runner/regress/data/extension_test/111/v15/ext_index.json

@@ -0,0 +1,11 @@
+{
+  "embedding": {
+    "control_file_content": "comment = 'hnsw index' \ndefault_version = '0.1.0' \nmodule_pathname = '$libdir/embedding' \nrelocatable = true \ntrusted = true",
+    "extension_archive": "111/v15/extensions/embedding.tar.zst"
+  },
+  "anon": {
+    "control_file_content": "# PostgreSQL Anonymizer (anon) extension \ncomment = 'Data anonymization tools' \ndefault_version = '1.1.0' \ndirectory='extension/anon' \nrelocatable = false \nrequires = 'pgcrypto' \nsuperuser = false \nmodule_pathname = '$libdir/anon' \ntrusted = true \n",
+    "extension_archive": "111/v15/extensions/anon.tar.zst"
+  }
+}
+

test_runner/regress/data/extension_test/112/v14/ext_index.json

@@ -0,0 +1,10 @@
+{
+  "embedding": {
+    "control_file_content": "comment = 'hnsw index' \ndefault_version = '0.1.0' \nmodule_pathname = '$libdir/embedding' \nrelocatable = true \ntrusted = true",
+    "extension_archive": "111/v14/extensions/embedding.tar.zst"
+  },
+  "anon": {
+    "control_file_content": "# PostgreSQL Anonymizer (anon) extension \ncomment = 'Data anonymization tools' \ndefault_version = '1.1.0' \ndirectory='extension/anon' \nrelocatable = false \nrequires = 'pgcrypto' \nsuperuser = false \nmodule_pathname = '$libdir/anon' \ntrusted = true \n",
+    "extension_archive": "112/v14/extensions/anon.tar.zst"
+  }
+}

test_runner/regress/data/extension_test/112/v15/ext_index.json

@@ -0,0 +1,11 @@
+{
+  "embedding": {
+    "control_file_content": "comment = 'hnsw index' \ndefault_version = '0.1.0' \nmodule_pathname = '$libdir/embedding' \nrelocatable = true \ntrusted = true",
+    "extension_archive": "111/v15/extensions/embedding.tar.zst"
+  },
+  "anon": {
+    "control_file_content": "# PostgreSQL Anonymizer (anon) extension \ncomment = 'Data anonymization tools' \ndefault_version = '1.1.0' \ndirectory='extension/anon' \nrelocatable = false \nrequires = 'pgcrypto' \nsuperuser = false \nmodule_pathname = '$libdir/anon' \ntrusted = true \n",
+    "extension_archive": "112/v15/extensions/anon.tar.zst"
+  }
+}
+

test_runner/regress/data/extension_test/113/v14/ext_index.json

@@ -0,0 +1,10 @@
+{
+  "embedding": {
+    "control_file_content": "comment = 'hnsw index' \ndefault_version = '0.1.0' \nmodule_pathname = '$libdir/embedding' \nrelocatable = true \ntrusted = true",
+    "extension_archive": "113/v14/extensions/embedding.tar.zst"
+  },
+  "anon": {
+    "control_file_content": "# PostgreSQL Anonymizer (anon) extension \ncomment = 'Data anonymization tools' \ndefault_version = '1.1.0' \ndirectory='extension/anon' \nrelocatable = false \nrequires = 'pgcrypto' \nsuperuser = false \nmodule_pathname = '$libdir/anon' \ntrusted = true \n",
+    "extension_archive": "111/v14/extensions/anon.tar.zst"
+  }
+}

test_runner/regress/data/extension_test/113/v15/ext_index.json

@@ -0,0 +1,11 @@
+{
+  "embedding": {
+    "control_file_content": "comment = 'hnsw index' \ndefault_version = '0.1.0' \nmodule_pathname = '$libdir/embedding' \nrelocatable = true \ntrusted = true",
+    "extension_archive": "113/v15/extensions/embedding.tar.zst"
+  },
+  "anon": {
+    "control_file_content": "# PostgreSQL Anonymizer (anon) extension \ncomment = 'Data anonymization tools' \ndefault_version = '1.1.0' \ndirectory='extension/anon' \nrelocatable = false \nrequires = 'pgcrypto' \nsuperuser = false \nmodule_pathname = '$libdir/anon' \ntrusted = true \n",
+    "extension_archive": "111/v15/extensions/anon.tar.zst"
+  }
+}
+

test_runner/regress/data/extension_test/v14/ext_index.json

@@ -1,14 +0,0 @@
-{
-  "enabled_extensions": {
-    "123454321": [
-      "anon"
-    ],
-    "public": [
-      "embedding"
-    ]
-  },
-  "control_data": {
-    "embedding": "comment = 'hnsw index' \ndefault_version = '0.1.0' \nmodule_pathname = '$libdir/embedding' \nrelocatable = true \ntrusted = true",
-    "anon": "# PostgreSQL Anonymizer (anon) extension \ncomment = 'Data anonymization tools' \ndefault_version = '1.1.0' \ndirectory='extension/anon' \nrelocatable = false \nrequires = 'pgcrypto' \nsuperuser = false \nmodule_pathname = '$libdir/anon' \ntrusted = true \n"
-  }
-}

test_runner/regress/data/extension_test/v15/ext_index.json

@@ -1,14 +0,0 @@
-{
-  "enabled_extensions": {
-    "123454321": [
-      "anon"
-    ],
-    "public": [
-      "embedding"
-    ]
-  },
-  "control_data": {
-    "embedding": "comment = 'hnsw index' \ndefault_version = '0.1.0' \nmodule_pathname = '$libdir/embedding' \nrelocatable = true \ntrusted = true",
-    "anon": "# PostgreSQL Anonymizer (anon) extension \ncomment = 'Data anonymization tools' \ndefault_version = '1.1.0' \ndirectory='extension/anon' \nrelocatable = false \nrequires = 'pgcrypto' \nsuperuser = false \nmodule_pathname = '$libdir/anon' \ntrusted = true \n"
-  }
-}

test_runner/regress/test_download_extensions.py

@@ -11,21 +11,16 @@ from fixtures.neon_fixtures import (
 )
 from fixtures.pg_version import PgVersion
 
-# Generate mock extension files and upload them to the mock bucket.
-#
-# NOTE: You must have appropriate AWS credentials to run REAL_S3 test.
-# It may also be necessary to set the following environment variables for MOCK_S3 test:
-#   export AWS_ACCESS_KEY_ID='test' #   export AWS_SECRET_ACCESS_KEY='test'
-#   export AWS_SECURITY_TOKEN='test' #   export AWS_SESSION_TOKEN='test'
-#   export AWS_DEFAULT_REGION='us-east-1'
-
 
+# Test downloading remote extension.
 @pytest.mark.parametrize("remote_storage_kind", available_s3_storages())
 def test_remote_extensions(
     neon_env_builder: NeonEnvBuilder,
     remote_storage_kind: RemoteStorageKind,
     pg_version: PgVersion,
 ):
+    if remote_storage_kind == RemoteStorageKind.REAL_S3:
+        return None
     neon_env_builder.enable_remote_storage(
         remote_storage_kind=remote_storage_kind,
         test_name="test_remote_extensions",
@@ -36,87 +31,84 @@ def test_remote_extensions(
     tenant_id, _ = env.neon_cli.create_tenant()
     env.neon_cli.create_timeline("test_remote_extensions", tenant_id=tenant_id)
 
+    assert env.ext_remote_storage is not None  # satisfy mypy
+    assert env.remote_storage_client is not None  # satisfy mypy
+
     # For MOCK_S3 we upload test files.
     # For REAL_S3 we use the files already in the bucket
     if remote_storage_kind == RemoteStorageKind.MOCK_S3:
         log.info("Uploading test files to mock bucket")
+        os.chdir("test_runner/regress/data/extension_test")
+        for path in os.walk("."):
+            prefix, _, files = path
+            for file in files:
+                # the [2:] is to remove the leading "./"
+                full_path = os.path.join(prefix, file)[2:]
 
-        def upload_test_file(from_path, to_path):
-            assert env.ext_remote_storage is not None  # satisfy mypy
-            assert env.remote_storage_client is not None  # satisfy mypy
-            with open(
-                f"test_runner/regress/data/extension_test/v{pg_version}/{from_path}", "rb"
-            ) as f:
-                env.remote_storage_client.upload_fileobj(
-                    f,
-                    env.ext_remote_storage.bucket_name,
-                    f"ext/v{pg_version}/{to_path}",
-                )
+                with open(full_path, "rb") as f:
+                    log.info(f"UPLOAD {full_path} to ext/{full_path}")
+                    env.remote_storage_client.upload_fileobj(
+                        f,
+                        env.ext_remote_storage.bucket_name,
+                        f"ext/{full_path}",
+                    )
+        os.chdir("../../../..")
+    # Start a compute node and check that it can download the extensions
+    # and use them to CREATE EXTENSION and LOAD
+    endpoint = env.endpoints.create_start(
+        "test_remote_extensions",
+        tenant_id=tenant_id,
+        remote_ext_config=env.ext_remote_storage.to_string(),
+        # config_lines=["log_min_messages=debug3"],
+    )
+    with closing(endpoint.connect()) as conn:
+        with conn.cursor() as cur:
+            # Check that appropriate control files were downloaded
+            cur.execute("SELECT * FROM pg_available_extensions")
+            all_extensions = [x[0] for x in cur.fetchall()]
+            log.info(all_extensions)
+            assert "anon" in all_extensions
+            assert "embedding" in all_extensions
+            # TODO: check that we cant't download custom extensions for other tenant ids
 
-        upload_test_file("ext_index.json", "ext_index.json")
-        upload_test_file("anon.tar.gz", "extensions/anon.tar.gz")
-        upload_test_file("embedding.tar.gz", "extensions/embedding.tar.gz")
+            # check that we can download public extension
+            cur.execute("CREATE EXTENSION embedding")
+            cur.execute("SELECT extname FROM pg_extension")
+            assert "embedding" in [x[0] for x in cur.fetchall()]
 
-    assert env.ext_remote_storage is not None  # satisfy mypy
-    assert env.remote_storage_client is not None  # satisfy mypy
-    try:
-        # Start a compute node and check that it can download the extensions
-        # and use them to CREATE EXTENSION and LOAD
-        endpoint = env.endpoints.create_start(
-            "test_remote_extensions",
-            tenant_id=tenant_id,
-            remote_ext_config=env.ext_remote_storage.to_string(),
-            # config_lines=["log_min_messages=debug3"],
-        )
-        with closing(endpoint.connect()) as conn:
-            with conn.cursor() as cur:
-                # Check that appropriate control files were downloaded
-                cur.execute("SELECT * FROM pg_available_extensions")
-                all_extensions = [x[0] for x in cur.fetchall()]
-                log.info(all_extensions)
-                assert "anon" in all_extensions
-                assert "embedding" in all_extensions
-                # TODO: check that we cant't download custom extensions for other tenant ids
-
-                # check that we can download public extension
-                cur.execute("CREATE EXTENSION embedding")
-                cur.execute("SELECT extname FROM pg_extension")
-                assert "embedding" in [x[0] for x in cur.fetchall()]
-
-                # check that we can download private extension
-                try:
-                    cur.execute("CREATE EXTENSION anon")
-                except Exception as err:
-                    log.info("error creating anon extension")
-                    assert "pgcrypto" in str(err), "unexpected error creating anon extension"
-
-                # TODO: try to load libraries as well
-
-    finally:
-        # Cleaning up downloaded files is important for local tests
-        # or else one test could reuse the files from another test or another test run
-        cleanup_files = [
-            "lib/postgresql/anon.so",
-            "lib/postgresql/embedding.so",
-            "share/postgresql/extension/anon.control",
-            "share/postgresql/extension/embedding--0.1.0.sql",
-            "share/postgresql/extension/embedding.control",
-        ]
-        cleanup_files = [f"pg_install/v{pg_version}/" + x for x in cleanup_files]
-        cleanup_folders = [
-            "extensions",
-            f"pg_install/v{pg_version}/share/postgresql/extension/anon",
-            f"pg_install/v{pg_version}/extensions",
-        ]
-        for file in cleanup_files:
+            # check that we can download private extension
             try:
-                os.remove(file)
-                log.info(f"removed file {file}")
+                cur.execute("CREATE EXTENSION anon")
             except Exception as err:
-                log.info(f"error removing file {file}: {err}")
-        for folder in cleanup_folders:
-            try:
-                shutil.rmtree(folder)
-                log.info(f"removed folder {folder}")
-            except Exception as err:
-                log.info(f"error removing file {file}: {err}")
+                log.info("error creating anon extension")
+                assert "pgcrypto" in str(err), "unexpected error creating anon extension"
+
+            # TODO: try to load libraries as well
+
+    # Cleaning up downloaded files is important for local tests
+    # or else one test could reuse the files from another test or another test run
+    cleanup_files = [
+        "lib/postgresql/anon.so",
+        "lib/postgresql/embedding.so",
+        "share/postgresql/extension/anon.control",
+        "share/postgresql/extension/embedding--0.1.0.sql",
+        "share/postgresql/extension/embedding.control",
+    ]
+    cleanup_files = [f"pg_install/v{pg_version}/" + x for x in cleanup_files]
+    cleanup_folders = [
+        "extensions",
+        f"pg_install/v{pg_version}/share/postgresql/extension/anon",
+        f"pg_install/v{pg_version}/extensions",
+    ]
+    for file in cleanup_files:
+        try:
+            os.remove(file)
+            log.info(f"removed file {file}")
+        except Exception as err:
+            log.info(f"error removing file {file}: {err}")
+    for folder in cleanup_folders:
+        try:
+            shutil.rmtree(folder)
+            log.info(f"removed folder {folder}")
+        except Exception as err:
+            log.info(f"error removing file {file}: {err}")