From aba3fafe2cb50ddb622a35c6189972943dcd70d6 Mon Sep 17 00:00:00 2001
From: Alek Westover <alek.westover@gmail.com>
Date: Thu, 20 Jul 2023 13:31:19 -0400
Subject: [PATCH] update rfc. run real s3 tests. note they are expected to
 fail, as real s3 does not yet have files uploaded to it

---
 compute_tools/src/extension_server.rs         |  5 +-
 docs/rfcs/024-extension-loading.md            | 67 +++++++++++++------
 .../regress/test_download_extensions.py       |  2 -
 3 files changed, 48 insertions(+), 26 deletions(-)

diff --git a/compute_tools/src/extension_server.rs b/compute_tools/src/extension_server.rs
index ee6580ae36..36c348d76a 100644
--- a/compute_tools/src/extension_server.rs
+++ b/compute_tools/src/extension_server.rs
@@ -1,8 +1,5 @@
-// AT LEAST download_extension / get_available_extensions need large changes
-// refactor custom_extensions to be clear that we have moved the access logic to control plane
-
 // Download extension files from the extension store
-// and put them in the right place in the postgres directory
+// and put them in the right place in the postgres directory (share / lib)
 /*
 The layout of the S3 bucket is as follows:
 ├── 111
diff --git a/docs/rfcs/024-extension-loading.md b/docs/rfcs/024-extension-loading.md
index 4ac187dfcc..f775e3372c 100644
--- a/docs/rfcs/024-extension-loading.md
+++ b/docs/rfcs/024-extension-loading.md
@@ -141,37 +141,64 @@ popular extensions.
 
 The layout of the S3 bucket is as follows:
 ```
-v14/ext_index.json
-    -- this contains information necessary to create control files
-v14/extensions/test_ext1.tar.gz
-    -- this contains the library files and sql files necessary to create this extension
-v14/extensions/custom_ext1.tar.gz
+├── 111
+    │   ├── v14
+    │   │   ├── extensions
+    │   │   │   ├── anon.tar.zst
+    │   │   │   └── embedding.tar.zst
+    │   │   └── ext_index.json
+    │   └── v15
+    │       ├── extensions
+    │       │   ├── anon.tar.zst
+    │       │   └── embedding.tar.zst
+    │       └── ext_index.json
+    ├── 112
+    │   ├── v14
+    │   │   ├── extensions
+    │   │   │   └── anon.tar.zst
+    │   │   └── ext_index.json
+    │   └── v15
+    │       ├── extensions
+    │       │   └── anon.tar.zst
+    │       └── ext_index.json
+    └── 113
+        ├── v14
+        │   ├── extensions
+        │   │   └── embedding.tar.zst
+        │   └── ext_index.json
+        └── v15
+            ├── extensions
+            │   └── embedding.tar.zst
+            └── ext_index.json
 ```
-The difference between private and public extensions is determined by who can
-load the extension. This is specified in `ext_index.json`.
-Speicially, `ext_index.json` has a list of public extensions, and a list of
-extensions enabled for specific tenant-ids. Here is an example `ext_index.json`:
+Note that build number cannot be part of prefix because we might need extensions
+from other build numbers.
+
+`ext_index.json` stores the control files and location of extension archives
+
+We do not duplicate `extension.tar.zst` files.
+We only upload a new one if it is updated.
+**access** is controlled by spec
+
+More specifically, here is an example `ext_index.json`
 ```
 {
-  "enabled_extensions": {
-    "123454321": [
-      "anon"
-    ],
-    "public": [
-      "embedding"
-    ]
+  "embedding": {
+    "control_file_content": "comment = 'hnsw index' \ndefault_version = '0.1.0' \nmodule_pathname = '$libdir/embedding' \nrelocatable = true \ntrusted = true",
+    "extension_archive": "111/v14/extensions/embedding.tar.zst"
   },
-  "control_data": {
-    "embedding": "comment = 'hnsw index' \ndefault_version = '0.1.0' \nmodule_pathname = '$libdir/embedding' \nrelocatable = true \ntrusted = true",
-    "anon": "# PostgreSQL Anonymizer (anon) extension \ncomment = 'Data anonymization tools' \ndefault_version = '1.1.0' \ndirectory='extension/anon' \nrelocatable = false \nrequires = 'pgcrypto' \nsuperuser = false \nmodule_pathname = '$libdir/anon' \ntrusted = true \n"
+  "anon": {
+    "control_file_content": "# PostgreSQL Anonymizer (anon) extension \ncomment = 'Data anonymization tools' \ndefault_version = '1.1.0' \ndirectory='extension/anon' \nrelocatable = false \nrequires = 'pgcrypto' \nsuperuser = false \nmodule_pathname = '$libdir/anon' \ntrusted = true \n",
+    "extension_archive": "111/v14/extensions/anon.tar.zst"
   }
 }
 ```
 
+
 ### How to add new extension to the Extension Storage?
 
 Simply upload build artifacts to the S3 bucket.
-Implement a CI step for that. Splitting it from ompute-node-image build.
+Implement a CI step for that. Splitting it from compute-node-image build.
 
 ### How do we deal with extension versions and updates?
 
diff --git a/test_runner/regress/test_download_extensions.py b/test_runner/regress/test_download_extensions.py
index 57c0f37bf7..4a5729df04 100644
--- a/test_runner/regress/test_download_extensions.py
+++ b/test_runner/regress/test_download_extensions.py
@@ -19,8 +19,6 @@ def test_remote_extensions(
     remote_storage_kind: RemoteStorageKind,
     pg_version: PgVersion,
 ):
-    if remote_storage_kind == RemoteStorageKind.REAL_S3:
-        return None
     neon_env_builder.enable_remote_storage(
         remote_storage_kind=remote_storage_kind,
         test_name="test_remote_extensions",