From 353a735acb4f1bcd1e644f47f5b8ef00649d74d1 Mon Sep 17 00:00:00 2001 From: Alek Westover Date: Fri, 14 Jul 2023 07:58:05 -0400 Subject: [PATCH 1/3] @arpad-m suggested using as_slice instead of creating a cursor --- compute_tools/src/extension_server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute_tools/src/extension_server.rs b/compute_tools/src/extension_server.rs index bff8fca303..4bdddaf544 100644 --- a/compute_tools/src/extension_server.rs +++ b/compute_tools/src/extension_server.rs @@ -157,7 +157,7 @@ pub async fn download_extension( .read_to_end(&mut write_data_buffer) .await?; let unzip_dest = pgbin.strip_suffix("/bin/postgres").expect("bad pgbin"); - let tar = GzDecoder::new(std::io::Cursor::new(write_data_buffer)); + let tar = GzDecoder::new(write_data_buffer.as_slice()); let mut archive = Archive::new(tar); archive.unpack(unzip_dest)?; info!("Download + unzip {:?} completed successfully", &ext_path); From 3cf83014d4138f538f37d95935155f29e65a8c4d Mon Sep 17 00:00:00 2001 From: Alek Westover Date: Fri, 14 Jul 2023 09:21:46 -0400 Subject: [PATCH 2/3] patch rfc --- docs/rfcs/024-extension-loading.md | 50 ++++++++++++++++-------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/docs/rfcs/024-extension-loading.md b/docs/rfcs/024-extension-loading.md index bc9ed4407b..874503e406 100644 --- a/docs/rfcs/024-extension-loading.md +++ b/docs/rfcs/024-extension-loading.md @@ -139,29 +139,33 @@ popular extensions. ## Extension Storage implementation -Extension Storage in our case is an S3 bucket with a "directory" per build and postgres version, -where extension files are stored as plain files in the bucket following the same directory structure as in the postgres. - -i.e. - -`s3://///lib/postgis-3.1.so` -`s3://///share/extension/postgis.control` -`s3://///share/extension/postgis--3.1.sql` - -To handle custom extensions, that available only to specific users, we use per-extension subdirectories: - -i.e. -`s3://////lib/ext-name.so`, etc. -`s3://////share/extension/ext-name.control`, etc. - -On compute start, `compute_ctl` accepts a list of custom_ext_prefixes. - -To get the list of available extensions,`compute_ctl` downloads control files from all prefixes: - -`s3://///share/extension/` -`s3://////share/extension/` -`s3://////share/extension/` - +The layout of the S3 bucket is as follows: +``` +v14/ext_index.json + -- this contains information necessary to create control files +v14/extensions/test_ext1.tar.gz + -- this contains the library files and sql files necessary to create this extension +v14/extensions/custom_ext1.tar.gz +``` +The difference between private and public extensions is determined by who can +load the extension. This is specified in `ext_index.json`. +Speicially, `ext_index.json` has a list of public extensions, and a list of +extensions enabled for specific tenant-ids. Here is an example `ext_index.json`: +``` +{ + "enabled_extensions": { + "123454321": [ + "anon" + ], + "public": [ + "embedding" + ] + }, + "control_data": { + "embedding": "comment = 'hnsw index' \ndefault_version = '0.1.0' \nmodule_pathname = '$libdir/embedding' \nrelocatable = true \ntrusted = true", + "anon": "# PostgreSQL Anonymizer (anon) extension \ncomment = 'Data anonymization tools' \ndefault_version = '1.1.0' \ndirectory='extension/anon' \nrelocatable = false \nrequires = 'pgcrypto' \nsuperuser = false \nmodule_pathname = '$libdir/anon' \ntrusted = true \n" + } +} ### How to add new extension to the Extension Storage? From 87eead5220014052af418a354ea30b89d433a739 Mon Sep 17 00:00:00 2001 From: Alek Westover Date: Fri, 14 Jul 2023 10:54:16 -0400 Subject: [PATCH 3/3] Update rfc --- docs/rfcs/024-extension-loading.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/rfcs/024-extension-loading.md b/docs/rfcs/024-extension-loading.md index 874503e406..4ac187dfcc 100644 --- a/docs/rfcs/024-extension-loading.md +++ b/docs/rfcs/024-extension-loading.md @@ -166,7 +166,7 @@ extensions enabled for specific tenant-ids. Here is an example `ext_index.json`: "anon": "# PostgreSQL Anonymizer (anon) extension \ncomment = 'Data anonymization tools' \ndefault_version = '1.1.0' \ndirectory='extension/anon' \nrelocatable = false \nrequires = 'pgcrypto' \nsuperuser = false \nmodule_pathname = '$libdir/anon' \ntrusted = true \n" } } - +``` ### How to add new extension to the Extension Storage?