diff --git a/.gitignore b/.gitignore index 3ebf5fbd3e..e2e26e0eda 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ compaction-suite-results.* # pgindent typedef lists *.list + +# various files for local testing +/proxy/.subzero diff --git a/proxy/SUBZERO.md b/proxy/SUBZERO.md deleted file mode 100644 index 38ac85fa80..0000000000 --- a/proxy/SUBZERO.md +++ /dev/null @@ -1,39 +0,0 @@ -# Subzero - -## Setup - -In the root of the repo folder, run: - -Let's create self-signed certificate by running: -```sh -openssl req -new -x509 -days 365 -nodes -text -out server.crt -keyout server.key -subj "/CN=*.local.neon.build" -``` - -bring up the database using docker compose -```sh -docker compose up -f proxy/subzero/docker-compose.yml -d -``` - -bring up the local proxy (but disable pg_session_jwt extension installation) -```sh -cargo run --bin local_proxy -- \ - --disable-pg-session-jwt \ - --http 0.0.0.0:7432 -``` - -bring up the proxy (auth broker) which also handles the /rest routes handled by subzero code -```sh -LOGFMT=text cargo run --bin proxy -- \ - --is-auth-broker true \ - --is-rest-broker true \ - -c server.crt -k server.key \ - --wss 0.0.0.0:8080 \ - --http 0.0.0.0:7002 \ - --auth-backend cplane-v1 -``` - -```sh -curl -k -i \ - -H "Authorization: Bearer $NEON_JWT" \ - "https://127.0.0.1:8080/rest/v1/items" -``` \ No newline at end of file diff --git a/proxy/src/serverless/rest.rs b/proxy/src/serverless/rest.rs index 4099c37e3a..ed8dba1994 100644 --- a/proxy/src/serverless/rest.rs +++ b/proxy/src/serverless/rest.rs @@ -63,7 +63,34 @@ use std::collections::HashMap; use jsonpath_lib::select; use url::form_urlencoded; - +static JSON_SCHEMA: &str = r#" + { + "schemas":[ + { + "name":"test", + "objects":[ + { + "kind":"table", + "name":"items", + "columns":[ + { + "name":"id", + "data_type":"integer", + "primary_key":true + }, + { + "name":"name", + "data_type":"text" + } + ], + "foreign_keys":[], + "permissions":[] + } + ] + } + ] + } +"#; pub(super) static NEON_REQUEST_ID: HeaderName = HeaderName::from_static("neon-request-id"); @@ -71,7 +98,7 @@ pub(super) static NEON_REQUEST_ID: HeaderName = HeaderName::from_static("neon-re static CONN_STRING: HeaderName = HeaderName::from_static("neon-connection-string"); //static RAW_TEXT_OUTPUT: HeaderName = HeaderName::from_static("neon-raw-text-output"); //static ARRAY_MODE: HeaderName = HeaderName::from_static("neon-array-mode"); -//static ALLOW_POOL: HeaderName = HeaderName::from_static("neon-pool-opt-in"); +static ALLOW_POOL: HeaderName = HeaderName::from_static("neon-pool-opt-in"); static TXN_ISOLATION_LEVEL: HeaderName = HeaderName::from_static("neon-batch-isolation-level"); static TXN_READ_ONLY: HeaderName = HeaderName::from_static("neon-batch-read-only"); //static TXN_DEFERRABLE: HeaderName = HeaderName::from_static("neon-batch-deferrable"); @@ -452,34 +479,7 @@ static HEADERS_TO_FORWARD: &[&HeaderName] = &[ &AUTHORIZATION, ]; -static JSON_SCHEMA: &str = r#" - { - "schemas":[ - { - "name":"test", - "objects":[ - { - "kind":"table", - "name":"items", - "columns":[ - { - "name":"id", - "data_type":"integer", - "primary_key":true - }, - { - "name":"name", - "data_type":"text" - } - ], - "foreign_keys":[], - "permissions":[] - } - ] - } - ] - } -"#; + fn content_range_header(lower: i64, upper: i64, total: Option) -> String { //debug!("content_range_header: lower: {}, upper: {}, total: {:?}", lower, upper, total); @@ -836,7 +836,7 @@ async fn handle_rest_inner( None } }; - + //TODO: check if the token is properly cached in the backend (should we cache the parsed claims?) // read the role from the jwt claims (and set it to the "anon" role if not present) let (role, authenticated) = match &jwt_claims { Some(claims) => match select(claims, &role_claim_path) { @@ -988,6 +988,7 @@ async fn handle_rest_inner( req = req.header(&NEON_REQUEST_ID, uuid_to_header_value(ctx.session_id())); req = req.header(&CONN_STRING, HeaderValue::from_str(connection_string).unwrap()); req = req.header(&TXN_ISOLATION_LEVEL, HeaderValue::from_str("ReadCommitted").unwrap()); + req = req.header(&ALLOW_POOL, HeaderValue::from_str("true").unwrap()); if api_request.read_only { req = req.header(&TXN_READ_ONLY, HeaderValue::from_str("true").unwrap()); } diff --git a/proxy/subzero/docker-compose.yml b/proxy/subzero/docker-compose.yml deleted file mode 100644 index 606a1e0d45..0000000000 --- a/proxy/subzero/docker-compose.yml +++ /dev/null @@ -1,12 +0,0 @@ -services: - proxy-postgres: - image: postgres:17-bookworm - container_name: proxy-postgres - environment: - POSTGRES_HOST_AUTH_METHOD: trust - POSTGRES_USER: superuser - POSTGRES_DB: database - ports: - - "5432:5432" - volumes: - - ./sql:/docker-entrypoint-initdb.d diff --git a/proxy/subzero/sql/include/neon_control_plane.sql b/proxy/subzero/sql/include/neon_control_plane.sql deleted file mode 100644 index 8b58b81b2b..0000000000 --- a/proxy/subzero/sql/include/neon_control_plane.sql +++ /dev/null @@ -1,12 +0,0 @@ --- docker exec -it proxy-postgres psql -U postgres -c "CREATE SCHEMA IF NOT EXISTS neon_control_plane" --- docker exec -it proxy-postgres psql -U postgres -c "CREATE TABLE neon_control_plane.endpoints (endpoint_id VARCHAR(255) PRIMARY KEY, allowed_ips VARCHAR(255))" --- docker exec -it proxy-postgres psql -U postgres -c "CREATE ROLE proxy WITH SUPERUSER LOGIN PASSWORD 'password';" - -CREATE SCHEMA IF NOT EXISTS neon_control_plane; - -CREATE TABLE IF NOT EXISTS neon_control_plane.endpoints ( - endpoint_id VARCHAR(255) PRIMARY KEY, - allowed_ips VARCHAR(255) -); - --- CREATE ROLE proxy WITH SUPERUSER LOGIN PASSWORD 'password'; \ No newline at end of file diff --git a/proxy/subzero/sql/include/pgrst.sql b/proxy/subzero/sql/include/pgrst.sql deleted file mode 100644 index f23dd4eb2f..0000000000 --- a/proxy/subzero/sql/include/pgrst.sql +++ /dev/null @@ -1,28 +0,0 @@ - --- code to monitor the last schema update -CREATE SCHEMA IF NOT EXISTS pgrst; - -ALTER ROLE authenticator SET pgrst.last_schema_updated = ''; --- Create an event trigger function -CREATE OR REPLACE FUNCTION pgrst.pgrst_watch() RETURNS event_trigger -LANGUAGE plpgsql -AS $$ -DECLARE - current_timestamp_text TEXT; -BEGIN - current_timestamp_text := now()::text; - EXECUTE 'ALTER ROLE authenticator SET pgrst.last_schema_updated = ' || quote_literal(current_timestamp_text); -END; -$$; - - -CREATE OR REPLACE FUNCTION pgrst.last_schema_updated() RETURNS text - LANGUAGE sql - AS $$ - SELECT current_setting('pgrst.last_schema_updated', true); -$$; - --- This event trigger will fire after every ddl_command_end event -CREATE EVENT TRIGGER pgrst_watch - ON ddl_command_end - EXECUTE PROCEDURE pgrst.pgrst_watch(); \ No newline at end of file diff --git a/proxy/subzero/sql/include/roles.sql b/proxy/subzero/sql/include/roles.sql deleted file mode 100644 index 78b77d330d..0000000000 --- a/proxy/subzero/sql/include/roles.sql +++ /dev/null @@ -1,124 +0,0 @@ -CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; -CREATE ROLE anon NOLOGIN; -GRANT anon TO authenticator; - --- reloadable config options --- these settings will override the values in configs/no-defaults.config, so they must be different --- ALTER ROLE authenticator SET pgrst.db_aggregates_enabled = 'false'; -ALTER ROLE authenticator SET pgrst.db_anon_role = 'anon'; -ALTER ROLE authenticator SET pgrst.db_extra_search_path = 'public, extensions'; -ALTER ROLE authenticator SET pgrst.db_max_rows = '500'; --- ALTER ROLE authenticator SET pgrst.db_plan_enabled = 'false'; --- ALTER ROLE authenticator SET pgrst.db_pre_config = 'postgrest.preconf'; --- ALTER ROLE authenticator SET pgrst.db_pre_request = 'test.custom_headers'; --- ALTER ROLE authenticator SET pgrst.db_prepared_statements = 'false'; --- ALTER ROLE authenticator SET pgrst.db_root_spec = 'root'; -ALTER ROLE authenticator SET pgrst.db_schemas = 'test, tenant1, tenant2'; --- ALTER ROLE authenticator SET pgrst.db_tx_end = 'commit-allow-override'; --- ALTER ROLE authenticator SET pgrst.jwt_aud = 'https://example.org'; --- ALTER ROLE authenticator SET pgrst.jwt_cache_max_lifetime = '3600'; -ALTER ROLE authenticator SET pgrst.jwt_role_claim_key = '."role"'; --- ALTER ROLE authenticator SET pgrst.jwt_secret = 'REALLY=REALLY=REALLY=REALLY=VERY=SAFE'; --- ALTER ROLE authenticator SET pgrst.jwt_secret_is_base64 = 'false'; -ALTER ROLE authenticator SET pgrst.not_existing = 'should be ignored'; --- ALTER ROLE authenticator SET pgrst.openapi_server_proxy_uri = 'https://example.org/api'; --- ALTER ROLE authenticator SET pgrst.server_cors_allowed_origins = 'http://origin.com'; --- ALTER ROLE authenticator SET pgrst.server_timing_enabled = 'false'; --- ALTER ROLE authenticator SET pgrst.server_trace_header = 'CF-Ray'; --- ALTER ROLE authenticator SET pgrst.db_hoisted_tx_settings = 'autovacuum_work_mem'; - --- override with database specific setting --- ALTER ROLE authenticator IN DATABASE :DBNAME SET pgrst.db_extra_search_path = 'public, extensions, private'; --- ALTER ROLE authenticator IN DATABASE :DBNAME SET pgrst.jwt_secret = 'OVERRIDE=REALLY=REALLY=REALLY=REALLY=VERY=SAFE'; --- ALTER ROLE authenticator IN DATABASE :DBNAME SET pgrst.not_existing = 'should be ignored'; - --- other database settings that should be ignored --- CREATE DATABASE other; --- ALTER ROLE authenticator IN DATABASE other SET pgrst.db_max_rows = '1111'; - --- non-reloadable configs --- ALTER ROLE authenticator SET pgrst.admin_server_host = 'ignored'; --- ALTER ROLE authenticator SET pgrst.admin_server_port = 'ignored'; --- ALTER ROLE authenticator SET pgrst.db_channel = 'ignored'; --- ALTER ROLE authenticator SET pgrst.db_channel_enabled = 'ignored'; --- ALTER ROLE authenticator SET pgrst.db_config = 'true'; --- ALTER ROLE authenticator SET pgrst.db_pool = 'ignored'; --- ALTER ROLE authenticator SET pgrst.db_pool_acquisition_timeout = 'ignored'; --- ALTER ROLE authenticator SET pgrst.db_pool_timeout = 'ignored'; --- ALTER ROLE authenticator SET pgrst.db_pool_max_idletime = 'ignored'; --- ALTER ROLE authenticator SET pgrst.db_pool_max_lifetime = 'ignored'; --- ALTER ROLE authenticator SET pgrst.db_uri = 'postgresql://ignored'; --- ALTER ROLE authenticator SET pgrst.log_level = 'ignored'; --- ALTER ROLE authenticator SET pgrst.log_query = 'ignored'; --- ALTER ROLE authenticator SET pgrst.server_host = 'ignored'; --- ALTER ROLE authenticator SET pgrst.server_port = 'ignored'; --- ALTER ROLE authenticator SET pgrst.server_unix_socket = 'ignored'; --- ALTER ROLE authenticator SET pgrst.server_unix_socket_mode = 'ignored'; - --- other authenticator reloadable config options --- these settings will override the values in configs/no-defaults.config, so they must be different --- CREATE ROLE other_authenticator LOGIN NOINHERIT; --- ALTER ROLE other_authenticator SET pgrst.db_aggregates_enabled = 'false'; --- ALTER ROLE other_authenticator SET pgrst.db_extra_search_path = 'public, extensions, other'; --- ALTER ROLE other_authenticator SET pgrst.db_max_rows = '100'; --- ALTER ROLE other_authenticator SET pgrst.db_plan_enabled = 'true'; --- ALTER ROLE other_authenticator SET pgrst.db_pre_config = 'postgrest.other_preconf'; --- ALTER ROLE other_authenticator SET pgrst.db_pre_request = 'test.other_custom_headers'; --- ALTER ROLE other_authenticator SET pgrst.db_prepared_statements = 'false'; --- ALTER ROLE other_authenticator SET pgrst.db_root_spec = 'other_root'; --- ALTER ROLE other_authenticator SET pgrst.db_schemas = 'test, other_tenant1, other_tenant2'; --- ALTER ROLE other_authenticator SET pgrst.jwt_aud = 'https://otherexample.org'; --- ALTER ROLE other_authenticator SET pgrst.jwt_secret = 'ODERREALLYREALLYREALLYREALLYVERYSAFE'; --- ALTER ROLE other_authenticator SET pgrst.jwt_secret_is_base64 = 'false'; --- ALTER ROLE other_authenticator SET pgrst.jwt_cache_max_lifetime = '7200'; --- ALTER ROLE other_authenticator SET pgrst.openapi_mode = 'disabled'; --- ALTER ROLE other_authenticator SET pgrst.openapi_security_active = 'false'; --- ALTER ROLE other_authenticator SET pgrst.openapi_server_proxy_uri = 'https://otherexample.org/api'; --- ALTER ROLE other_authenticator SET pgrst.server_cors_allowed_origins = 'http://otherorigin.com'; --- ALTER ROLE other_authenticator SET pgrst.server_timing_enabled = 'true'; --- ALTER ROLE other_authenticator SET pgrst.server_trace_header = 'traceparent'; --- ALTER ROLE other_authenticator SET pgrst.db_hoisted_tx_settings = 'maintenance_work_mem'; - --- create schema postgrest; --- grant usage on schema postgrest to authenticator; --- grant usage on schema postgrest to other_authenticator; - --- -- pre-config hook --- create or replace function postgrest.pre_config() --- returns void as $$ --- begin --- if current_user = 'other_authenticator' then --- perform --- set_config('pgrst.jwt_role_claim_key', '."other"."pre_config_role"', true) --- , set_config('pgrst.db_anon_role', 'pre_config_role', true) --- , set_config('pgrst.db_schemas', 'will be overriden with the above ALTER ROLE.. db_schemas', true) --- , set_config('pgrst.db_tx_end', 'rollback-allow-override', true); --- else --- null; --- end if; --- end $$ language plpgsql; - --- create or replace function postgrest.preconf() --- returns void as $$ --- begin --- null; --- end $$ language plpgsql; - --- create or replace function postgrest.other_preconf() --- returns void as $$ --- begin --- perform postgrest.pre_config(); --- end $$ language plpgsql; - --- -- authenticator used for tests that manipulate statement timeout --- CREATE ROLE timeout_authenticator LOGIN NOINHERIT; - --- create function set_statement_timeout(role text, milliseconds int) returns void as $_$ --- begin --- execute format($$ --- alter role %I set statement_timeout to %L; --- $$, role, milliseconds); --- end $_$ volatile security definer language plpgsql; - --- -- authenticator used for test-independent database manipulation --- CREATE ROLE meta_authenticator LOGIN NOINHERIT; diff --git a/proxy/subzero/sql/include/tenant1.sql b/proxy/subzero/sql/include/tenant1.sql deleted file mode 100644 index 19ee717b76..0000000000 --- a/proxy/subzero/sql/include/tenant1.sql +++ /dev/null @@ -1,18 +0,0 @@ -CREATE SCHEMA IF NOT EXISTS tenant1; - -CREATE TABLE IF NOT EXISTS tenant1.items ( - id SERIAL PRIMARY KEY, - name VARCHAR(255) NOT NULL -); - -INSERT INTO tenant1.items (name) VALUES - ('tenant1 item 1'), - ('tenant1 item 2'), - ('tenant1 item 3'); - - -CREATE ROLE tenant1_role NOLOGIN; -GRANT tenant1_role TO authenticator; - -GRANT USAGE ON SCHEMA tenant1 TO tenant1_role; -GRANT ALL ON ALL TABLES IN SCHEMA tenant1 TO tenant1_role; \ No newline at end of file diff --git a/proxy/subzero/sql/include/tenant2.sql b/proxy/subzero/sql/include/tenant2.sql deleted file mode 100644 index b0b6290482..0000000000 --- a/proxy/subzero/sql/include/tenant2.sql +++ /dev/null @@ -1,18 +0,0 @@ -CREATE SCHEMA IF NOT EXISTS tenant2; - -CREATE TABLE IF NOT EXISTS tenant2.items ( - id SERIAL PRIMARY KEY, - name VARCHAR(255) NOT NULL -); - -INSERT INTO tenant2.items (name) VALUES - ('tenant2 item 1'), - ('tenant2 item 2'), - ('tenant2 item 3'); - - -CREATE ROLE tenant2_role NOLOGIN; -GRANT tenant2_role TO authenticator; - -GRANT USAGE ON SCHEMA tenant2 TO tenant2_role; -GRANT ALL ON ALL TABLES IN SCHEMA tenant2 TO tenant2_role; \ No newline at end of file diff --git a/proxy/subzero/sql/include/test.sql b/proxy/subzero/sql/include/test.sql deleted file mode 100644 index 7bf3781505..0000000000 --- a/proxy/subzero/sql/include/test.sql +++ /dev/null @@ -1,17 +0,0 @@ -CREATE SCHEMA IF NOT EXISTS test; - -CREATE TABLE IF NOT EXISTS test.items ( - id SERIAL PRIMARY KEY, - name VARCHAR(255) NOT NULL -); - -INSERT INTO test.items (name) VALUES - ('test item 1'), - ('test item 2'), - ('test item 3'); - -CREATE ROLE authenticated NOLOGIN; -GRANT authenticated TO authenticator; - -GRANT USAGE ON SCHEMA test TO authenticated; -GRANT ALL ON ALL TABLES IN SCHEMA test TO authenticated; \ No newline at end of file diff --git a/proxy/subzero/sql/init.sql b/proxy/subzero/sql/init.sql deleted file mode 100644 index 3b9c585d9c..0000000000 --- a/proxy/subzero/sql/init.sql +++ /dev/null @@ -1,6 +0,0 @@ -\ir include/neon_control_plane.sql -\ir include/roles.sql -\ir include/test.sql -\ir include/tenant1.sql -\ir include/tenant2.sql -\ir include/pgrst.sql \ No newline at end of file