From 86be2747d0be9235a11c8926649152e9eb0969db Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Sun, 5 Nov 2023 10:16:56 -0800 Subject: [PATCH 1/6] [DRAFT] test upcoming changes to vm-builder --- .github/workflows/build_and_test.yml | 6 +- vm-image-spec.yaml | 124 +++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 vm-image-spec.yaml diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index e0f8afc954..f48670af21 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -857,7 +857,8 @@ jobs: - name: Downloading vm-builder run: | - curl -fL https://github.com/neondatabase/autoscaling/releases/download/$VM_BUILDER_VERSION/vm-builder -o vm-builder + # TEMPORARY: Pull custom version of vm-builder built from neondatabase/autoscaling#600. + curl -fL https://sharnoff.io/misc/8706cca48732d4f9/vm-builder-9d18874 -o vm-builder chmod +x vm-builder # Note: we need a separate pull step here because otherwise vm-builder will try to pull, and @@ -869,8 +870,7 @@ jobs: - name: Build vm image run: | ./vm-builder \ - -enable-file-cache \ - -cgroup-uid=postgres \ + -spec=vm-image-spec.yaml -src=369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} \ -dst=369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} diff --git a/vm-image-spec.yaml b/vm-image-spec.yaml new file mode 100644 index 0000000000..60385f9fa9 --- /dev/null +++ b/vm-image-spec.yaml @@ -0,0 +1,124 @@ +# Supplemental file for neondatabase/autoscaling's vm-builder, for producing the VM compute image. +--- +commands: + - name: cgconfigparser + user: root + sysvInitAction: sysinit + shell: 'cgconfigparser -l /etc/cgconfig.conf -s 1664' + - name: pgbouncer + user: nobody + sysvInitAction: respawn + shell: '/usr/local/bin/pgbouncer /etc/pgbouncer.ini' + - name: postgres-exporter + user: nobody + sysvInitAction: respawn + shell: 'DATA_SOURCE_NAME="user=cloud_admin sslmode=disable dbname=postgres" /bin/postgres_exporter' +files: + - name: pgbouncer.ini + content: | + [databases] + *=host=localhost port=5432 auth_user=cloud_admin + [pgbouncer] + listen_port=6432 + listen_addr=0.0.0.0 + auth_type=scram-sha-256 + auth_user=cloud_admin + auth_dbname=postgres + client_tls_sslmode=disable + server_tls_sslmode=disable + pool_mode=transaction + max_client_conn=10000 + default_pool_size=16 + max_prepared_statements=0 + - name: cgconfig.conf + content: | + # Configuration for cgroups in VM compute nodes + group neon-postgres { + perm { + admin { + uid = postgres; + } + task { + gid = users; + } + } + memory {} + } +build: | + # Build cgroup-tools + # + # At time of writing (2023-03-14), debian bullseye has a version of cgroup-tools (technically + # libcgroup) that doesn't support cgroup v2 (version 0.41-11). Unfortunately, the vm-monitor + # requires cgroup v2, so we'll build cgroup-tools ourselves. + FROM debian:bullseye-slim as libcgroup-builder + ENV LIBCGROUP_VERSION v2.0.3 + + RUN set -exu \ + && apt update \ + && apt install --no-install-recommends -y \ + git \ + ca-certificates \ + automake \ + cmake \ + make \ + gcc \ + byacc \ + flex \ + libtool \ + libpam0g-dev \ + && git clone --depth 1 -b $LIBCGROUP_VERSION https://github.com/libcgroup/libcgroup \ + && INSTALL_DIR="/libcgroup-install" \ + && mkdir -p "$INSTALL_DIR/bin" "$INSTALL_DIR/include" \ + && cd libcgroup \ + # extracted from bootstrap.sh, with modified flags: + && (test -d m4 || mkdir m4) \ + && autoreconf -fi \ + && rm -rf autom4te.cache \ + && CFLAGS="-O3" ./configure --prefix="$INSTALL_DIR" --sysconfdir=/etc --localstatedir=/var --enable-opaque-hierarchy="name=systemd" \ + # actually build the thing... + && make install + + FROM quay.io/prometheuscommunity/postgres-exporter:v0.12.0 AS postgres-exporter + + # Build pgbouncer + # + FROM debian:bullseye-slim AS pgbouncer + RUN set -e \ + && apt-get update \ + && apt-get install -y \ + curl \ + build-essential \ + pkg-config \ + libevent-dev \ + libssl-dev + + ENV PGBOUNCER_VERSION 1.21.0 + ENV PGBOUNCER_GITPATH 1_21_0 + RUN set -e \ + && curl -sfSL https://github.com/pgbouncer/pgbouncer/releases/download/pgbouncer_${PGBOUNCER_GITPATH}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz -o pgbouncer-${PGBOUNCER_VERSION}.tar.gz \ + && tar xzvf pgbouncer-${PGBOUNCER_VERSION}.tar.gz \ + && cd pgbouncer-${PGBOUNCER_VERSION} \ + && LDFLAGS=-static ./configure --prefix=/usr/local/pgbouncer --without-openssl \ + && make -j $(nproc) \ + && make install +merge: | + # tweak nofile limits + RUN set -e \ + && echo 'fs.file-max = 1048576' >>/etc/sysctl.conf \ + && test ! -e /etc/security || ( \ + echo '* - nofile 1048576' >>/etc/security/limits.conf \ + && echo 'root - nofile 1048576' >>/etc/security/limits.conf \ + ) + + COPY cgconfig.conf /etc/cgconfig.conf + COPY pgbouncer.ini /etc/pgbouncer.ini + RUN set -e \ + && chown postgres:postgres /etc/pgbouncer.ini \ + && chmod 0644 /etc/pgbouncer.ini \ + && chmod 0644 /etc/cgconfig.conf + + COPY --from=libcgroup-builder /libcgroup-install/bin/* /usr/bin/ + COPY --from=libcgroup-builder /libcgroup-install/lib/* /usr/lib/ + COPY --from=libcgroup-builder /libcgroup-install/sbin/* /usr/sbin/ + COPY --from=postgres-exporter /bin/postgres_exporter /bin/postgres_exporter + COPY --from=pgbouncer /usr/local/pgbouncer/bin/pgbouncer /usr/local/bin/pgbouncer From 03753e277efa8ccf4e792c0a65c66a45e50a6e48 Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Sun, 5 Nov 2023 12:01:14 -0800 Subject: [PATCH 2/6] fix missing '\' --- .github/workflows/build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index f48670af21..220ff90a39 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -870,7 +870,7 @@ jobs: - name: Build vm image run: | ./vm-builder \ - -spec=vm-image-spec.yaml + -spec=vm-image-spec.yaml \ -src=369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} \ -dst=369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} From ba252c951311aedc8bff74ab3f1a91d5717060e4 Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Sun, 5 Nov 2023 12:20:29 -0800 Subject: [PATCH 3/6] fix field name --- vm-image-spec.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm-image-spec.yaml b/vm-image-spec.yaml index 60385f9fa9..6e7bb6d50e 100644 --- a/vm-image-spec.yaml +++ b/vm-image-spec.yaml @@ -14,7 +14,7 @@ commands: sysvInitAction: respawn shell: 'DATA_SOURCE_NAME="user=cloud_admin sslmode=disable dbname=postgres" /bin/postgres_exporter' files: - - name: pgbouncer.ini + - filename: pgbouncer.ini content: | [databases] *=host=localhost port=5432 auth_user=cloud_admin @@ -30,7 +30,7 @@ files: max_client_conn=10000 default_pool_size=16 max_prepared_statements=0 - - name: cgconfig.conf + - filename: cgconfig.conf content: | # Configuration for cgroups in VM compute nodes group neon-postgres { From 4ffe43b8f778837bb6b4aa862ff0b6628c87de57 Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Sun, 5 Nov 2023 14:57:24 -0800 Subject: [PATCH 4/6] update to explicitly set shutdown hook --- .github/workflows/build_and_test.yml | 2 +- vm-image-spec.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 220ff90a39..b5e46f506b 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -858,7 +858,7 @@ jobs: - name: Downloading vm-builder run: | # TEMPORARY: Pull custom version of vm-builder built from neondatabase/autoscaling#600. - curl -fL https://sharnoff.io/misc/8706cca48732d4f9/vm-builder-9d18874 -o vm-builder + curl -fL https://sharnoff.io/misc/8706cca48732d4f9/vm-builder-29ef2a1 -o vm-builder chmod +x vm-builder # Note: we need a separate pull step here because otherwise vm-builder will try to pull, and diff --git a/vm-image-spec.yaml b/vm-image-spec.yaml index 6e7bb6d50e..2aa935fac6 100644 --- a/vm-image-spec.yaml +++ b/vm-image-spec.yaml @@ -13,6 +13,8 @@ commands: user: nobody sysvInitAction: respawn shell: 'DATA_SOURCE_NAME="user=cloud_admin sslmode=disable dbname=postgres" /bin/postgres_exporter' +shutdownHook: | + su -p postgres --session-command '/usr/local/bin/pg_ctl stop -D /var/db/postgres/compute/pgdata -m fast --wait -t 10' files: - filename: pgbouncer.ini content: | From 6b82f22ada28c72797235df0cd6a8c2104e15df3 Mon Sep 17 00:00:00 2001 From: khanova <32508607+khanova@users.noreply.github.com> Date: Thu, 16 Nov 2023 13:19:13 +0100 Subject: [PATCH 5/6] Collect number of connections by sni type (#5867) ## Problem We don't know the number of users with the different kind of authentication: ["sni", "endpoint in options" (A and B from [here](https://neon.tech/docs/connect/connection-errors)), "password_hack"] ## Summary of changes Collect metrics by sni kind. --- proxy/src/auth/credentials.rs | 20 +++++++++++++++++++- proxy/src/proxy.rs | 9 +++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/proxy/src/auth/credentials.rs b/proxy/src/auth/credentials.rs index d7a8edca79..9fe9c26f0c 100644 --- a/proxy/src/auth/credentials.rs +++ b/proxy/src/auth/credentials.rs @@ -1,7 +1,9 @@ //! User credentials used in authentication. use crate::{ - auth::password_hack::parse_endpoint_param, error::UserFacingError, proxy::neon_options, + auth::password_hack::parse_endpoint_param, + error::UserFacingError, + proxy::{neon_options, NUM_CONNECTION_ACCEPTED_BY_SNI}, }; use itertools::Itertools; use pq_proto::StartupMessageParams; @@ -124,6 +126,22 @@ impl<'a> ClientCredentials<'a> { .transpose()?; info!(user, project = project.as_deref(), "credentials"); + if sni.is_some() { + info!("Connection with sni"); + NUM_CONNECTION_ACCEPTED_BY_SNI + .with_label_values(&["sni"]) + .inc(); + } else if project.is_some() { + NUM_CONNECTION_ACCEPTED_BY_SNI + .with_label_values(&["no_sni"]) + .inc(); + info!("Connection without sni"); + } else { + NUM_CONNECTION_ACCEPTED_BY_SNI + .with_label_values(&["password_hack"]) + .inc(); + info!("Connection with password hack"); + } let cache_key = format!( "{}{}", diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index 2d38acd05b..b03f95dc5f 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -129,6 +129,15 @@ pub static RATE_LIMITER_LIMIT: Lazy = Lazy::new(|| { .unwrap() }); +pub static NUM_CONNECTION_ACCEPTED_BY_SNI: Lazy = Lazy::new(|| { + register_int_counter_vec!( + "proxy_accepted_connections_by_sni", + "Number of connections (per sni).", + &["kind"], + ) + .unwrap() +}); + pub struct LatencyTimer { // time since the stopwatch was started start: Option, From 6fae789e88eccf145583863f58a006500774f512 Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Thu, 16 Nov 2023 15:04:21 +0100 Subject: [PATCH 6/6] update to v0.19.0 --- .github/workflows/build_and_test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index b5e46f506b..e986aced8e 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -847,7 +847,7 @@ jobs: run: shell: sh -eu {0} env: - VM_BUILDER_VERSION: v0.18.5 + VM_BUILDER_VERSION: v0.19.0 steps: - name: Checkout @@ -857,8 +857,7 @@ jobs: - name: Downloading vm-builder run: | - # TEMPORARY: Pull custom version of vm-builder built from neondatabase/autoscaling#600. - curl -fL https://sharnoff.io/misc/8706cca48732d4f9/vm-builder-29ef2a1 -o vm-builder + curl -fL https://github.com/neondatabase/autoscaling/releases/download/$VM_BUILDER_VERSION/vm-builder -o vm-builder chmod +x vm-builder # Note: we need a separate pull step here because otherwise vm-builder will try to pull, and