Calm clippy

Dockerfile.compute-node-v14

@@ -57,6 +57,31 @@ RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.0.tar.gz && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_tiger_geocoder.control && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_topology.control
 
+# Layer "pg-stat-contribs-build"
+# Build pg_stat_statements, pg_wait_sampling, pg_query_state
+
+FROM build-deps AS pg-query-state-build
+COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
+
+RUN wget https://github.com/postgrespro/pg_query_state/archive/refs/heads/master.tar.gz && \
+    tar xvzf master.tar.gz && \
+    cd pg_query_state-master && \
+    export PATH="/usr/local/pgsql/bin:$PATH" && \
+    USE_PGXS=true PG_CONFIG=/usr/local/pgsql/bin/pg_config make -j $(getconf _NPROCESSORS_ONLN) && \
+    USE_PGXS=true PG_CONFIG=/usr/local/pgsql/bin/pg_config make -j $(getconf _NPROCESSORS_ONLN) install && \
+    echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_query_state.control
+
+FROM build-deps AS pg-wait-sampling-build
+COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
+
+RUN wget https://github.com/postgrespro/pg_wait_sampling/archive/refs/heads/master.tar.gz && \
+    tar xvzf master.tar.gz && \
+    cd pg_wait_sampling-master && \
+    export PATH="/usr/local/pgsql/bin:$PATH" && \
+    USE_PGXS=true PG_CONFIG=/usr/local/pgsql/bin/pg_config make -j $(getconf _NPROCESSORS_ONLN) && \
+    USE_PGXS=true PG_CONFIG=/usr/local/pgsql/bin/pg_config make -j $(getconf _NPROCESSORS_ONLN) install && \
+    echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_wait_sampling.control
+
 #
 # Layer "plv8-build"
 # Build plv8
@@ -116,6 +141,8 @@ RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.0.1.tar.gz -O h3
 #
 FROM build-deps AS neon-pg-ext-build
 COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/
+COPY --from=pg-query-state-build /usr/local/pgsql/ /usr/local/pgsql/
+COPY --from=pg-wait-sampling-build /usr/local/pgsql/ /usr/local/pgsql/
 # plv8 still sometimes crashes during the creation
 # COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=h3-pg-build /usr/local/pgsql/ /usr/local/pgsql/

Makefile

@@ -116,6 +116,21 @@ postgres-v14: postgres-v14-configure \
 	+@echo "Compiling pageinspect v14"
 	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pageinspect install
 
+.PHONY: pg_wait_sampling-v14
+pg_wait_sampling-v14: postgres-v14-configure
+	+@echo "Compiling pg_wait_sampling v14"
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pg_wait_sampling install
+
+.PHONY: pg_stat_statements-v14
+pg_stat_statements-v14: postgres-v14-configure
+	+@echo "Compiling pg_stat_statements v14"
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pg_stat_statements install
+
+.PHONY: pg_query_state-v14
+pg_query_state-v14: postgres-v14-configure
+	+@echo "Compiling pg_query_state v14"
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pg_query_state install
+
 .PHONY: postgres-v15
 postgres-v15: postgres-v15-configure \
 		  postgres-v15-headers # to prevent `make install` conflicts with neon's `postgres-headers`

compute_tools/src/compute.rs

@@ -26,6 +26,7 @@ use chrono::{DateTime, Utc};
 use log::info;
 use postgres::{Client, NoTls};
 use serde::{Serialize, Serializer};
+use tokio_postgres;
 
 use crate::checker::create_writablity_check_data;
 use crate::config;
@@ -50,6 +51,19 @@ pub struct ComputeNode {
     pub state: RwLock<ComputeState>,
 }
 
+#[derive(Debug, Serialize)]
+pub struct InsightsRow {
+    pub query: String,
+    pub total_runtime: String,
+    pub mean_runtime: String,
+}
+
+#[derive(Serialize)]
+pub struct Insights {
+    count: u64,
+    statements: Vec<InsightsRow>,
+}
+
 fn rfc3339_serialize<S>(x: &DateTime<Utc>, s: S) -> Result<S::Ok, S::Error>
 where
     S: Serializer,
@@ -296,6 +310,7 @@ impl ComputeNode {
             Ok(client) => client,
         };
 
+        create_system_extensions(&mut client)?;
         handle_roles(&self.spec, &mut client)?;
         handle_databases(&self.spec, &mut client)?;
         handle_role_deletions(self, &mut client)?;
@@ -351,4 +366,38 @@ impl ComputeNode {
         self.prepare_pgdata()?;
         self.run()
     }
+
+    pub async fn collect_insights(&self) -> String {
+        let mut result_rows: Vec<String> = Vec::new();
+        let connect_result = tokio_postgres::connect(self.connstr.as_str(), NoTls).await;
+        let (client, connection) = connect_result.unwrap();
+        tokio::spawn(async move {
+            if let Err(e) = connection.await {
+                eprintln!("connection error: {}", e);
+            }
+        });
+        for message in (client
+            .simple_query(
+                "
+SELECT
+    row_to_json(pg_stat_statements)
+FROM
+    pg_stat_statements
+WHERE
+    userid != 'cloud_admin'::regrole::oid
+ORDER BY
+    (mean_exec_time + mean_plan_time) DESC
+LIMIT 100",
+            )
+            .await)
+            .unwrap()
+            .iter()
+        {
+            if let postgres::SimpleQueryMessage::Row(row) = message {
+                result_rows.push(row.get(0).unwrap().to_string());
+            }
+        }
+
+        format!("{{\"pg_stat_statements\": [{}]}}", result_rows.join(","))
+    }
 }

compute_tools/src/http/api.rs

@@ -46,6 +46,13 @@ async fn routes(req: Request<Body>, compute: Arc<ComputeNode>) -> Response<Body>
             Response::new(Body::from(serde_json::to_string(&compute.metrics).unwrap()))
         }
 
+        // Collect Postgres current usage insights
+        (&Method::GET, "/insights") => {
+            info!("serving /insights GET request");
+            let insights = compute.collect_insights().await;
+            Response::new(Body::from(insights))
+        }
+
         // DEPRECATED, use POST instead
         (&Method::GET, "/check_writability") => {
             info!("serving /check_writability GET request");

compute_tools/src/pg_helpers.rs

@@ -61,7 +61,9 @@ impl GenericOption {
 
     /// Represent `GenericOption` as configuration option.
     pub fn to_pg_setting(&self) -> String {
-        if let Some(val) = &self.value {
+        if let Some(val) = self.value.as_ref() {
+            // XXX: such overrides don't look as they should be here, but I cannot find a better place now
+            // Probably, we need to add some override pass before the first compute start steps.
             let name = match self.name.as_str() {
                 "safekeepers" => "neon.safekeepers",
                 "wal_acceptor_reconnect" => "neon.safekeeper_reconnect_timeout",
@@ -69,9 +71,14 @@ impl GenericOption {
                 it => it,
             };
 
+            let mut res_val = val.to_owned();
+            if self.name == "shared_preload_libraries" && !res_val.contains("pg_stat_statements") {
+                res_val = format!("{},pg_stat_statements", res_val)
+            }
+
             match self.vartype.as_ref() {
-                "string" => format!("{} = '{}'", name, val),
-                _ => format!("{} = {}", name, val),
+                "string" => format!("{} = '{}'", name, res_val),
+                _ => format!("{} = {}", name, res_val),
             }
         } else {
             self.name.to_owned()

compute_tools/src/spec.rs

@@ -426,3 +426,14 @@ pub fn handle_grants(node: &ComputeNode, client: &mut Client) -> Result<()> {
 
     Ok(())
 }
+
+pub fn create_system_extensions(client: &mut Client) -> Result<()> {
+    for ext in ["pg_stat_statements", "neon"].iter() {
+        let query: String = format!("CREATE EXTENSION IF NOT EXISTS {}", ext);
+
+        info!("creating extension '{}'", ext);
+        client.execute(query.as_str(), &[])?;
+    }
+
+    Ok(())
+}

compute_tools/tests/pg_helpers_tests.rs

@@ -28,8 +28,7 @@ mod pg_helpers_tests {
 
         assert_eq!(
             spec.cluster.settings.as_pg_settings(),
-            "fsync = off\nwal_level = replica\nhot_standby = on\nneon.safekeepers = '127.0.0.1:6502,127.0.0.1:6503,127.0.0.1:6501'\nwal_log_hints = on\nlog_connections = on\nshared_buffers = 32768\nport = 55432\nmax_connections = 100\nmax_wal_senders = 10\nlisten_addresses = '0.0.0.0'\nwal_sender_timeout = 0\npassword_encryption = md5\nmaintenance_work_mem = 65536\nmax_parallel_workers = 8\nmax_worker_processes = 8\nneon.tenant_id = 'b0554b632bd4d547a63b86c3630317e8'\nmax_replication_slots = 10\nneon.timeline_id = '2414a61ffc94e428f14b5758fe308e13'\nshared_preload_libraries = 'neon'\nsynchronous_standby_names = 'walproposer'\nneon.pageserver_connstring = 'host=127.0.0.1 port=6400'"
-        );
+            "fsync = off\nwal_level = replica\nhot_standby = on\nneon.safekeepers = '127.0.0.1:6502,127.0.0.1:6503,127.0.0.1:6501'\nwal_log_hints = on\nlog_connections = on\nshared_buffers = 32768\nport = 55432\nmax_connections = 100\nmax_wal_senders = 10\nlisten_addresses = '0.0.0.0'\nwal_sender_timeout = 0\npassword_encryption = md5\nmaintenance_work_mem = 65536\nmax_parallel_workers = 8\nmax_worker_processes = 8\nneon.tenant_id = 'b0554b632bd4d547a63b86c3630317e8'\nmax_replication_slots = 10\nneon.timeline_id = '2414a61ffc94e428f14b5758fe308e13'\nshared_preload_libraries = 'neon,pg_stat_statements'\nsynchronous_standby_names = 'walproposer'\nneon.pageserver_connstring = 'host=127.0.0.1 port=6400'"        );
     }
 
     #[test]

control_plane/src/compute.rs

@@ -321,7 +321,10 @@ impl PostgresNode {
             // uses only needed variables namely host, port, user, password.
             format!("postgresql://no_user:{password}@{host}:{port}")
         };
-        conf.append("shared_preload_libraries", "neon");
+        conf.append(
+            "shared_preload_libraries",
+            "neon, pg_stat_statements, pg_wait_sampling, pg_query_state",
+        );
         conf.append_line("");
         conf.append("neon.pageserver_connstring", &pageserver_connstr);
         conf.append("neon.tenant_id", &self.tenant_id.to_string());