feat: label every Go service's Postgres connections with application_name warmbly-<binary> and add a read-only warmblyctl db load report of connections by service, longest sessions, costliest pg_stat_statements queries and largest tables

This commit is contained in:
Matthew Meszaros
2026-11-04 11:52:08 +01:00
parent 1effe5ea52
commit 67bb118866
6 changed files with 313 additions and 0 deletions
+267
View File
@@ -0,0 +1,267 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"strings"
"text/tabwriter"
"time"
"github.com/jackc/pgx/v5/pgxpool"
)
func runDB(ctx context.Context, args []string) error {
if len(args) == 0 {
dbUsage(os.Stderr)
return errors.New("`db` needs a subcommand. Pick one from the list above.")
}
switch args[0] {
case "help", "-h", "--help":
dbUsage(os.Stdout)
return nil
case "load":
return runDBLoad(ctx, args[1:])
}
dbUsage(os.Stderr)
return fmt.Errorf("unknown db subcommand %q", args[0])
}
func dbUsage(w io.Writer) {
fmt.Fprintln(w, "Usage:\n warmblyctl db load [--limit 15] [--json]")
}
// dbLoad is what the database is busy with. Read-only: it reads the statistics views and changes nothing.
type dbLoad struct {
MaxConnections int `json:"max_connections"`
Connections []dbConnections `json:"connections"`
Busy []dbBusy `json:"busy"`
Statements []dbStatement `json:"statements"`
StatementsNote string `json:"statements_note,omitempty"`
Tables []dbTable `json:"tables"`
Since *time.Time `json:"statements_since,omitempty"`
Settings map[string]string `json:"settings"`
}
type dbConnections struct {
Application string `json:"application"`
State string `json:"state"`
Count int `json:"count"`
}
// dbBusy carries no query text: pg_stat_activity holds literal values, pg_stat_statements does not.
type dbBusy struct {
PID int `json:"pid"`
Application string `json:"application"`
State string `json:"state"`
Seconds float64 `json:"seconds"`
Wait string `json:"wait"`
}
type dbStatement struct {
Calls int64 `json:"calls"`
TotalMS float64 `json:"total_ms"`
MeanMS float64 `json:"mean_ms"`
Rows int64 `json:"rows"`
Query string `json:"query"`
}
type dbTable struct {
Name string `json:"name"`
Bytes int64 `json:"bytes"`
Live int64 `json:"live_rows"`
Dead int64 `json:"dead_rows"`
SeqScans int64 `json:"seq_scans"`
LastVacuum *time.Time `json:"last_autovacuum,omitempty"`
}
func runDBLoad(ctx context.Context, args []string) error {
fs := newFlagSet("db load")
limit := fs.Int("limit", 15, "rows to show per section")
asJSON := fs.Bool("json", false, "print the report as JSON")
if err := fs.Parse(args); err != nil {
return err
}
if err := noExtraArgs(fs); err != nil {
return err
}
if *limit < 1 || *limit > 200 {
return errors.New("--limit must be between 1 and 200")
}
c, err := connect(ctx)
if err != nil {
return err
}
defer c.close()
load, err := readDBLoad(ctx, c.db.Pool, *limit)
if err != nil {
return err
}
if *asJSON {
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
return enc.Encode(load)
}
printDBLoad(os.Stdout, load)
return nil
}
func readDBLoad(ctx context.Context, pool *pgxpool.Pool, limit int) (*dbLoad, error) {
load := &dbLoad{Settings: map[string]string{}, Connections: []dbConnections{}, Busy: []dbBusy{}, Statements: []dbStatement{}, Tables: []dbTable{}}
if err := pool.QueryRow(ctx, `SELECT current_setting('max_connections')::int`).Scan(&load.MaxConnections); err != nil {
return nil, fmt.Errorf("read max_connections: %w", err)
}
for _, name := range []string{"shared_buffers", "work_mem", "statement_timeout", "idle_in_transaction_session_timeout", "autovacuum_vacuum_scale_factor"} {
var v string
if err := pool.QueryRow(ctx, `SELECT current_setting($1, true)`, name).Scan(&v); err == nil {
load.Settings[name] = v
}
}
rows, err := pool.Query(ctx, `SELECT COALESCE(NULLIF(application_name, ''), '(unnamed)'), COALESCE(state, '(background)'), count(*)::int
FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid()
GROUP BY 1, 2 ORDER BY 3 DESC, 1, 2`)
if err != nil {
return nil, fmt.Errorf("read connections: %w", err)
}
for rows.Next() {
var r dbConnections
if err := rows.Scan(&r.Application, &r.State, &r.Count); err != nil {
return nil, err
}
load.Connections = append(load.Connections, r)
}
if err := rows.Err(); err != nil {
return nil, err
}
rows, err = pool.Query(ctx, `SELECT pid, COALESCE(NULLIF(application_name, ''), '(unnamed)'), state,
EXTRACT(EPOCH FROM now() - COALESCE(xact_start, query_start))::float8,
COALESCE(wait_event_type || ':' || wait_event, '')
FROM pg_stat_activity
WHERE datname = current_database() AND pid <> pg_backend_pid() AND state IS NOT NULL AND state <> 'idle'
ORDER BY COALESCE(xact_start, query_start) LIMIT $1`, limit)
if err != nil {
return nil, fmt.Errorf("read active sessions: %w", err)
}
for rows.Next() {
var r dbBusy
if err := rows.Scan(&r.PID, &r.Application, &r.State, &r.Seconds, &r.Wait); err != nil {
return nil, err
}
load.Busy = append(load.Busy, r)
}
if err := rows.Err(); err != nil {
return nil, err
}
if err := readStatements(ctx, pool, limit, load); err != nil {
return nil, err
}
rows, err = pool.Query(ctx, `SELECT relname, pg_total_relation_size(relid), n_live_tup, n_dead_tup, seq_scan, last_autovacuum
FROM pg_stat_user_tables ORDER BY pg_total_relation_size(relid) DESC LIMIT $1`, limit)
if err != nil {
return nil, fmt.Errorf("read tables: %w", err)
}
for rows.Next() {
var r dbTable
if err := rows.Scan(&r.Name, &r.Bytes, &r.Live, &r.Dead, &r.SeqScans, &r.LastVacuum); err != nil {
return nil, err
}
load.Tables = append(load.Tables, r)
}
return load, rows.Err()
}
// readStatements reads pg_stat_statements when the server has it; its text is normalized, so it holds no values.
func readStatements(ctx context.Context, pool *pgxpool.Pool, limit int, load *dbLoad) error {
var installed bool
if err := pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements')`).Scan(&installed); err != nil {
return fmt.Errorf("check pg_stat_statements: %w", err)
}
if !installed {
load.StatementsNote = "pg_stat_statements is not installed; run CREATE EXTENSION pg_stat_statements as a superuser to see the costliest queries"
return nil
}
rows, err := pool.Query(ctx, `SELECT calls, total_exec_time, mean_exec_time, rows, left(regexp_replace(query, '\s+', ' ', 'g'), 200)
FROM pg_stat_statements WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database())
ORDER BY total_exec_time DESC LIMIT $1`, limit)
if err != nil {
load.StatementsNote = "pg_stat_statements could not be read by this role: " + err.Error()
return nil
}
for rows.Next() {
var r dbStatement
if err := rows.Scan(&r.Calls, &r.TotalMS, &r.MeanMS, &r.Rows, &r.Query); err != nil {
return err
}
load.Statements = append(load.Statements, r)
}
if err := rows.Err(); err != nil {
return err
}
var since time.Time
if err := pool.QueryRow(ctx, `SELECT stats_reset FROM pg_stat_statements_info`).Scan(&since); err == nil {
load.Since = &since
}
return nil
}
func printDBLoad(w io.Writer, load *dbLoad) {
used := 0
for _, c := range load.Connections {
used += c.Count
}
fmt.Fprintf(w, "Connections: %d of max_connections %d\n", used, load.MaxConnections)
tw := tabwriter.NewWriter(w, 2, 4, 2, ' ', 0)
fmt.Fprintln(tw, " APPLICATION\tSTATE\tCOUNT")
for _, c := range load.Connections {
fmt.Fprintf(tw, " %s\t%s\t%d\n", c.Application, c.State, c.Count)
}
tw.Flush()
fmt.Fprintln(w, "\nLongest-running sessions:")
if len(load.Busy) == 0 {
fmt.Fprintln(w, " none")
} else {
tw = tabwriter.NewWriter(w, 2, 4, 2, ' ', 0)
fmt.Fprintln(tw, " PID\tAPPLICATION\tSTATE\tSECONDS\tWAITING ON")
for _, b := range load.Busy {
fmt.Fprintf(tw, " %d\t%s\t%s\t%.1f\t%s\n", b.PID, b.Application, b.State, b.Seconds, b.Wait)
}
tw.Flush()
}
fmt.Fprint(w, "\nCostliest statements")
if load.Since != nil {
fmt.Fprintf(w, " since %s", load.Since.Format(time.RFC3339))
}
fmt.Fprintln(w, ":")
if load.StatementsNote != "" {
fmt.Fprintf(w, " %s\n", load.StatementsNote)
}
tw = tabwriter.NewWriter(w, 2, 4, 2, ' ', 0)
if len(load.Statements) > 0 {
fmt.Fprintln(tw, " TOTAL MS\tCALLS\tMEAN MS\tROWS\tQUERY")
}
for _, s := range load.Statements {
fmt.Fprintf(tw, " %.0f\t%d\t%.2f\t%d\t%s\n", s.TotalMS, s.Calls, s.MeanMS, s.Rows, strings.TrimSpace(s.Query))
}
tw.Flush()
fmt.Fprintln(w, "\nLargest tables:")
tw = tabwriter.NewWriter(w, 2, 4, 2, ' ', 0)
fmt.Fprintln(tw, " TABLE\tSIZE\tLIVE ROWS\tDEAD ROWS\tSEQ SCANS\tLAST AUTOVACUUM")
for _, t := range load.Tables {
vacuum := "never"
if t.LastVacuum != nil {
vacuum = t.LastVacuum.Format(time.RFC3339)
}
fmt.Fprintf(tw, " %s\t%s\t%d\t%d\t%d\t%s\n", t.Name, humanBytes(t.Bytes), t.Live, t.Dead, t.SeqScans, vacuum)
}
tw.Flush()
}
+3
View File
@@ -90,6 +90,8 @@ func dispatch(ctx context.Context, args []string) error {
return runInboxTag(ctx, args[1:])
case "keys":
return runKeys(ctx, args[1:])
case "db":
return runDB(ctx, args[1:])
}
if _, ok := apiFamilies[args[0]]; ok {
return runAPIResource(ctx, args[0], args[1:])
@@ -138,6 +140,7 @@ var commands = []command{
{"fleet rollout start", "Move the fleet to a tag in stages: one canary, then waves", composeExec + "fleet rollout start v1.4.2 --bake 30m"},
{"fleet rollout status", "Show a staged rollout's stage, admitted nodes and halt reason", composeExec + "fleet rollout status"},
{"fleet env-status", "Whether every live node has applied the current rendered env", composeExec + "fleet env-status"},
{"db load", "Show what the database is busy with: connections by service, costliest queries, largest tables", composeExec + "db load"},
}
func usage(w *os.File) {
@@ -247,6 +247,7 @@ The [shared authentication reference](/api/authentication/) remains the backend
| [`org list`](#org-list) | Lists the workspaces on this instance with their id, owner, and size |
| [`org export`](#org-export) | Writes a whole workspace to a portable archive file |
| [`org import`](#org-import) | Applies an archive to a workspace on this instance |
| [`db load`](#db-load) | Shows what the database is busy with: connections by service, costliest queries, largest tables |
`warmblyctl --help` lists them, and `warmblyctl <command> --help` prints one command's flags with an example. The [API commands](#the-api-commands) are further down, because they authenticate differently.
@@ -695,6 +696,22 @@ Every one of these runs `warmblyctl` inside the backend container of a compose i
| `make grant-admin EMAIL=... ROLE=...` | `user grant-admin` |
| `make revoke-admin EMAIL=...` | `user revoke-admin` |
## db load
Shows what the database is busy with. It only reads the statistics views and changes nothing.
```bash
warmblyctl db load
warmblyctl db load --limit 30 --json
```
It prints four sections:
- **Connections** grouped by the service that opened them and their state, against `max_connections`. Every Go service names its connections `warmbly-<binary>` (`warmbly-backend`, `warmbly-consumer`, `warmbly-warmblyctl`), unless the connection string sets its own `application_name`.
- **Longest-running sessions**, with how long each has been in its transaction and what it is waiting on. Query text is left out, because it can hold literal values.
- **Costliest statements** by total execution time, read from `pg_stat_statements`. Its text is normalized, so it holds no values. When the extension is not installed, the section says how to enable it.
- **Largest tables** with live and dead rows, sequential scans and the last autovacuum.
## See also
- [First run](/development/first-run/) for claiming a fresh instance and provisioning the owner unattended
+12
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"github.com/jackc/pgx/v5"
@@ -68,6 +69,7 @@ func connectionConfigs(endpoint, directEndpoint string) (*pgxpool.Config, *pgx.C
return nil, nil, false, errors.New("invalid PRIMARY_DB_DIRECT connection configuration")
}
direct := directPool.ConnConfig
nameConnections(pool.ConnConfig, direct)
direct.ConnectTimeout = defaultConnectTimeout
direct.RuntimeParams["idle_in_transaction_session_timeout"] = idleInTxnTimeoutMs
direct.RuntimeParams["statement_timeout"] = statementTimeoutMs
@@ -82,6 +84,16 @@ func connectionConfigs(endpoint, directEndpoint string) (*pgxpool.Config, *pgx.C
return pool, direct, pooled, nil
}
// nameConnections labels each connection with the binary that opened it, so pg_stat_activity shows who holds the slots; a DSN's own application_name wins.
func nameConnections(configs ...*pgx.ConnConfig) {
name := "warmbly-" + filepath.Base(os.Args[0])
for _, c := range configs {
if c.RuntimeParams["application_name"] == "" {
c.RuntimeParams["application_name"] = name
}
}
}
// Transaction pooling relies on role defaults, which must apply to every server connection.
func verifyPooledTimeouts(ctx context.Context, conn *pgx.Conn) error {
var statement, idle int64
@@ -25,6 +25,19 @@ func TestTransactionConnectionConfiguration(t *testing.T) {
if pool.ConnConfig.TLSConfig == nil || session.TLSConfig == nil || pool.ConnConfig.TLSConfig.InsecureSkipVerify || session.TLSConfig.InsecureSkipVerify {
t.Fatal("both connection paths must preserve TLS verification")
}
if name := pool.ConnConfig.RuntimeParams["application_name"]; !strings.HasPrefix(name, "warmbly-") || session.RuntimeParams["application_name"] != name {
t.Fatalf("connections are not labelled with the process that opened them: %q", name)
}
}
func TestApplicationNameFromTheDSNWins(t *testing.T) {
pool, session, _, err := connectionConfigs("postgres://app:fixture@db.example:5432/app?application_name=ops", "")
if err != nil {
t.Fatal(err)
}
if pool.ConnConfig.RuntimeParams["application_name"] != "ops" || session.RuntimeParams["application_name"] != "ops" {
t.Fatal("an operator's application_name was replaced")
}
}
func TestDirectConnectionDefaults(t *testing.T) {
+1
View File
@@ -44,6 +44,7 @@ for. Prefer parsing that over reasoning from the prose.
| Back the instance up | `warmblyctl backup --out /data/blobs/warmbly.tar.gz`, then copy it out and delete it |
| Restore one onto this instance | `warmblyctl restore --file /data/blobs/warmbly.tar.gz` |
| See which key opens each stored secret | `warmblyctl keys status [--json]` |
| See what the database is busy with: connections by service, costliest queries, largest tables | `warmblyctl db load [--limit N] [--json]` |
| Move secrets onto a new primary key | `warmblyctl keys reseal` (dry run), then `keys reseal --apply` |
| Mint replacement keys and tokens | `warmblyctl keys generate [NAME...]` |