From 67bb118866fecd9747963bbbb6bef858cb908fa6 Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Wed, 4 Nov 2026 11:52:08 +0100 Subject: [PATCH] feat: label every Go service's Postgres connections with application_name warmbly- and add a read-only warmblyctl db load report of connections by service, longest sessions, costliest pg_stat_statements queries and largest tables --- cmd/warmblyctl/db.go | 267 ++++++++++++++++++ cmd/warmblyctl/main.go | 3 + docs/content/docs/development/warmblyctl.mdx | 17 ++ internal/infrastructure/db/connections.go | 12 + .../infrastructure/db/connections_test.go | 13 + skills/warmbly-ops/SKILL.md | 1 + 6 files changed, 313 insertions(+) create mode 100644 cmd/warmblyctl/db.go diff --git a/cmd/warmblyctl/db.go b/cmd/warmblyctl/db.go new file mode 100644 index 000000000..349da08ae --- /dev/null +++ b/cmd/warmblyctl/db.go @@ -0,0 +1,267 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + "text/tabwriter" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func runDB(ctx context.Context, args []string) error { + if len(args) == 0 { + dbUsage(os.Stderr) + return errors.New("`db` needs a subcommand. Pick one from the list above.") + } + switch args[0] { + case "help", "-h", "--help": + dbUsage(os.Stdout) + return nil + case "load": + return runDBLoad(ctx, args[1:]) + } + dbUsage(os.Stderr) + return fmt.Errorf("unknown db subcommand %q", args[0]) +} + +func dbUsage(w io.Writer) { + fmt.Fprintln(w, "Usage:\n warmblyctl db load [--limit 15] [--json]") +} + +// dbLoad is what the database is busy with. Read-only: it reads the statistics views and changes nothing. +type dbLoad struct { + MaxConnections int `json:"max_connections"` + Connections []dbConnections `json:"connections"` + Busy []dbBusy `json:"busy"` + Statements []dbStatement `json:"statements"` + StatementsNote string `json:"statements_note,omitempty"` + Tables []dbTable `json:"tables"` + Since *time.Time `json:"statements_since,omitempty"` + Settings map[string]string `json:"settings"` +} + +type dbConnections struct { + Application string `json:"application"` + State string `json:"state"` + Count int `json:"count"` +} + +// dbBusy carries no query text: pg_stat_activity holds literal values, pg_stat_statements does not. +type dbBusy struct { + PID int `json:"pid"` + Application string `json:"application"` + State string `json:"state"` + Seconds float64 `json:"seconds"` + Wait string `json:"wait"` +} + +type dbStatement struct { + Calls int64 `json:"calls"` + TotalMS float64 `json:"total_ms"` + MeanMS float64 `json:"mean_ms"` + Rows int64 `json:"rows"` + Query string `json:"query"` +} + +type dbTable struct { + Name string `json:"name"` + Bytes int64 `json:"bytes"` + Live int64 `json:"live_rows"` + Dead int64 `json:"dead_rows"` + SeqScans int64 `json:"seq_scans"` + LastVacuum *time.Time `json:"last_autovacuum,omitempty"` +} + +func runDBLoad(ctx context.Context, args []string) error { + fs := newFlagSet("db load") + limit := fs.Int("limit", 15, "rows to show per section") + asJSON := fs.Bool("json", false, "print the report as JSON") + if err := fs.Parse(args); err != nil { + return err + } + if err := noExtraArgs(fs); err != nil { + return err + } + if *limit < 1 || *limit > 200 { + return errors.New("--limit must be between 1 and 200") + } + c, err := connect(ctx) + if err != nil { + return err + } + defer c.close() + load, err := readDBLoad(ctx, c.db.Pool, *limit) + if err != nil { + return err + } + if *asJSON { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(load) + } + printDBLoad(os.Stdout, load) + return nil +} + +func readDBLoad(ctx context.Context, pool *pgxpool.Pool, limit int) (*dbLoad, error) { + load := &dbLoad{Settings: map[string]string{}, Connections: []dbConnections{}, Busy: []dbBusy{}, Statements: []dbStatement{}, Tables: []dbTable{}} + if err := pool.QueryRow(ctx, `SELECT current_setting('max_connections')::int`).Scan(&load.MaxConnections); err != nil { + return nil, fmt.Errorf("read max_connections: %w", err) + } + for _, name := range []string{"shared_buffers", "work_mem", "statement_timeout", "idle_in_transaction_session_timeout", "autovacuum_vacuum_scale_factor"} { + var v string + if err := pool.QueryRow(ctx, `SELECT current_setting($1, true)`, name).Scan(&v); err == nil { + load.Settings[name] = v + } + } + + rows, err := pool.Query(ctx, `SELECT COALESCE(NULLIF(application_name, ''), '(unnamed)'), COALESCE(state, '(background)'), count(*)::int + FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid() + GROUP BY 1, 2 ORDER BY 3 DESC, 1, 2`) + if err != nil { + return nil, fmt.Errorf("read connections: %w", err) + } + for rows.Next() { + var r dbConnections + if err := rows.Scan(&r.Application, &r.State, &r.Count); err != nil { + return nil, err + } + load.Connections = append(load.Connections, r) + } + if err := rows.Err(); err != nil { + return nil, err + } + + rows, err = pool.Query(ctx, `SELECT pid, COALESCE(NULLIF(application_name, ''), '(unnamed)'), state, + EXTRACT(EPOCH FROM now() - COALESCE(xact_start, query_start))::float8, + COALESCE(wait_event_type || ':' || wait_event, '') + FROM pg_stat_activity + WHERE datname = current_database() AND pid <> pg_backend_pid() AND state IS NOT NULL AND state <> 'idle' + ORDER BY COALESCE(xact_start, query_start) LIMIT $1`, limit) + if err != nil { + return nil, fmt.Errorf("read active sessions: %w", err) + } + for rows.Next() { + var r dbBusy + if err := rows.Scan(&r.PID, &r.Application, &r.State, &r.Seconds, &r.Wait); err != nil { + return nil, err + } + load.Busy = append(load.Busy, r) + } + if err := rows.Err(); err != nil { + return nil, err + } + + if err := readStatements(ctx, pool, limit, load); err != nil { + return nil, err + } + + rows, err = pool.Query(ctx, `SELECT relname, pg_total_relation_size(relid), n_live_tup, n_dead_tup, seq_scan, last_autovacuum + FROM pg_stat_user_tables ORDER BY pg_total_relation_size(relid) DESC LIMIT $1`, limit) + if err != nil { + return nil, fmt.Errorf("read tables: %w", err) + } + for rows.Next() { + var r dbTable + if err := rows.Scan(&r.Name, &r.Bytes, &r.Live, &r.Dead, &r.SeqScans, &r.LastVacuum); err != nil { + return nil, err + } + load.Tables = append(load.Tables, r) + } + return load, rows.Err() +} + +// readStatements reads pg_stat_statements when the server has it; its text is normalized, so it holds no values. +func readStatements(ctx context.Context, pool *pgxpool.Pool, limit int, load *dbLoad) error { + var installed bool + if err := pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements')`).Scan(&installed); err != nil { + return fmt.Errorf("check pg_stat_statements: %w", err) + } + if !installed { + load.StatementsNote = "pg_stat_statements is not installed; run CREATE EXTENSION pg_stat_statements as a superuser to see the costliest queries" + return nil + } + rows, err := pool.Query(ctx, `SELECT calls, total_exec_time, mean_exec_time, rows, left(regexp_replace(query, '\s+', ' ', 'g'), 200) + FROM pg_stat_statements WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) + ORDER BY total_exec_time DESC LIMIT $1`, limit) + if err != nil { + load.StatementsNote = "pg_stat_statements could not be read by this role: " + err.Error() + return nil + } + for rows.Next() { + var r dbStatement + if err := rows.Scan(&r.Calls, &r.TotalMS, &r.MeanMS, &r.Rows, &r.Query); err != nil { + return err + } + load.Statements = append(load.Statements, r) + } + if err := rows.Err(); err != nil { + return err + } + var since time.Time + if err := pool.QueryRow(ctx, `SELECT stats_reset FROM pg_stat_statements_info`).Scan(&since); err == nil { + load.Since = &since + } + return nil +} + +func printDBLoad(w io.Writer, load *dbLoad) { + used := 0 + for _, c := range load.Connections { + used += c.Count + } + fmt.Fprintf(w, "Connections: %d of max_connections %d\n", used, load.MaxConnections) + tw := tabwriter.NewWriter(w, 2, 4, 2, ' ', 0) + fmt.Fprintln(tw, " APPLICATION\tSTATE\tCOUNT") + for _, c := range load.Connections { + fmt.Fprintf(tw, " %s\t%s\t%d\n", c.Application, c.State, c.Count) + } + tw.Flush() + + fmt.Fprintln(w, "\nLongest-running sessions:") + if len(load.Busy) == 0 { + fmt.Fprintln(w, " none") + } else { + tw = tabwriter.NewWriter(w, 2, 4, 2, ' ', 0) + fmt.Fprintln(tw, " PID\tAPPLICATION\tSTATE\tSECONDS\tWAITING ON") + for _, b := range load.Busy { + fmt.Fprintf(tw, " %d\t%s\t%s\t%.1f\t%s\n", b.PID, b.Application, b.State, b.Seconds, b.Wait) + } + tw.Flush() + } + + fmt.Fprint(w, "\nCostliest statements") + if load.Since != nil { + fmt.Fprintf(w, " since %s", load.Since.Format(time.RFC3339)) + } + fmt.Fprintln(w, ":") + if load.StatementsNote != "" { + fmt.Fprintf(w, " %s\n", load.StatementsNote) + } + tw = tabwriter.NewWriter(w, 2, 4, 2, ' ', 0) + if len(load.Statements) > 0 { + fmt.Fprintln(tw, " TOTAL MS\tCALLS\tMEAN MS\tROWS\tQUERY") + } + for _, s := range load.Statements { + fmt.Fprintf(tw, " %.0f\t%d\t%.2f\t%d\t%s\n", s.TotalMS, s.Calls, s.MeanMS, s.Rows, strings.TrimSpace(s.Query)) + } + tw.Flush() + + fmt.Fprintln(w, "\nLargest tables:") + tw = tabwriter.NewWriter(w, 2, 4, 2, ' ', 0) + fmt.Fprintln(tw, " TABLE\tSIZE\tLIVE ROWS\tDEAD ROWS\tSEQ SCANS\tLAST AUTOVACUUM") + for _, t := range load.Tables { + vacuum := "never" + if t.LastVacuum != nil { + vacuum = t.LastVacuum.Format(time.RFC3339) + } + fmt.Fprintf(tw, " %s\t%s\t%d\t%d\t%d\t%s\n", t.Name, humanBytes(t.Bytes), t.Live, t.Dead, t.SeqScans, vacuum) + } + tw.Flush() +} diff --git a/cmd/warmblyctl/main.go b/cmd/warmblyctl/main.go index 10df8b64c..671a68300 100644 --- a/cmd/warmblyctl/main.go +++ b/cmd/warmblyctl/main.go @@ -90,6 +90,8 @@ func dispatch(ctx context.Context, args []string) error { return runInboxTag(ctx, args[1:]) case "keys": return runKeys(ctx, args[1:]) + case "db": + return runDB(ctx, args[1:]) } if _, ok := apiFamilies[args[0]]; ok { return runAPIResource(ctx, args[0], args[1:]) @@ -138,6 +140,7 @@ var commands = []command{ {"fleet rollout start", "Move the fleet to a tag in stages: one canary, then waves", composeExec + "fleet rollout start v1.4.2 --bake 30m"}, {"fleet rollout status", "Show a staged rollout's stage, admitted nodes and halt reason", composeExec + "fleet rollout status"}, {"fleet env-status", "Whether every live node has applied the current rendered env", composeExec + "fleet env-status"}, + {"db load", "Show what the database is busy with: connections by service, costliest queries, largest tables", composeExec + "db load"}, } func usage(w *os.File) { diff --git a/docs/content/docs/development/warmblyctl.mdx b/docs/content/docs/development/warmblyctl.mdx index 91afabfe2..bf69c40fb 100644 --- a/docs/content/docs/development/warmblyctl.mdx +++ b/docs/content/docs/development/warmblyctl.mdx @@ -247,6 +247,7 @@ The [shared authentication reference](/api/authentication/) remains the backend | [`org list`](#org-list) | Lists the workspaces on this instance with their id, owner, and size | | [`org export`](#org-export) | Writes a whole workspace to a portable archive file | | [`org import`](#org-import) | Applies an archive to a workspace on this instance | +| [`db load`](#db-load) | Shows what the database is busy with: connections by service, costliest queries, largest tables | `warmblyctl --help` lists them, and `warmblyctl --help` prints one command's flags with an example. The [API commands](#the-api-commands) are further down, because they authenticate differently. @@ -695,6 +696,22 @@ Every one of these runs `warmblyctl` inside the backend container of a compose i | `make grant-admin EMAIL=... ROLE=...` | `user grant-admin` | | `make revoke-admin EMAIL=...` | `user revoke-admin` | +## db load + +Shows what the database is busy with. It only reads the statistics views and changes nothing. + +```bash +warmblyctl db load +warmblyctl db load --limit 30 --json +``` + +It prints four sections: + +- **Connections** grouped by the service that opened them and their state, against `max_connections`. Every Go service names its connections `warmbly-` (`warmbly-backend`, `warmbly-consumer`, `warmbly-warmblyctl`), unless the connection string sets its own `application_name`. +- **Longest-running sessions**, with how long each has been in its transaction and what it is waiting on. Query text is left out, because it can hold literal values. +- **Costliest statements** by total execution time, read from `pg_stat_statements`. Its text is normalized, so it holds no values. When the extension is not installed, the section says how to enable it. +- **Largest tables** with live and dead rows, sequential scans and the last autovacuum. + ## See also - [First run](/development/first-run/) for claiming a fresh instance and provisioning the owner unattended diff --git a/internal/infrastructure/db/connections.go b/internal/infrastructure/db/connections.go index a869ded23..9f6e651dc 100644 --- a/internal/infrastructure/db/connections.go +++ b/internal/infrastructure/db/connections.go @@ -4,6 +4,7 @@ import ( "context" "errors" "os" + "path/filepath" "strings" "github.com/jackc/pgx/v5" @@ -68,6 +69,7 @@ func connectionConfigs(endpoint, directEndpoint string) (*pgxpool.Config, *pgx.C return nil, nil, false, errors.New("invalid PRIMARY_DB_DIRECT connection configuration") } direct := directPool.ConnConfig + nameConnections(pool.ConnConfig, direct) direct.ConnectTimeout = defaultConnectTimeout direct.RuntimeParams["idle_in_transaction_session_timeout"] = idleInTxnTimeoutMs direct.RuntimeParams["statement_timeout"] = statementTimeoutMs @@ -82,6 +84,16 @@ func connectionConfigs(endpoint, directEndpoint string) (*pgxpool.Config, *pgx.C return pool, direct, pooled, nil } +// nameConnections labels each connection with the binary that opened it, so pg_stat_activity shows who holds the slots; a DSN's own application_name wins. +func nameConnections(configs ...*pgx.ConnConfig) { + name := "warmbly-" + filepath.Base(os.Args[0]) + for _, c := range configs { + if c.RuntimeParams["application_name"] == "" { + c.RuntimeParams["application_name"] = name + } + } +} + // Transaction pooling relies on role defaults, which must apply to every server connection. func verifyPooledTimeouts(ctx context.Context, conn *pgx.Conn) error { var statement, idle int64 diff --git a/internal/infrastructure/db/connections_test.go b/internal/infrastructure/db/connections_test.go index ad37b3b6f..fd4a573eb 100644 --- a/internal/infrastructure/db/connections_test.go +++ b/internal/infrastructure/db/connections_test.go @@ -25,6 +25,19 @@ func TestTransactionConnectionConfiguration(t *testing.T) { if pool.ConnConfig.TLSConfig == nil || session.TLSConfig == nil || pool.ConnConfig.TLSConfig.InsecureSkipVerify || session.TLSConfig.InsecureSkipVerify { t.Fatal("both connection paths must preserve TLS verification") } + if name := pool.ConnConfig.RuntimeParams["application_name"]; !strings.HasPrefix(name, "warmbly-") || session.RuntimeParams["application_name"] != name { + t.Fatalf("connections are not labelled with the process that opened them: %q", name) + } +} + +func TestApplicationNameFromTheDSNWins(t *testing.T) { + pool, session, _, err := connectionConfigs("postgres://app:fixture@db.example:5432/app?application_name=ops", "") + if err != nil { + t.Fatal(err) + } + if pool.ConnConfig.RuntimeParams["application_name"] != "ops" || session.RuntimeParams["application_name"] != "ops" { + t.Fatal("an operator's application_name was replaced") + } } func TestDirectConnectionDefaults(t *testing.T) { diff --git a/skills/warmbly-ops/SKILL.md b/skills/warmbly-ops/SKILL.md index 62b2fe0cd..42ebb2ce4 100644 --- a/skills/warmbly-ops/SKILL.md +++ b/skills/warmbly-ops/SKILL.md @@ -44,6 +44,7 @@ for. Prefer parsing that over reasoning from the prose. | Back the instance up | `warmblyctl backup --out /data/blobs/warmbly.tar.gz`, then copy it out and delete it | | Restore one onto this instance | `warmblyctl restore --file /data/blobs/warmbly.tar.gz` | | See which key opens each stored secret | `warmblyctl keys status [--json]` | +| See what the database is busy with: connections by service, costliest queries, largest tables | `warmblyctl db load [--limit N] [--json]` | | Move secrets onto a new primary key | `warmblyctl keys reseal` (dry run), then `keys reseal --apply` | | Mint replacement keys and tokens | `warmblyctl keys generate [NAME...]` |