mirror of
https://github.com/warmbly/warmbly.git
synced 2026-11-18 08:19:19 +00:00
feat: label every Go service's Postgres connections with application_name warmbly-<binary> and add a read-only warmblyctl db load report of connections by service, longest sessions, costliest pg_stat_statements queries and largest tables
This commit is contained in:
@@ -0,0 +1,267 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func runDB(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
dbUsage(os.Stderr)
|
||||
return errors.New("`db` needs a subcommand. Pick one from the list above.")
|
||||
}
|
||||
switch args[0] {
|
||||
case "help", "-h", "--help":
|
||||
dbUsage(os.Stdout)
|
||||
return nil
|
||||
case "load":
|
||||
return runDBLoad(ctx, args[1:])
|
||||
}
|
||||
dbUsage(os.Stderr)
|
||||
return fmt.Errorf("unknown db subcommand %q", args[0])
|
||||
}
|
||||
|
||||
func dbUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, "Usage:\n warmblyctl db load [--limit 15] [--json]")
|
||||
}
|
||||
|
||||
// dbLoad is what the database is busy with. Read-only: it reads the statistics views and changes nothing.
|
||||
type dbLoad struct {
|
||||
MaxConnections int `json:"max_connections"`
|
||||
Connections []dbConnections `json:"connections"`
|
||||
Busy []dbBusy `json:"busy"`
|
||||
Statements []dbStatement `json:"statements"`
|
||||
StatementsNote string `json:"statements_note,omitempty"`
|
||||
Tables []dbTable `json:"tables"`
|
||||
Since *time.Time `json:"statements_since,omitempty"`
|
||||
Settings map[string]string `json:"settings"`
|
||||
}
|
||||
|
||||
type dbConnections struct {
|
||||
Application string `json:"application"`
|
||||
State string `json:"state"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
// dbBusy carries no query text: pg_stat_activity holds literal values, pg_stat_statements does not.
|
||||
type dbBusy struct {
|
||||
PID int `json:"pid"`
|
||||
Application string `json:"application"`
|
||||
State string `json:"state"`
|
||||
Seconds float64 `json:"seconds"`
|
||||
Wait string `json:"wait"`
|
||||
}
|
||||
|
||||
type dbStatement struct {
|
||||
Calls int64 `json:"calls"`
|
||||
TotalMS float64 `json:"total_ms"`
|
||||
MeanMS float64 `json:"mean_ms"`
|
||||
Rows int64 `json:"rows"`
|
||||
Query string `json:"query"`
|
||||
}
|
||||
|
||||
type dbTable struct {
|
||||
Name string `json:"name"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Live int64 `json:"live_rows"`
|
||||
Dead int64 `json:"dead_rows"`
|
||||
SeqScans int64 `json:"seq_scans"`
|
||||
LastVacuum *time.Time `json:"last_autovacuum,omitempty"`
|
||||
}
|
||||
|
||||
func runDBLoad(ctx context.Context, args []string) error {
|
||||
fs := newFlagSet("db load")
|
||||
limit := fs.Int("limit", 15, "rows to show per section")
|
||||
asJSON := fs.Bool("json", false, "print the report as JSON")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := noExtraArgs(fs); err != nil {
|
||||
return err
|
||||
}
|
||||
if *limit < 1 || *limit > 200 {
|
||||
return errors.New("--limit must be between 1 and 200")
|
||||
}
|
||||
c, err := connect(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer c.close()
|
||||
load, err := readDBLoad(ctx, c.db.Pool, *limit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(load)
|
||||
}
|
||||
printDBLoad(os.Stdout, load)
|
||||
return nil
|
||||
}
|
||||
|
||||
func readDBLoad(ctx context.Context, pool *pgxpool.Pool, limit int) (*dbLoad, error) {
|
||||
load := &dbLoad{Settings: map[string]string{}, Connections: []dbConnections{}, Busy: []dbBusy{}, Statements: []dbStatement{}, Tables: []dbTable{}}
|
||||
if err := pool.QueryRow(ctx, `SELECT current_setting('max_connections')::int`).Scan(&load.MaxConnections); err != nil {
|
||||
return nil, fmt.Errorf("read max_connections: %w", err)
|
||||
}
|
||||
for _, name := range []string{"shared_buffers", "work_mem", "statement_timeout", "idle_in_transaction_session_timeout", "autovacuum_vacuum_scale_factor"} {
|
||||
var v string
|
||||
if err := pool.QueryRow(ctx, `SELECT current_setting($1, true)`, name).Scan(&v); err == nil {
|
||||
load.Settings[name] = v
|
||||
}
|
||||
}
|
||||
|
||||
rows, err := pool.Query(ctx, `SELECT COALESCE(NULLIF(application_name, ''), '(unnamed)'), COALESCE(state, '(background)'), count(*)::int
|
||||
FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid()
|
||||
GROUP BY 1, 2 ORDER BY 3 DESC, 1, 2`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read connections: %w", err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var r dbConnections
|
||||
if err := rows.Scan(&r.Application, &r.State, &r.Count); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
load.Connections = append(load.Connections, r)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rows, err = pool.Query(ctx, `SELECT pid, COALESCE(NULLIF(application_name, ''), '(unnamed)'), state,
|
||||
EXTRACT(EPOCH FROM now() - COALESCE(xact_start, query_start))::float8,
|
||||
COALESCE(wait_event_type || ':' || wait_event, '')
|
||||
FROM pg_stat_activity
|
||||
WHERE datname = current_database() AND pid <> pg_backend_pid() AND state IS NOT NULL AND state <> 'idle'
|
||||
ORDER BY COALESCE(xact_start, query_start) LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read active sessions: %w", err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var r dbBusy
|
||||
if err := rows.Scan(&r.PID, &r.Application, &r.State, &r.Seconds, &r.Wait); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
load.Busy = append(load.Busy, r)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := readStatements(ctx, pool, limit, load); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rows, err = pool.Query(ctx, `SELECT relname, pg_total_relation_size(relid), n_live_tup, n_dead_tup, seq_scan, last_autovacuum
|
||||
FROM pg_stat_user_tables ORDER BY pg_total_relation_size(relid) DESC LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read tables: %w", err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var r dbTable
|
||||
if err := rows.Scan(&r.Name, &r.Bytes, &r.Live, &r.Dead, &r.SeqScans, &r.LastVacuum); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
load.Tables = append(load.Tables, r)
|
||||
}
|
||||
return load, rows.Err()
|
||||
}
|
||||
|
||||
// readStatements reads pg_stat_statements when the server has it; its text is normalized, so it holds no values.
|
||||
func readStatements(ctx context.Context, pool *pgxpool.Pool, limit int, load *dbLoad) error {
|
||||
var installed bool
|
||||
if err := pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements')`).Scan(&installed); err != nil {
|
||||
return fmt.Errorf("check pg_stat_statements: %w", err)
|
||||
}
|
||||
if !installed {
|
||||
load.StatementsNote = "pg_stat_statements is not installed; run CREATE EXTENSION pg_stat_statements as a superuser to see the costliest queries"
|
||||
return nil
|
||||
}
|
||||
rows, err := pool.Query(ctx, `SELECT calls, total_exec_time, mean_exec_time, rows, left(regexp_replace(query, '\s+', ' ', 'g'), 200)
|
||||
FROM pg_stat_statements WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database())
|
||||
ORDER BY total_exec_time DESC LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
load.StatementsNote = "pg_stat_statements could not be read by this role: " + err.Error()
|
||||
return nil
|
||||
}
|
||||
for rows.Next() {
|
||||
var r dbStatement
|
||||
if err := rows.Scan(&r.Calls, &r.TotalMS, &r.MeanMS, &r.Rows, &r.Query); err != nil {
|
||||
return err
|
||||
}
|
||||
load.Statements = append(load.Statements, r)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
var since time.Time
|
||||
if err := pool.QueryRow(ctx, `SELECT stats_reset FROM pg_stat_statements_info`).Scan(&since); err == nil {
|
||||
load.Since = &since
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func printDBLoad(w io.Writer, load *dbLoad) {
|
||||
used := 0
|
||||
for _, c := range load.Connections {
|
||||
used += c.Count
|
||||
}
|
||||
fmt.Fprintf(w, "Connections: %d of max_connections %d\n", used, load.MaxConnections)
|
||||
tw := tabwriter.NewWriter(w, 2, 4, 2, ' ', 0)
|
||||
fmt.Fprintln(tw, " APPLICATION\tSTATE\tCOUNT")
|
||||
for _, c := range load.Connections {
|
||||
fmt.Fprintf(tw, " %s\t%s\t%d\n", c.Application, c.State, c.Count)
|
||||
}
|
||||
tw.Flush()
|
||||
|
||||
fmt.Fprintln(w, "\nLongest-running sessions:")
|
||||
if len(load.Busy) == 0 {
|
||||
fmt.Fprintln(w, " none")
|
||||
} else {
|
||||
tw = tabwriter.NewWriter(w, 2, 4, 2, ' ', 0)
|
||||
fmt.Fprintln(tw, " PID\tAPPLICATION\tSTATE\tSECONDS\tWAITING ON")
|
||||
for _, b := range load.Busy {
|
||||
fmt.Fprintf(tw, " %d\t%s\t%s\t%.1f\t%s\n", b.PID, b.Application, b.State, b.Seconds, b.Wait)
|
||||
}
|
||||
tw.Flush()
|
||||
}
|
||||
|
||||
fmt.Fprint(w, "\nCostliest statements")
|
||||
if load.Since != nil {
|
||||
fmt.Fprintf(w, " since %s", load.Since.Format(time.RFC3339))
|
||||
}
|
||||
fmt.Fprintln(w, ":")
|
||||
if load.StatementsNote != "" {
|
||||
fmt.Fprintf(w, " %s\n", load.StatementsNote)
|
||||
}
|
||||
tw = tabwriter.NewWriter(w, 2, 4, 2, ' ', 0)
|
||||
if len(load.Statements) > 0 {
|
||||
fmt.Fprintln(tw, " TOTAL MS\tCALLS\tMEAN MS\tROWS\tQUERY")
|
||||
}
|
||||
for _, s := range load.Statements {
|
||||
fmt.Fprintf(tw, " %.0f\t%d\t%.2f\t%d\t%s\n", s.TotalMS, s.Calls, s.MeanMS, s.Rows, strings.TrimSpace(s.Query))
|
||||
}
|
||||
tw.Flush()
|
||||
|
||||
fmt.Fprintln(w, "\nLargest tables:")
|
||||
tw = tabwriter.NewWriter(w, 2, 4, 2, ' ', 0)
|
||||
fmt.Fprintln(tw, " TABLE\tSIZE\tLIVE ROWS\tDEAD ROWS\tSEQ SCANS\tLAST AUTOVACUUM")
|
||||
for _, t := range load.Tables {
|
||||
vacuum := "never"
|
||||
if t.LastVacuum != nil {
|
||||
vacuum = t.LastVacuum.Format(time.RFC3339)
|
||||
}
|
||||
fmt.Fprintf(tw, " %s\t%s\t%d\t%d\t%d\t%s\n", t.Name, humanBytes(t.Bytes), t.Live, t.Dead, t.SeqScans, vacuum)
|
||||
}
|
||||
tw.Flush()
|
||||
}
|
||||
@@ -90,6 +90,8 @@ func dispatch(ctx context.Context, args []string) error {
|
||||
return runInboxTag(ctx, args[1:])
|
||||
case "keys":
|
||||
return runKeys(ctx, args[1:])
|
||||
case "db":
|
||||
return runDB(ctx, args[1:])
|
||||
}
|
||||
if _, ok := apiFamilies[args[0]]; ok {
|
||||
return runAPIResource(ctx, args[0], args[1:])
|
||||
@@ -138,6 +140,7 @@ var commands = []command{
|
||||
{"fleet rollout start", "Move the fleet to a tag in stages: one canary, then waves", composeExec + "fleet rollout start v1.4.2 --bake 30m"},
|
||||
{"fleet rollout status", "Show a staged rollout's stage, admitted nodes and halt reason", composeExec + "fleet rollout status"},
|
||||
{"fleet env-status", "Whether every live node has applied the current rendered env", composeExec + "fleet env-status"},
|
||||
{"db load", "Show what the database is busy with: connections by service, costliest queries, largest tables", composeExec + "db load"},
|
||||
}
|
||||
|
||||
func usage(w *os.File) {
|
||||
|
||||
@@ -247,6 +247,7 @@ The [shared authentication reference](/api/authentication/) remains the backend
|
||||
| [`org list`](#org-list) | Lists the workspaces on this instance with their id, owner, and size |
|
||||
| [`org export`](#org-export) | Writes a whole workspace to a portable archive file |
|
||||
| [`org import`](#org-import) | Applies an archive to a workspace on this instance |
|
||||
| [`db load`](#db-load) | Shows what the database is busy with: connections by service, costliest queries, largest tables |
|
||||
|
||||
`warmblyctl --help` lists them, and `warmblyctl <command> --help` prints one command's flags with an example. The [API commands](#the-api-commands) are further down, because they authenticate differently.
|
||||
|
||||
@@ -695,6 +696,22 @@ Every one of these runs `warmblyctl` inside the backend container of a compose i
|
||||
| `make grant-admin EMAIL=... ROLE=...` | `user grant-admin` |
|
||||
| `make revoke-admin EMAIL=...` | `user revoke-admin` |
|
||||
|
||||
## db load
|
||||
|
||||
Shows what the database is busy with. It only reads the statistics views and changes nothing.
|
||||
|
||||
```bash
|
||||
warmblyctl db load
|
||||
warmblyctl db load --limit 30 --json
|
||||
```
|
||||
|
||||
It prints four sections:
|
||||
|
||||
- **Connections** grouped by the service that opened them and their state, against `max_connections`. Every Go service names its connections `warmbly-<binary>` (`warmbly-backend`, `warmbly-consumer`, `warmbly-warmblyctl`), unless the connection string sets its own `application_name`.
|
||||
- **Longest-running sessions**, with how long each has been in its transaction and what it is waiting on. Query text is left out, because it can hold literal values.
|
||||
- **Costliest statements** by total execution time, read from `pg_stat_statements`. Its text is normalized, so it holds no values. When the extension is not installed, the section says how to enable it.
|
||||
- **Largest tables** with live and dead rows, sequential scans and the last autovacuum.
|
||||
|
||||
## See also
|
||||
|
||||
- [First run](/development/first-run/) for claiming a fresh instance and provisioning the owner unattended
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
@@ -68,6 +69,7 @@ func connectionConfigs(endpoint, directEndpoint string) (*pgxpool.Config, *pgx.C
|
||||
return nil, nil, false, errors.New("invalid PRIMARY_DB_DIRECT connection configuration")
|
||||
}
|
||||
direct := directPool.ConnConfig
|
||||
nameConnections(pool.ConnConfig, direct)
|
||||
direct.ConnectTimeout = defaultConnectTimeout
|
||||
direct.RuntimeParams["idle_in_transaction_session_timeout"] = idleInTxnTimeoutMs
|
||||
direct.RuntimeParams["statement_timeout"] = statementTimeoutMs
|
||||
@@ -82,6 +84,16 @@ func connectionConfigs(endpoint, directEndpoint string) (*pgxpool.Config, *pgx.C
|
||||
return pool, direct, pooled, nil
|
||||
}
|
||||
|
||||
// nameConnections labels each connection with the binary that opened it, so pg_stat_activity shows who holds the slots; a DSN's own application_name wins.
|
||||
func nameConnections(configs ...*pgx.ConnConfig) {
|
||||
name := "warmbly-" + filepath.Base(os.Args[0])
|
||||
for _, c := range configs {
|
||||
if c.RuntimeParams["application_name"] == "" {
|
||||
c.RuntimeParams["application_name"] = name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Transaction pooling relies on role defaults, which must apply to every server connection.
|
||||
func verifyPooledTimeouts(ctx context.Context, conn *pgx.Conn) error {
|
||||
var statement, idle int64
|
||||
|
||||
@@ -25,6 +25,19 @@ func TestTransactionConnectionConfiguration(t *testing.T) {
|
||||
if pool.ConnConfig.TLSConfig == nil || session.TLSConfig == nil || pool.ConnConfig.TLSConfig.InsecureSkipVerify || session.TLSConfig.InsecureSkipVerify {
|
||||
t.Fatal("both connection paths must preserve TLS verification")
|
||||
}
|
||||
if name := pool.ConnConfig.RuntimeParams["application_name"]; !strings.HasPrefix(name, "warmbly-") || session.RuntimeParams["application_name"] != name {
|
||||
t.Fatalf("connections are not labelled with the process that opened them: %q", name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationNameFromTheDSNWins(t *testing.T) {
|
||||
pool, session, _, err := connectionConfigs("postgres://app:fixture@db.example:5432/app?application_name=ops", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pool.ConnConfig.RuntimeParams["application_name"] != "ops" || session.RuntimeParams["application_name"] != "ops" {
|
||||
t.Fatal("an operator's application_name was replaced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectConnectionDefaults(t *testing.T) {
|
||||
|
||||
@@ -44,6 +44,7 @@ for. Prefer parsing that over reasoning from the prose.
|
||||
| Back the instance up | `warmblyctl backup --out /data/blobs/warmbly.tar.gz`, then copy it out and delete it |
|
||||
| Restore one onto this instance | `warmblyctl restore --file /data/blobs/warmbly.tar.gz` |
|
||||
| See which key opens each stored secret | `warmblyctl keys status [--json]` |
|
||||
| See what the database is busy with: connections by service, costliest queries, largest tables | `warmblyctl db load [--limit N] [--json]` |
|
||||
| Move secrets onto a new primary key | `warmblyctl keys reseal` (dry run), then `keys reseal --apply` |
|
||||
| Mint replacement keys and tokens | `warmblyctl keys generate [NAME...]` |
|
||||
|
||||
|
||||
Reference in New Issue
Block a user