rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-10 15:02:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add basic metrics to page cache, and show them in the TUI.

Browse Source 
In the passing, tefactor the code to render the widgets slightly.
This commit is contained in:
Heikki Linnakangas
2021-03-24 19:59:11 +02:00
committed by Stas Kelvich
parent bfc2ed26cf
commit 53b31f60be
3 changed files with 191 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
src/page_cache.rs
View File

@@ -12,6 +12,8 @@ use std::error::Error;
use std::sync::Arc;
use std::sync::Condvar;
use std::sync::Mutex;
use std::sync::atomic::AtomicU64;
use std::sync::atomic::Ordering;
use bytes::Bytes;
use lazy_static::lazy_static;
use rand::Rng;
@@ -26,6 +28,26 @@ pub struct PageCache {
// Channel for communicating with the WAL redo process here.
pub walredo_sender: Sender<Arc<CacheEntry>>,
pub walredo_receiver: Receiver<Arc<CacheEntry>>,
// Counters, for metrics collection.
pub num_entries: AtomicU64,
pub num_page_images: AtomicU64,
pub num_wal_records: AtomicU64,
pub num_getpage_requests: AtomicU64,
// copies of shared.first/last_valid_lsn fields (copied here so
// that they can be read without acquiring the mutex).
pub first_valid_lsn: AtomicU64,
pub last_valid_lsn: AtomicU64,
}
pub struct PageCacheStats {
pub num_entries: u64,
pub num_page_images: u64,
pub num_wal_records: u64,
pub num_getpage_requests: u64,
pub first_valid_lsn: u64,
pub last_valid_lsn: u64,
}
//
@@ -49,7 +71,7 @@ struct PageCacheShared {
// we receive all the WAL up to the request.
//
first_valid_lsn: u64,
last_valid_lsn: u64
last_valid_lsn: u64,
}
lazy_static! {
@@ -71,6 +93,14 @@ fn init_page_cache() -> PageCache
walredo_sender: s,
walredo_receiver: r,
num_entries: AtomicU64::new(0),
num_page_images: AtomicU64::new(0),
num_wal_records: AtomicU64::new(0),
num_getpage_requests: AtomicU64::new(0),
first_valid_lsn: AtomicU64::new(0),
last_valid_lsn: AtomicU64::new(0),
}
}
@@ -163,21 +193,12 @@ pub struct WALRecord {
//
pub fn get_page_at_lsn(tag: BufferTag, lsn: u64) -> Result<Bytes, Box<dyn Error>>
{
// TODO:
//
// Look up cache entry
// If it's a page image, return that. If it's a WAL record, walk backwards
// to the latest page image. Then apply all the WAL records up until the
// given LSN.
//
let minkey = CacheKey {
tag: tag,
lsn: 0
};
let maxkey = CacheKey {
tag: tag,
lsn: lsn
};
PAGECACHE.num_getpage_requests.fetch_add(1, Ordering::Relaxed);
// Look up cache entry. If it's a page image, return that. If it's a WAL record,
// ask the WAL redo service to reconstruct the page image from the WAL records.
let minkey = CacheKey { tag: tag, lsn: 0 };
let maxkey = CacheKey { tag: tag, lsn: lsn };
let entry_rc: Arc<CacheEntry>;
{
@@ -329,7 +350,10 @@ pub fn put_wal_record(tag: BufferTag, rec: WALRecord)
}
let oldentry = shared.pagecache.insert(key, Arc::new(entry));
PAGECACHE.num_entries.fetch_add(1, Ordering::Relaxed);
assert!(oldentry.is_none());
PAGECACHE.num_wal_records.fetch_add(1, Ordering::Relaxed);
}
//
@@ -349,10 +373,13 @@ pub fn put_page_image(tag: BufferTag, lsn: u64, img: Bytes)
let pagecache = &mut shared.pagecache;
let oldentry = pagecache.insert(key, Arc::new(entry));
PAGECACHE.num_entries.fetch_add(1, Ordering::Relaxed);
assert!(oldentry.is_none());
debug!("inserted page image for {}/{}/{}_{} blk {} at {}",
tag.spcnode, tag.dbnode, tag.relnode, tag.forknum, tag.blknum, lsn);
PAGECACHE.num_page_images.fetch_add(1, Ordering::Relaxed);
}
//
@@ -364,6 +391,7 @@ pub fn advance_last_valid_lsn(lsn: u64)
assert!(lsn >= shared.last_valid_lsn);
shared.last_valid_lsn = lsn;
PAGECACHE.last_valid_lsn.store(lsn, Ordering::Relaxed);
}
//
@@ -379,6 +407,7 @@ pub fn _advance_first_valid_lsn(lsn: u64)
assert!(shared.last_valid_lsn == 0 || lsn < shared.last_valid_lsn);
shared.first_valid_lsn = lsn;
PAGECACHE.first_valid_lsn.store(lsn, Ordering::Relaxed);
}
pub fn init_valid_lsn(lsn: u64)
@@ -390,6 +419,8 @@ pub fn init_valid_lsn(lsn: u64)
shared.first_valid_lsn = lsn;
shared.last_valid_lsn = lsn;
PAGECACHE.first_valid_lsn.store(lsn, Ordering::Relaxed);
PAGECACHE.last_valid_lsn.store(lsn, Ordering::Relaxed);
}
pub fn get_last_valid_lsn() -> u64
@@ -399,7 +430,6 @@ pub fn get_last_valid_lsn() -> u64
return shared.last_valid_lsn;
}
//
// Simple test function for the WAL redo code:
//
@@ -478,3 +508,15 @@ pub fn relsize_exist(rel: &RelTag) -> bool
let relsize_cache = &shared.relsize_cache;
relsize_cache.contains_key(rel)
}
pub fn get_stats() -> PageCacheStats
{
PageCacheStats {
num_entries: PAGECACHE.num_entries.load(Ordering::Relaxed),
num_page_images: PAGECACHE.num_page_images.load(Ordering::Relaxed),
num_wal_records: PAGECACHE.num_wal_records.load(Ordering::Relaxed),
num_getpage_requests: PAGECACHE.num_getpage_requests.load(Ordering::Relaxed),
first_valid_lsn: PAGECACHE.first_valid_lsn.load(Ordering::Relaxed),
last_valid_lsn: PAGECACHE.last_valid_lsn.load(Ordering::Relaxed),
}
}

187
src/tui.rs
View File

@@ -6,10 +6,12 @@ use std::{error::Error, io};
use std::sync::Arc;
use termion::{event::Key, input::MouseTerminal, raw::IntoRawMode, screen::AlternateScreen};
use tui::backend::TermionBackend;
use tui::style::{Color, Style};
use tui::buffer::Buffer;
use tui::style::{Color, Style, Modifier};
use tui::Terminal;
use tui::widgets::{Block, Borders, BorderType};
use tui::layout::{Layout, Direction, Constraint};
use tui::text::{Text, Span, Spans};
use tui::widgets::{Widget, Block, Borders, BorderType, Paragraph};
use tui::layout::{Layout, Direction, Constraint, Rect};
use lazy_static::lazy_static;
use slog;
@@ -100,19 +102,19 @@ pub fn ui_main<'b>() -> Result<(), Box<dyn Error>> {
terminal.draw(|f| {
let size = f.size();
// +---------------+---------------+
// | | |
// | top_top_left | |
// | | |
// +---------------+ top_right |
// | | |
// | top_bot_left | |
// | | |
// +---------------+---------------+
// | |
// | bottom |
// | |
// +---------------+---------------+
// +----------------+----------------+
// | | |
// | top_top_left | top_top_right |
// | | |
// +----------------+----------------|
// | | |
// | top_bot_left | top_left_right |
// | | |
// +----------------+----------------+
// | |
// | bottom |
// | |
// +---------------------------------+
let chunks = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Percentage(70), Constraint::Percentage(30)].as_ref())
@@ -134,49 +136,27 @@ pub fn ui_main<'b>() -> Result<(), Box<dyn Error>> {
let top_top_left_chunk = c[0];
let top_bot_left_chunk = c[1];
let w = TuiLoggerWidget::default(PAGESERVICE_DRAIN.as_ref())
.block(Block::default()
.borders(Borders::ALL)
.title("Page Service")
.border_type(BorderType::Rounded))
.show_module(false)
.style_error(Style::default().fg(Color::Red))
.style_warn(Style::default().fg(Color::Yellow))
.style_info(Style::default().fg(Color::Green));
f.render_widget(w, top_top_left_chunk);
let c = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref())
.split(top_right_chunk);
let top_top_right_chunk = c[0];
let top_bot_right_chunk = c[1];
let w = TuiLoggerWidget::default(WALREDO_DRAIN.as_ref())
.block(Block::default()
.borders(Borders::ALL)
.title("WAL Redo")
.border_type(BorderType::Rounded))
.show_module(false)
.style_error(Style::default().fg(Color::Red))
.style_warn(Style::default().fg(Color::Yellow))
.style_info(Style::default().fg(Color::Green));
f.render_widget(w, top_bot_left_chunk);
f.render_widget(LogWidget::new(PAGESERVICE_DRAIN.as_ref(),"Page Service"),
top_top_left_chunk);
let w = TuiLoggerWidget::default(WALRECEIVER_DRAIN.as_ref())
.block(Block::default()
.borders(Borders::ALL)
.title("WAL Receiver")
.border_type(BorderType::Rounded))
.show_module(false)
.style_error(Style::default().fg(Color::Red))
.style_warn(Style::default().fg(Color::Yellow))
.style_info(Style::default().fg(Color::Green));
f.render_widget(w, top_right_chunk);
f.render_widget(LogWidget::new(WALREDO_DRAIN.as_ref(), "WAL Redo"),
top_bot_left_chunk);
let w = TuiLoggerWidget::default(CATCHALL_DRAIN.as_ref())
.block(Block::default()
.borders(Borders::ALL)
.title("Other log")
.border_type(BorderType::Rounded))
.show_module(true)
.style_error(Style::default().fg(Color::Red))
.style_warn(Style::default().fg(Color::Yellow))
.style_info(Style::default().fg(Color::Green));
f.render_widget(w, bottom_chunk);
f.render_widget(LogWidget::new(WALRECEIVER_DRAIN.as_ref(), "WAL Receiver"),
top_top_right_chunk);
f.render_widget(MetricsWidget {}, top_bot_right_chunk);
f.render_widget(LogWidget::new(CATCHALL_DRAIN.as_ref(), "All Log")
.show_module(true),
bottom_chunk);
})?;
@@ -196,3 +176,98 @@ pub fn ui_main<'b>() -> Result<(), Box<dyn Error>> {
Ok(())
}
struct LogWidget<'a> {
logger: &'a TuiLogger,
title: &'a str,
show_module: bool,
}
impl<'a> LogWidget<'a> {
fn new(logger: &'a TuiLogger, title: &'a str) -> LogWidget<'a> {
LogWidget { logger, title, show_module: false }
}
fn show_module(mut self, b: bool) -> LogWidget<'a> {
self.show_module = b;
self
}
}
impl<'a> Widget for LogWidget<'a> {
fn render(self, area: Rect, buf: &mut Buffer) {
let w = TuiLoggerWidget::default(self.logger)
.block(Block::default()
.borders(Borders::ALL)
.title(self.title)
.border_type(BorderType::Rounded))
.show_module(true)
.style_error(Style::default().fg(Color::Red))
.style_warn(Style::default().fg(Color::Yellow))
.style_info(Style::default().fg(Color::Green));
w.render(area, buf);
}
}
// Render a widget to show some metrics
struct MetricsWidget {
}
fn get_metric_u64<'a>(title: &'a str, value: u64) -> Spans<'a> {
Spans::from(vec![
Span::styled(format!("{:<20}", title), Style::default()),
Span::raw(": "),
Span::styled(value.to_string(), Style::default().add_modifier(Modifier::BOLD)),
])
}
fn get_metric_str<'a>(title: &'a str, value: &'a str) -> Spans<'a> {
Spans::from(vec![
Span::styled(format!("{:<20}", title), Style::default()),
Span::raw(": "),
Span::styled(value, Style::default().add_modifier(Modifier::BOLD)),
])
}
// FIXME: We really should define a datatype for LSNs, with Display trait and
// helper functions. There's one in tokio-postgres, but I don't think we want
// to rely on that.
fn format_lsn(lsn: u64) -> String
{
return format!("{:X}/{:X}", lsn >> 32, lsn & 0xffff_ffff)
}
impl tui::widgets::Widget for MetricsWidget {
fn render(self, area: Rect, buf: &mut Buffer) {
let block = Block::default()
.borders(Borders::ALL)
.title("Page Cache Metrics")
.border_type(BorderType::Rounded);
let inner_area = block.inner(area);
block.render(area, buf);
let mut lines: Vec<Spans> = Vec::new();
let page_cache_stats = crate::page_cache::get_stats();
let lsnrange = format!("{} - {}",
format_lsn(page_cache_stats.first_valid_lsn),
format_lsn(page_cache_stats.last_valid_lsn));
lines.push(get_metric_str("Valid LSN range", &lsnrange));
lines.push(get_metric_u64("# of cache entries", page_cache_stats.num_entries));
lines.push(get_metric_u64("# of page images", page_cache_stats.num_page_images));
lines.push(get_metric_u64("# of WAL records", page_cache_stats.num_wal_records));
lines.push(get_metric_u64("# of GetPage@LSN calls", page_cache_stats.num_getpage_requests));
let text = Text::from(lines);
Paragraph::new(text).render(inner_area, buf);
}
}

1
src/walredo.rs
View File

@@ -99,6 +99,7 @@ fn handle_apply_request(process: &WalRedoProcess, runtime: &Runtime, entry_rc: A
error!("could not apply WAL records: {}", e);
} else {
entry.page_image = Some(apply_result.unwrap());
page_cache::PAGECACHE.num_page_images.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
// Wake up the requester, whether the operation succeeded or not.