rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-06-02 13:00:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add exponential histogram to estiumate optimal cache size

Browse Source
This commit is contained in:
Konstantin Knizhnik
2024-09-13 10:26:02 +01:00
parent d0cbfda15c
commit 61f827364f
5 changed files with 111 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pgxn/neon/Makefile
View File

@@ -23,7 +23,7 @@ SHLIB_LINK_INTERNAL = $(libpq)
SHLIB_LINK = -lcurl
EXTENSION = neon
DATA = neon--1.0.sql neon--1.0--1.1.sql neon--1.1--1.2.sql neon--1.2--1.3.sql neon--1.3--1.2.sql neon--1.2--1.1.sql neon--1.1--1.0.sql neon--1.3--1.4.sql neon--1.4--1.3.sql
DATA = neon--1.0.sql neon--1.0--1.1.sql neon--1.1--1.2.sql neon--1.2--1.3.sql neon--1.3--1.2.sql neon--1.2--1.1.sql neon--1.1--1.0.sql neon--1.3--1.4.sql neon--1.4--1.3.sql neon--1.4--1.5.sql neon--1.5--1.4.sql
PGFILEDESC = "neon - cloud storage for PostgreSQL"
EXTRA_CLEAN = \

22
pgxn/neon/file_cache.c
View File

@@ -1263,7 +1263,7 @@ approximate_working_set_size_seconds(PG_FUNCTION_ARGS)
int32 dc;
time_t duration = PG_ARGISNULL(0) ? (time_t)-1 : PG_GETARG_INT32(0);
LWLockAcquire(lfc_lock, LW_SHARED);
dc = (int32) estimateSHLL(&lfc_ctl->wss_estimation, duration);
dc = (int32) estimateSHLL(&lfc_ctl->wss_estimation, duration, 0);
LWLockRelease(lfc_lock);
PG_RETURN_INT32(dc);
}
@@ -1280,7 +1280,7 @@ approximate_working_set_size(PG_FUNCTION_ARGS)
int32 dc;
bool reset = PG_GETARG_BOOL(0);
LWLockAcquire(lfc_lock, reset ? LW_EXCLUSIVE : LW_SHARED);
dc = (int32) estimateSHLL(&lfc_ctl->wss_estimation, (time_t)-1);
dc = (int32) estimateSHLL(&lfc_ctl->wss_estimation, (time_t)-1, 0);
if (reset)
memset(lfc_ctl->wss_estimation.regs, 0, sizeof lfc_ctl->wss_estimation.regs);
LWLockRelease(lfc_lock);
@@ -1288,3 +1288,21 @@ approximate_working_set_size(PG_FUNCTION_ARGS)
}
PG_RETURN_NULL();
}
PG_FUNCTION_INFO_V1(approximate_optimal_cache_size);
Datum
approximate_optimal_cache_size(PG_FUNCTION_ARGS)
{
if (lfc_size_limit != 0)
{
int32 dc;
time_t duration = PG_ARGISNULL(0) ? (time_t)-1 : PG_GETARG_INT32(0);
double min_hit_ratio = PG_ARGISNULL(1) ? 0 : PG_GETARG_FLOAT8(1);
LWLockAcquire(lfc_lock, LW_SHARED);
dc = (int32) estimateSHLL(&lfc_ctl->wss_estimation, duration, min_hit_ratio);
LWLockRelease(lfc_lock);
PG_RETURN_INT32(dc);
}
PG_RETURN_NULL();
}

47
pgxn/neon/hll.c
View File

@@ -126,19 +126,50 @@ addSHLL(HyperLogLogState *cState, uint32 hash)
/* Compute the rank of the remaining 32 - "k" (registerWidth) bits */
count = rho(hash << HLL_BIT_WIDTH, HLL_C_BITS);
cState->regs[index][count] = now;
if (cState->regs[index][count].ts)
{
/* update histgoram */
int64_t delta = (now - cState->regs[index][count].ts)/USECS_PER_SEC;
uint32_t new_histogram[HIST_SIZE] = {0};
for (int i = 0; i < HIST_SIZE; i++) {
/* Use average point of interval */
uint32 interval_log2 = pg_ceil_log2_32((delta + (HIST_MIN_INTERVAL*((i+1) + ((1<<i)/2))/2)) / HIST_MIN_INTERVAL);
uint32 cell = Min(interval_log2, HIST_SIZE-1);
new_histogram[cell] += cState->regs[index][count].histogram[i];
}
memcpy(cState->regs[index][count].histogram, new_histogram, sizeof new_hostogram);
}
cState->regs[index][count].ts = now;
cState->regs[index][count].histogram[0] += 1;
}
static uint32_t
getAccessCount(const HyperLogLogRegister* reg, time_t duration)
{
uint32_t count = 0;
for (size_t i = 0; i < HIST_SIZE && (HIST_MIN_INTERVAL << i) <= duration; i++) {
count += reg->histogram[i];
}
return count;
}
static uint8
getMaximum(const TimestampTz* reg, TimestampTz since)
getMaximum(const HyperLogLogRegister* reg, TimestampTz since, time_t duration, double min_hit_ratio)
{
uint8 max = 0;
for (size_t i = 0; i < HLL_C_BITS + 1; i++)
size_t i, j;
uint32_t total_count = 0;
for (i = 0; i < HIST_SIZE && (HIST_MIN_INTERVAL << i) <= duration; i++) {
total_count += getAccessCount(reg, duration);
}
if (total_count != 0)
{
if (reg[i] >= since)
for (i = 0; i < HLL_C_BITS + 1; i++)
{
max = i;
if (reg[i].ts >= since && 1.0 - getAccessCount(reg, duration) / total_count >= min_hit_ration)
{
max = i;
}
}
}
@@ -150,7 +181,7 @@ getMaximum(const TimestampTz* reg, TimestampTz since)
* Estimates cardinality, based on elements added so far
*/
double
estimateSHLL(HyperLogLogState *cState, time_t duration)
estimateSHLL(HyperLogLogState *cState, time_t duration, double min_hit_ratio)
{
double result;
double sum = 0.0;
@@ -161,7 +192,7 @@ estimateSHLL(HyperLogLogState *cState, time_t duration)
for (i = 0; i < HLL_N_REGISTERS; i++)
{
R[i] = getMaximum(cState->regs[i], since);
R[i] = getMaximum(cState->regs[i], since, duration, min_hit_ratio);
sum += 1.0 / pow(2.0, R[i]);
}

18
pgxn/neon/hll.h
View File

@@ -53,6 +53,14 @@
#define HLL_C_BITS (32 - HLL_BIT_WIDTH)
#define HLL_N_REGISTERS (1 << HLL_BIT_WIDTH)
/*
* Number of histogram cells. We use exponential histogram with first interval
* equals to one minutes. Autoscaler request LFC statistic with intervals 1,2,...,60 seconds,
* so 1^8=64 seems to be enough for our needs.
*/
#define HIST_SIZE 8
#define HIST_MIN_INTERVAL 60 /* seconds */
/*
* HyperLogLog is an approximate technique for computing the number of distinct
* entries in a set. Importantly, it does this by using a fixed amount of
@@ -74,13 +82,19 @@
* precision; as 32 bits in second precision should be enough for statistics.
* However, that is not yet implemented.
*/
typedef struct
{
TimestampTz ts; /* last access timestamp */
uint32_t histogram[HIST_SIZE]; /* access counter histogram */
} HyperLogLogRegister;
typedef struct HyperLogLogState
{
TimestampTz regs[HLL_N_REGISTERS][HLL_C_BITS + 1];
HyperLogLogRegister regs[HLL_N_REGISTERS][HLL_C_BITS + 1];
} HyperLogLogState;
extern void initSHLL(HyperLogLogState *cState);
extern void addSHLL(HyperLogLogState *cState, uint32 hash);
extern double estimateSHLL(HyperLogLogState *cState, time_t dutration);
extern double estimateSHLL(HyperLogLogState *cState, time_t dutration, double min_hit_ratio);
#endif

35
test_runner/regress/test_lfc_working_set_approximation.py
View File

@@ -114,3 +114,38 @@ def test_sliding_working_set_approximation(neon_simple_env: NeonEnv):
assert estimation_1k >= 20 and estimation_1k <= 40
assert estimation_10k >= 200 and estimation_10k <= 400
def test_optimal_cache_size_approximation(neon_simple_env: NeonEnv):
env = neon_simple_env
endpoint = env.endpoints.create_start(
branch_name="main",
config_lines=[
"autovacuum = off",
"shared_buffers=1MB",
"neon.max_file_cache_size=256MB",
"neon.file_cache_size_limit=245MB",
],
)
conn = endpoint.connect()
cur = conn.cursor()
cur.execute("create extension neon")
cur.execute(
"create table t_huge(pk integer primary key, count integer default 0, payload text default repeat('?', 128))"
)
cur.execute(
"create table t_small(pk integer primary key, count integer default 0, payload text default repeat('?', 128))"
)
cur.execute("insert into t_huge(pk) values (generate_series(1,1000000))")
cur.execute("insert into t_small (pk) values (generate_series(1,100000))")
time.sleep(2)
before = time.monotonic()
for _ in 1..100:
cur.execute("select sum(count) from t_small")
cur.execute("select sum(count) from t_huge")
after = time.monotonic()
cur.execute(f"select approximate_optimal_cache_size({int(after - before + 1, 0.99)})")
estimation = cur.fetchall()[0][0]
log.info(f"Working set size for selecting 1k records {estimation}")
assert estimation_1k >= 20 and estimation_1k <= 40