From 9fdf5fbb7ed3452526622d780ab9ebad1d896718 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 3 Jun 2025 17:41:22 +0300 Subject: [PATCH 01/24] Use a separate freelist to track LFC "holes" When the LFC is shrunk, we punch holes in the underlying file to release the disk space to the OS. We tracked it in the same hash table as the in-use entries, because that was convenient. However, I'm working on being able to shrink the hash table too, and once we do that, we'll need some other place to track the holes. Implement a simple scheme of an in-memory array and a chain of on-disk blocks for that. --- pgxn/neon/file_cache.c | 289 ++++++++++++++++++++++++++++++++--------- 1 file changed, 230 insertions(+), 59 deletions(-) diff --git a/pgxn/neon/file_cache.c b/pgxn/neon/file_cache.c index 45a4695495..ec8c5db8aa 100644 --- a/pgxn/neon/file_cache.c +++ b/pgxn/neon/file_cache.c @@ -21,6 +21,7 @@ #include "access/xlog.h" #include "funcapi.h" #include "miscadmin.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "pgstat.h" #include "port/pg_iovec.h" @@ -64,7 +65,7 @@ * * Cache is always reconstructed at node startup, so we do not need to save mapping somewhere and worry about * its consistency. - + * * * ## Holes * @@ -76,13 +77,15 @@ * fallocate(FALLOC_FL_PUNCH_HOLE) call. The nominal size of the file doesn't * shrink, but the disk space it uses does. * - * Each hole is tracked by a dummy FileCacheEntry, which are kept in the - * 'holes' linked list. They are entered into the chunk hash table, with a - * special key where the blockNumber is used to store the 'offset' of the - * hole, and all other fields are zero. Holes are never looked up in the hash - * table, we only enter them there to have a FileCacheEntry that we can keep - * in the linked list. If the soft limit is raised again, we reuse the holes - * before extending the nominal size of the file. + * Each hole is tracked in a freelist. The freelist consists of two parts: a + * fixed-size array in shared memory, and a linked chain of on-disk + * blocks. When the in-memory array fills up, it's flushed to a new on-disk + * chunk. If the soft limit is raised again, we reuse the holes before + * extending the nominal size of the file. + * + * The in-memory freelist array is protected by 'lfc_lock', while the on-disk + * chain is protected by a separate 'lfc_freelist_lock'. Locking rule to + * avoid deadlocks: always acquire lfc_freelist_lock first, then lfc_lock. */ /* Local file storage allocation chunk. @@ -100,6 +103,8 @@ #define SIZE_MB_TO_CHUNKS(size) ((uint32)((size) * MB / BLCKSZ >> lfc_chunk_size_log)) #define BLOCK_TO_CHUNK_OFF(blkno) ((blkno) & (lfc_blocks_per_chunk-1)) +#define INVALID_OFFSET (0xffffffff) + /* * Blocks are read or written to LFC file outside LFC critical section. * To synchronize access to such block, writer set state of such block to PENDING. @@ -123,11 +128,11 @@ typedef struct FileCacheEntry uint32 hash; uint32 offset; uint32 access_count; - dlist_node list_node; /* LRU/holes list node */ + dlist_node list_node; /* LRU list node */ uint32 state[FLEXIBLE_ARRAY_MEMBER]; /* two bits per block */ } FileCacheEntry; -#define FILE_CACHE_ENRTY_SIZE MAXALIGN(offsetof(FileCacheEntry, state) + (lfc_blocks_per_chunk*2+31)/32*4) +#define FILE_CACHE_ENTRY_SIZE MAXALIGN(offsetof(FileCacheEntry, state) + (lfc_blocks_per_chunk*2+31)/32*4) #define GET_STATE(entry, i) (((entry)->state[(i) / 16] >> ((i) % 16 * 2)) & 3) #define SET_STATE(entry, i, new_state) (entry)->state[(i) / 16] = ((entry)->state[(i) / 16] & ~(3 << ((i) % 16 * 2))) | ((new_state) << ((i) % 16 * 2)) @@ -161,7 +166,6 @@ typedef struct FileCacheControl uint64 evicted_pages; /* number of evicted pages */ dlist_head lru; /* double linked list for LRU replacement * algorithm */ - dlist_head holes; /* double linked list of punched holes */ HyperLogLogState wss_estimation; /* estimation of working set size */ ConditionVariable cv[N_COND_VARS]; /* turnstile of condition variables */ PrewarmWorkerState prewarm_workers[MAX_PREWARM_WORKERS]; @@ -172,17 +176,35 @@ typedef struct FileCacheControl bool prewarm_active; bool prewarm_canceled; dsm_handle prewarm_lfc_state_handle; + + /* + * Free list. This is large enough to hold one chunks worth of entries. + */ + uint32 freelist_size; + uint32 freelist_head; + uint32 num_free_pages; + uint32 free_pages[FLEXIBLE_ARRAY_MEMBER]; } FileCacheControl; +typedef struct FreeListChunk +{ + uint32 next; + uint32 num_free_pages; + uint32 free_pages[FLEXIBLE_ARRAY_MEMBER]; +} FreeListChunk; + #define FILE_CACHE_STATE_MAGIC 0xfcfcfcfc #define FILE_CACHE_STATE_BITMAP(fcs) ((uint8*)&(fcs)->chunks[(fcs)->n_chunks]) #define FILE_CACHE_STATE_SIZE_FOR_CHUNKS(n_chunks) (sizeof(FileCacheState) + (n_chunks)*sizeof(BufferTag) + (((n_chunks) * lfc_blocks_per_chunk)+7)/8) #define FILE_CACHE_STATE_SIZE(fcs) (sizeof(FileCacheState) + (fcs->n_chunks)*sizeof(BufferTag) + (((fcs->n_chunks) << fcs->chunk_size_log)+7)/8) +#define FREELIST_ENTRIES_PER_CHUNK(c) ((c) * BLCKSZ / sizeof(uint32) - 2) + static HTAB *lfc_hash; static int lfc_desc = -1; static LWLockId lfc_lock; +static LWLockId lfc_freelist_lock; static int lfc_max_size; static int lfc_size_limit; static int lfc_prewarm_limit; @@ -205,6 +227,11 @@ bool AmPrewarmWorker; #define LFC_ENABLED() (lfc_ctl->limit != 0) +static bool freelist_push(uint32 offset); +static bool freelist_prepare_pop(void); +static uint32 freelist_pop(void); +static bool freelist_is_empty(void); + /* * Close LFC file if opened. * All backends should close their LFC files once LFC is disabled. @@ -248,7 +275,9 @@ lfc_switch_off(void) lfc_ctl->used_pages = 0; lfc_ctl->limit = 0; dlist_init(&lfc_ctl->lru); - dlist_init(&lfc_ctl->holes); + + lfc_ctl->freelist_head = INVALID_OFFSET; + lfc_ctl->num_free_pages = 0; /* * We need to use unlink to to avoid races in LFC write, because it is not @@ -317,6 +346,7 @@ lfc_ensure_opened(void) static void lfc_shmem_startup(void) { + size_t size; bool found; static HASHCTL info; @@ -327,15 +357,19 @@ lfc_shmem_startup(void) LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); - lfc_ctl = (FileCacheControl *) ShmemInitStruct("lfc", sizeof(FileCacheControl), &found); + size = offsetof(FileCacheControl, free_pages); + size += FREELIST_ENTRIES_PER_CHUNK(lfc_blocks_per_chunk) * sizeof(uint32); + + lfc_ctl = (FileCacheControl *) ShmemInitStruct("lfc", size, &found); if (!found) { int fd; uint32 n_chunks = SIZE_MB_TO_CHUNKS(lfc_max_size); lfc_lock = (LWLockId) GetNamedLWLockTranche("lfc_lock"); + lfc_freelist_lock = (LWLockId) GetNamedLWLockTranche("lfc_freelist_lock"); info.keysize = sizeof(BufferTag); - info.entrysize = FILE_CACHE_ENRTY_SIZE; + info.entrysize = FILE_CACHE_ENTRY_SIZE; /* * n_chunks+1 because we add new element to hash table before eviction @@ -345,9 +379,12 @@ lfc_shmem_startup(void) n_chunks + 1, n_chunks + 1, &info, HASH_ELEM | HASH_BLOBS); - memset(lfc_ctl, 0, sizeof(FileCacheControl)); + memset(lfc_ctl, 0, offsetof(FileCacheControl, free_pages)); dlist_init(&lfc_ctl->lru); - dlist_init(&lfc_ctl->holes); + + lfc_ctl->freelist_size = FREELIST_ENTRIES_PER_CHUNK(lfc_blocks_per_chunk); + lfc_ctl->freelist_head = INVALID_OFFSET; + lfc_ctl->num_free_pages = 0; /* Initialize hyper-log-log structure for estimating working set size */ initSHLL(&lfc_ctl->wss_estimation); @@ -376,13 +413,20 @@ lfc_shmem_startup(void) static void lfc_shmem_request(void) { + size_t size; + #if PG_VERSION_NUM>=150000 if (prev_shmem_request_hook) prev_shmem_request_hook(); #endif - RequestAddinShmemSpace(sizeof(FileCacheControl) + hash_estimate_size(SIZE_MB_TO_CHUNKS(lfc_max_size) + 1, FILE_CACHE_ENRTY_SIZE)); + size = offsetof(FileCacheControl, free_pages); + size += FREELIST_ENTRIES_PER_CHUNK(lfc_blocks_per_chunk) * sizeof(uint32); + size += hash_estimate_size(SIZE_MB_TO_CHUNKS(lfc_max_size) + 1, FILE_CACHE_ENTRY_SIZE); + + RequestAddinShmemSpace(size); RequestNamedLWLockTranche("lfc_lock", 1); + RequestNamedLWLockTranche("lfc_freelist_lock", 2); } static bool @@ -435,12 +479,14 @@ lfc_change_limit_hook(int newval, void *extra) if (!lfc_ctl || !is_normal_backend()) return; + LWLockAcquire(lfc_freelist_lock, LW_EXCLUSIVE); LWLockAcquire(lfc_lock, LW_EXCLUSIVE); /* Open LFC file only if LFC was enabled or we are going to reenable it */ if (newval == 0 && !LFC_ENABLED()) { LWLockRelease(lfc_lock); + LWLockRelease(lfc_freelist_lock); /* File should be reopened if LFC is reenabled */ lfc_close_file(); return; @@ -449,6 +495,7 @@ lfc_change_limit_hook(int newval, void *extra) if (!lfc_ensure_opened()) { LWLockRelease(lfc_lock); + LWLockRelease(lfc_freelist_lock); return; } @@ -464,18 +511,14 @@ lfc_change_limit_hook(int newval, void *extra) * returning their space to file system */ FileCacheEntry *victim = dlist_container(FileCacheEntry, list_node, dlist_pop_head_node(&lfc_ctl->lru)); - FileCacheEntry *hole; uint32 offset = victim->offset; - uint32 hash; - bool found; - BufferTag holetag; CriticalAssert(victim->access_count == 0); #ifdef FALLOC_FL_PUNCH_HOLE if (fallocate(lfc_desc, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, (off_t) victim->offset * lfc_blocks_per_chunk * BLCKSZ, lfc_blocks_per_chunk * BLCKSZ) < 0) neon_log(LOG, "Failed to punch hole in file: %m"); #endif - /* We remove the old entry, and re-enter a hole to the hash table */ + /* We remove the entry, and enter a hole to the freelist */ for (int i = 0; i < lfc_blocks_per_chunk; i++) { bool is_page_cached = GET_STATE(victim, i) == AVAILABLE; @@ -484,15 +527,14 @@ lfc_change_limit_hook(int newval, void *extra) } hash_search_with_hash_value(lfc_hash, &victim->key, victim->hash, HASH_REMOVE, NULL); - memset(&holetag, 0, sizeof(holetag)); - holetag.blockNum = offset; - hash = get_hash_value(lfc_hash, &holetag); - hole = hash_search_with_hash_value(lfc_hash, &holetag, hash, HASH_ENTER, &found); - hole->hash = hash; - hole->offset = offset; - hole->access_count = 0; - CriticalAssert(!found); - dlist_push_tail(&lfc_ctl->holes, &hole->list_node); + if (!freelist_push(offset)) + { + /* freelist_push already logged the error */ + lfc_switch_off(); + LWLockRelease(lfc_lock); + LWLockRelease(lfc_freelist_lock); + return; + } lfc_ctl->used -= 1; } @@ -504,6 +546,7 @@ lfc_change_limit_hook(int newval, void *extra) neon_log(DEBUG1, "set local file cache limit to %d", new_size); LWLockRelease(lfc_lock); + LWLockRelease(lfc_freelist_lock); } void @@ -1380,7 +1423,7 @@ lfc_init_new_entry(FileCacheEntry* entry, uint32 hash) * options, in order of preference: * * Unless there is no space available, we can: - * 1. Use an entry from the `holes` list, and + * 1. Use an entry from the freelist, and * 2. Create a new entry. * We can always, regardless of space in the LFC: * 3. evict an entry from LRU, and @@ -1388,17 +1431,10 @@ lfc_init_new_entry(FileCacheEntry* entry, uint32 hash) */ if (lfc_ctl->used < lfc_ctl->limit) { - if (!dlist_is_empty(&lfc_ctl->holes)) + if (!freelist_is_empty()) { /* We can reuse a hole that was left behind when the LFC was shrunk previously */ - FileCacheEntry *hole = dlist_container(FileCacheEntry, list_node, - dlist_pop_head_node(&lfc_ctl->holes)); - uint32 offset = hole->offset; - bool hole_found; - - hash_search_with_hash_value(lfc_hash, &hole->key, - hole->hash, HASH_REMOVE, &hole_found); - CriticalAssert(hole_found); + uint32 offset = freelist_pop(); lfc_ctl->used += 1; entry->offset = offset; /* reuse the hole */ @@ -1512,6 +1548,7 @@ lfc_prefetch(NRelFileInfo rinfo, ForkNumber forknum, BlockNumber blkno, hash = get_hash_value(lfc_hash, &tag); cv = &lfc_ctl->cv[hash % N_COND_VARS]; + retry: LWLockAcquire(lfc_lock, LW_EXCLUSIVE); if (!LFC_ENABLED() || !lfc_ensure_opened()) @@ -1520,6 +1557,9 @@ lfc_prefetch(NRelFileInfo rinfo, ForkNumber forknum, BlockNumber blkno, return false; } + if (!freelist_prepare_pop()) + goto retry; + lwlsn = neon_get_lwlsn(rinfo, forknum, blkno); if (lwlsn > lsn) @@ -1653,6 +1693,7 @@ lfc_writev(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, CriticalAssert(BufTagGetRelNumber(&tag) != InvalidRelFileNumber); + retry: LWLockAcquire(lfc_lock, LW_EXCLUSIVE); if (!LFC_ENABLED() || !lfc_ensure_opened()) @@ -1662,6 +1703,9 @@ lfc_writev(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, } generation = lfc_ctl->generation; + if (!freelist_prepare_pop()) + goto retry; + /* * For every chunk that has blocks we're interested in, we * 1. get the chunk header @@ -1823,6 +1867,140 @@ lfc_writev(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, LWLockRelease(lfc_lock); } +/**** freelist management ****/ + + +/* + * Prerequisites: + * - The caller is holding 'lfc_lock'. XXX + */ +static bool +freelist_prepare_pop(void) +{ + /* + * If the in-memory freelist is empty, but there are more blocks available, load them. + * + * TODO: if there + */ + if (lfc_ctl->num_free_pages == 0 && lfc_ctl->freelist_head != INVALID_OFFSET) + { + uint32 freelist_head; + FreeListChunk *freelist_chunk; + size_t bytes_read; + + LWLockRelease(lfc_lock); + LWLockAcquire(lfc_freelist_lock, LW_EXCLUSIVE); + + if (!(lfc_ctl->num_free_pages == 0 && lfc_ctl->freelist_head != INVALID_OFFSET)) + { + /* someone else did the work for us while we were not holding the lock */ + LWLockRelease(lfc_freelist_lock); + return false; + } + + freelist_head = lfc_ctl->freelist_head; + freelist_chunk = palloc(lfc_blocks_per_chunk * BLCKSZ); + + bytes_read = 0; + while (bytes_read < lfc_blocks_per_chunk * BLCKSZ) + { + ssize_t rc; + + rc = pread(lfc_desc, freelist_chunk, lfc_blocks_per_chunk * BLCKSZ - bytes_read, (off_t) freelist_head * lfc_blocks_per_chunk * BLCKSZ + bytes_read); + if (rc < 0) + { + lfc_disable("read freelist page"); + return false; + } + bytes_read += rc; + } + + LWLockAcquire(lfc_lock, LW_EXCLUSIVE); + if (lfc_generation != lfc_ctl->generation) + { + LWLockRelease(lfc_lock); + return false; + } + + Assert(lfc_ctl->freelist_head == freelist_head); + Assert(lfc_ctl->num_free_pages == 0); + lfc_ctl->freelist_head = freelist_chunk->next; + lfc_ctl->num_free_pages = freelist_chunk->num_free_pages; + memcpy(lfc_ctl->free_pages, freelist_chunk->free_pages, lfc_ctl->num_free_pages * sizeof(uint32)); + pfree(freelist_chunk); + + LWLockRelease(lfc_lock); + LWLockRelease(lfc_freelist_lock); + return false; + } + + return true; +} + +/* + * Prerequisites: + * - The caller is holding 'lfc_lock' and 'lfc_freelist_lock'. + * + * Returns 'false' on error. + */ +static bool +freelist_push(uint32 offset) +{ + Assert(lfc_ctl->freelist_size == FREELIST_ENTRIES_PER_CHUNK(lfc_blocks_per_chunk)); + if (lfc_ctl->num_free_pages == lfc_ctl->freelist_size) + { + FreeListChunk *freelist_chunk; + struct iovec iov; + ssize_t rc; + + freelist_chunk = palloc(lfc_blocks_per_chunk * BLCKSZ); + + /* write the existing entries to the chunk on disk */ + freelist_chunk->next = lfc_ctl->freelist_head; + freelist_chunk->num_free_pages = lfc_ctl->num_free_pages; + memcpy(freelist_chunk->free_pages, lfc_ctl->free_pages, lfc_ctl->num_free_pages * sizeof(uint32)); + + /* Use the passed-in offset to hold the freelist chunk itself */ + iov.iov_base = freelist_chunk; + iov.iov_len = lfc_blocks_per_chunk * BLCKSZ; + rc = pg_pwritev_with_retry(lfc_desc, &iov, 1, (off_t) offset * lfc_blocks_per_chunk * BLCKSZ); + + pfree(freelist_chunk); + + if (rc < 0) + return false; + + lfc_ctl->freelist_head = offset; + lfc_ctl->num_free_pages = 0; + } + else + { + lfc_ctl->free_pages[lfc_ctl->num_free_pages] = offset; + lfc_ctl->num_free_pages++; + } + return true; +} + +static uint32 +freelist_pop(void) +{ + uint32 result; + + /* The caller should've checked that the list is not empty */ + Assert(lfc_ctl->num_free_pages > 0); + + result = lfc_ctl->free_pages[lfc_ctl->num_free_pages - 1]; + lfc_ctl->num_free_pages--; + + return result; +} + +static bool +freelist_is_empty(void) +{ + return lfc_ctl->num_free_pages == 0; +} + typedef struct { TupleDesc tupdesc; @@ -2049,12 +2227,8 @@ local_cache_pages(PG_FUNCTION_ARGS) hash_seq_init(&status, lfc_hash); while ((entry = hash_seq_search(&status)) != NULL) { - /* Skip hole tags */ - if (NInfoGetRelNumber(BufTagGetNRelFileInfo(entry->key)) != 0) - { - for (int i = 0; i < lfc_blocks_per_chunk; i++) - n_pages += GET_STATE(entry, i) == AVAILABLE; - } + for (int i = 0; i < lfc_blocks_per_chunk; i++) + n_pages += GET_STATE(entry, i) == AVAILABLE; } } } @@ -2082,19 +2256,16 @@ local_cache_pages(PG_FUNCTION_ARGS) { for (int i = 0; i < lfc_blocks_per_chunk; i++) { - if (NInfoGetRelNumber(BufTagGetNRelFileInfo(entry->key)) != 0) + if (GET_STATE(entry, i) == AVAILABLE) { - if (GET_STATE(entry, i) == AVAILABLE) - { - fctx->record[n].pageoffs = entry->offset * lfc_blocks_per_chunk + i; - fctx->record[n].relfilenode = NInfoGetRelNumber(BufTagGetNRelFileInfo(entry->key)); - fctx->record[n].reltablespace = NInfoGetSpcOid(BufTagGetNRelFileInfo(entry->key)); - fctx->record[n].reldatabase = NInfoGetDbOid(BufTagGetNRelFileInfo(entry->key)); - fctx->record[n].forknum = entry->key.forkNum; - fctx->record[n].blocknum = entry->key.blockNum + i; - fctx->record[n].accesscount = entry->access_count; - n += 1; - } + fctx->record[n].pageoffs = entry->offset * lfc_blocks_per_chunk + i; + fctx->record[n].relfilenode = NInfoGetRelNumber(BufTagGetNRelFileInfo(entry->key)); + fctx->record[n].reltablespace = NInfoGetSpcOid(BufTagGetNRelFileInfo(entry->key)); + fctx->record[n].reldatabase = NInfoGetDbOid(BufTagGetNRelFileInfo(entry->key)); + fctx->record[n].forknum = entry->key.forkNum; + fctx->record[n].blocknum = entry->key.blockNum + i; + fctx->record[n].accesscount = entry->access_count; + n += 1; } } } From 96b4de1de6bc4295ab53312e1e181fb5da649ef8 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 4 Jun 2025 18:02:05 +0300 Subject: [PATCH 02/24] Make LFC chunk size a compile-time constant A runtime setting is nicer, but the next commit will replace the hash table with a different implementation that requires the value size to be a compile-time constant. --- pgxn/neon/file_cache.c | 129 +++++++++++++++++------------------------ 1 file changed, 54 insertions(+), 75 deletions(-) diff --git a/pgxn/neon/file_cache.c b/pgxn/neon/file_cache.c index ec8c5db8aa..485c282414 100644 --- a/pgxn/neon/file_cache.c +++ b/pgxn/neon/file_cache.c @@ -95,13 +95,13 @@ * 1Mb chunks can reduce hash map size to 320Mb. * 2. Improve access locality, subsequent pages will be allocated together improving seqscan speed */ -#define MAX_BLOCKS_PER_CHUNK_LOG 7 /* 1Mb chunk */ -#define MAX_BLOCKS_PER_CHUNK (1 << MAX_BLOCKS_PER_CHUNK_LOG) +#define BLOCKS_PER_CHUNK_LOG 7 /* 1Mb chunk */ +#define BLOCKS_PER_CHUNK (1 << BLOCKS_PER_CHUNK_LOG) #define MB ((uint64)1024*1024) -#define SIZE_MB_TO_CHUNKS(size) ((uint32)((size) * MB / BLCKSZ >> lfc_chunk_size_log)) -#define BLOCK_TO_CHUNK_OFF(blkno) ((blkno) & (lfc_blocks_per_chunk-1)) +#define SIZE_MB_TO_CHUNKS(size) ((uint32)((size) * MB / BLCKSZ >> BLOCKS_PER_CHUNK_LOG)) +#define BLOCK_TO_CHUNK_OFF(blkno) ((blkno) & (BLOCKS_PER_CHUNK-1)) #define INVALID_OFFSET (0xffffffff) @@ -129,10 +129,9 @@ typedef struct FileCacheEntry uint32 offset; uint32 access_count; dlist_node list_node; /* LRU list node */ - uint32 state[FLEXIBLE_ARRAY_MEMBER]; /* two bits per block */ + uint32 state[(BLOCKS_PER_CHUNK * 2 + 31) / 32]; /* two bits per block */ } FileCacheEntry; -#define FILE_CACHE_ENTRY_SIZE MAXALIGN(offsetof(FileCacheEntry, state) + (lfc_blocks_per_chunk*2+31)/32*4) #define GET_STATE(entry, i) (((entry)->state[(i) / 16] >> ((i) % 16 * 2)) & 3) #define SET_STATE(entry, i, new_state) (entry)->state[(i) / 16] = ((entry)->state[(i) / 16] & ~(3 << ((i) % 16 * 2))) | ((new_state) << ((i) % 16 * 2)) @@ -141,6 +140,9 @@ typedef struct FileCacheEntry #define MAX_PREWARM_WORKERS 8 + +#define FREELIST_ENTRIES_PER_CHUNK (BLOCKS_PER_CHUNK * BLCKSZ / sizeof(uint32) - 2) + typedef struct PrewarmWorkerState { uint32 prewarmed_pages; @@ -183,24 +185,22 @@ typedef struct FileCacheControl uint32 freelist_size; uint32 freelist_head; uint32 num_free_pages; - uint32 free_pages[FLEXIBLE_ARRAY_MEMBER]; + uint32 free_pages[FREELIST_ENTRIES_PER_CHUNK]; } FileCacheControl; typedef struct FreeListChunk { uint32 next; uint32 num_free_pages; - uint32 free_pages[FLEXIBLE_ARRAY_MEMBER]; + uint32 free_pages[FREELIST_ENTRIES_PER_CHUNK]; } FreeListChunk; #define FILE_CACHE_STATE_MAGIC 0xfcfcfcfc #define FILE_CACHE_STATE_BITMAP(fcs) ((uint8*)&(fcs)->chunks[(fcs)->n_chunks]) -#define FILE_CACHE_STATE_SIZE_FOR_CHUNKS(n_chunks) (sizeof(FileCacheState) + (n_chunks)*sizeof(BufferTag) + (((n_chunks) * lfc_blocks_per_chunk)+7)/8) +#define FILE_CACHE_STATE_SIZE_FOR_CHUNKS(n_chunks) (sizeof(FileCacheState) + (n_chunks)*sizeof(BufferTag) + (((n_chunks) * BLOCKS_PER_CHUNK)+7)/8) #define FILE_CACHE_STATE_SIZE(fcs) (sizeof(FileCacheState) + (fcs->n_chunks)*sizeof(BufferTag) + (((fcs->n_chunks) << fcs->chunk_size_log)+7)/8) -#define FREELIST_ENTRIES_PER_CHUNK(c) ((c) * BLCKSZ / sizeof(uint32) - 2) - static HTAB *lfc_hash; static int lfc_desc = -1; static LWLockId lfc_lock; @@ -209,8 +209,7 @@ static int lfc_max_size; static int lfc_size_limit; static int lfc_prewarm_limit; static int lfc_prewarm_batch; -static int lfc_chunk_size_log = MAX_BLOCKS_PER_CHUNK_LOG; -static int lfc_blocks_per_chunk = MAX_BLOCKS_PER_CHUNK; +static int lfc_blocks_per_chunk_ro = BLOCKS_PER_CHUNK; static char *lfc_path; static uint64 lfc_generation; static FileCacheControl *lfc_ctl; @@ -357,8 +356,7 @@ lfc_shmem_startup(void) LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); - size = offsetof(FileCacheControl, free_pages); - size += FREELIST_ENTRIES_PER_CHUNK(lfc_blocks_per_chunk) * sizeof(uint32); + size = sizeof(FileCacheControl); lfc_ctl = (FileCacheControl *) ShmemInitStruct("lfc", size, &found); if (!found) @@ -369,7 +367,7 @@ lfc_shmem_startup(void) lfc_lock = (LWLockId) GetNamedLWLockTranche("lfc_lock"); lfc_freelist_lock = (LWLockId) GetNamedLWLockTranche("lfc_freelist_lock"); info.keysize = sizeof(BufferTag); - info.entrysize = FILE_CACHE_ENTRY_SIZE; + info.entrysize = sizeof(FileCacheEntry); /* * n_chunks+1 because we add new element to hash table before eviction @@ -382,7 +380,7 @@ lfc_shmem_startup(void) memset(lfc_ctl, 0, offsetof(FileCacheControl, free_pages)); dlist_init(&lfc_ctl->lru); - lfc_ctl->freelist_size = FREELIST_ENTRIES_PER_CHUNK(lfc_blocks_per_chunk); + lfc_ctl->freelist_size = FREELIST_ENTRIES_PER_CHUNK; lfc_ctl->freelist_head = INVALID_OFFSET; lfc_ctl->num_free_pages = 0; @@ -420,9 +418,8 @@ lfc_shmem_request(void) prev_shmem_request_hook(); #endif - size = offsetof(FileCacheControl, free_pages); - size += FREELIST_ENTRIES_PER_CHUNK(lfc_blocks_per_chunk) * sizeof(uint32); - size += hash_estimate_size(SIZE_MB_TO_CHUNKS(lfc_max_size) + 1, FILE_CACHE_ENTRY_SIZE); + size = sizeof(FileCacheControl); + size += hash_estimate_size(SIZE_MB_TO_CHUNKS(lfc_max_size) + 1, sizeof(FileCacheEntry)); RequestAddinShmemSpace(size); RequestNamedLWLockTranche("lfc_lock", 1); @@ -442,24 +439,6 @@ is_normal_backend(void) return lfc_ctl && MyProc && UsedShmemSegAddr && !IsParallelWorker(); } -static bool -lfc_check_chunk_size(int *newval, void **extra, GucSource source) -{ - if (*newval & (*newval - 1)) - { - elog(ERROR, "LFC chunk size should be power of two"); - return false; - } - return true; -} - -static void -lfc_change_chunk_size(int newval, void* extra) -{ - lfc_chunk_size_log = pg_ceil_log2_32(newval); -} - - static bool lfc_check_limit_hook(int *newval, void **extra, GucSource source) { @@ -515,11 +494,11 @@ lfc_change_limit_hook(int newval, void *extra) CriticalAssert(victim->access_count == 0); #ifdef FALLOC_FL_PUNCH_HOLE - if (fallocate(lfc_desc, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, (off_t) victim->offset * lfc_blocks_per_chunk * BLCKSZ, lfc_blocks_per_chunk * BLCKSZ) < 0) + if (fallocate(lfc_desc, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, (off_t) victim->offset * BLOCKS_PER_CHUNK * BLCKSZ, BLOCKS_PER_CHUNK * BLCKSZ) < 0) neon_log(LOG, "Failed to punch hole in file: %m"); #endif /* We remove the entry, and enter a hole to the freelist */ - for (int i = 0; i < lfc_blocks_per_chunk; i++) + for (int i = 0; i < BLOCKS_PER_CHUNK; i++) { bool is_page_cached = GET_STATE(victim, i) == AVAILABLE; lfc_ctl->used_pages -= is_page_cached; @@ -622,14 +601,14 @@ lfc_init(void) DefineCustomIntVariable("neon.file_cache_chunk_size", "LFC chunk size in blocks (should be power of two)", NULL, - &lfc_blocks_per_chunk, - MAX_BLOCKS_PER_CHUNK, - 1, - MAX_BLOCKS_PER_CHUNK, - PGC_POSTMASTER, + &lfc_blocks_per_chunk_ro, + BLOCKS_PER_CHUNK, + BLOCKS_PER_CHUNK, + BLOCKS_PER_CHUNK, + PGC_INTERNAL, GUC_UNIT_BLOCKS, - lfc_check_chunk_size, - lfc_change_chunk_size, + NULL, + NULL, NULL); DefineCustomIntVariable("neon.file_cache_prewarm_limit", @@ -692,7 +671,7 @@ lfc_get_state(size_t max_entries) fcs = (FileCacheState*)palloc0(state_size); SET_VARSIZE(fcs, state_size); fcs->magic = FILE_CACHE_STATE_MAGIC; - fcs->chunk_size_log = lfc_chunk_size_log; + fcs->chunk_size_log = BLOCKS_PER_CHUNK_LOG; fcs->n_chunks = n_entries; bitmap = FILE_CACHE_STATE_BITMAP(fcs); @@ -700,11 +679,11 @@ lfc_get_state(size_t max_entries) { FileCacheEntry *entry = dlist_container(FileCacheEntry, list_node, iter.cur); fcs->chunks[i] = entry->key; - for (int j = 0; j < lfc_blocks_per_chunk; j++) + for (int j = 0; j < BLOCKS_PER_CHUNK; j++) { if (GET_STATE(entry, j) != UNAVAILABLE) { - BITMAP_SET(bitmap, i*lfc_blocks_per_chunk + j); + BITMAP_SET(bitmap, i*BLOCKS_PER_CHUNK + j); n_pages += 1; } } @@ -713,7 +692,7 @@ lfc_get_state(size_t max_entries) } Assert(i == n_entries); fcs->n_pages = n_pages; - Assert(pg_popcount((char*)bitmap, ((n_entries << lfc_chunk_size_log) + 7)/8) == n_pages); + Assert(pg_popcount((char*)bitmap, ((n_entries << BLOCKS_PER_CHUNK_LOG) + 7)/8) == n_pages); elog(LOG, "LFC: save state of %d chunks %d pages", (int)n_entries, (int)n_pages); } @@ -769,7 +748,7 @@ lfc_prewarm(FileCacheState* fcs, uint32 n_workers) } fcs_chunk_size_log = fcs->chunk_size_log; - if (fcs_chunk_size_log > MAX_BLOCKS_PER_CHUNK_LOG) + if (fcs_chunk_size_log > BLOCKS_PER_CHUNK_LOG) { elog(ERROR, "LFC: Invalid chunk size log: %u", fcs->chunk_size_log); } @@ -1001,14 +980,14 @@ lfc_invalidate(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber nblocks) LWLockAcquire(lfc_lock, LW_EXCLUSIVE); if (LFC_ENABLED()) { - for (BlockNumber blkno = 0; blkno < nblocks; blkno += lfc_blocks_per_chunk) + for (BlockNumber blkno = 0; blkno < nblocks; blkno += BLOCKS_PER_CHUNK) { tag.blockNum = blkno; hash = get_hash_value(lfc_hash, &tag); entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_FIND, NULL); if (entry != NULL) { - for (int i = 0; i < lfc_blocks_per_chunk; i++) + for (int i = 0; i < BLOCKS_PER_CHUNK; i++) { if (GET_STATE(entry, i) == AVAILABLE) { @@ -1091,12 +1070,12 @@ lfc_cache_containsv(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, } while (true) { - int this_chunk = Min(nblocks - i, lfc_blocks_per_chunk - chunk_offs); + int this_chunk = Min(nblocks - i, BLOCKS_PER_CHUNK - chunk_offs); entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_FIND, NULL); if (entry != NULL) { - for (; chunk_offs < lfc_blocks_per_chunk && i < nblocks; chunk_offs++, i++) + for (; chunk_offs < BLOCKS_PER_CHUNK && i < nblocks; chunk_offs++, i++) { if (GET_STATE(entry, chunk_offs) != UNAVAILABLE) { @@ -1197,9 +1176,9 @@ lfc_readv_select(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, while (nblocks > 0) { struct iovec iov[PG_IOV_MAX]; - uint8 chunk_mask[MAX_BLOCKS_PER_CHUNK / 8] = {0}; + uint8 chunk_mask[BLOCKS_PER_CHUNK / 8] = {0}; int chunk_offs = BLOCK_TO_CHUNK_OFF(blkno); - int blocks_in_chunk = Min(nblocks, lfc_blocks_per_chunk - chunk_offs); + int blocks_in_chunk = Min(nblocks, BLOCKS_PER_CHUNK - chunk_offs); int iteration_hits = 0; int iteration_misses = 0; uint64 io_time_us = 0; @@ -1339,7 +1318,7 @@ lfc_readv_select(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, if (iteration_hits != 0) { /* chunk offset (# of pages) into the LFC file */ - off_t first_read_offset = (off_t) entry_offset * lfc_blocks_per_chunk; + off_t first_read_offset = (off_t) entry_offset * BLOCKS_PER_CHUNK; int nwrite = iov_last_used - first_block_in_chunk_read; /* offset of first IOV */ first_read_offset += chunk_offs + first_block_in_chunk_read; @@ -1463,7 +1442,7 @@ lfc_init_new_entry(FileCacheEntry* entry, uint32 hash) FileCacheEntry *victim = dlist_container(FileCacheEntry, list_node, dlist_pop_head_node(&lfc_ctl->lru)); - for (int i = 0; i < lfc_blocks_per_chunk; i++) + for (int i = 0; i < BLOCKS_PER_CHUNK; i++) { bool is_page_cached = GET_STATE(victim, i) == AVAILABLE; lfc_ctl->used_pages -= is_page_cached; @@ -1489,7 +1468,7 @@ lfc_init_new_entry(FileCacheEntry* entry, uint32 hash) entry->hash = hash; lfc_ctl->pinned += 1; - for (int i = 0; i < lfc_blocks_per_chunk; i++) + for (int i = 0; i < BLOCKS_PER_CHUNK; i++) SET_STATE(entry, i, UNAVAILABLE); return true; @@ -1618,7 +1597,7 @@ lfc_prefetch(NRelFileInfo rinfo, ForkNumber forknum, BlockNumber blkno, pgstat_report_wait_start(WAIT_EVENT_NEON_LFC_WRITE); INSTR_TIME_SET_CURRENT(io_start); rc = pwrite(lfc_desc, buffer, BLCKSZ, - ((off_t) entry_offset * lfc_blocks_per_chunk + chunk_offs) * BLCKSZ); + ((off_t) entry_offset * BLOCKS_PER_CHUNK + chunk_offs) * BLCKSZ); INSTR_TIME_SET_CURRENT(io_end); pgstat_report_wait_end(); @@ -1719,7 +1698,7 @@ lfc_writev(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, { struct iovec iov[PG_IOV_MAX]; int chunk_offs = BLOCK_TO_CHUNK_OFF(blkno); - int blocks_in_chunk = Min(nblocks, lfc_blocks_per_chunk - chunk_offs); + int blocks_in_chunk = Min(nblocks, BLOCKS_PER_CHUNK - chunk_offs); instr_time io_start, io_end; ConditionVariable* cv; @@ -1807,7 +1786,7 @@ lfc_writev(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, pgstat_report_wait_start(WAIT_EVENT_NEON_LFC_WRITE); INSTR_TIME_SET_CURRENT(io_start); rc = pwritev(lfc_desc, iov, blocks_in_chunk, - ((off_t) entry_offset * lfc_blocks_per_chunk + chunk_offs) * BLCKSZ); + ((off_t) entry_offset * BLOCKS_PER_CHUNK + chunk_offs) * BLCKSZ); INSTR_TIME_SET_CURRENT(io_end); pgstat_report_wait_end(); @@ -1899,14 +1878,14 @@ freelist_prepare_pop(void) } freelist_head = lfc_ctl->freelist_head; - freelist_chunk = palloc(lfc_blocks_per_chunk * BLCKSZ); + freelist_chunk = palloc(BLOCKS_PER_CHUNK * BLCKSZ); bytes_read = 0; - while (bytes_read < lfc_blocks_per_chunk * BLCKSZ) + while (bytes_read < BLOCKS_PER_CHUNK * BLCKSZ) { ssize_t rc; - rc = pread(lfc_desc, freelist_chunk, lfc_blocks_per_chunk * BLCKSZ - bytes_read, (off_t) freelist_head * lfc_blocks_per_chunk * BLCKSZ + bytes_read); + rc = pread(lfc_desc, freelist_chunk, BLOCKS_PER_CHUNK * BLCKSZ - bytes_read, (off_t) freelist_head * BLOCKS_PER_CHUNK * BLCKSZ + bytes_read); if (rc < 0) { lfc_disable("read freelist page"); @@ -1946,14 +1925,14 @@ freelist_prepare_pop(void) static bool freelist_push(uint32 offset) { - Assert(lfc_ctl->freelist_size == FREELIST_ENTRIES_PER_CHUNK(lfc_blocks_per_chunk)); + Assert(lfc_ctl->freelist_size == FREELIST_ENTRIES_PER_CHUNK); if (lfc_ctl->num_free_pages == lfc_ctl->freelist_size) { FreeListChunk *freelist_chunk; struct iovec iov; ssize_t rc; - freelist_chunk = palloc(lfc_blocks_per_chunk * BLCKSZ); + freelist_chunk = palloc(BLOCKS_PER_CHUNK * BLCKSZ); /* write the existing entries to the chunk on disk */ freelist_chunk->next = lfc_ctl->freelist_head; @@ -1962,8 +1941,8 @@ freelist_push(uint32 offset) /* Use the passed-in offset to hold the freelist chunk itself */ iov.iov_base = freelist_chunk; - iov.iov_len = lfc_blocks_per_chunk * BLCKSZ; - rc = pg_pwritev_with_retry(lfc_desc, &iov, 1, (off_t) offset * lfc_blocks_per_chunk * BLCKSZ); + iov.iov_len = BLOCKS_PER_CHUNK * BLCKSZ; + rc = pg_pwritev_with_retry(lfc_desc, &iov, 1, (off_t) offset * BLOCKS_PER_CHUNK * BLCKSZ); pfree(freelist_chunk); @@ -2097,7 +2076,7 @@ neon_get_lfc_stats(PG_FUNCTION_ARGS) break; case 8: key = "file_cache_chunk_size_pages"; - value = lfc_blocks_per_chunk; + value = BLOCKS_PER_CHUNK; break; case 9: key = "file_cache_chunks_pinned"; @@ -2227,7 +2206,7 @@ local_cache_pages(PG_FUNCTION_ARGS) hash_seq_init(&status, lfc_hash); while ((entry = hash_seq_search(&status)) != NULL) { - for (int i = 0; i < lfc_blocks_per_chunk; i++) + for (int i = 0; i < BLOCKS_PER_CHUNK; i++) n_pages += GET_STATE(entry, i) == AVAILABLE; } } @@ -2254,11 +2233,11 @@ local_cache_pages(PG_FUNCTION_ARGS) hash_seq_init(&status, lfc_hash); while ((entry = hash_seq_search(&status)) != NULL) { - for (int i = 0; i < lfc_blocks_per_chunk; i++) + for (int i = 0; i < BLOCKS_PER_CHUNK; i++) { if (GET_STATE(entry, i) == AVAILABLE) { - fctx->record[n].pageoffs = entry->offset * lfc_blocks_per_chunk + i; + fctx->record[n].pageoffs = entry->offset * BLOCKS_PER_CHUNK + i; fctx->record[n].relfilenode = NInfoGetRelNumber(BufTagGetNRelFileInfo(entry->key)); fctx->record[n].reltablespace = NInfoGetSpcOid(BufTagGetNRelFileInfo(entry->key)); fctx->record[n].reldatabase = NInfoGetDbOid(BufTagGetNRelFileInfo(entry->key)); From 6145cfd1c2be08d4df737e06b172f4eaa18e3a8a Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 5 Jun 2025 18:13:03 +0300 Subject: [PATCH 03/24] Move neon-shmem facility to separate module within the crate --- libs/neon-shmem/src/lib.rs | 417 +--------------------------------- libs/neon-shmem/src/shmem.rs | 418 +++++++++++++++++++++++++++++++++++ 2 files changed, 419 insertions(+), 416 deletions(-) create mode 100644 libs/neon-shmem/src/shmem.rs diff --git a/libs/neon-shmem/src/lib.rs b/libs/neon-shmem/src/lib.rs index e1b14b1371..d4f171ed66 100644 --- a/libs/neon-shmem/src/lib.rs +++ b/libs/neon-shmem/src/lib.rs @@ -1,418 +1,3 @@ //! Shared memory utilities for neon communicator -use std::num::NonZeroUsize; -use std::os::fd::{AsFd, BorrowedFd, OwnedFd}; -use std::ptr::NonNull; -use std::sync::atomic::{AtomicUsize, Ordering}; - -use nix::errno::Errno; -use nix::sys::mman::MapFlags; -use nix::sys::mman::ProtFlags; -use nix::sys::mman::mmap as nix_mmap; -use nix::sys::mman::munmap as nix_munmap; -use nix::unistd::ftruncate as nix_ftruncate; - -/// ShmemHandle represents a shared memory area that can be shared by processes over fork(). -/// Unlike shared memory allocated by Postgres, this area is resizable, up to 'max_size' that's -/// specified at creation. -/// -/// The area is backed by an anonymous file created with memfd_create(). The full address space for -/// 'max_size' is reserved up-front with mmap(), but whenever you call [`ShmemHandle::set_size`], -/// the underlying file is resized. Do not access the area beyond the current size. Currently, that -/// will cause the file to be expanded, but we might use mprotect() etc. to enforce that in the -/// future. -pub struct ShmemHandle { - /// memfd file descriptor - fd: OwnedFd, - - max_size: usize, - - // Pointer to the beginning of the shared memory area. The header is stored there. - shared_ptr: NonNull, - - // Pointer to the beginning of the user data - pub data_ptr: NonNull, -} - -/// This is stored at the beginning in the shared memory area. -struct SharedStruct { - max_size: usize, - - /// Current size of the backing file. The high-order bit is used for the RESIZE_IN_PROGRESS flag - current_size: AtomicUsize, -} - -const RESIZE_IN_PROGRESS: usize = 1 << 63; - -const HEADER_SIZE: usize = std::mem::size_of::(); - -/// Error type returned by the ShmemHandle functions. -#[derive(thiserror::Error, Debug)] -#[error("{msg}: {errno}")] -pub struct Error { - pub msg: String, - pub errno: Errno, -} - -impl Error { - fn new(msg: &str, errno: Errno) -> Error { - Error { - msg: msg.to_string(), - errno, - } - } -} - -impl ShmemHandle { - /// Create a new shared memory area. To communicate between processes, the processes need to be - /// fork()'d after calling this, so that the ShmemHandle is inherited by all processes. - /// - /// If the ShmemHandle is dropped, the memory is unmapped from the current process. Other - /// processes can continue using it, however. - pub fn new(name: &str, initial_size: usize, max_size: usize) -> Result { - // create the backing anonymous file. - let fd = create_backing_file(name)?; - - Self::new_with_fd(fd, initial_size, max_size) - } - - fn new_with_fd( - fd: OwnedFd, - initial_size: usize, - max_size: usize, - ) -> Result { - // We reserve the high-order bit for the RESIZE_IN_PROGRESS flag, and the actual size - // is a little larger than this because of the SharedStruct header. Make the upper limit - // somewhat smaller than that, because with anything close to that, you'll run out of - // memory anyway. - if max_size >= 1 << 48 { - panic!("max size {} too large", max_size); - } - if initial_size > max_size { - panic!("initial size {initial_size} larger than max size {max_size}"); - } - - // The actual initial / max size is the one given by the caller, plus the size of - // 'SharedStruct'. - let initial_size = HEADER_SIZE + initial_size; - let max_size = NonZeroUsize::new(HEADER_SIZE + max_size).unwrap(); - - // Reserve address space for it with mmap - // - // TODO: Use MAP_HUGETLB if possible - let start_ptr = unsafe { - nix_mmap( - None, - max_size, - ProtFlags::PROT_READ | ProtFlags::PROT_WRITE, - MapFlags::MAP_SHARED, - &fd, - 0, - ) - } - .map_err(|e| Error::new("mmap failed: {e}", e))?; - - // Reserve space for the initial size - enlarge_file(fd.as_fd(), initial_size as u64)?; - - // Initialize the header - let shared: NonNull = start_ptr.cast(); - unsafe { - shared.write(SharedStruct { - max_size: max_size.into(), - current_size: AtomicUsize::new(initial_size), - }) - }; - - // The user data begins after the header - let data_ptr = unsafe { start_ptr.cast().add(HEADER_SIZE) }; - - Ok(ShmemHandle { - fd, - max_size: max_size.into(), - shared_ptr: shared, - data_ptr, - }) - } - - // return reference to the header - fn shared(&self) -> &SharedStruct { - unsafe { self.shared_ptr.as_ref() } - } - - /// Resize the shared memory area. 'new_size' must not be larger than the 'max_size' specified - /// when creating the area. - /// - /// This may only be called from one process/thread concurrently. We detect that case - /// and return an Error. - pub fn set_size(&self, new_size: usize) -> Result<(), Error> { - let new_size = new_size + HEADER_SIZE; - let shared = self.shared(); - - if new_size > self.max_size { - panic!( - "new size ({} is greater than max size ({})", - new_size, self.max_size - ); - } - assert_eq!(self.max_size, shared.max_size); - - // Lock the area by setting the bit in 'current_size' - // - // Ordering::Relaxed would probably be sufficient here, as we don't access any other memory - // and the posix_fallocate/ftruncate call is surely a synchronization point anyway. But - // since this is not performance-critical, better safe than sorry . - let mut old_size = shared.current_size.load(Ordering::Acquire); - loop { - if (old_size & RESIZE_IN_PROGRESS) != 0 { - return Err(Error::new( - "concurrent resize detected", - Errno::UnknownErrno, - )); - } - match shared.current_size.compare_exchange( - old_size, - new_size, - Ordering::Acquire, - Ordering::Relaxed, - ) { - Ok(_) => break, - Err(x) => old_size = x, - } - } - - // Ok, we got the lock. - // - // NB: If anything goes wrong, we *must* clear the bit! - let result = { - use std::cmp::Ordering::{Equal, Greater, Less}; - match new_size.cmp(&old_size) { - Less => nix_ftruncate(&self.fd, new_size as i64).map_err(|e| { - Error::new("could not shrink shmem segment, ftruncate failed: {e}", e) - }), - Equal => Ok(()), - Greater => enlarge_file(self.fd.as_fd(), new_size as u64), - } - }; - - // Unlock - shared.current_size.store( - if result.is_ok() { new_size } else { old_size }, - Ordering::Release, - ); - - result - } - - /// Returns the current user-visible size of the shared memory segment. - /// - /// NOTE: a concurrent set_size() call can change the size at any time. It is the caller's - /// responsibility not to access the area beyond the current size. - pub fn current_size(&self) -> usize { - let total_current_size = - self.shared().current_size.load(Ordering::Relaxed) & !RESIZE_IN_PROGRESS; - total_current_size - HEADER_SIZE - } -} - -impl Drop for ShmemHandle { - fn drop(&mut self) { - // SAFETY: The pointer was obtained from mmap() with the given size. - // We unmap the entire region. - let _ = unsafe { nix_munmap(self.shared_ptr.cast(), self.max_size) }; - // The fd is dropped automatically by OwnedFd. - } -} - -/// Create a "backing file" for the shared memory area. On Linux, use memfd_create(), to create an -/// anonymous in-memory file. One macos, fall back to a regular file. That's good enough for -/// development and testing, but in production we want the file to stay in memory. -/// -/// disable 'unused_variables' warnings, because in the macos path, 'name' is unused. -#[allow(unused_variables)] -fn create_backing_file(name: &str) -> Result { - #[cfg(not(target_os = "macos"))] - { - nix::sys::memfd::memfd_create(name, nix::sys::memfd::MFdFlags::empty()) - .map_err(|e| Error::new("memfd_create failed: {e}", e)) - } - #[cfg(target_os = "macos")] - { - let file = tempfile::tempfile().map_err(|e| { - Error::new( - "could not create temporary file to back shmem area: {e}", - nix::errno::Errno::from_raw(e.raw_os_error().unwrap_or(0)), - ) - })?; - Ok(OwnedFd::from(file)) - } -} - -fn enlarge_file(fd: BorrowedFd, size: u64) -> Result<(), Error> { - // Use posix_fallocate() to enlarge the file. It reserves the space correctly, so that - // we don't get a segfault later when trying to actually use it. - #[cfg(not(target_os = "macos"))] - { - nix::fcntl::posix_fallocate(fd, 0, size as i64).map_err(|e| { - Error::new( - "could not grow shmem segment, posix_fallocate failed: {e}", - e, - ) - }) - } - // As a fallback on macos, which doesn't have posix_fallocate, use plain 'fallocate' - #[cfg(target_os = "macos")] - { - nix::unistd::ftruncate(fd, size as i64) - .map_err(|e| Error::new("could not grow shmem segment, ftruncate failed: {e}", e)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - use nix::unistd::ForkResult; - use std::ops::Range; - - /// check that all bytes in given range have the expected value. - fn assert_range(ptr: *const u8, expected: u8, range: Range) { - for i in range { - let b = unsafe { *(ptr.add(i)) }; - assert_eq!(expected, b, "unexpected byte at offset {}", i); - } - } - - /// Write 'b' to all bytes in the given range - fn write_range(ptr: *mut u8, b: u8, range: Range) { - unsafe { std::ptr::write_bytes(ptr.add(range.start), b, range.end - range.start) }; - } - - // simple single-process test of growing and shrinking - #[test] - fn test_shmem_resize() -> Result<(), Error> { - let max_size = 1024 * 1024; - let init_struct = ShmemHandle::new("test_shmem_resize", 0, max_size)?; - - assert_eq!(init_struct.current_size(), 0); - - // Initial grow - let size1 = 10000; - init_struct.set_size(size1).unwrap(); - assert_eq!(init_struct.current_size(), size1); - - // Write some data - let data_ptr = init_struct.data_ptr.as_ptr(); - write_range(data_ptr, 0xAA, 0..size1); - assert_range(data_ptr, 0xAA, 0..size1); - - // Shrink - let size2 = 5000; - init_struct.set_size(size2).unwrap(); - assert_eq!(init_struct.current_size(), size2); - - // Grow again - let size3 = 20000; - init_struct.set_size(size3).unwrap(); - assert_eq!(init_struct.current_size(), size3); - - // Try to read it. The area that was shrunk and grown again should read as all zeros now - assert_range(data_ptr, 0xAA, 0..5000); - assert_range(data_ptr, 0, 5000..size1); - - // Try to grow beyond max_size - //let size4 = max_size + 1; - //assert!(init_struct.set_size(size4).is_err()); - - // Dropping init_struct should unmap the memory - drop(init_struct); - - Ok(()) - } - - /// This is used in tests to coordinate between test processes. It's like std::sync::Barrier, - /// but is stored in the shared memory area and works across processes. It's implemented by - /// polling, because e.g. standard rust mutexes are not guaranteed to work across processes. - struct SimpleBarrier { - num_procs: usize, - count: AtomicUsize, - } - - impl SimpleBarrier { - unsafe fn init(ptr: *mut SimpleBarrier, num_procs: usize) { - unsafe { - *ptr = SimpleBarrier { - num_procs, - count: AtomicUsize::new(0), - } - } - } - - pub fn wait(&self) { - let old = self.count.fetch_add(1, Ordering::Relaxed); - - let generation = old / self.num_procs; - - let mut current = old + 1; - while current < (generation + 1) * self.num_procs { - std::thread::sleep(std::time::Duration::from_millis(10)); - current = self.count.load(Ordering::Relaxed); - } - } - } - - #[test] - fn test_multi_process() { - // Initialize - let max_size = 1_000_000_000_000; - let init_struct = ShmemHandle::new("test_multi_process", 0, max_size).unwrap(); - let ptr = init_struct.data_ptr.as_ptr(); - - // Store the SimpleBarrier in the first 1k of the area. - init_struct.set_size(10000).unwrap(); - let barrier_ptr: *mut SimpleBarrier = unsafe { - ptr.add(ptr.align_offset(std::mem::align_of::())) - .cast() - }; - unsafe { SimpleBarrier::init(barrier_ptr, 2) }; - let barrier = unsafe { barrier_ptr.as_ref().unwrap() }; - - // Fork another test process. The code after this runs in both processes concurrently. - let fork_result = unsafe { nix::unistd::fork().unwrap() }; - - // In the parent, fill bytes between 1000..2000. In the child, between 2000..3000 - if fork_result.is_parent() { - write_range(ptr, 0xAA, 1000..2000); - } else { - write_range(ptr, 0xBB, 2000..3000); - } - barrier.wait(); - // Verify the contents. (in both processes) - assert_range(ptr, 0xAA, 1000..2000); - assert_range(ptr, 0xBB, 2000..3000); - - // Grow, from the child this time - let size = 10_000_000; - if !fork_result.is_parent() { - init_struct.set_size(size).unwrap(); - } - barrier.wait(); - - // make some writes at the end - if fork_result.is_parent() { - write_range(ptr, 0xAA, (size - 10)..size); - } else { - write_range(ptr, 0xBB, (size - 20)..(size - 10)); - } - barrier.wait(); - - // Verify the contents. (This runs in both processes) - assert_range(ptr, 0, (size - 1000)..(size - 20)); - assert_range(ptr, 0xBB, (size - 20)..(size - 10)); - assert_range(ptr, 0xAA, (size - 10)..size); - - if let ForkResult::Parent { child } = fork_result { - nix::sys::wait::waitpid(child, None).unwrap(); - } - } -} +pub mod shmem; diff --git a/libs/neon-shmem/src/shmem.rs b/libs/neon-shmem/src/shmem.rs new file mode 100644 index 0000000000..21b1454b10 --- /dev/null +++ b/libs/neon-shmem/src/shmem.rs @@ -0,0 +1,418 @@ +//! Dynamically resizable contiguous chunk of shared memory + +use std::num::NonZeroUsize; +use std::os::fd::{AsFd, BorrowedFd, OwnedFd}; +use std::ptr::NonNull; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use nix::errno::Errno; +use nix::sys::mman::MapFlags; +use nix::sys::mman::ProtFlags; +use nix::sys::mman::mmap as nix_mmap; +use nix::sys::mman::munmap as nix_munmap; +use nix::unistd::ftruncate as nix_ftruncate; + +/// ShmemHandle represents a shared memory area that can be shared by processes over fork(). +/// Unlike shared memory allocated by Postgres, this area is resizable, up to 'max_size' that's +/// specified at creation. +/// +/// The area is backed by an anonymous file created with memfd_create(). The full address space for +/// 'max_size' is reserved up-front with mmap(), but whenever you call [`ShmemHandle::set_size`], +/// the underlying file is resized. Do not access the area beyond the current size. Currently, that +/// will cause the file to be expanded, but we might use mprotect() etc. to enforce that in the +/// future. +pub struct ShmemHandle { + /// memfd file descriptor + fd: OwnedFd, + + max_size: usize, + + // Pointer to the beginning of the shared memory area. The header is stored there. + shared_ptr: NonNull, + + // Pointer to the beginning of the user data + pub data_ptr: NonNull, +} + +/// This is stored at the beginning in the shared memory area. +struct SharedStruct { + max_size: usize, + + /// Current size of the backing file. The high-order bit is used for the RESIZE_IN_PROGRESS flag + current_size: AtomicUsize, +} + +const RESIZE_IN_PROGRESS: usize = 1 << 63; + +const HEADER_SIZE: usize = std::mem::size_of::(); + +/// Error type returned by the ShmemHandle functions. +#[derive(thiserror::Error, Debug)] +#[error("{msg}: {errno}")] +pub struct Error { + pub msg: String, + pub errno: Errno, +} + +impl Error { + fn new(msg: &str, errno: Errno) -> Error { + Error { + msg: msg.to_string(), + errno, + } + } +} + +impl ShmemHandle { + /// Create a new shared memory area. To communicate between processes, the processes need to be + /// fork()'d after calling this, so that the ShmemHandle is inherited by all processes. + /// + /// If the ShmemHandle is dropped, the memory is unmapped from the current process. Other + /// processes can continue using it, however. + pub fn new(name: &str, initial_size: usize, max_size: usize) -> Result { + // create the backing anonymous file. + let fd = create_backing_file(name)?; + + Self::new_with_fd(fd, initial_size, max_size) + } + + fn new_with_fd( + fd: OwnedFd, + initial_size: usize, + max_size: usize, + ) -> Result { + // We reserve the high-order bit for the RESIZE_IN_PROGRESS flag, and the actual size + // is a little larger than this because of the SharedStruct header. Make the upper limit + // somewhat smaller than that, because with anything close to that, you'll run out of + // memory anyway. + if max_size >= 1 << 48 { + panic!("max size {} too large", max_size); + } + if initial_size > max_size { + panic!("initial size {initial_size} larger than max size {max_size}"); + } + + // The actual initial / max size is the one given by the caller, plus the size of + // 'SharedStruct'. + let initial_size = HEADER_SIZE + initial_size; + let max_size = NonZeroUsize::new(HEADER_SIZE + max_size).unwrap(); + + // Reserve address space for it with mmap + // + // TODO: Use MAP_HUGETLB if possible + let start_ptr = unsafe { + nix_mmap( + None, + max_size, + ProtFlags::PROT_READ | ProtFlags::PROT_WRITE, + MapFlags::MAP_SHARED, + &fd, + 0, + ) + } + .map_err(|e| Error::new("mmap failed: {e}", e))?; + + // Reserve space for the initial size + enlarge_file(fd.as_fd(), initial_size as u64)?; + + // Initialize the header + let shared: NonNull = start_ptr.cast(); + unsafe { + shared.write(SharedStruct { + max_size: max_size.into(), + current_size: AtomicUsize::new(initial_size), + }) + }; + + // The user data begins after the header + let data_ptr = unsafe { start_ptr.cast().add(HEADER_SIZE) }; + + Ok(ShmemHandle { + fd, + max_size: max_size.into(), + shared_ptr: shared, + data_ptr, + }) + } + + // return reference to the header + fn shared(&self) -> &SharedStruct { + unsafe { self.shared_ptr.as_ref() } + } + + /// Resize the shared memory area. 'new_size' must not be larger than the 'max_size' specified + /// when creating the area. + /// + /// This may only be called from one process/thread concurrently. We detect that case + /// and return an Error. + pub fn set_size(&self, new_size: usize) -> Result<(), Error> { + let new_size = new_size + HEADER_SIZE; + let shared = self.shared(); + + if new_size > self.max_size { + panic!( + "new size ({} is greater than max size ({})", + new_size, self.max_size + ); + } + assert_eq!(self.max_size, shared.max_size); + + // Lock the area by setting the bit in 'current_size' + // + // Ordering::Relaxed would probably be sufficient here, as we don't access any other memory + // and the posix_fallocate/ftruncate call is surely a synchronization point anyway. But + // since this is not performance-critical, better safe than sorry . + let mut old_size = shared.current_size.load(Ordering::Acquire); + loop { + if (old_size & RESIZE_IN_PROGRESS) != 0 { + return Err(Error::new( + "concurrent resize detected", + Errno::UnknownErrno, + )); + } + match shared.current_size.compare_exchange( + old_size, + new_size, + Ordering::Acquire, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(x) => old_size = x, + } + } + + // Ok, we got the lock. + // + // NB: If anything goes wrong, we *must* clear the bit! + let result = { + use std::cmp::Ordering::{Equal, Greater, Less}; + match new_size.cmp(&old_size) { + Less => nix_ftruncate(&self.fd, new_size as i64).map_err(|e| { + Error::new("could not shrink shmem segment, ftruncate failed: {e}", e) + }), + Equal => Ok(()), + Greater => enlarge_file(self.fd.as_fd(), new_size as u64), + } + }; + + // Unlock + shared.current_size.store( + if result.is_ok() { new_size } else { old_size }, + Ordering::Release, + ); + + result + } + + /// Returns the current user-visible size of the shared memory segment. + /// + /// NOTE: a concurrent set_size() call can change the size at any time. It is the caller's + /// responsibility not to access the area beyond the current size. + pub fn current_size(&self) -> usize { + let total_current_size = + self.shared().current_size.load(Ordering::Relaxed) & !RESIZE_IN_PROGRESS; + total_current_size - HEADER_SIZE + } +} + +impl Drop for ShmemHandle { + fn drop(&mut self) { + // SAFETY: The pointer was obtained from mmap() with the given size. + // We unmap the entire region. + let _ = unsafe { nix_munmap(self.shared_ptr.cast(), self.max_size) }; + // The fd is dropped automatically by OwnedFd. + } +} + +/// Create a "backing file" for the shared memory area. On Linux, use memfd_create(), to create an +/// anonymous in-memory file. One macos, fall back to a regular file. That's good enough for +/// development and testing, but in production we want the file to stay in memory. +/// +/// disable 'unused_variables' warnings, because in the macos path, 'name' is unused. +#[allow(unused_variables)] +fn create_backing_file(name: &str) -> Result { + #[cfg(not(target_os = "macos"))] + { + nix::sys::memfd::memfd_create(name, nix::sys::memfd::MFdFlags::empty()) + .map_err(|e| Error::new("memfd_create failed: {e}", e)) + } + #[cfg(target_os = "macos")] + { + let file = tempfile::tempfile().map_err(|e| { + Error::new( + "could not create temporary file to back shmem area: {e}", + nix::errno::Errno::from_raw(e.raw_os_error().unwrap_or(0)), + ) + })?; + Ok(OwnedFd::from(file)) + } +} + +fn enlarge_file(fd: BorrowedFd, size: u64) -> Result<(), Error> { + // Use posix_fallocate() to enlarge the file. It reserves the space correctly, so that + // we don't get a segfault later when trying to actually use it. + #[cfg(not(target_os = "macos"))] + { + nix::fcntl::posix_fallocate(fd, 0, size as i64).map_err(|e| { + Error::new( + "could not grow shmem segment, posix_fallocate failed: {e}", + e, + ) + }) + } + // As a fallback on macos, which doesn't have posix_fallocate, use plain 'fallocate' + #[cfg(target_os = "macos")] + { + nix::unistd::ftruncate(fd, size as i64) + .map_err(|e| Error::new("could not grow shmem segment, ftruncate failed: {e}", e)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use nix::unistd::ForkResult; + use std::ops::Range; + + /// check that all bytes in given range have the expected value. + fn assert_range(ptr: *const u8, expected: u8, range: Range) { + for i in range { + let b = unsafe { *(ptr.add(i)) }; + assert_eq!(expected, b, "unexpected byte at offset {}", i); + } + } + + /// Write 'b' to all bytes in the given range + fn write_range(ptr: *mut u8, b: u8, range: Range) { + unsafe { std::ptr::write_bytes(ptr.add(range.start), b, range.end - range.start) }; + } + + // simple single-process test of growing and shrinking + #[test] + fn test_shmem_resize() -> Result<(), Error> { + let max_size = 1024 * 1024; + let init_struct = ShmemHandle::new("test_shmem_resize", 0, max_size)?; + + assert_eq!(init_struct.current_size(), 0); + + // Initial grow + let size1 = 10000; + init_struct.set_size(size1).unwrap(); + assert_eq!(init_struct.current_size(), size1); + + // Write some data + let data_ptr = init_struct.data_ptr.as_ptr(); + write_range(data_ptr, 0xAA, 0..size1); + assert_range(data_ptr, 0xAA, 0..size1); + + // Shrink + let size2 = 5000; + init_struct.set_size(size2).unwrap(); + assert_eq!(init_struct.current_size(), size2); + + // Grow again + let size3 = 20000; + init_struct.set_size(size3).unwrap(); + assert_eq!(init_struct.current_size(), size3); + + // Try to read it. The area that was shrunk and grown again should read as all zeros now + assert_range(data_ptr, 0xAA, 0..5000); + assert_range(data_ptr, 0, 5000..size1); + + // Try to grow beyond max_size + //let size4 = max_size + 1; + //assert!(init_struct.set_size(size4).is_err()); + + // Dropping init_struct should unmap the memory + drop(init_struct); + + Ok(()) + } + + /// This is used in tests to coordinate between test processes. It's like std::sync::Barrier, + /// but is stored in the shared memory area and works across processes. It's implemented by + /// polling, because e.g. standard rust mutexes are not guaranteed to work across processes. + struct SimpleBarrier { + num_procs: usize, + count: AtomicUsize, + } + + impl SimpleBarrier { + unsafe fn init(ptr: *mut SimpleBarrier, num_procs: usize) { + unsafe { + *ptr = SimpleBarrier { + num_procs, + count: AtomicUsize::new(0), + } + } + } + + pub fn wait(&self) { + let old = self.count.fetch_add(1, Ordering::Relaxed); + + let generation = old / self.num_procs; + + let mut current = old + 1; + while current < (generation + 1) * self.num_procs { + std::thread::sleep(std::time::Duration::from_millis(10)); + current = self.count.load(Ordering::Relaxed); + } + } + } + + #[test] + fn test_multi_process() { + // Initialize + let max_size = 1_000_000_000_000; + let init_struct = ShmemHandle::new("test_multi_process", 0, max_size).unwrap(); + let ptr = init_struct.data_ptr.as_ptr(); + + // Store the SimpleBarrier in the first 1k of the area. + init_struct.set_size(10000).unwrap(); + let barrier_ptr: *mut SimpleBarrier = unsafe { + ptr.add(ptr.align_offset(std::mem::align_of::())) + .cast() + }; + unsafe { SimpleBarrier::init(barrier_ptr, 2) }; + let barrier = unsafe { barrier_ptr.as_ref().unwrap() }; + + // Fork another test process. The code after this runs in both processes concurrently. + let fork_result = unsafe { nix::unistd::fork().unwrap() }; + + // In the parent, fill bytes between 1000..2000. In the child, between 2000..3000 + if fork_result.is_parent() { + write_range(ptr, 0xAA, 1000..2000); + } else { + write_range(ptr, 0xBB, 2000..3000); + } + barrier.wait(); + // Verify the contents. (in both processes) + assert_range(ptr, 0xAA, 1000..2000); + assert_range(ptr, 0xBB, 2000..3000); + + // Grow, from the child this time + let size = 10_000_000; + if !fork_result.is_parent() { + init_struct.set_size(size).unwrap(); + } + barrier.wait(); + + // make some writes at the end + if fork_result.is_parent() { + write_range(ptr, 0xAA, (size - 10)..size); + } else { + write_range(ptr, 0xBB, (size - 20)..(size - 10)); + } + barrier.wait(); + + // Verify the contents. (This runs in both processes) + assert_range(ptr, 0, (size - 1000)..(size - 20)); + assert_range(ptr, 0xBB, (size - 20)..(size - 10)); + assert_range(ptr, 0xAA, (size - 10)..size); + + if let ForkResult::Parent { child } = fork_result { + nix::sys::wait::waitpid(child, None).unwrap(); + } + } +} From 10b936bf031d0cd94420ae31227ff64c2f0bd8d4 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 5 Jun 2025 18:31:29 +0300 Subject: [PATCH 04/24] Use a custom Rust implementation to replace the LFC hash table The new implementation lives in a separately allocated shared memory area, which could be resized. Resizing it isn't actually implemented yet, though. It would require some co-operation from the LFC code. --- Cargo.lock | 126 +++++++- Cargo.toml | 3 + Makefile | 7 + libs/neon-shmem/Cargo.toml | 6 +- libs/neon-shmem/src/hash.rs | 304 ++++++++++++++++++ libs/neon-shmem/src/hash/core.rs | 174 ++++++++++ libs/neon-shmem/src/hash/tests.rs | 220 +++++++++++++ libs/neon-shmem/src/lib.rs | 1 + pgxn/neon/Makefile | 4 +- pgxn/neon/communicator/Cargo.toml | 13 + pgxn/neon/communicator/README.md | 8 + pgxn/neon/communicator/build.rs | 22 ++ pgxn/neon/communicator/cbindgen.toml | 4 + .../communicator/src/file_cache_hashmap.rs | 240 ++++++++++++++ pgxn/neon/communicator/src/lib.rs | 1 + pgxn/neon/file_cache.c | 127 ++++---- 16 files changed, 1186 insertions(+), 74 deletions(-) create mode 100644 libs/neon-shmem/src/hash.rs create mode 100644 libs/neon-shmem/src/hash/core.rs create mode 100644 libs/neon-shmem/src/hash/tests.rs create mode 100644 pgxn/neon/communicator/Cargo.toml create mode 100644 pgxn/neon/communicator/README.md create mode 100644 pgxn/neon/communicator/build.rs create mode 100644 pgxn/neon/communicator/cbindgen.toml create mode 100644 pgxn/neon/communicator/src/file_cache_hashmap.rs create mode 100644 pgxn/neon/communicator/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 588a63b6a3..bafcaea594 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1086,6 +1086,25 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbindgen" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadd868a2ce9ca38de7eeafdcec9c7065ef89b42b32f0839278d55f35c54d1ff" +dependencies = [ + "clap", + "heck 0.4.1", + "indexmap 2.9.0", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.100", + "tempfile", + "toml", +] + [[package]] name = "cc" version = "1.2.16" @@ -1212,7 +1231,7 @@ version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.100", @@ -1270,6 +1289,14 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "communicator" +version = "0.1.0" +dependencies = [ + "cbindgen", + "neon-shmem", +] + [[package]] name = "compute_api" version = "0.1.0" @@ -1936,7 +1963,7 @@ checksum = "0892a17df262a24294c382f0d5997571006e7a4348b4327557c4ff1cd4a8bccc" dependencies = [ "darling", "either", - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.100", @@ -2500,6 +2527,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + [[package]] name = "gettid" version = "0.1.3" @@ -2712,6 +2751,12 @@ dependencies = [ "http 1.1.0", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" @@ -3648,7 +3693,7 @@ version = "0.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e6777fc80a575f9503d908c8b498782a6c3ee88a06cb416dc3941401e43b94" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.100", @@ -3710,7 +3755,7 @@ dependencies = [ "procfs", "prometheus", "rand 0.8.5", - "rand_distr", + "rand_distr 0.4.3", "twox-hash", ] @@ -3799,6 +3844,8 @@ name = "neon-shmem" version = "0.1.0" dependencies = [ "nix 0.30.1", + "rand 0.9.1", + "rand_distr 0.5.1", "tempfile", "thiserror 1.0.69", "workspace_hack", @@ -5092,7 +5139,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ "bytes", - "heck", + "heck 0.5.0", "itertools 0.12.1", "log", "multimap", @@ -5113,7 +5160,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c1318b19085f08681016926435853bbf7858f9c082d0999b80550ff5d9abe15" dependencies = [ "bytes", - "heck", + "heck 0.5.0", "itertools 0.12.1", "log", "multimap", @@ -5238,7 +5285,7 @@ dependencies = [ "postgres_backend", "pq_proto", "rand 0.8.5", - "rand_distr", + "rand_distr 0.4.3", "rcgen", "redis", "regex", @@ -5342,6 +5389,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + [[package]] name = "rand" version = "0.7.3" @@ -5366,6 +5419,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", +] + [[package]] name = "rand_chacha" version = "0.2.2" @@ -5386,6 +5449,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", +] + [[package]] name = "rand_core" version = "0.5.1" @@ -5404,6 +5477,15 @@ dependencies = [ "getrandom 0.2.11", ] +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.3", +] + [[package]] name = "rand_distr" version = "0.4.3" @@ -5414,6 +5496,16 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.1", +] + [[package]] name = "rand_hc" version = "0.2.0" @@ -6900,7 +6992,7 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "rustversion", @@ -8199,6 +8291,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasite" version = "0.1.0" @@ -8556,6 +8657,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags 2.8.0", +] + [[package]] name = "workspace_hack" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index a040010fb7..df0ab04fb6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ members = [ "libs/proxy/postgres-types2", "libs/proxy/tokio-postgres2", "endpoint_storage", + "pgxn/neon/communicator", ] [workspace.package] @@ -251,6 +252,7 @@ desim = { version = "0.1", path = "./libs/desim" } endpoint_storage = { version = "0.0.1", path = "./endpoint_storage/" } http-utils = { version = "0.1", path = "./libs/http-utils/" } metrics = { version = "0.1", path = "./libs/metrics/" } +neon-shmem = { version = "0.1", path = "./libs/neon-shmem/" } pageserver = { path = "./pageserver" } pageserver_api = { version = "0.1", path = "./libs/pageserver_api/" } pageserver_client = { path = "./pageserver/client" } @@ -278,6 +280,7 @@ walproposer = { version = "0.1", path = "./libs/walproposer/" } workspace_hack = { version = "0.1", path = "./workspace_hack/" } ## Build dependencies +cbindgen = "0.28.0" criterion = "0.5.1" rcgen = "0.13" rstest = "0.18" diff --git a/Makefile b/Makefile index 0911465fb8..820f3c20f1 100644 --- a/Makefile +++ b/Makefile @@ -18,10 +18,12 @@ ifeq ($(BUILD_TYPE),release) PG_LDFLAGS = $(LDFLAGS) # Unfortunately, `--profile=...` is a nightly feature CARGO_BUILD_FLAGS += --release + NEON_CARGO_ARTIFACT_TARGET_DIR = $(ROOT_PROJECT_DIR)/target/release else ifeq ($(BUILD_TYPE),debug) PG_CONFIGURE_OPTS = --enable-debug --with-openssl --enable-cassert --enable-depend PG_CFLAGS += -O0 -g3 $(CFLAGS) PG_LDFLAGS = $(LDFLAGS) + NEON_CARGO_ARTIFACT_TARGET_DIR = $(ROOT_PROJECT_DIR)/target/debug else $(error Bad build type '$(BUILD_TYPE)', see Makefile for options) endif @@ -180,11 +182,16 @@ postgres-check-%: postgres-% .PHONY: neon-pg-ext-% neon-pg-ext-%: postgres-% + +@echo "Compiling communicator $*" + $(CARGO_CMD_PREFIX) cargo build -p communicator $(CARGO_BUILD_FLAGS) + +@echo "Compiling neon $*" mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-$* $(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/$*/bin/pg_config COPT='$(COPT)' \ + LIBCOMMUNICATOR_PATH=$(NEON_CARGO_ARTIFACT_TARGET_DIR) \ -C $(POSTGRES_INSTALL_DIR)/build/neon-$* \ -f $(ROOT_PROJECT_DIR)/pgxn/neon/Makefile install + +@echo "Compiling neon_walredo $*" mkdir -p $(POSTGRES_INSTALL_DIR)/build/neon-walredo-$* $(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/$*/bin/pg_config COPT='$(COPT)' \ diff --git a/libs/neon-shmem/Cargo.toml b/libs/neon-shmem/Cargo.toml index 2a636bec40..de65f3e8cc 100644 --- a/libs/neon-shmem/Cargo.toml +++ b/libs/neon-shmem/Cargo.toml @@ -6,8 +6,12 @@ license.workspace = true [dependencies] thiserror.workspace = true -nix.workspace=true +nix.workspace = true workspace_hack = { version = "0.1", path = "../../workspace_hack" } +[dev-dependencies] +rand = "0.9.1" +rand_distr = "0.5.1" + [target.'cfg(target_os = "macos")'.dependencies] tempfile = "3.14.0" diff --git a/libs/neon-shmem/src/hash.rs b/libs/neon-shmem/src/hash.rs new file mode 100644 index 0000000000..ae53e2ec41 --- /dev/null +++ b/libs/neon-shmem/src/hash.rs @@ -0,0 +1,304 @@ +//! Hash table implementation on top of 'shmem' +//! +//! Features required in the long run by the communicator project: +//! +//! [X] Accessible from both Postgres processes and rust threads in the communicator process +//! [X] Low latency +//! [ ] Scalable to lots of concurrent accesses (currently relies on caller for locking) +//! [ ] Resizable + +use std::fmt::Debug; +use std::hash::{DefaultHasher, Hash, Hasher}; +use std::mem::MaybeUninit; + +use crate::shmem::ShmemHandle; + +mod core; +pub mod entry; + +#[cfg(test)] +mod tests; + +use core::CoreHashMap; +use entry::{Entry, OccupiedEntry}; + +#[derive(Debug)] +pub struct OutOfMemoryError(); + +pub struct HashMapInit<'a, K, V> { + // Hash table can be allocated in a fixed memory area, or in a resizeable ShmemHandle. + shmem_handle: Option, + shared_ptr: *mut HashMapShared<'a, K, V>, +} + +pub struct HashMapAccess<'a, K, V> { + shmem_handle: Option, + shared_ptr: *mut HashMapShared<'a, K, V>, +} + +unsafe impl<'a, K: Sync, V: Sync> Sync for HashMapAccess<'a, K, V> {} +unsafe impl<'a, K: Send, V: Send> Send for HashMapAccess<'a, K, V> {} + +impl<'a, K, V> HashMapInit<'a, K, V> { + pub fn attach_writer(self) -> HashMapAccess<'a, K, V> { + HashMapAccess { + shmem_handle: self.shmem_handle, + shared_ptr: self.shared_ptr, + } + } + + pub fn attach_reader(self) -> HashMapAccess<'a, K, V> { + // no difference to attach_writer currently + self.attach_writer() + } +} + +/// This is stored in the shared memory area +/// +/// NOTE: We carve out the parts from a contiguous chunk. Growing and shrinking the hash table +/// relies on the memory layout! The data structures are laid out in the contiguous shared memory +/// area as follows: +/// +/// HashMapShared +/// [buckets] +/// [dictionary] +/// +/// In between the above parts, there can be padding bytes to align the parts correctly. +struct HashMapShared<'a, K, V> { + inner: CoreHashMap<'a, K, V>, +} + +impl<'a, K, V> HashMapInit<'a, K, V> +where + K: Clone + Hash + Eq, +{ + pub fn estimate_size(num_buckets: u32) -> usize { + // add some margin to cover alignment etc. + CoreHashMap::::estimate_size(num_buckets) + size_of::>() + 1000 + } + + pub fn init_in_fixed_area( + num_buckets: u32, + area: &'a mut [MaybeUninit], + ) -> HashMapInit<'a, K, V> { + Self::init_common(num_buckets, None, area.as_mut_ptr().cast(), area.len()) + } + + /// Initialize a new hash map in the given shared memory area + pub fn init_in_shmem(num_buckets: u32, mut shmem: ShmemHandle) -> HashMapInit<'a, K, V> { + let size = Self::estimate_size(num_buckets); + shmem + .set_size(size) + .expect("could not resize shared memory area"); + + let ptr = unsafe { shmem.data_ptr.as_mut() }; + Self::init_common(num_buckets, Some(shmem), ptr, size) + } + + fn init_common( + num_buckets: u32, + shmem_handle: Option, + area_ptr: *mut u8, + area_len: usize, + ) -> HashMapInit<'a, K, V> { + // carve out the HashMapShared struct from the area. + let mut ptr: *mut u8 = area_ptr; + let end_ptr: *mut u8 = unsafe { area_ptr.add(area_len) }; + ptr = unsafe { ptr.add(ptr.align_offset(align_of::>())) }; + let shared_ptr: *mut HashMapShared = ptr.cast(); + ptr = unsafe { ptr.add(size_of::>()) }; + + // carve out the buckets + ptr = unsafe { ptr.byte_add(ptr.align_offset(align_of::>())) }; + let buckets_ptr = ptr; + ptr = unsafe { ptr.add(size_of::>() * num_buckets as usize) }; + + // use remaining space for the dictionary + ptr = unsafe { ptr.byte_add(ptr.align_offset(align_of::())) }; + assert!(ptr.addr() < end_ptr.addr()); + let dictionary_ptr = ptr; + let dictionary_size = unsafe { end_ptr.byte_offset_from(ptr) / size_of::() as isize }; + assert!(dictionary_size > 0); + + let buckets = + unsafe { std::slice::from_raw_parts_mut(buckets_ptr.cast(), num_buckets as usize) }; + let dictionary = unsafe { + std::slice::from_raw_parts_mut(dictionary_ptr.cast(), dictionary_size as usize) + }; + let hashmap = CoreHashMap::new(buckets, dictionary); + unsafe { + std::ptr::write(shared_ptr, HashMapShared { inner: hashmap }); + } + + HashMapInit { + shmem_handle: shmem_handle, + shared_ptr, + } + } +} + +impl<'a, K, V> HashMapAccess<'a, K, V> +where + K: Clone + Hash + Eq, +{ + pub fn get_hash_value(&self, key: &K) -> u64 { + let mut hasher = DefaultHasher::new(); + key.hash(&mut hasher); + hasher.finish() + } + + pub fn get_with_hash<'e>(&'e self, key: &K, hash: u64) -> Option<&'e V> { + let map = unsafe { self.shared_ptr.as_ref() }.unwrap(); + + map.inner.get_with_hash(key, hash) + } + + pub fn entry_with_hash(&mut self, key: K, hash: u64) -> Entry<'a, '_, K, V> { + let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); + + map.inner.entry_with_hash(key, hash) + } + + pub fn remove_with_hash(&mut self, key: &K, hash: u64) { + let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); + + match map.inner.entry_with_hash(key.clone(), hash) { + Entry::Occupied(e) => { + e.remove(); + } + Entry::Vacant(_) => {} + }; + } + + pub fn entry_at_bucket(&mut self, pos: usize) -> Option> { + let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); + map.inner.entry_at_bucket(pos) + } + + pub fn get_num_buckets(&self) -> usize { + let map = unsafe { self.shared_ptr.as_ref() }.unwrap(); + map.inner.get_num_buckets() + } + + /// Return the key and value stored in bucket with given index. This can be used to + /// iterate through the hash map. (An Iterator might be nicer. The communicator's + /// clock algorithm needs to _slowly_ iterate through all buckets with its clock hand, + /// without holding a lock. If we switch to an Iterator, it must not hold the lock.) + pub fn get_at_bucket(&self, pos: usize) -> Option<&(K, V)> { + let map = unsafe { self.shared_ptr.as_ref() }.unwrap(); + + if pos >= map.inner.buckets.len() { + return None; + } + let bucket = &map.inner.buckets[pos]; + bucket.inner.as_ref() + } + + pub fn get_bucket_for_value(&self, val_ptr: *const V) -> usize { + let map = unsafe { self.shared_ptr.as_ref() }.unwrap(); + + let origin = map.inner.buckets.as_ptr(); + let idx = (val_ptr as usize - origin as usize) / (size_of::() as usize); + assert!(idx < map.inner.buckets.len()); + + idx + } + + // for metrics + pub fn get_num_buckets_in_use(&self) -> usize { + let map = unsafe { self.shared_ptr.as_ref() }.unwrap(); + map.inner.buckets_in_use as usize + } + + /// Grow + /// + /// 1. grow the underlying shared memory area + /// 2. Initialize new buckets. This overwrites the current dictionary + /// 3. Recalculate the dictionary + pub fn grow(&mut self, num_buckets: u32) -> Result<(), crate::shmem::Error> { + let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); + let inner = &mut map.inner; + let old_num_buckets = inner.buckets.len() as u32; + + if num_buckets < old_num_buckets { + panic!("grow called with a smaller number of buckets"); + } + if num_buckets == old_num_buckets { + return Ok(()); + } + let shmem_handle = self + .shmem_handle + .as_ref() + .expect("grow called on a fixed-size hash table"); + + let size_bytes = HashMapInit::::estimate_size(num_buckets); + shmem_handle.set_size(size_bytes)?; + let end_ptr: *mut u8 = unsafe { shmem_handle.data_ptr.as_ptr().add(size_bytes) }; + + // Initialize new buckets. The new buckets are linked to the free list. NB: This overwrites + // the dictionary! + let buckets_ptr = inner.buckets.as_mut_ptr(); + unsafe { + for i in old_num_buckets..num_buckets { + let bucket_ptr = buckets_ptr.add(i as usize); + bucket_ptr.write(core::Bucket { + next: if i < num_buckets { + i as u32 + 1 + } else { + inner.free_head + }, + inner: None, + }); + } + } + + // Recalculate the dictionary + let buckets; + let dictionary; + unsafe { + let buckets_end_ptr = buckets_ptr.add(num_buckets as usize); + let dictionary_ptr: *mut u32 = buckets_end_ptr + .byte_add(buckets_end_ptr.align_offset(align_of::())) + .cast(); + let dictionary_size: usize = + end_ptr.byte_offset_from(buckets_end_ptr) as usize / size_of::(); + + buckets = std::slice::from_raw_parts_mut(buckets_ptr, num_buckets as usize); + dictionary = std::slice::from_raw_parts_mut(dictionary_ptr, dictionary_size); + } + for i in 0..dictionary.len() { + dictionary[i] = core::INVALID_POS; + } + + for i in 0..old_num_buckets as usize { + if buckets[i].inner.is_none() { + continue; + } + + let mut hasher = DefaultHasher::new(); + buckets[i].inner.as_ref().unwrap().0.hash(&mut hasher); + let hash = hasher.finish(); + + let pos: usize = (hash % dictionary.len() as u64) as usize; + buckets[i].next = dictionary[pos]; + dictionary[pos] = i as u32; + } + + // Finally, update the CoreHashMap struct + inner.dictionary = dictionary; + inner.buckets = buckets; + inner.free_head = old_num_buckets; + + Ok(()) + } + + // TODO: Shrinking is a multi-step process that requires co-operation from the caller + // + // 1. The caller must first call begin_shrink(). That forbids allocation of higher-numbered + // buckets. + // + // 2. Next, the caller must evict all entries in higher-numbered buckets. + // + // 3. Finally, call finish_shrink(). This recomputes the dictionary and shrinks the underlying + // shmem area +} diff --git a/libs/neon-shmem/src/hash/core.rs b/libs/neon-shmem/src/hash/core.rs new file mode 100644 index 0000000000..f4ff8ed2c4 --- /dev/null +++ b/libs/neon-shmem/src/hash/core.rs @@ -0,0 +1,174 @@ +//! Simple hash table with chaining +//! +//! # Resizing +//! + +use std::hash::Hash; +use std::mem::MaybeUninit; + +use crate::hash::entry::{Entry, OccupiedEntry, PrevPos, VacantEntry}; + +pub(crate) const INVALID_POS: u32 = u32::MAX; + +// Bucket +pub(crate) struct Bucket { + pub(crate) next: u32, + pub(crate) inner: Option<(K, V)>, +} + +pub(crate) struct CoreHashMap<'a, K, V> { + pub(crate) dictionary: &'a mut [u32], + pub(crate) buckets: &'a mut [Bucket], + pub(crate) free_head: u32, + + pub(crate) _user_list_head: u32, + + // metrics + pub(crate) buckets_in_use: u32, +} + +#[derive(Debug)] +pub struct FullError(); + +impl<'a, K: Hash + Eq, V> CoreHashMap<'a, K, V> +where + K: Clone + Hash + Eq, +{ + const FILL_FACTOR: f32 = 0.60; + + pub fn estimate_size(num_buckets: u32) -> usize { + let mut size = 0; + + // buckets + size += size_of::>() * num_buckets as usize; + + // dictionary + size += (f32::ceil((size_of::() * num_buckets as usize) as f32 / Self::FILL_FACTOR)) + as usize; + + size + } + + pub fn new( + buckets: &'a mut [MaybeUninit>], + dictionary: &'a mut [MaybeUninit], + ) -> CoreHashMap<'a, K, V> { + // Initialize the buckets + for i in 0..buckets.len() { + buckets[i].write(Bucket { + next: if i < buckets.len() - 1 { + i as u32 + 1 + } else { + INVALID_POS + }, + inner: None, + }); + } + + // Initialize the dictionary + for i in 0..dictionary.len() { + dictionary[i].write(INVALID_POS); + } + + // TODO: use std::slice::assume_init_mut() once it stabilizes + let buckets = + unsafe { std::slice::from_raw_parts_mut(buckets.as_mut_ptr().cast(), buckets.len()) }; + let dictionary = unsafe { + std::slice::from_raw_parts_mut(dictionary.as_mut_ptr().cast(), dictionary.len()) + }; + + CoreHashMap { + dictionary, + buckets, + free_head: 0, + buckets_in_use: 0, + _user_list_head: INVALID_POS, + } + } + + pub fn get_with_hash(&self, key: &K, hash: u64) -> Option<&V> { + let mut next = self.dictionary[hash as usize % self.dictionary.len()]; + loop { + if next == INVALID_POS { + return None; + } + + let bucket = &self.buckets[next as usize]; + let (bucket_key, bucket_value) = bucket.inner.as_ref().expect("entry is in use"); + if bucket_key == key { + return Some(&bucket_value); + } + next = bucket.next; + } + } + + // all updates are done through Entry + pub fn entry_with_hash(&mut self, key: K, hash: u64) -> Entry<'a, '_, K, V> { + let dict_pos = hash as usize % self.dictionary.len(); + let first = self.dictionary[dict_pos]; + if first == INVALID_POS { + // no existing entry + return Entry::Vacant(VacantEntry { + map: self, + key, + dict_pos: dict_pos as u32, + }); + } + + let mut prev_pos = PrevPos::First(dict_pos as u32); + let mut next = first; + loop { + let bucket = &mut self.buckets[next as usize]; + let (bucket_key, _bucket_value) = bucket.inner.as_mut().expect("entry is in use"); + if *bucket_key == key { + // found existing entry + return Entry::Occupied(OccupiedEntry { + map: self, + _key: key, + prev_pos, + bucket_pos: next, + }); + } + + if bucket.next == INVALID_POS { + // No existing entry + return Entry::Vacant(VacantEntry { + map: self, + key, + dict_pos: dict_pos as u32, + }); + } + prev_pos = PrevPos::Chained(next); + next = bucket.next; + } + } + + pub fn get_num_buckets(&self) -> usize { + self.buckets.len() + } + + pub fn entry_at_bucket(&mut self, pos: usize) -> Option> { + if pos >= self.buckets.len() { + return None; + } + + todo!() + //self.buckets[pos].inner.as_ref() + } + + pub(crate) fn alloc_bucket(&mut self, key: K, value: V) -> Result { + let pos = self.free_head; + if pos == INVALID_POS { + return Err(FullError()); + } + + let bucket = &mut self.buckets[pos as usize]; + self.free_head = bucket.next; + self.buckets_in_use += 1; + + bucket.next = INVALID_POS; + bucket.inner = Some((key, value)); + + return Ok(pos); + } +} diff --git a/libs/neon-shmem/src/hash/tests.rs b/libs/neon-shmem/src/hash/tests.rs new file mode 100644 index 0000000000..073aea5220 --- /dev/null +++ b/libs/neon-shmem/src/hash/tests.rs @@ -0,0 +1,220 @@ +use std::collections::BTreeMap; +use std::collections::HashSet; +use std::fmt::{Debug, Formatter}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use crate::hash::HashMapAccess; +use crate::hash::HashMapInit; +use crate::hash::UpdateAction; +use crate::shmem::ShmemHandle; + +use rand::seq::SliceRandom; +use rand::{Rng, RngCore}; +use rand_distr::Zipf; + +const TEST_KEY_LEN: usize = 16; + +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] +struct TestKey([u8; TEST_KEY_LEN]); + +impl From<&TestKey> for u128 { + fn from(val: &TestKey) -> u128 { + u128::from_be_bytes(val.0) + } +} + +impl From for TestKey { + fn from(val: u128) -> TestKey { + TestKey(val.to_be_bytes()) + } +} + +impl<'a> From<&'a [u8]> for TestKey { + fn from(bytes: &'a [u8]) -> TestKey { + TestKey(bytes.try_into().unwrap()) + } +} + +fn test_inserts + Copy>(keys: &[K]) { + const MAX_MEM_SIZE: usize = 10000000; + let shmem = ShmemHandle::new("test_inserts", 0, MAX_MEM_SIZE).unwrap(); + + let init_struct = HashMapInit::::init_in_shmem(100000, shmem); + let w = init_struct.attach_writer(); + + for (idx, k) in keys.iter().enumerate() { + let res = w.insert(&(*k).into(), idx); + assert!(res.is_ok()); + } + + for (idx, k) in keys.iter().enumerate() { + let x = w.get(&(*k).into()); + let value = x.as_deref().copied(); + assert_eq!(value, Some(idx)); + } + + //eprintln!("stats: {:?}", tree_writer.get_statistics()); +} + +#[test] +fn dense() { + // This exercises splitting a node with prefix + let keys: &[u128] = &[0, 1, 2, 3, 256]; + test_inserts(keys); + + // Dense keys + let mut keys: Vec = (0..10000).collect(); + test_inserts(&keys); + + // Do the same in random orders + for _ in 1..10 { + keys.shuffle(&mut rand::rng()); + test_inserts(&keys); + } +} + +#[test] +fn sparse() { + // sparse keys + let mut keys: Vec = Vec::new(); + let mut used_keys = HashSet::new(); + for _ in 0..10000 { + loop { + let key = rand::random::(); + if used_keys.get(&key).is_some() { + continue; + } + used_keys.insert(key); + keys.push(key.into()); + break; + } + } + test_inserts(&keys); +} + +struct TestValue(AtomicUsize); + +impl TestValue { + fn new(val: usize) -> TestValue { + TestValue(AtomicUsize::new(val)) + } + + fn load(&self) -> usize { + self.0.load(Ordering::Relaxed) + } +} + +impl Clone for TestValue { + fn clone(&self) -> TestValue { + TestValue::new(self.load()) + } +} + +impl Debug for TestValue { + fn fmt(&self, fmt: &mut Formatter<'_>) -> Result<(), std::fmt::Error> { + write!(fmt, "{:?}", self.load()) + } +} + +#[derive(Clone, Debug)] +struct TestOp(TestKey, Option); + +fn apply_op( + op: &TestOp, + sut: &HashMapAccess, + shadow: &mut BTreeMap, +) { + eprintln!("applying op: {op:?}"); + + // apply the change to the shadow tree first + let shadow_existing = if let Some(v) = op.1 { + shadow.insert(op.0, v) + } else { + shadow.remove(&op.0) + }; + + // apply to Art tree + sut.update_with_fn(&op.0, |existing| { + assert_eq!(existing.map(TestValue::load), shadow_existing); + + match (existing, op.1) { + (None, None) => UpdateAction::Nothing, + (None, Some(new_val)) => UpdateAction::Insert(TestValue::new(new_val)), + (Some(_old_val), None) => UpdateAction::Remove, + (Some(old_val), Some(new_val)) => { + old_val.0.store(new_val, Ordering::Relaxed); + UpdateAction::Nothing + } + } + }) + .expect("out of memory"); +} + +#[test] +fn random_ops() { + const MAX_MEM_SIZE: usize = 10000000; + let shmem = ShmemHandle::new("test_inserts", 0, MAX_MEM_SIZE).unwrap(); + + let init_struct = HashMapInit::::init_in_shmem(100000, shmem); + let writer = init_struct.attach_writer(); + + let mut shadow: std::collections::BTreeMap = BTreeMap::new(); + + let distribution = Zipf::new(u128::MAX as f64, 1.1).unwrap(); + let mut rng = rand::rng(); + for i in 0..100000 { + let key: TestKey = (rng.sample(distribution) as u128).into(); + + let op = TestOp(key, if rng.random_bool(0.75) { Some(i) } else { None }); + + apply_op(&op, &writer, &mut shadow); + + if i % 1000 == 0 { + eprintln!("{i} ops processed"); + //eprintln!("stats: {:?}", tree_writer.get_statistics()); + //test_iter(&tree_writer, &shadow); + } + } +} + +#[test] +fn test_grow() { + const MEM_SIZE: usize = 10000000; + let shmem = ShmemHandle::new("test_grow", 0, MEM_SIZE).unwrap(); + + let init_struct = HashMapInit::::init_in_shmem(1000, shmem); + let writer = init_struct.attach_writer(); + + let mut shadow: std::collections::BTreeMap = BTreeMap::new(); + + let mut rng = rand::rng(); + for i in 0..10000 { + let key: TestKey = ((rng.next_u32() % 1000) as u128).into(); + + let op = TestOp(key, if rng.random_bool(0.75) { Some(i) } else { None }); + + apply_op(&op, &writer, &mut shadow); + + if i % 1000 == 0 { + eprintln!("{i} ops processed"); + //eprintln!("stats: {:?}", tree_writer.get_statistics()); + //test_iter(&tree_writer, &shadow); + } + } + + writer.grow(1500).unwrap(); + + for i in 0..10000 { + let key: TestKey = ((rng.next_u32() % 1500) as u128).into(); + + let op = TestOp(key, if rng.random_bool(0.75) { Some(i) } else { None }); + + apply_op(&op, &writer, &mut shadow); + + if i % 1000 == 0 { + eprintln!("{i} ops processed"); + //eprintln!("stats: {:?}", tree_writer.get_statistics()); + //test_iter(&tree_writer, &shadow); + } + } +} diff --git a/libs/neon-shmem/src/lib.rs b/libs/neon-shmem/src/lib.rs index d4f171ed66..f601010122 100644 --- a/libs/neon-shmem/src/lib.rs +++ b/libs/neon-shmem/src/lib.rs @@ -1,3 +1,4 @@ //! Shared memory utilities for neon communicator +pub mod hash; pub mod shmem; diff --git a/pgxn/neon/Makefile b/pgxn/neon/Makefile index 8bcc6bf924..bc0d3cdeb7 100644 --- a/pgxn/neon/Makefile +++ b/pgxn/neon/Makefile @@ -1,6 +1,5 @@ # pgxs/neon/Makefile - MODULE_big = neon OBJS = \ $(WIN32RES) \ @@ -22,7 +21,8 @@ OBJS = \ walproposer.o \ walproposer_pg.o \ control_plane_connector.o \ - walsender_hooks.o + walsender_hooks.o \ + $(LIBCOMMUNICATOR_PATH)/libcommunicator.a PG_CPPFLAGS = -I$(libpq_srcdir) SHLIB_LINK_INTERNAL = $(libpq) diff --git a/pgxn/neon/communicator/Cargo.toml b/pgxn/neon/communicator/Cargo.toml new file mode 100644 index 0000000000..f09b9d7a14 --- /dev/null +++ b/pgxn/neon/communicator/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "communicator" +version = "0.1.0" +edition = "2024" + +[lib] +crate-type = ["staticlib"] + +[dependencies] +neon-shmem.workspace = true + +[build-dependencies] +cbindgen.workspace = true diff --git a/pgxn/neon/communicator/README.md b/pgxn/neon/communicator/README.md new file mode 100644 index 0000000000..48fda68721 --- /dev/null +++ b/pgxn/neon/communicator/README.md @@ -0,0 +1,8 @@ +This package will evolve into a "compute-pageserver communicator" +process and machinery. For now, it just provides wrappers on the +neon-shmem Rust crate, to allow using it in the C implementation of +the LFC. + +At compilation time, pgxn/neon/communicator/ produces a static +library, libcommunicator.a. It is linked to the neon.so extension +library. diff --git a/pgxn/neon/communicator/build.rs b/pgxn/neon/communicator/build.rs new file mode 100644 index 0000000000..ef570c3d0a --- /dev/null +++ b/pgxn/neon/communicator/build.rs @@ -0,0 +1,22 @@ +use std::env; + +fn main() -> Result<(), Box> { + let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); + + cbindgen::generate(crate_dir).map_or_else( + |error| match error { + cbindgen::Error::ParseSyntaxError { .. } => { + // This means there was a syntax error in the Rust sources. Don't panic, because + // we want the build to continue and the Rust compiler to hit the error. The + // Rust compiler produces a better error message than cbindgen. + eprintln!("Generating C bindings failed because of a Rust syntax error"); + } + e => panic!("Unable to generate C bindings: {:?}", e), + }, + |bindings| { + bindings.write_to_file("communicator_bindings.h"); + }, + ); + + Ok(()) +} diff --git a/pgxn/neon/communicator/cbindgen.toml b/pgxn/neon/communicator/cbindgen.toml new file mode 100644 index 0000000000..72e0c8174a --- /dev/null +++ b/pgxn/neon/communicator/cbindgen.toml @@ -0,0 +1,4 @@ +language = "C" + +[enum] +prefix_with_name = true diff --git a/pgxn/neon/communicator/src/file_cache_hashmap.rs b/pgxn/neon/communicator/src/file_cache_hashmap.rs new file mode 100644 index 0000000000..0a9ec3db31 --- /dev/null +++ b/pgxn/neon/communicator/src/file_cache_hashmap.rs @@ -0,0 +1,240 @@ +//! Glue code to allow using the Rust shmem hash map implementation from C code +//! +//! For convience of adapting existing code, the interface provided somewhat resembles the dynahash +//! interface. +//! +//! NOTE: The caller is responsible for locking! The caller is expected to hold the PostgreSQL +//! LWLock, 'lfc_lock', while accessing the hash table, in shared or exclusive mode as appropriate. + +use std::ffi::c_void; +use std::marker::PhantomData; + +use neon_shmem::hash::entry::Entry; +use neon_shmem::hash::{HashMapAccess, HashMapInit}; +use neon_shmem::shmem::ShmemHandle; + +/// NB: This must match the definition of BufferTag in Postgres C headers. We could use bindgen to +/// generate this from the C headers, but prefer to not introduce dependency on bindgen for now. +/// +/// Note that there are no padding bytes. If the corresponding C struct has padding bytes, the C C +/// code must clear them. +#[derive(Clone, Debug, Hash, Eq, PartialEq)] +#[repr(C)] +pub struct FileCacheKey { + pub _spc_id: u32, + pub _db_id: u32, + pub _rel_number: u32, + pub _fork_num: u32, + pub _block_num: u32, +} + +/// Like with FileCacheKey, this must match the definition of FileCacheEntry in file_cache.c. We +/// don't look at the contents here though, it's sufficent that the size and alignment matches. +#[derive(Clone, Debug, Default)] +#[repr(C)] +pub struct FileCacheEntry { + pub _offset: u32, + pub _access_count: u32, + pub _prev: *mut FileCacheEntry, + pub _next: *mut FileCacheEntry, + pub _state: [u32; 8], +} + +/// XXX: This could be just: +/// +/// ```ignore +/// type FileCacheHashMapHandle = HashMapInit<'a, FileCacheKey, FileCacheEntry> +/// ``` +/// +/// but with that, cbindgen generates a broken typedef in the C header file which doesn't +/// compile. It apparently gets confused by the generics. +#[repr(transparent)] +pub struct FileCacheHashMapHandle<'a>( + pub *mut c_void, + PhantomData>, +); +impl<'a> From>> for FileCacheHashMapHandle<'a> { + fn from(x: Box>) -> Self { + FileCacheHashMapHandle(Box::into_raw(x) as *mut c_void, PhantomData::default()) + } +} +impl<'a> From> for Box> { + fn from(x: FileCacheHashMapHandle) -> Self { + unsafe { Box::from_raw(x.0.cast()) } + } +} + +/// XXX: same for this +#[repr(transparent)] +pub struct FileCacheHashMapAccess<'a>( + pub *mut c_void, + PhantomData>, +); +impl<'a> From>> for FileCacheHashMapAccess<'a> { + fn from(x: Box>) -> Self { + // Convert the Box into a raw mutable pointer to the HashMapAccess itself. + // This transfers ownership of the HashMapAccess (and its contained ShmemHandle) + // to the raw pointer. The C caller is now responsible for managing this memory. + FileCacheHashMapAccess(Box::into_raw(x) as *mut c_void, PhantomData::default()) + } +} +impl<'a> FileCacheHashMapAccess<'a> { + fn as_ref(self) -> &'a HashMapAccess<'a, FileCacheKey, FileCacheEntry> { + let ptr: *mut HashMapAccess<'_, FileCacheKey, FileCacheEntry> = self.0.cast(); + unsafe { ptr.as_ref().unwrap() } + } + fn as_mut(self) -> &'a mut HashMapAccess<'a, FileCacheKey, FileCacheEntry> { + let ptr: *mut HashMapAccess<'_, FileCacheKey, FileCacheEntry> = self.0.cast(); + unsafe { ptr.as_mut().unwrap() } + } +} + +/// Initialize the shared memory area at postmaster startup. The returned handle is inherited +/// by all the backend processes across fork() +#[unsafe(no_mangle)] +pub extern "C" fn bcomm_file_cache_shmem_init<'a>( + initial_num_buckets: u32, + max_num_buckets: u32, +) -> FileCacheHashMapHandle<'a> { + let max_bytes = HashMapInit::::estimate_size(max_num_buckets); + let shmem_handle = + ShmemHandle::new("lfc mapping", 0, max_bytes).expect("shmem initialization failed"); + + let handle = HashMapInit::::init_in_shmem( + initial_num_buckets, + shmem_handle, + ); + + Box::new(handle).into() +} + +/// Initialize the access to the shared memory area in a backend process. +/// +/// XXX: I'm not sure if this actually gets called in each process, or if the returned struct +/// is also inherited across fork(). It currently works either way but if this did more +/// initialization that needed to be done after fork(), then it would matter. +#[unsafe(no_mangle)] +pub extern "C" fn bcomm_file_cache_shmem_access<'a>( + handle: FileCacheHashMapHandle<'a>, +) -> FileCacheHashMapAccess<'a> { + let handle: Box> = handle.into(); + Box::new(handle.attach_writer()).into() +} + +/// Return the current number of buckets in the hash table +#[unsafe(no_mangle)] +pub extern "C" fn bcomm_file_cache_get_num_buckets<'a>( + map: FileCacheHashMapAccess<'static>, +) -> u32 { + let map = map.as_ref(); + map.get_num_buckets().try_into().unwrap() +} + +/// Look up the entry with given key and hash. +/// +/// This is similar to dynahash's hash_search(... , HASH_FIND) +#[unsafe(no_mangle)] +pub extern "C" fn bcomm_file_cache_hash_find<'a>( + map: FileCacheHashMapAccess<'static>, + key: &FileCacheKey, + hash: u64, +) -> Option<&'static FileCacheEntry> { + let map = map.as_ref(); + map.get_with_hash(key, hash) +} + +/// Look up the entry at given bucket position +/// +/// This has no direct equivalent in the dynahash interface, but can be used to +/// iterate through all entries in the hash table. +#[unsafe(no_mangle)] +pub extern "C" fn bcomm_file_cache_hash_get_at_pos<'a>( + map: FileCacheHashMapAccess<'static>, + pos: u32, +) -> Option<&'static FileCacheEntry> { + let map = map.as_ref(); + map.get_at_bucket(pos as usize).map(|(_k, v)| v) +} + +/// Remove entry, given a pointer to the value. +/// +/// This is equivalent to dynahash hash_search(entry->key, HASH_REMOVE), where 'entry' +/// is an entry you have previously looked up +#[unsafe(no_mangle)] +pub extern "C" fn bcomm_file_cache_hash_remove_entry<'a, 'b>( + map: FileCacheHashMapAccess, + entry: *mut FileCacheEntry, +) { + let map = map.as_mut(); + let pos = map.get_bucket_for_value(entry); + match map.entry_at_bucket(pos) { + Some(e) => { + e.remove(); + } + None => { + // todo: shouldn't happen, panic? + } + } +} + +/// Compute the hash for given key +/// +/// This is equivalent to dynahash get_hash_value() function. We use Rust's default hasher +/// for calculating the hash though. +#[unsafe(no_mangle)] +pub extern "C" fn bcomm_file_cache_get_hash_value<'a, 'b>( + map: FileCacheHashMapAccess<'static>, + key: &FileCacheKey, +) -> u64 { + map.as_ref().get_hash_value(key) +} + +/// Insert a new entry to the hash table +/// +/// This is equivalent to dynahash hash_search(..., HASH_ENTER). +#[unsafe(no_mangle)] +pub extern "C" fn bcomm_file_cache_hash_enter<'a, 'b>( + map: FileCacheHashMapAccess, + key: &FileCacheKey, + hash: u64, + found: &mut bool, +) -> *mut FileCacheEntry { + match map.as_mut().entry_with_hash(key.clone(), hash) { + Entry::Occupied(mut e) => { + *found = true; + e.get_mut() + } + Entry::Vacant(e) => { + *found = false; + let initial_value = FileCacheEntry::default(); + e.insert(initial_value).expect("TODO: hash table full") + } + } +} + +/// Get the key for a given entry, which must be present in the hash table. +/// +/// Dynahash requires the key to be part of the "value" struct, so you can always +/// access the key with something like `entry->key`. The Rust implementation however +/// stores the key separately. This function extracts the separately stored key. +#[unsafe(no_mangle)] +pub extern "C" fn bcomm_file_cache_hash_get_key_for_entry<'a, 'b>( + map: FileCacheHashMapAccess, + entry: *const FileCacheEntry, +) -> Option<&FileCacheKey> { + let map = map.as_ref(); + let pos = map.get_bucket_for_value(entry); + map.get_at_bucket(pos as usize).map(|(k, _v)| k) +} + +/// Remove all entries from the hash table +#[unsafe(no_mangle)] +pub extern "C" fn bcomm_file_cache_hash_reset<'a, 'b>(map: FileCacheHashMapAccess) { + let map = map.as_mut(); + let num_buckets = map.get_num_buckets(); + for i in 0..num_buckets { + if let Some(e) = map.entry_at_bucket(i) { + e.remove(); + } + } +} diff --git a/pgxn/neon/communicator/src/lib.rs b/pgxn/neon/communicator/src/lib.rs new file mode 100644 index 0000000000..4120ce0d38 --- /dev/null +++ b/pgxn/neon/communicator/src/lib.rs @@ -0,0 +1 @@ +pub mod file_cache_hashmap; diff --git a/pgxn/neon/file_cache.c b/pgxn/neon/file_cache.c index 485c282414..5d199716d2 100644 --- a/pgxn/neon/file_cache.c +++ b/pgxn/neon/file_cache.c @@ -22,7 +22,6 @@ #include "funcapi.h" #include "miscadmin.h" #include "common/file_utils.h" -#include "common/hashfn.h" #include "pgstat.h" #include "port/pg_iovec.h" #include "postmaster/bgworker.h" @@ -37,7 +36,6 @@ #include "storage/procsignal.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" -#include "utils/dynahash.h" #include "utils/guc.h" #if PG_VERSION_NUM >= 150000 @@ -47,6 +45,7 @@ #include "hll.h" #include "bitmap.h" #include "file_cache.h" +#include "file_cache_rust_hash.h" #include "neon.h" #include "neon_lwlsncache.h" #include "neon_perf_counters.h" @@ -124,14 +123,18 @@ typedef enum FileCacheBlockState typedef struct FileCacheEntry { - BufferTag key; - uint32 hash; uint32 offset; uint32 access_count; dlist_node list_node; /* LRU list node */ uint32 state[(BLOCKS_PER_CHUNK * 2 + 31) / 32]; /* two bits per block */ } FileCacheEntry; +/* Todo: alignment must be the same too */ +StaticAssertDecl(sizeof(FileCacheEntry) == sizeof(RustFileCacheEntry), + "Rust and C declarations of FileCacheEntry are incompatible"); +StaticAssertDecl(sizeof(BufferTag) == sizeof(RustFileCacheKey), + "Rust and C declarations of FileCacheKey are incompatible"); + #define GET_STATE(entry, i) (((entry)->state[(i) / 16] >> ((i) % 16 * 2)) & 3) #define SET_STATE(entry, i, new_state) (entry)->state[(i) / 16] = ((entry)->state[(i) / 16] & ~(3 << ((i) % 16 * 2))) | ((new_state) << ((i) % 16 * 2)) @@ -201,7 +204,8 @@ typedef struct FreeListChunk #define FILE_CACHE_STATE_SIZE_FOR_CHUNKS(n_chunks) (sizeof(FileCacheState) + (n_chunks)*sizeof(BufferTag) + (((n_chunks) * BLOCKS_PER_CHUNK)+7)/8) #define FILE_CACHE_STATE_SIZE(fcs) (sizeof(FileCacheState) + (fcs->n_chunks)*sizeof(BufferTag) + (((fcs->n_chunks) << fcs->chunk_size_log)+7)/8) -static HTAB *lfc_hash; +static FileCacheHashMapHandle lfc_hash_handle; +static FileCacheHashMapAccess lfc_hash; static int lfc_desc = -1; static LWLockId lfc_lock; static LWLockId lfc_freelist_lock; @@ -258,15 +262,9 @@ lfc_switch_off(void) if (LFC_ENABLED()) { - HASH_SEQ_STATUS status; - FileCacheEntry *entry; - /* Invalidate hash */ - hash_seq_init(&status, lfc_hash); - while ((entry = hash_seq_search(&status)) != NULL) - { - hash_search_with_hash_value(lfc_hash, &entry->key, entry->hash, HASH_REMOVE, NULL); - } + file_cache_hash_reset(lfc_hash); + lfc_ctl->generation += 1; lfc_ctl->size = 0; lfc_ctl->pinned = 0; @@ -347,7 +345,6 @@ lfc_shmem_startup(void) { size_t size; bool found; - static HASHCTL info; if (prev_shmem_startup_hook) { @@ -366,17 +363,13 @@ lfc_shmem_startup(void) lfc_lock = (LWLockId) GetNamedLWLockTranche("lfc_lock"); lfc_freelist_lock = (LWLockId) GetNamedLWLockTranche("lfc_freelist_lock"); - info.keysize = sizeof(BufferTag); - info.entrysize = sizeof(FileCacheEntry); /* * n_chunks+1 because we add new element to hash table before eviction * of victim */ - lfc_hash = ShmemInitHash("lfc_hash", - n_chunks + 1, n_chunks + 1, - &info, - HASH_ELEM | HASH_BLOBS); + lfc_hash_handle = file_cache_hash_shmem_init(n_chunks + 1, n_chunks + 1); + memset(lfc_ctl, 0, offsetof(FileCacheControl, free_pages)); dlist_init(&lfc_ctl->lru); @@ -406,6 +399,8 @@ lfc_shmem_startup(void) } LWLockRelease(AddinShmemInitLock); + + lfc_hash = file_cache_hash_shmem_access(lfc_hash_handle); } static void @@ -419,7 +414,6 @@ lfc_shmem_request(void) #endif size = sizeof(FileCacheControl); - size += hash_estimate_size(SIZE_MB_TO_CHUNKS(lfc_max_size) + 1, sizeof(FileCacheEntry)); RequestAddinShmemSpace(size); RequestNamedLWLockTranche("lfc_lock", 1); @@ -504,7 +498,7 @@ lfc_change_limit_hook(int newval, void *extra) lfc_ctl->used_pages -= is_page_cached; lfc_ctl->evicted_pages += is_page_cached; } - hash_search_with_hash_value(lfc_hash, &victim->key, victim->hash, HASH_REMOVE, NULL); + file_cache_hash_remove_entry(lfc_hash, victim); if (!freelist_push(offset)) { @@ -678,7 +672,7 @@ lfc_get_state(size_t max_entries) dlist_reverse_foreach(iter, &lfc_ctl->lru) { FileCacheEntry *entry = dlist_container(FileCacheEntry, list_node, iter.cur); - fcs->chunks[i] = entry->key; + fcs->chunks[i] = *file_cache_hash_get_key_for_entry(lfc_hash, entry); for (int j = 0; j < BLOCKS_PER_CHUNK; j++) { if (GET_STATE(entry, j) != UNAVAILABLE) @@ -967,7 +961,7 @@ lfc_invalidate(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber nblocks) { BufferTag tag; FileCacheEntry *entry; - uint32 hash; + uint64 hash; if (lfc_maybe_disabled()) /* fast exit if file cache is disabled */ return; @@ -983,8 +977,8 @@ lfc_invalidate(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber nblocks) for (BlockNumber blkno = 0; blkno < nblocks; blkno += BLOCKS_PER_CHUNK) { tag.blockNum = blkno; - hash = get_hash_value(lfc_hash, &tag); - entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_FIND, NULL); + hash = file_cache_hash_get_hash_value(lfc_hash, &tag); + entry = file_cache_hash_find(lfc_hash, &tag, hash); if (entry != NULL) { for (int i = 0; i < BLOCKS_PER_CHUNK; i++) @@ -1012,7 +1006,7 @@ lfc_cache_contains(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno) FileCacheEntry *entry; int chunk_offs = BLOCK_TO_CHUNK_OFF(blkno); bool found = false; - uint32 hash; + uint64 hash; if (lfc_maybe_disabled()) /* fast exit if file cache is disabled */ return false; @@ -1022,12 +1016,12 @@ lfc_cache_contains(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno) tag.blockNum = blkno - chunk_offs; CriticalAssert(BufTagGetRelNumber(&tag) != InvalidRelFileNumber); - hash = get_hash_value(lfc_hash, &tag); + hash = file_cache_hash_get_hash_value(lfc_hash, &tag); LWLockAcquire(lfc_lock, LW_SHARED); if (LFC_ENABLED()) { - entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_FIND, NULL); + entry = file_cache_hash_find(lfc_hash, &tag, hash); found = entry != NULL && GET_STATE(entry, chunk_offs) != UNAVAILABLE; } LWLockRelease(lfc_lock); @@ -1046,7 +1040,7 @@ lfc_cache_containsv(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, FileCacheEntry *entry; uint32 chunk_offs; int found = 0; - uint32 hash; + uint64 hash; int i = 0; if (lfc_maybe_disabled()) /* fast exit if file cache is disabled */ @@ -1059,7 +1053,7 @@ lfc_cache_containsv(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, chunk_offs = BLOCK_TO_CHUNK_OFF(blkno); tag.blockNum = blkno - chunk_offs; - hash = get_hash_value(lfc_hash, &tag); + hash = file_cache_hash_get_hash_value(lfc_hash, &tag); LWLockAcquire(lfc_lock, LW_SHARED); @@ -1071,7 +1065,7 @@ lfc_cache_containsv(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, while (true) { int this_chunk = Min(nblocks - i, BLOCKS_PER_CHUNK - chunk_offs); - entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_FIND, NULL); + entry = file_cache_hash_find(lfc_hash, &tag, hash); if (entry != NULL) { @@ -1101,7 +1095,7 @@ lfc_cache_containsv(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, */ chunk_offs = BLOCK_TO_CHUNK_OFF(blkno + i); tag.blockNum = (blkno + i) - chunk_offs; - hash = get_hash_value(lfc_hash, &tag); + hash = file_cache_hash_get_hash_value(lfc_hash, &tag); } LWLockRelease(lfc_lock); @@ -1150,7 +1144,7 @@ lfc_readv_select(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, BufferTag tag; FileCacheEntry *entry; ssize_t rc; - uint32 hash; + uint64 hash; uint64 generation; uint32 entry_offset; int blocks_read = 0; @@ -1228,7 +1222,7 @@ lfc_readv_select(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, Assert(iov_last_used - first_block_in_chunk_read >= n_blocks_to_read); tag.blockNum = blkno - chunk_offs; - hash = get_hash_value(lfc_hash, &tag); + hash = file_cache_hash_get_hash_value(lfc_hash, &tag); cv = &lfc_ctl->cv[hash % N_COND_VARS]; LWLockAcquire(lfc_lock, LW_EXCLUSIVE); @@ -1241,13 +1235,13 @@ lfc_readv_select(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, return blocks_read; } - entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_FIND, NULL); + entry = file_cache_hash_find(lfc_hash, &tag, hash); /* Approximate working set for the blocks assumed in this entry */ for (int i = 0; i < blocks_in_chunk; i++) { tag.blockNum = blkno + i; - addSHLL(&lfc_ctl->wss_estimation, hash_bytes((uint8_t const*)&tag, sizeof(tag))); + addSHLL(&lfc_ctl->wss_estimation, file_cache_hash_get_hash_value(lfc_hash, &tag)); } if (entry == NULL) @@ -1395,7 +1389,7 @@ lfc_readv_select(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, * Returns false if there are no unpinned entries and chunk can not be added. */ static bool -lfc_init_new_entry(FileCacheEntry* entry, uint32 hash) +lfc_init_new_entry(FileCacheEntry *entry) { /*----------- * If the chunk wasn't already in the LFC then we have these @@ -1451,21 +1445,18 @@ lfc_init_new_entry(FileCacheEntry* entry, uint32 hash) CriticalAssert(victim->access_count == 0); entry->offset = victim->offset; /* grab victim's chunk */ - hash_search_with_hash_value(lfc_hash, &victim->key, - victim->hash, HASH_REMOVE, NULL); + file_cache_hash_remove_entry(lfc_hash, victim); neon_log(DEBUG2, "Swap file cache page"); } else { /* Can't add this chunk - we don't have the space for it */ - hash_search_with_hash_value(lfc_hash, &entry->key, hash, - HASH_REMOVE, NULL); + file_cache_hash_remove_entry(lfc_hash, entry); lfc_ctl->prewarm_canceled = true; /* cancel prewarm if LFC limit is reached */ return false; } entry->access_count = 1; - entry->hash = hash; lfc_ctl->pinned += 1; for (int i = 0; i < BLOCKS_PER_CHUNK; i++) @@ -1505,7 +1496,7 @@ lfc_prefetch(NRelFileInfo rinfo, ForkNumber forknum, BlockNumber blkno, FileCacheEntry *entry; ssize_t rc; bool found; - uint32 hash; + uint64 hash; uint64 generation; uint32 entry_offset; instr_time io_start, io_end; @@ -1524,7 +1515,7 @@ lfc_prefetch(NRelFileInfo rinfo, ForkNumber forknum, BlockNumber blkno, CriticalAssert(BufTagGetRelNumber(&tag) != InvalidRelFileNumber); tag.blockNum = blkno - chunk_offs; - hash = get_hash_value(lfc_hash, &tag); + hash = file_cache_hash_get_hash_value(lfc_hash, &tag); cv = &lfc_ctl->cv[hash % N_COND_VARS]; retry: @@ -1549,12 +1540,12 @@ lfc_prefetch(NRelFileInfo rinfo, ForkNumber forknum, BlockNumber blkno, return false; } - entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_ENTER, &found); + entry = file_cache_hash_enter(lfc_hash, &tag, hash, &found); if (lfc_prewarm_update_ws_estimation) { tag.blockNum = blkno; - addSHLL(&lfc_ctl->wss_estimation, hash_bytes((uint8_t const*)&tag, sizeof(tag))); + addSHLL(&lfc_ctl->wss_estimation, file_cache_hash_get_hash_value(lfc_hash, &tag)); } if (found) { @@ -1576,7 +1567,7 @@ lfc_prefetch(NRelFileInfo rinfo, ForkNumber forknum, BlockNumber blkno, } else { - if (!lfc_init_new_entry(entry, hash)) + if (!lfc_init_new_entry(entry)) { /* * We can't process this chunk due to lack of space in LFC, @@ -1659,7 +1650,7 @@ lfc_writev(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, FileCacheEntry *entry; ssize_t rc; bool found; - uint32 hash; + uint64 hash; uint64 generation; uint32 entry_offset; int buf_offset = 0; @@ -1711,16 +1702,16 @@ lfc_writev(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, } tag.blockNum = blkno - chunk_offs; - hash = get_hash_value(lfc_hash, &tag); + hash = file_cache_hash_get_hash_value(lfc_hash, &tag); cv = &lfc_ctl->cv[hash % N_COND_VARS]; - entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_ENTER, &found); + entry = file_cache_hash_enter(lfc_hash, &tag, hash, &found); /* Approximate working set for the blocks assumed in this entry */ for (int i = 0; i < blocks_in_chunk; i++) { tag.blockNum = blkno + i; - addSHLL(&lfc_ctl->wss_estimation, hash_bytes((uint8_t const*)&tag, sizeof(tag))); + addSHLL(&lfc_ctl->wss_estimation, file_cache_hash_get_hash_value(lfc_hash, &tag)); } if (found) @@ -1737,7 +1728,7 @@ lfc_writev(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, } else { - if (!lfc_init_new_entry(entry, hash)) + if (!lfc_init_new_entry(entry)) { /* * We can't process this chunk due to lack of space in LFC, @@ -2147,7 +2138,6 @@ local_cache_pages(PG_FUNCTION_ARGS) if (SRF_IS_FIRSTCALL()) { - HASH_SEQ_STATUS status; FileCacheEntry *entry; uint32 n_pages = 0; @@ -2203,9 +2193,14 @@ local_cache_pages(PG_FUNCTION_ARGS) if (LFC_ENABLED()) { - hash_seq_init(&status, lfc_hash); - while ((entry = hash_seq_search(&status)) != NULL) + uint32 num_buckets = file_cache_hash_get_num_buckets(lfc_hash); + + for (uint32 pos = 0; pos < num_buckets; pos++) { + entry = file_cache_hash_get_at_pos(lfc_hash, pos); + if (entry == NULL) + continue; + for (int i = 0; i < BLOCKS_PER_CHUNK; i++) n_pages += GET_STATE(entry, i) == AVAILABLE; } @@ -2229,20 +2224,26 @@ local_cache_pages(PG_FUNCTION_ARGS) * in the fctx->record structure. */ uint32 n = 0; + uint32 num_buckets = file_cache_hash_get_num_buckets(lfc_hash); - hash_seq_init(&status, lfc_hash); - while ((entry = hash_seq_search(&status)) != NULL) + for (uint32 pos = 0; pos < num_buckets; pos++) { + entry = file_cache_hash_get_at_pos(lfc_hash, pos); + if (entry == NULL) + continue; + for (int i = 0; i < BLOCKS_PER_CHUNK; i++) { + const BufferTag *key = file_cache_hash_get_key_for_entry(lfc_hash, entry); + if (GET_STATE(entry, i) == AVAILABLE) { fctx->record[n].pageoffs = entry->offset * BLOCKS_PER_CHUNK + i; - fctx->record[n].relfilenode = NInfoGetRelNumber(BufTagGetNRelFileInfo(entry->key)); - fctx->record[n].reltablespace = NInfoGetSpcOid(BufTagGetNRelFileInfo(entry->key)); - fctx->record[n].reldatabase = NInfoGetDbOid(BufTagGetNRelFileInfo(entry->key)); - fctx->record[n].forknum = entry->key.forkNum; - fctx->record[n].blocknum = entry->key.blockNum + i; + fctx->record[n].relfilenode = NInfoGetRelNumber(BufTagGetNRelFileInfo(*key)); + fctx->record[n].reltablespace = NInfoGetSpcOid(BufTagGetNRelFileInfo(*key)); + fctx->record[n].reldatabase = NInfoGetDbOid(BufTagGetNRelFileInfo(*key)); + fctx->record[n].forknum = key->forkNum; + fctx->record[n].blocknum = key->blockNum + i; fctx->record[n].accesscount = entry->access_count; n += 1; } From 16d6898e448c42870d1e782bfad70afa564761f1 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 12 Jun 2025 02:37:59 +0300 Subject: [PATCH 05/24] git add missing file --- libs/neon-shmem/src/hash/entry.rs | 91 +++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 libs/neon-shmem/src/hash/entry.rs diff --git a/libs/neon-shmem/src/hash/entry.rs b/libs/neon-shmem/src/hash/entry.rs new file mode 100644 index 0000000000..cb9efcf742 --- /dev/null +++ b/libs/neon-shmem/src/hash/entry.rs @@ -0,0 +1,91 @@ +//! Like std::collections::hash_map::Entry; + +use crate::hash::core::{CoreHashMap, FullError, INVALID_POS}; + +use std::hash::Hash; +use std::mem; + +pub enum Entry<'a, 'b, K, V> { + Occupied(OccupiedEntry<'a, 'b, K, V>), + Vacant(VacantEntry<'a, 'b, K, V>), +} + +pub(crate) enum PrevPos { + First(u32), + Chained(u32), +} + +pub struct OccupiedEntry<'a, 'b, K, V> { + pub(crate) map: &'b mut CoreHashMap<'a, K, V>, + pub(crate) _key: K, // The key of the occupied entry + pub(crate) prev_pos: PrevPos, + pub(crate) bucket_pos: u32, // The position of the bucket in the CoreHashMap's buckets array +} + +impl<'a, 'b, K, V> OccupiedEntry<'a, 'b, K, V> { + pub fn get(&self) -> &V { + &self.map.buckets[self.bucket_pos as usize] + .inner + .as_ref() + .unwrap() + .1 + } + + pub fn get_mut(&mut self) -> &mut V { + &mut self.map.buckets[self.bucket_pos as usize] + .inner + .as_mut() + .unwrap() + .1 + } + + pub fn insert(&mut self, value: V) -> V { + let bucket = &mut self.map.buckets[self.bucket_pos as usize]; + // This assumes inner is Some, which it must be for an OccupiedEntry + let old_value = mem::replace(&mut bucket.inner.as_mut().unwrap().1, value); + old_value + } + + pub fn remove(self) -> V { + // CoreHashMap::remove returns Option<(K, V)>. We know it's Some for an OccupiedEntry. + let bucket = &mut self.map.buckets[self.bucket_pos as usize]; + + // unlink it from the chain + match self.prev_pos { + PrevPos::First(dict_pos) => self.map.dictionary[dict_pos as usize] = bucket.next, + PrevPos::Chained(bucket_pos) => { + self.map.buckets[bucket_pos as usize].next = bucket.next + } + } + + // and add it to the freelist + let bucket = &mut self.map.buckets[self.bucket_pos as usize]; + let old_value = bucket.inner.take(); + bucket.next = self.map.free_head; + self.map.free_head = self.bucket_pos; + self.map.buckets_in_use -= 1; + + return old_value.unwrap().1; + } +} + +pub struct VacantEntry<'a, 'b, K, V> { + pub(crate) map: &'b mut CoreHashMap<'a, K, V>, + pub(crate) key: K, // The key to insert + pub(crate) dict_pos: u32, +} + +impl<'a, 'b, K: Clone + Hash + Eq, V> VacantEntry<'a, 'b, K, V> { + pub fn insert(self, value: V) -> Result<&'b mut V, FullError> { + let pos = self.map.alloc_bucket(self.key, value)?; + if pos == INVALID_POS { + return Err(FullError()); + } + let bucket = &mut self.map.buckets[pos as usize]; + bucket.next = self.map.dictionary[self.dict_pos as usize]; + self.map.dictionary[self.dict_pos as usize] = pos; + + let result = &mut self.map.buckets[pos as usize].inner.as_mut().unwrap().1; + return Ok(result); + } +} From b6b122e07b3e20f57684046c7202a2ed5daab821 Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Mon, 16 Jun 2025 10:20:30 -0700 Subject: [PATCH 06/24] nw: add shrinking and deletion skeletons --- libs/neon-shmem/src/hash.rs | 48 ++++++++++++++++++++++++++++++++ libs/neon-shmem/src/hash/core.rs | 45 ++++++++++++++++++++++++------ 2 files changed, 84 insertions(+), 9 deletions(-) diff --git a/libs/neon-shmem/src/hash.rs b/libs/neon-shmem/src/hash.rs index ae53e2ec41..9b1c1cee89 100644 --- a/libs/neon-shmem/src/hash.rs +++ b/libs/neon-shmem/src/hash.rs @@ -292,6 +292,54 @@ where Ok(()) } + fn begin_shrink(&mut self, num_buckets: u32) { + let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); + if num_buckets < map.inner.get_num_buckets() as u32 { + panic!("shrink called with a larger number of buckets"); + } + map.inner.alloc_limit = num_buckets; + } + + fn finish_shrink(&mut self) -> Result<(), crate::shmem::Error> { + let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); + let inner = &mut map.inner; + if !inner.is_shrinking() { + panic!("called finish_shrink when no shrink is in progress"); + } + + let new_num_buckets = inner.alloc_limit; + + if inner.get_num_buckets() == new_num_buckets as usize { + return Ok(()); + } + + for b in &inner.buckets[new_num_buckets as usize..] { + if b.inner.is_some() { + // TODO(quantumish) Do we want to treat this as a violation of an invariant + // or a legitimate error the caller can run into? Originally I thought this + // could return something like a UnevictedError(index) as soon as it runs + // into something (that way a caller could clear their soon-to-be-shrinked + // buckets by repeatedly trying to call `finish_shrink`). + // + // Would require making a wider error type enum with this and shmem errors. + panic!("unevicted entries in shrinked space") + } + } + + let shmem_handle = self + .shmem_handle + .as_ref() + .expect("shrink called on a fixed-size hash table"); + + let size_bytes = HashMapInit::::estimate_size(new_num_buckets); + shmem_handle.set_size(size_bytes)?; + let end_ptr: *mut u8 = unsafe { shmem_handle.data_ptr.as_ptr().add(size_bytes) }; + + let buckets_ptr = inner.buckets.as_mut_ptr(); + + Ok(()) + } + // TODO: Shrinking is a multi-step process that requires co-operation from the caller // // 1. The caller must first call begin_shrink(). That forbids allocation of higher-numbered diff --git a/libs/neon-shmem/src/hash/core.rs b/libs/neon-shmem/src/hash/core.rs index f4ff8ed2c4..eb60e21bad 100644 --- a/libs/neon-shmem/src/hash/core.rs +++ b/libs/neon-shmem/src/hash/core.rs @@ -22,6 +22,7 @@ pub(crate) struct CoreHashMap<'a, K, V> { pub(crate) free_head: u32, pub(crate) _user_list_head: u32, + pub(crate) alloc_limit: u32, // metrics pub(crate) buckets_in_use: u32, @@ -83,6 +84,7 @@ where free_head: 0, buckets_in_use: 0, _user_list_head: INVALID_POS, + alloc_limit: INVALID_POS, } } @@ -147,25 +149,50 @@ where self.buckets.len() } + pub fn is_shrinking(&self) -> bool { + self.alloc_limit != INVALID_POS + } + pub fn entry_at_bucket(&mut self, pos: usize) -> Option> { if pos >= self.buckets.len() { return None; } - todo!() - //self.buckets[pos].inner.as_ref() + let entry = self.buckets[pos].inner.as_ref(); + if entry.is_none() { + return None; + } + + let (key, _) = entry.unwrap(); + Some(OccupiedEntry { + _key: key.clone(), // TODO(quantumish): clone unavoidable? + bucket_pos: pos as u32, + map: self, + prev_pos: todo!(), // TODO(quantumish): possibly needs O(n) traversals to rediscover - costly! + }) } pub(crate) fn alloc_bucket(&mut self, key: K, value: V) -> Result { - let pos = self.free_head; - if pos == INVALID_POS { - return Err(FullError()); - } + let mut pos = self.free_head; - let bucket = &mut self.buckets[pos as usize]; - self.free_head = bucket.next; - self.buckets_in_use += 1; + // TODO(quantumish): relies on INVALID_POS being u32::MAX by default! + // instead add a clause `pos != INVALID_POS`? + let mut prev = PrevPos::First(self.free_head); + while pos < self.alloc_limit { + if pos == INVALID_POS { + return Err(FullError()); + } + let bucket = &mut self.buckets[pos as usize]; + prev = PrevPos::Chained(pos); + pos = bucket.next; + } + let bucket = &mut self.buckets[pos as usize]; + match prev { + PrevPos::First(_) => self.free_head = bucket.next, + PrevPos::Chained(p) => self.buckets[p].next = bucket.next, + } + self.buckets_in_use += 1; bucket.next = INVALID_POS; bucket.inner = Some((key, value)); From ac87544e79b6080948f1ba2bf6d16c2c9018d178 Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Mon, 16 Jun 2025 13:13:38 -0700 Subject: [PATCH 07/24] Implement shrinking, add basic tests for core operations --- libs/neon-shmem/src/hash.rs | 117 +++++++++++++++++------------- libs/neon-shmem/src/hash/core.rs | 61 +++++++++++----- libs/neon-shmem/src/hash/tests.rs | 113 ++++++++++++++++++++++------- 3 files changed, 193 insertions(+), 98 deletions(-) diff --git a/libs/neon-shmem/src/hash.rs b/libs/neon-shmem/src/hash.rs index 9b1c1cee89..907a32bfca 100644 --- a/libs/neon-shmem/src/hash.rs +++ b/libs/neon-shmem/src/hash.rs @@ -19,7 +19,7 @@ pub mod entry; #[cfg(test)] mod tests; -use core::CoreHashMap; +use core::{CoreHashMap, INVALID_POS}; use entry::{Entry, OccupiedEntry}; #[derive(Debug)] @@ -210,6 +210,53 @@ where map.inner.buckets_in_use as usize } + /// Helper function that abstracts the common logic between growing and shrinking. + /// The only significant difference in the rehashing step is how many buckets to rehash! + fn rehash_dict( + &mut self, + inner: &mut CoreHashMap<'a, K, V>, + buckets_ptr: *mut core::Bucket, + end_ptr: *mut u8, + num_buckets: u32, + rehash_buckets: u32, + ) { + // Recalculate the dictionary + let buckets; + let dictionary; + unsafe { + let buckets_end_ptr = buckets_ptr.add(num_buckets as usize); + let dictionary_ptr: *mut u32 = buckets_end_ptr + .byte_add(buckets_end_ptr.align_offset(align_of::())) + .cast(); + let dictionary_size: usize = + end_ptr.byte_offset_from(buckets_end_ptr) as usize / size_of::(); + + buckets = std::slice::from_raw_parts_mut(buckets_ptr, num_buckets as usize); + dictionary = std::slice::from_raw_parts_mut(dictionary_ptr, dictionary_size); + } + for i in 0..dictionary.len() { + dictionary[i] = INVALID_POS; + } + + for i in 0..rehash_buckets as usize { + if buckets[i].inner.is_none() { + continue; + } + + let mut hasher = DefaultHasher::new(); + buckets[i].inner.as_ref().unwrap().0.hash(&mut hasher); + let hash = hasher.finish(); + + let pos: usize = (hash % dictionary.len() as u64) as usize; + buckets[i].next = dictionary[pos]; + dictionary[pos] = i as u32; + } + + // Finally, update the CoreHashMap struct + inner.dictionary = dictionary; + inner.buckets = buckets; + } + /// Grow /// /// 1. grow the underlying shared memory area @@ -247,46 +294,17 @@ where } else { inner.free_head }, + prev: if i > 0 { + i as u32 - 1 + } else { + INVALID_POS + }, inner: None, }); } } - // Recalculate the dictionary - let buckets; - let dictionary; - unsafe { - let buckets_end_ptr = buckets_ptr.add(num_buckets as usize); - let dictionary_ptr: *mut u32 = buckets_end_ptr - .byte_add(buckets_end_ptr.align_offset(align_of::())) - .cast(); - let dictionary_size: usize = - end_ptr.byte_offset_from(buckets_end_ptr) as usize / size_of::(); - - buckets = std::slice::from_raw_parts_mut(buckets_ptr, num_buckets as usize); - dictionary = std::slice::from_raw_parts_mut(dictionary_ptr, dictionary_size); - } - for i in 0..dictionary.len() { - dictionary[i] = core::INVALID_POS; - } - - for i in 0..old_num_buckets as usize { - if buckets[i].inner.is_none() { - continue; - } - - let mut hasher = DefaultHasher::new(); - buckets[i].inner.as_ref().unwrap().0.hash(&mut hasher); - let hash = hasher.finish(); - - let pos: usize = (hash % dictionary.len() as u64) as usize; - buckets[i].next = dictionary[pos]; - dictionary[pos] = i as u32; - } - - // Finally, update the CoreHashMap struct - inner.dictionary = dictionary; - inner.buckets = buckets; + self.rehash_dict(inner, buckets_ptr, end_ptr, num_buckets, old_num_buckets); inner.free_head = old_num_buckets; Ok(()) @@ -294,7 +312,7 @@ where fn begin_shrink(&mut self, num_buckets: u32) { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); - if num_buckets < map.inner.get_num_buckets() as u32 { + if num_buckets > map.inner.get_num_buckets() as u32 { panic!("shrink called with a larger number of buckets"); } map.inner.alloc_limit = num_buckets; @@ -307,14 +325,14 @@ where panic!("called finish_shrink when no shrink is in progress"); } - let new_num_buckets = inner.alloc_limit; + let num_buckets = inner.alloc_limit; - if inner.get_num_buckets() == new_num_buckets as usize { + if inner.get_num_buckets() == num_buckets as usize { return Ok(()); } - for b in &inner.buckets[new_num_buckets as usize..] { - if b.inner.is_some() { + for i in (num_buckets as usize)..inner.buckets.len() { + if inner.buckets[i].inner.is_some() { // TODO(quantumish) Do we want to treat this as a violation of an invariant // or a legitimate error the caller can run into? Originally I thought this // could return something like a UnevictedError(index) as soon as it runs @@ -324,6 +342,10 @@ where // Would require making a wider error type enum with this and shmem errors. panic!("unevicted entries in shrinked space") } + let prev_pos = inner.buckets[i].prev; + if prev_pos != INVALID_POS { + inner.buckets[prev_pos as usize].next = inner.buckets[i].next; + } } let shmem_handle = self @@ -331,22 +353,13 @@ where .as_ref() .expect("shrink called on a fixed-size hash table"); - let size_bytes = HashMapInit::::estimate_size(new_num_buckets); + let size_bytes = HashMapInit::::estimate_size(num_buckets); shmem_handle.set_size(size_bytes)?; let end_ptr: *mut u8 = unsafe { shmem_handle.data_ptr.as_ptr().add(size_bytes) }; - let buckets_ptr = inner.buckets.as_mut_ptr(); + self.rehash_dict(inner, buckets_ptr, end_ptr, num_buckets, num_buckets); Ok(()) } - // TODO: Shrinking is a multi-step process that requires co-operation from the caller - // - // 1. The caller must first call begin_shrink(). That forbids allocation of higher-numbered - // buckets. - // - // 2. Next, the caller must evict all entries in higher-numbered buckets. - // - // 3. Finally, call finish_shrink(). This recomputes the dictionary and shrinks the underlying - // shmem area } diff --git a/libs/neon-shmem/src/hash/core.rs b/libs/neon-shmem/src/hash/core.rs index eb60e21bad..1e0ebede4a 100644 --- a/libs/neon-shmem/src/hash/core.rs +++ b/libs/neon-shmem/src/hash/core.rs @@ -13,6 +13,7 @@ pub(crate) const INVALID_POS: u32 = u32::MAX; // Bucket pub(crate) struct Bucket { pub(crate) next: u32, + pub(crate) prev: u32, pub(crate) inner: Option<(K, V)>, } @@ -22,6 +23,7 @@ pub(crate) struct CoreHashMap<'a, K, V> { pub(crate) free_head: u32, pub(crate) _user_list_head: u32, + /// Maximum index of a bucket allowed to be allocated. INVALID_POS if no limit. pub(crate) alloc_limit: u32, // metrics @@ -62,6 +64,11 @@ where } else { INVALID_POS }, + prev: if i > 0 { + i as u32 - 1 + } else { + INVALID_POS + }, inner: None, }); } @@ -153,45 +160,61 @@ where self.alloc_limit != INVALID_POS } - pub fn entry_at_bucket(&mut self, pos: usize) -> Option> { - if pos >= self.buckets.len() { - return None; - } + pub fn entry_at_bucket(&mut self, pos: usize) -> Option> { + if pos >= self.buckets.len() { + return None; + } + let prev = self.buckets[pos].prev; let entry = self.buckets[pos].inner.as_ref(); if entry.is_none() { return None; - } - - let (key, _) = entry.unwrap(); + } + + let (key, _) = entry.unwrap(); Some(OccupiedEntry { _key: key.clone(), // TODO(quantumish): clone unavoidable? bucket_pos: pos as u32, map: self, - prev_pos: todo!(), // TODO(quantumish): possibly needs O(n) traversals to rediscover - costly! + prev_pos: if prev == INVALID_POS { + // TODO(quantumish): populating this correctly would require an O(n) scan over the dictionary + // (perhaps not if we refactored the prev field to be itself something like PrevPos). The real + // question though is whether this even needs to be populated correctly? All downstream uses of + // this function so far are just for deletion, which isn't really concerned with the dictionary. + // Then again, it's unintuitive to appear to return a normal OccupiedEntry which really is fake. + PrevPos::First(todo!("unclear what to do here")) + } else { + PrevPos::Chained(prev) + } }) } pub(crate) fn alloc_bucket(&mut self, key: K, value: V) -> Result { - let mut pos = self.free_head; + let mut pos = self.free_head; - // TODO(quantumish): relies on INVALID_POS being u32::MAX by default! - // instead add a clause `pos != INVALID_POS`? let mut prev = PrevPos::First(self.free_head); - while pos < self.alloc_limit { - if pos == INVALID_POS { - return Err(FullError()); - } + while pos!= INVALID_POS && pos >= self.alloc_limit { let bucket = &mut self.buckets[pos as usize]; prev = PrevPos::Chained(pos); pos = bucket.next; } - let bucket = &mut self.buckets[pos as usize]; + if pos == INVALID_POS { + return Err(FullError()); + } match prev { - PrevPos::First(_) => self.free_head = bucket.next, - PrevPos::Chained(p) => self.buckets[p].next = bucket.next, + PrevPos::First(_) => { + let next_pos = self.buckets[pos as usize].next; + self.free_head = next_pos; + self.buckets[next_pos as usize].prev = INVALID_POS; + } + PrevPos::Chained(p) => if p != INVALID_POS { + let next_pos = self.buckets[pos as usize].next; + self.buckets[p as usize].next = next_pos; + self.buckets[next_pos as usize].prev = p; + }, } + let bucket = &mut self.buckets[pos as usize]; self.buckets_in_use += 1; bucket.next = INVALID_POS; bucket.inner = Some((key, value)); @@ -199,3 +222,5 @@ where return Ok(pos); } } + + diff --git a/libs/neon-shmem/src/hash/tests.rs b/libs/neon-shmem/src/hash/tests.rs index 073aea5220..c207e35a56 100644 --- a/libs/neon-shmem/src/hash/tests.rs +++ b/libs/neon-shmem/src/hash/tests.rs @@ -5,7 +5,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use crate::hash::HashMapAccess; use crate::hash::HashMapInit; -use crate::hash::UpdateAction; +use crate::hash::Entry; use crate::shmem::ShmemHandle; use rand::seq::SliceRandom; @@ -35,20 +35,28 @@ impl<'a> From<&'a [u8]> for TestKey { } } -fn test_inserts + Copy>(keys: &[K]) { +fn test_inserts + Copy>(keys: &[K]) { const MAX_MEM_SIZE: usize = 10000000; let shmem = ShmemHandle::new("test_inserts", 0, MAX_MEM_SIZE).unwrap(); let init_struct = HashMapInit::::init_in_shmem(100000, shmem); - let w = init_struct.attach_writer(); + let mut w = init_struct.attach_writer(); for (idx, k) in keys.iter().enumerate() { - let res = w.insert(&(*k).into(), idx); - assert!(res.is_ok()); + let hash = w.get_hash_value(&(*k).into()); + let res = w.entry_with_hash((*k).into(), hash); + match res { + Entry::Occupied(mut e) => { e.insert(idx); } + Entry::Vacant(e) => { + let res = e.insert(idx); + assert!(res.is_ok()); + }, + }; } for (idx, k) in keys.iter().enumerate() { - let x = w.get(&(*k).into()); + let hash = w.get_hash_value(&(*k).into()); + let x = w.get_with_hash(&(*k).into(), hash); let value = x.as_deref().copied(); assert_eq!(value, Some(idx)); } @@ -121,7 +129,7 @@ struct TestOp(TestKey, Option); fn apply_op( op: &TestOp, - sut: &HashMapAccess, + map: &mut HashMapAccess, shadow: &mut BTreeMap, ) { eprintln!("applying op: {op:?}"); @@ -133,21 +141,24 @@ fn apply_op( shadow.remove(&op.0) }; - // apply to Art tree - sut.update_with_fn(&op.0, |existing| { - assert_eq!(existing.map(TestValue::load), shadow_existing); + let hash = map.get_hash_value(&op.0); + let entry = map.entry_with_hash(op.0, hash); + let hash_existing = match op.1 { + Some(new) => { + match entry { + Entry::Occupied(mut e) => Some(e.insert(new)), + Entry::Vacant(e) => { e.insert(new).unwrap(); None }, + } + }, + None => { + match entry { + Entry::Occupied(e) => Some(e.remove()), + Entry::Vacant(_) => None, + } + }, + }; - match (existing, op.1) { - (None, None) => UpdateAction::Nothing, - (None, Some(new_val)) => UpdateAction::Insert(TestValue::new(new_val)), - (Some(_old_val), None) => UpdateAction::Remove, - (Some(old_val), Some(new_val)) => { - old_val.0.store(new_val, Ordering::Relaxed); - UpdateAction::Nothing - } - } - }) - .expect("out of memory"); + assert_eq!(shadow_existing, hash_existing); } #[test] @@ -155,8 +166,8 @@ fn random_ops() { const MAX_MEM_SIZE: usize = 10000000; let shmem = ShmemHandle::new("test_inserts", 0, MAX_MEM_SIZE).unwrap(); - let init_struct = HashMapInit::::init_in_shmem(100000, shmem); - let writer = init_struct.attach_writer(); + let init_struct = HashMapInit::::init_in_shmem(100000, shmem); + let mut writer = init_struct.attach_writer(); let mut shadow: std::collections::BTreeMap = BTreeMap::new(); @@ -167,7 +178,7 @@ fn random_ops() { let op = TestOp(key, if rng.random_bool(0.75) { Some(i) } else { None }); - apply_op(&op, &writer, &mut shadow); + apply_op(&op, &mut writer, &mut shadow); if i % 1000 == 0 { eprintln!("{i} ops processed"); @@ -182,8 +193,8 @@ fn test_grow() { const MEM_SIZE: usize = 10000000; let shmem = ShmemHandle::new("test_grow", 0, MEM_SIZE).unwrap(); - let init_struct = HashMapInit::::init_in_shmem(1000, shmem); - let writer = init_struct.attach_writer(); + let init_struct = HashMapInit::::init_in_shmem(1000, shmem); + let mut writer = init_struct.attach_writer(); let mut shadow: std::collections::BTreeMap = BTreeMap::new(); @@ -193,7 +204,7 @@ fn test_grow() { let op = TestOp(key, if rng.random_bool(0.75) { Some(i) } else { None }); - apply_op(&op, &writer, &mut shadow); + apply_op(&op, &mut writer, &mut shadow); if i % 1000 == 0 { eprintln!("{i} ops processed"); @@ -209,7 +220,7 @@ fn test_grow() { let op = TestOp(key, if rng.random_bool(0.75) { Some(i) } else { None }); - apply_op(&op, &writer, &mut shadow); + apply_op(&op, &mut writer, &mut shadow); if i % 1000 == 0 { eprintln!("{i} ops processed"); @@ -218,3 +229,49 @@ fn test_grow() { } } } + + +#[test] +fn test_shrink() { + const MEM_SIZE: usize = 10000000; + let shmem = ShmemHandle::new("test_shrink", 0, MEM_SIZE).unwrap(); + + let init_struct = HashMapInit::::init_in_shmem(1500, shmem); + let mut writer = init_struct.attach_writer(); + + let mut shadow: std::collections::BTreeMap = BTreeMap::new(); + + let mut rng = rand::rng(); + for i in 0..100 { + let key: TestKey = ((rng.next_u32() % 1500) as u128).into(); + + let op = TestOp(key, if rng.random_bool(0.75) { Some(i) } else { None }); + + apply_op(&op, &mut writer, &mut shadow); + + if i % 1000 == 0 { + eprintln!("{i} ops processed"); + } + } + + writer.begin_shrink(1000); + for i in 1000..1500 { + if let Some(entry) = writer.entry_at_bucket(i) { + shadow.remove(&entry._key); + entry.remove(); + } + } + writer.finish_shrink().unwrap(); + + for i in 0..10000 { + let key: TestKey = ((rng.next_u32() % 1000) as u128).into(); + + let op = TestOp(key, if rng.random_bool(0.75) { Some(i) } else { None }); + + apply_op(&op, &mut writer, &mut shadow); + + if i % 1000 == 0 { + eprintln!("{i} ops processed"); + } + } +} From bb1e359872ca19b311466d8e3a363fb41b849cdf Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Mon, 16 Jun 2025 16:01:46 -0700 Subject: [PATCH 08/24] Add testing utilities for hash map, freelist bugfixes --- libs/neon-shmem/src/hash.rs | 31 +++-- libs/neon-shmem/src/hash/core.rs | 36 +++--- libs/neon-shmem/src/hash/entry.rs | 12 +- libs/neon-shmem/src/hash/tests.rs | 185 ++++++++++++++++++------------ 4 files changed, 161 insertions(+), 103 deletions(-) diff --git a/libs/neon-shmem/src/hash.rs b/libs/neon-shmem/src/hash.rs index 907a32bfca..45e593fc48 100644 --- a/libs/neon-shmem/src/hash.rs +++ b/libs/neon-shmem/src/hash.rs @@ -20,7 +20,7 @@ pub mod entry; mod tests; use core::{CoreHashMap, INVALID_POS}; -use entry::{Entry, OccupiedEntry}; +use entry::{Entry, OccupiedEntry, PrevPos}; #[derive(Debug)] pub struct OutOfMemoryError(); @@ -289,15 +289,15 @@ where for i in old_num_buckets..num_buckets { let bucket_ptr = buckets_ptr.add(i as usize); bucket_ptr.write(core::Bucket { - next: if i < num_buckets { + next: if i < num_buckets-1 { i as u32 + 1 } else { inner.free_head }, prev: if i > 0 { - i as u32 - 1 + PrevPos::Chained(i as u32 - 1) } else { - INVALID_POS + PrevPos::First(INVALID_POS) }, inner: None, }); @@ -315,6 +315,10 @@ where if num_buckets > map.inner.get_num_buckets() as u32 { panic!("shrink called with a larger number of buckets"); } + _ = self + .shmem_handle + .as_ref() + .expect("shrink called on a fixed-size hash table"); map.inner.alloc_limit = num_buckets; } @@ -342,9 +346,21 @@ where // Would require making a wider error type enum with this and shmem errors. panic!("unevicted entries in shrinked space") } - let prev_pos = inner.buckets[i].prev; - if prev_pos != INVALID_POS { - inner.buckets[prev_pos as usize].next = inner.buckets[i].next; + match inner.buckets[i].prev { + PrevPos::First(_) => { + let next_pos = inner.buckets[i].next; + inner.free_head = next_pos; + if next_pos != INVALID_POS { + inner.buckets[next_pos as usize].prev = PrevPos::First(INVALID_POS); + } + }, + PrevPos::Chained(j) => { + let next_pos = inner.buckets[i].next; + inner.buckets[j as usize].next = next_pos; + if next_pos != INVALID_POS { + inner.buckets[next_pos as usize].prev = PrevPos::Chained(j); + } + } } } @@ -358,6 +374,7 @@ where let end_ptr: *mut u8 = unsafe { shmem_handle.data_ptr.as_ptr().add(size_bytes) }; let buckets_ptr = inner.buckets.as_mut_ptr(); self.rehash_dict(inner, buckets_ptr, end_ptr, num_buckets, num_buckets); + inner.alloc_limit = INVALID_POS; Ok(()) } diff --git a/libs/neon-shmem/src/hash/core.rs b/libs/neon-shmem/src/hash/core.rs index 1e0ebede4a..d02d234a87 100644 --- a/libs/neon-shmem/src/hash/core.rs +++ b/libs/neon-shmem/src/hash/core.rs @@ -13,7 +13,7 @@ pub(crate) const INVALID_POS: u32 = u32::MAX; // Bucket pub(crate) struct Bucket { pub(crate) next: u32, - pub(crate) prev: u32, + pub(crate) prev: PrevPos, pub(crate) inner: Option<(K, V)>, } @@ -65,9 +65,9 @@ where INVALID_POS }, prev: if i > 0 { - i as u32 - 1 + PrevPos::Chained(i as u32 - 1) } else { - INVALID_POS + PrevPos::First(INVALID_POS) }, inner: None, }); @@ -176,24 +176,15 @@ where _key: key.clone(), // TODO(quantumish): clone unavoidable? bucket_pos: pos as u32, map: self, - prev_pos: if prev == INVALID_POS { - // TODO(quantumish): populating this correctly would require an O(n) scan over the dictionary - // (perhaps not if we refactored the prev field to be itself something like PrevPos). The real - // question though is whether this even needs to be populated correctly? All downstream uses of - // this function so far are just for deletion, which isn't really concerned with the dictionary. - // Then again, it's unintuitive to appear to return a normal OccupiedEntry which really is fake. - PrevPos::First(todo!("unclear what to do here")) - } else { - PrevPos::Chained(prev) - } + prev_pos: prev, }) } - + pub(crate) fn alloc_bucket(&mut self, key: K, value: V) -> Result { let mut pos = self.free_head; let mut prev = PrevPos::First(self.free_head); - while pos!= INVALID_POS && pos >= self.alloc_limit { + while pos != INVALID_POS && pos >= self.alloc_limit { let bucket = &mut self.buckets[pos as usize]; prev = PrevPos::Chained(pos); pos = bucket.next; @@ -204,16 +195,23 @@ where match prev { PrevPos::First(_) => { let next_pos = self.buckets[pos as usize].next; - self.free_head = next_pos; - self.buckets[next_pos as usize].prev = INVALID_POS; + self.free_head = next_pos; + // HACK(quantumish): Really, the INVALID_POS should be the position within the dictionary. + // This isn't passed into this function, though, and so for now rather than changing that + // we can just check it from `alloc_bucket`. Not a great solution. + if next_pos != INVALID_POS { + self.buckets[next_pos as usize].prev = PrevPos::First(INVALID_POS); + } } PrevPos::Chained(p) => if p != INVALID_POS { let next_pos = self.buckets[pos as usize].next; self.buckets[p as usize].next = next_pos; - self.buckets[next_pos as usize].prev = p; + if next_pos != INVALID_POS { + self.buckets[next_pos as usize].prev = PrevPos::Chained(p); + } }, } - + let bucket = &mut self.buckets[pos as usize]; self.buckets_in_use += 1; bucket.next = INVALID_POS; diff --git a/libs/neon-shmem/src/hash/entry.rs b/libs/neon-shmem/src/hash/entry.rs index cb9efcf742..147a464745 100644 --- a/libs/neon-shmem/src/hash/entry.rs +++ b/libs/neon-shmem/src/hash/entry.rs @@ -10,6 +10,7 @@ pub enum Entry<'a, 'b, K, V> { Vacant(VacantEntry<'a, 'b, K, V>), } +#[derive(Clone, Copy)] pub(crate) enum PrevPos { First(u32), Chained(u32), @@ -58,10 +59,14 @@ impl<'a, 'b, K, V> OccupiedEntry<'a, 'b, K, V> { } } - // and add it to the freelist + // and add it to the freelist + if self.map.free_head != INVALID_POS { + self.map.buckets[self.map.free_head as usize].prev = PrevPos::Chained(self.bucket_pos); + } let bucket = &mut self.map.buckets[self.bucket_pos as usize]; let old_value = bucket.inner.take(); - bucket.next = self.map.free_head; + bucket.next = self.map.free_head; + bucket.prev = PrevPos::First(INVALID_POS); self.map.free_head = self.bucket_pos; self.map.buckets_in_use -= 1; @@ -82,6 +87,9 @@ impl<'a, 'b, K: Clone + Hash + Eq, V> VacantEntry<'a, 'b, K, V> { return Err(FullError()); } let bucket = &mut self.map.buckets[pos as usize]; + if let PrevPos::First(INVALID_POS) = bucket.prev { + bucket.prev = PrevPos::First(self.dict_pos); + } bucket.next = self.map.dictionary[self.dict_pos as usize]; self.map.dictionary[self.dict_pos as usize] = pos; diff --git a/libs/neon-shmem/src/hash/tests.rs b/libs/neon-shmem/src/hash/tests.rs index c207e35a56..b95c33e578 100644 --- a/libs/neon-shmem/src/hash/tests.rs +++ b/libs/neon-shmem/src/hash/tests.rs @@ -1,6 +1,8 @@ use std::collections::BTreeMap; use std::collections::HashSet; use std::fmt::{Debug, Formatter}; +use std::mem::uninitialized; +use std::mem::MaybeUninit; use std::sync::atomic::{AtomicUsize, Ordering}; use crate::hash::HashMapAccess; @@ -132,8 +134,6 @@ fn apply_op( map: &mut HashMapAccess, shadow: &mut BTreeMap, ) { - eprintln!("applying op: {op:?}"); - // apply the change to the shadow tree first let shadow_existing = if let Some(v) = op.1 { shadow.insert(op.0, v) @@ -161,16 +161,42 @@ fn apply_op( assert_eq!(shadow_existing, hash_existing); } +fn do_random_ops( + num_ops: usize, + size: u32, + del_prob: f64, + writer: &mut HashMapAccess, + shadow: &mut BTreeMap, + rng: &mut rand::rngs::ThreadRng, +) { + for i in 0..num_ops { + let key: TestKey = ((rng.next_u32() % size) as u128).into(); + let op = TestOp(key, if rng.random_bool(del_prob) { Some(i) } else { None }); + apply_op(&op, writer, shadow); + } +} + +fn do_deletes( + num_ops: usize, + writer: &mut HashMapAccess, + shadow: &mut BTreeMap, +) { + for i in 0..num_ops { + let (k, _) = shadow.pop_first().unwrap(); + let hash = writer.get_hash_value(&k); + writer.remove_with_hash(&k, hash); + } +} + #[test] fn random_ops() { - const MAX_MEM_SIZE: usize = 10000000; - let shmem = ShmemHandle::new("test_inserts", 0, MAX_MEM_SIZE).unwrap(); + let shmem = ShmemHandle::new("test_inserts", 0, 10000000).unwrap(); let init_struct = HashMapInit::::init_in_shmem(100000, shmem); let mut writer = init_struct.attach_writer(); - let mut shadow: std::collections::BTreeMap = BTreeMap::new(); + let distribution = Zipf::new(u128::MAX as f64, 1.1).unwrap(); let mut rng = rand::rng(); for i in 0..100000 { @@ -190,88 +216,97 @@ fn random_ops() { #[test] fn test_grow() { - const MEM_SIZE: usize = 10000000; - let shmem = ShmemHandle::new("test_grow", 0, MEM_SIZE).unwrap(); - + let shmem = ShmemHandle::new("test_grow", 0, 10000000).unwrap(); let init_struct = HashMapInit::::init_in_shmem(1000, shmem); let mut writer = init_struct.attach_writer(); - let mut shadow: std::collections::BTreeMap = BTreeMap::new(); - let mut rng = rand::rng(); - for i in 0..10000 { - let key: TestKey = ((rng.next_u32() % 1000) as u128).into(); - - let op = TestOp(key, if rng.random_bool(0.75) { Some(i) } else { None }); - - apply_op(&op, &mut writer, &mut shadow); - - if i % 1000 == 0 { - eprintln!("{i} ops processed"); - //eprintln!("stats: {:?}", tree_writer.get_statistics()); - //test_iter(&tree_writer, &shadow); - } - } + do_random_ops(10000, 1000, 0.75, &mut writer, &mut shadow, &mut rng); writer.grow(1500).unwrap(); - - for i in 0..10000 { - let key: TestKey = ((rng.next_u32() % 1500) as u128).into(); - - let op = TestOp(key, if rng.random_bool(0.75) { Some(i) } else { None }); - - apply_op(&op, &mut writer, &mut shadow); - - if i % 1000 == 0 { - eprintln!("{i} ops processed"); - //eprintln!("stats: {:?}", tree_writer.get_statistics()); - //test_iter(&tree_writer, &shadow); - } - } + do_random_ops(10000, 1500, 0.75, &mut writer, &mut shadow, &mut rng); } - -#[test] -fn test_shrink() { - const MEM_SIZE: usize = 10000000; - let shmem = ShmemHandle::new("test_shrink", 0, MEM_SIZE).unwrap(); - - let init_struct = HashMapInit::::init_in_shmem(1500, shmem); - let mut writer = init_struct.attach_writer(); - - let mut shadow: std::collections::BTreeMap = BTreeMap::new(); - - let mut rng = rand::rng(); - for i in 0..100 { - let key: TestKey = ((rng.next_u32() % 1500) as u128).into(); - - let op = TestOp(key, if rng.random_bool(0.75) { Some(i) } else { None }); - - apply_op(&op, &mut writer, &mut shadow); - - if i % 1000 == 0 { - eprintln!("{i} ops processed"); - } - } - - writer.begin_shrink(1000); - for i in 1000..1500 { - if let Some(entry) = writer.entry_at_bucket(i) { +fn do_shrink( + writer: &mut HashMapAccess, + shadow: &mut BTreeMap, + from: u32, + to: u32 +) { + writer.begin_shrink(to); + for i in to..from { + if let Some(entry) = writer.entry_at_bucket(i as usize) { shadow.remove(&entry._key); entry.remove(); } } writer.finish_shrink().unwrap(); - - for i in 0..10000 { - let key: TestKey = ((rng.next_u32() % 1000) as u128).into(); - let op = TestOp(key, if rng.random_bool(0.75) { Some(i) } else { None }); - - apply_op(&op, &mut writer, &mut shadow); - - if i % 1000 == 0 { - eprintln!("{i} ops processed"); - } - } +} + +#[test] +fn test_shrink() { + let shmem = ShmemHandle::new("test_shrink", 0, 10000000).unwrap(); + let init_struct = HashMapInit::::init_in_shmem(1500, shmem); + let mut writer = init_struct.attach_writer(); + let mut shadow: std::collections::BTreeMap = BTreeMap::new(); + let mut rng = rand::rng(); + + do_random_ops(10000, 1500, 0.75, &mut writer, &mut shadow, &mut rng); + do_shrink(&mut writer, &mut shadow, 1500, 1000); + do_deletes(500, &mut writer, &mut shadow); + do_random_ops(10000, 500, 0.75, &mut writer, &mut shadow, &mut rng); + assert!(writer.get_num_buckets_in_use() <= 1000); +} + +#[test] +fn test_shrink_grow_seq() { + let shmem = ShmemHandle::new("test_shrink", 0, 10000000).unwrap(); + let init_struct = HashMapInit::::init_in_shmem(1500, shmem); + let mut writer = init_struct.attach_writer(); + let mut shadow: std::collections::BTreeMap = BTreeMap::new(); + let mut rng = rand::rng(); + + do_random_ops(500, 1000, 0.1, &mut writer, &mut shadow, &mut rng); + eprintln!("Shrinking to 750"); + do_shrink(&mut writer, &mut shadow, 1000, 750); + do_random_ops(200, 1000, 0.5, &mut writer, &mut shadow, &mut rng); + eprintln!("Growing to 1500"); + writer.grow(1500).unwrap(); + do_random_ops(600, 1500, 0.1, &mut writer, &mut shadow, &mut rng); + eprintln!("Shrinking to 200"); + do_shrink(&mut writer, &mut shadow, 1500, 200); + do_deletes(100, &mut writer, &mut shadow); + do_random_ops(50, 1500, 0.25, &mut writer, &mut shadow, &mut rng); + eprintln!("Growing to 10k"); + writer.grow(10000).unwrap(); + do_random_ops(10000, 5000, 0.25, &mut writer, &mut shadow, &mut rng); +} + + +#[test] +#[should_panic] +fn test_shrink_bigger() { + let shmem = ShmemHandle::new("test_shrink", 0, 10000000).unwrap(); + let init_struct = HashMapInit::::init_in_shmem(1500, shmem); + let mut writer = init_struct.attach_writer(); + writer.begin_shrink(2000); +} + +#[test] +#[should_panic] +fn test_shrink_early_finish() { + let shmem = ShmemHandle::new("test_shrink", 0, 10000000).unwrap(); + let init_struct = HashMapInit::::init_in_shmem(1500, shmem); + let mut writer = init_struct.attach_writer(); + writer.finish_shrink().unwrap(); +} + +#[test] +#[should_panic] +fn test_shrink_fixed_size() { + let mut area = [MaybeUninit::uninit(); 10000]; + let init_struct = HashMapInit::::init_in_fixed_area(3, &mut area); + let mut writer = init_struct.attach_writer(); + writer.begin_shrink(1); } From 477648b8cdf67600c2a6efeae572c4e348f092e3 Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Tue, 17 Jun 2025 11:23:10 -0700 Subject: [PATCH 09/24] Clean up hashmap implementation, add bucket tests --- libs/neon-shmem/src/hash.rs | 10 +-- libs/neon-shmem/src/hash/core.rs | 36 +++++----- libs/neon-shmem/src/hash/entry.rs | 2 +- libs/neon-shmem/src/hash/tests.rs | 107 +++++++++++++++++++++++------- 4 files changed, 108 insertions(+), 47 deletions(-) diff --git a/libs/neon-shmem/src/hash.rs b/libs/neon-shmem/src/hash.rs index 45e593fc48..f173f42a6d 100644 --- a/libs/neon-shmem/src/hash.rs +++ b/libs/neon-shmem/src/hash.rs @@ -131,7 +131,7 @@ where } HashMapInit { - shmem_handle: shmem_handle, + shmem_handle, shared_ptr, } } @@ -211,7 +211,7 @@ where } /// Helper function that abstracts the common logic between growing and shrinking. - /// The only significant difference in the rehashing step is how many buckets to rehash! + /// The only significant difference in the rehashing step is how many buckets to rehash. fn rehash_dict( &mut self, inner: &mut CoreHashMap<'a, K, V>, @@ -310,7 +310,8 @@ where Ok(()) } - fn begin_shrink(&mut self, num_buckets: u32) { + /// Begin a shrink, limiting all new allocations to be in buckets with index less than `num_buckets`. + pub fn begin_shrink(&mut self, num_buckets: u32) { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); if num_buckets > map.inner.get_num_buckets() as u32 { panic!("shrink called with a larger number of buckets"); @@ -322,7 +323,8 @@ where map.inner.alloc_limit = num_buckets; } - fn finish_shrink(&mut self) -> Result<(), crate::shmem::Error> { + /// Complete a shrink after caller has evicted entries, removing the unused buckets and rehashing. + pub fn finish_shrink(&mut self) -> Result<(), crate::shmem::Error> { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); let inner = &mut map.inner; if !inner.is_shrinking() { diff --git a/libs/neon-shmem/src/hash/core.rs b/libs/neon-shmem/src/hash/core.rs index d02d234a87..80f41fd8d4 100644 --- a/libs/neon-shmem/src/hash/core.rs +++ b/libs/neon-shmem/src/hash/core.rs @@ -159,7 +159,7 @@ where pub fn is_shrinking(&self) -> bool { self.alloc_limit != INVALID_POS } - + pub fn entry_at_bucket(&mut self, pos: usize) -> Option> { if pos >= self.buckets.len() { return None; @@ -167,22 +167,22 @@ where let prev = self.buckets[pos].prev; let entry = self.buckets[pos].inner.as_ref(); - if entry.is_none() { - return None; + match entry { + Some((key, _)) => Some(OccupiedEntry { + _key: key.clone(), + bucket_pos: pos as u32, + prev_pos: prev, + map: self, + }), + _ => None, } - - let (key, _) = entry.unwrap(); - Some(OccupiedEntry { - _key: key.clone(), // TODO(quantumish): clone unavoidable? - bucket_pos: pos as u32, - map: self, - prev_pos: prev, - }) } - - pub(crate) fn alloc_bucket(&mut self, key: K, value: V) -> Result { + + /// Find the position of an unused bucket via the freelist and initialize it. + pub(crate) fn alloc_bucket(&mut self, key: K, value: V, dict_pos: u32) -> Result { let mut pos = self.free_head; + // Find the first bucket we're *allowed* to use. let mut prev = PrevPos::First(self.free_head); while pos != INVALID_POS && pos >= self.alloc_limit { let bucket = &mut self.buckets[pos as usize]; @@ -192,15 +192,14 @@ where if pos == INVALID_POS { return Err(FullError()); } + + // Repair the freelist. match prev { PrevPos::First(_) => { let next_pos = self.buckets[pos as usize].next; self.free_head = next_pos; - // HACK(quantumish): Really, the INVALID_POS should be the position within the dictionary. - // This isn't passed into this function, though, and so for now rather than changing that - // we can just check it from `alloc_bucket`. Not a great solution. if next_pos != INVALID_POS { - self.buckets[next_pos as usize].prev = PrevPos::First(INVALID_POS); + self.buckets[next_pos as usize].prev = PrevPos::First(dict_pos); } } PrevPos::Chained(p) => if p != INVALID_POS { @@ -211,7 +210,8 @@ where } }, } - + + // Initialize the bucket. let bucket = &mut self.buckets[pos as usize]; self.buckets_in_use += 1; bucket.next = INVALID_POS; diff --git a/libs/neon-shmem/src/hash/entry.rs b/libs/neon-shmem/src/hash/entry.rs index 147a464745..64820b3d7b 100644 --- a/libs/neon-shmem/src/hash/entry.rs +++ b/libs/neon-shmem/src/hash/entry.rs @@ -82,7 +82,7 @@ pub struct VacantEntry<'a, 'b, K, V> { impl<'a, 'b, K: Clone + Hash + Eq, V> VacantEntry<'a, 'b, K, V> { pub fn insert(self, value: V) -> Result<&'b mut V, FullError> { - let pos = self.map.alloc_bucket(self.key, value)?; + let pos = self.map.alloc_bucket(self.key, value, self.dict_pos)?; if pos == INVALID_POS { return Err(FullError()); } diff --git a/libs/neon-shmem/src/hash/tests.rs b/libs/neon-shmem/src/hash/tests.rs index b95c33e578..ee6acfc144 100644 --- a/libs/neon-shmem/src/hash/tests.rs +++ b/libs/neon-shmem/src/hash/tests.rs @@ -62,8 +62,6 @@ fn test_inserts + Copy>(keys: &[K]) { let value = x.as_deref().copied(); assert_eq!(value, Some(idx)); } - - //eprintln!("stats: {:?}", tree_writer.get_statistics()); } #[test] @@ -188,6 +186,23 @@ fn do_deletes( } } +fn do_shrink( + writer: &mut HashMapAccess, + shadow: &mut BTreeMap, + from: u32, + to: u32 +) { + writer.begin_shrink(to); + for i in to..from { + if let Some(entry) = writer.entry_at_bucket(i as usize) { + shadow.remove(&entry._key); + entry.remove(); + } + } + writer.finish_shrink().unwrap(); + +} + #[test] fn random_ops() { let shmem = ShmemHandle::new("test_inserts", 0, 10000000).unwrap(); @@ -208,8 +223,6 @@ fn random_ops() { if i % 1000 == 0 { eprintln!("{i} ops processed"); - //eprintln!("stats: {:?}", tree_writer.get_statistics()); - //test_iter(&tree_writer, &shadow); } } } @@ -227,23 +240,6 @@ fn test_grow() { do_random_ops(10000, 1500, 0.75, &mut writer, &mut shadow, &mut rng); } -fn do_shrink( - writer: &mut HashMapAccess, - shadow: &mut BTreeMap, - from: u32, - to: u32 -) { - writer.begin_shrink(to); - for i in to..from { - if let Some(entry) = writer.entry_at_bucket(i as usize) { - shadow.remove(&entry._key); - entry.remove(); - } - } - writer.finish_shrink().unwrap(); - -} - #[test] fn test_shrink() { let shmem = ShmemHandle::new("test_shrink", 0, 10000000).unwrap(); @@ -261,7 +257,7 @@ fn test_shrink() { #[test] fn test_shrink_grow_seq() { - let shmem = ShmemHandle::new("test_shrink", 0, 10000000).unwrap(); + let shmem = ShmemHandle::new("test_shrink_grow_seq", 0, 10000000).unwrap(); let init_struct = HashMapInit::::init_in_shmem(1500, shmem); let mut writer = init_struct.attach_writer(); let mut shadow: std::collections::BTreeMap = BTreeMap::new(); @@ -283,11 +279,73 @@ fn test_shrink_grow_seq() { do_random_ops(10000, 5000, 0.25, &mut writer, &mut shadow, &mut rng); } +#[test] +fn test_bucket_ops() { + let shmem = ShmemHandle::new("test_bucket_ops", 0, 10000000).unwrap(); + let init_struct = HashMapInit::::init_in_shmem(1000, shmem); + let mut writer = init_struct.attach_writer(); + let hash = writer.get_hash_value(&1.into()); + match writer.entry_with_hash(1.into(), hash) { + Entry::Occupied(mut e) => { e.insert(2); }, + Entry::Vacant(e) => { e.insert(2).unwrap(); }, + } + assert_eq!(writer.get_num_buckets_in_use(), 1); + assert_eq!(writer.get_num_buckets(), 1000); + assert_eq!(writer.get_with_hash(&1.into(), hash), Some(&2)); + match writer.entry_with_hash(1.into(), hash) { + Entry::Occupied(e) => { + assert_eq!(e._key, 1.into()); + let pos = e.bucket_pos as usize; + assert_eq!(writer.entry_at_bucket(pos).unwrap()._key, 1.into()); + assert_eq!(writer.get_at_bucket(pos), Some(&(1.into(), 2))); + }, + Entry::Vacant(_) => { panic!("Insert didn't affect entry"); }, + } + writer.remove_with_hash(&1.into(), hash); + assert_eq!(writer.get_with_hash(&1.into(), hash), None); +} + +#[test] +fn test_shrink_zero() { + let shmem = ShmemHandle::new("test_shrink_zero", 0, 10000000).unwrap(); + let init_struct = HashMapInit::::init_in_shmem(1500, shmem); + let mut writer = init_struct.attach_writer(); + writer.begin_shrink(0); + for i in 0..1500 { + writer.entry_at_bucket(i).map(|x| x.remove()); + } + writer.finish_shrink().unwrap(); + assert_eq!(writer.get_num_buckets_in_use(), 0); + let hash = writer.get_hash_value(&1.into()); + let entry = writer.entry_with_hash(1.into(), hash); + if let Entry::Vacant(v) = entry { + assert!(v.insert(2).is_err()); + } else { + panic!("Somehow got non-vacant entry in empty map.") + } + writer.grow(50).unwrap(); + let entry = writer.entry_with_hash(1.into(), hash); + if let Entry::Vacant(v) = entry { + assert!(v.insert(2).is_ok()); + } else { + panic!("Somehow got non-vacant entry in empty map.") + } + assert_eq!(writer.get_num_buckets_in_use(), 1); +} + +#[test] +#[should_panic] +fn test_grow_oom() { + let shmem = ShmemHandle::new("test_grow_oom", 0, 500).unwrap(); + let init_struct = HashMapInit::::init_in_shmem(5, shmem); + let mut writer = init_struct.attach_writer(); + writer.grow(20000).unwrap(); +} #[test] #[should_panic] fn test_shrink_bigger() { - let shmem = ShmemHandle::new("test_shrink", 0, 10000000).unwrap(); + let shmem = ShmemHandle::new("test_shrink_bigger", 0, 10000000).unwrap(); let init_struct = HashMapInit::::init_in_shmem(1500, shmem); let mut writer = init_struct.attach_writer(); writer.begin_shrink(2000); @@ -296,7 +354,7 @@ fn test_shrink_bigger() { #[test] #[should_panic] fn test_shrink_early_finish() { - let shmem = ShmemHandle::new("test_shrink", 0, 10000000).unwrap(); + let shmem = ShmemHandle::new("test_shrink_early_finish", 0, 10000000).unwrap(); let init_struct = HashMapInit::::init_in_shmem(1500, shmem); let mut writer = init_struct.attach_writer(); writer.finish_shrink().unwrap(); @@ -310,3 +368,4 @@ fn test_shrink_fixed_size() { let mut writer = init_struct.attach_writer(); writer.begin_shrink(1); } + From 610ea22c46dab4a59825434b7ac0b4f6aaa0338f Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Fri, 20 Jun 2025 11:46:02 -0700 Subject: [PATCH 10/24] Generalize map to allow arbitrary hash fns, add clear() helper method --- Cargo.lock | 2 + libs/neon-shmem/Cargo.toml | 12 ++++ libs/neon-shmem/src/hash.rs | 108 +++++++++++++++++++++---------- libs/neon-shmem/src/hash/core.rs | 29 ++++++++- 4 files changed, 117 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bafcaea594..2f50f11cee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3843,9 +3843,11 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" name = "neon-shmem" version = "0.1.0" dependencies = [ + "criterion", "nix 0.30.1", "rand 0.9.1", "rand_distr 0.5.1", + "rustc-hash 1.1.0", "tempfile", "thiserror 1.0.69", "workspace_hack", diff --git a/libs/neon-shmem/Cargo.toml b/libs/neon-shmem/Cargo.toml index de65f3e8cc..284a19d55d 100644 --- a/libs/neon-shmem/Cargo.toml +++ b/libs/neon-shmem/Cargo.toml @@ -8,10 +8,22 @@ license.workspace = true thiserror.workspace = true nix.workspace = true workspace_hack = { version = "0.1", path = "../../workspace_hack" } +rustc-hash = { version = "2.1.1" } [dev-dependencies] +criterion = { workspace = true, features = ["html_reports"] } rand = "0.9.1" rand_distr = "0.5.1" +xxhash-rust = { version = "0.8.15", features = ["xxh3"] } +ahash.workspace = true [target.'cfg(target_os = "macos")'.dependencies] tempfile = "3.14.0" + +[[bench]] +name = "hmap_resize" +harness = false + +[[bin]] +name = "hmap_test" +path = "main.rs" diff --git a/libs/neon-shmem/src/hash.rs b/libs/neon-shmem/src/hash.rs index f173f42a6d..364787e2b7 100644 --- a/libs/neon-shmem/src/hash.rs +++ b/libs/neon-shmem/src/hash.rs @@ -8,9 +8,11 @@ //! [ ] Resizable use std::fmt::Debug; -use std::hash::{DefaultHasher, Hash, Hasher}; +use std::hash::{Hash, Hasher, BuildHasher}; use std::mem::MaybeUninit; +use rustc_hash::FxBuildHasher; + use crate::shmem::ShmemHandle; mod core; @@ -25,29 +27,32 @@ use entry::{Entry, OccupiedEntry, PrevPos}; #[derive(Debug)] pub struct OutOfMemoryError(); -pub struct HashMapInit<'a, K, V> { +pub struct HashMapInit<'a, K, V, S = rustc_hash::FxBuildHasher> { // Hash table can be allocated in a fixed memory area, or in a resizeable ShmemHandle. shmem_handle: Option, shared_ptr: *mut HashMapShared<'a, K, V>, + hasher: S, } -pub struct HashMapAccess<'a, K, V> { +pub struct HashMapAccess<'a, K, V, S = rustc_hash::FxBuildHasher> { shmem_handle: Option, shared_ptr: *mut HashMapShared<'a, K, V>, + hasher: S, } -unsafe impl<'a, K: Sync, V: Sync> Sync for HashMapAccess<'a, K, V> {} -unsafe impl<'a, K: Send, V: Send> Send for HashMapAccess<'a, K, V> {} +unsafe impl<'a, K: Sync, V: Sync, S> Sync for HashMapAccess<'a, K, V, S> {} +unsafe impl<'a, K: Send, V: Send, S> Send for HashMapAccess<'a, K, V, S> {} -impl<'a, K, V> HashMapInit<'a, K, V> { - pub fn attach_writer(self) -> HashMapAccess<'a, K, V> { +impl<'a, K, V, S> HashMapInit<'a, K, V, S> { + pub fn attach_writer(self) -> HashMapAccess<'a, K, V, S> { HashMapAccess { shmem_handle: self.shmem_handle, shared_ptr: self.shared_ptr, + hasher: self.hasher, } } - pub fn attach_reader(self) -> HashMapAccess<'a, K, V> { + pub fn attach_reader(self) -> HashMapAccess<'a, K, V, S> { // no difference to attach_writer currently self.attach_writer() } @@ -65,42 +70,60 @@ impl<'a, K, V> HashMapInit<'a, K, V> { /// /// In between the above parts, there can be padding bytes to align the parts correctly. struct HashMapShared<'a, K, V> { - inner: CoreHashMap<'a, K, V>, + inner: CoreHashMap<'a, K, V> } -impl<'a, K, V> HashMapInit<'a, K, V> +impl<'a, K, V> HashMapInit<'a, K, V, rustc_hash::FxBuildHasher> where - K: Clone + Hash + Eq, + K: Clone + Hash + Eq +{ + pub fn init_in_fixed_area( + num_buckets: u32, + area: &'a mut [MaybeUninit], + ) -> HashMapInit<'a, K, V> { + Self::init_in_fixed_area_with_hasher(num_buckets, area, rustc_hash::FxBuildHasher::default()) + } + + /// Initialize a new hash map in the given shared memory area + pub fn init_in_shmem(num_buckets: u32, shmem: ShmemHandle) -> HashMapInit<'a, K, V> { + Self::init_in_shmem_with_hasher(num_buckets, shmem, rustc_hash::FxBuildHasher::default()) + } +} + +impl<'a, K, V, S: BuildHasher> HashMapInit<'a, K, V, S> +where + K: Clone + Hash + Eq { pub fn estimate_size(num_buckets: u32) -> usize { // add some margin to cover alignment etc. CoreHashMap::::estimate_size(num_buckets) + size_of::>() + 1000 } - - pub fn init_in_fixed_area( - num_buckets: u32, - area: &'a mut [MaybeUninit], - ) -> HashMapInit<'a, K, V> { - Self::init_common(num_buckets, None, area.as_mut_ptr().cast(), area.len()) - } - - /// Initialize a new hash map in the given shared memory area - pub fn init_in_shmem(num_buckets: u32, mut shmem: ShmemHandle) -> HashMapInit<'a, K, V> { + + pub fn init_in_shmem_with_hasher(num_buckets: u32, mut shmem: ShmemHandle, hasher: S) -> HashMapInit<'a, K, V, S> { let size = Self::estimate_size(num_buckets); shmem .set_size(size) .expect("could not resize shared memory area"); let ptr = unsafe { shmem.data_ptr.as_mut() }; - Self::init_common(num_buckets, Some(shmem), ptr, size) + Self::init_common(num_buckets, Some(shmem), ptr, size, hasher) } + pub fn init_in_fixed_area_with_hasher( + num_buckets: u32, + area: &'a mut [MaybeUninit], + hasher: S, + ) -> HashMapInit<'a, K, V, S> { + Self::init_common(num_buckets, None, area.as_mut_ptr().cast(), area.len(), hasher) + } + fn init_common( num_buckets: u32, shmem_handle: Option, area_ptr: *mut u8, area_len: usize, - ) -> HashMapInit<'a, K, V> { + hasher: S, + ) -> HashMapInit<'a, K, V, S> { // carve out the HashMapShared struct from the area. let mut ptr: *mut u8 = area_ptr; let end_ptr: *mut u8 = unsafe { area_ptr.add(area_len) }; @@ -133,18 +156,17 @@ where HashMapInit { shmem_handle, shared_ptr, + hasher, } } } -impl<'a, K, V> HashMapAccess<'a, K, V> +impl<'a, K, V, S: BuildHasher> HashMapAccess<'a, K, V, S> where K: Clone + Hash + Eq, { pub fn get_hash_value(&self, key: &K) -> u64 { - let mut hasher = DefaultHasher::new(); - key.hash(&mut hasher); - hasher.finish() + self.hasher.hash_one(key) } pub fn get_with_hash<'e>(&'e self, key: &K, hash: u64) -> Option<&'e V> { @@ -210,6 +232,12 @@ where map.inner.buckets_in_use as usize } + pub fn clear(&mut self) { + let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); + let inner = &mut map.inner; + inner.clear() + } + /// Helper function that abstracts the common logic between growing and shrinking. /// The only significant difference in the rehashing step is how many buckets to rehash. fn rehash_dict( @@ -243,10 +271,7 @@ where continue; } - let mut hasher = DefaultHasher::new(); - buckets[i].inner.as_ref().unwrap().0.hash(&mut hasher); - let hash = hasher.finish(); - + let hash = self.hasher.hash_one(&buckets[i].inner.as_ref().unwrap().0); let pos: usize = (hash % dictionary.len() as u64) as usize; buckets[i].next = dictionary[pos]; dictionary[pos] = i as u32; @@ -256,6 +281,23 @@ where inner.dictionary = dictionary; inner.buckets = buckets; } + + /// Rehash the map. Intended for benchmarking only. + pub fn shuffle(&mut self) { + let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); + let inner = &mut map.inner; + + let shmem_handle = self + .shmem_handle + .as_ref() + .expect("TODO(quantumish): make shuffle work w/ fixed-size table"); + let num_buckets = inner.get_num_buckets() as u32; + let size_bytes = HashMapInit::::estimate_size(num_buckets); + let end_ptr: *mut u8 = unsafe { shmem_handle.data_ptr.as_ptr().add(size_bytes) }; + let buckets_ptr = inner.buckets.as_mut_ptr(); + self.rehash_dict(inner, buckets_ptr, end_ptr, num_buckets, num_buckets); + } + /// Grow /// @@ -278,7 +320,7 @@ where .as_ref() .expect("grow called on a fixed-size hash table"); - let size_bytes = HashMapInit::::estimate_size(num_buckets); + let size_bytes = HashMapInit::::estimate_size(num_buckets); shmem_handle.set_size(size_bytes)?; let end_ptr: *mut u8 = unsafe { shmem_handle.data_ptr.as_ptr().add(size_bytes) }; @@ -371,7 +413,7 @@ where .as_ref() .expect("shrink called on a fixed-size hash table"); - let size_bytes = HashMapInit::::estimate_size(num_buckets); + let size_bytes = HashMapInit::::estimate_size(num_buckets); shmem_handle.set_size(size_bytes)?; let end_ptr: *mut u8 = unsafe { shmem_handle.data_ptr.as_ptr().add(size_bytes) }; let buckets_ptr = inner.buckets.as_mut_ptr(); diff --git a/libs/neon-shmem/src/hash/core.rs b/libs/neon-shmem/src/hash/core.rs index 80f41fd8d4..049600e538 100644 --- a/libs/neon-shmem/src/hash/core.rs +++ b/libs/neon-shmem/src/hash/core.rs @@ -50,7 +50,7 @@ where as usize; size - } + } pub fn new( buckets: &'a mut [MaybeUninit>], @@ -160,6 +160,33 @@ where self.alloc_limit != INVALID_POS } + + // TODO(quantumish): How does this interact with an ongoing shrink? + pub fn clear(&mut self) { + for i in 0..self.buckets.len() { + self.buckets[i] = Bucket { + next: if i < self.buckets.len() - 1 { + i as u32 + 1 + } else { + INVALID_POS + }, + prev: if i > 0 { + PrevPos::Chained(i as u32 - 1) + } else { + PrevPos::First(INVALID_POS) + }, + inner: None, + } + } + + for i in 0..self.dictionary.len() { + self.dictionary[i] = INVALID_POS; + } + + self.buckets_in_use = 0; + self.alloc_limit = INVALID_POS; + } + pub fn entry_at_bucket(&mut self, pos: usize) -> Option> { if pos >= self.buckets.len() { return None; From 93a45708ff86c7a82ee4fced35f29bd3fdf964cc Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Mon, 23 Jun 2025 16:15:43 -0700 Subject: [PATCH 11/24] Change `finish_shrink` to remap entries in shrunk space --- libs/neon-shmem/src/hash.rs | 51 ++++++++++++++----------------- libs/neon-shmem/src/hash/entry.rs | 9 ++++++ 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/libs/neon-shmem/src/hash.rs b/libs/neon-shmem/src/hash.rs index 364787e2b7..efe7c96cb3 100644 --- a/libs/neon-shmem/src/hash.rs +++ b/libs/neon-shmem/src/hash.rs @@ -282,7 +282,7 @@ where inner.buckets = buckets; } - /// Rehash the map. Intended for benchmarking only. + /// Rehash the map. Intended for benchmarking only. pub fn shuffle(&mut self) { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); let inner = &mut map.inner; @@ -352,7 +352,7 @@ where Ok(()) } - /// Begin a shrink, limiting all new allocations to be in buckets with index less than `num_buckets`. + /// Begin a shrink, limiting all new allocations to be in buckets with index less than `num_buckets`. pub fn begin_shrink(&mut self, num_buckets: u32) { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); if num_buckets > map.inner.get_num_buckets() as u32 { @@ -377,32 +377,28 @@ where if inner.get_num_buckets() == num_buckets as usize { return Ok(()); - } + } else if inner.get_num_buckets() > num_buckets as usize { + panic!("called finish_shrink before enough entries were removed"); + } for i in (num_buckets as usize)..inner.buckets.len() { - if inner.buckets[i].inner.is_some() { - // TODO(quantumish) Do we want to treat this as a violation of an invariant - // or a legitimate error the caller can run into? Originally I thought this - // could return something like a UnevictedError(index) as soon as it runs - // into something (that way a caller could clear their soon-to-be-shrinked - // buckets by repeatedly trying to call `finish_shrink`). - // - // Would require making a wider error type enum with this and shmem errors. - panic!("unevicted entries in shrinked space") - } - match inner.buckets[i].prev { - PrevPos::First(_) => { - let next_pos = inner.buckets[i].next; - inner.free_head = next_pos; - if next_pos != INVALID_POS { - inner.buckets[next_pos as usize].prev = PrevPos::First(INVALID_POS); - } - }, - PrevPos::Chained(j) => { - let next_pos = inner.buckets[i].next; - inner.buckets[j as usize].next = next_pos; - if next_pos != INVALID_POS { - inner.buckets[next_pos as usize].prev = PrevPos::Chained(j); + if let Some((k, v)) = inner.buckets[i].inner.take() { + inner.alloc_bucket(k, v, inner.buckets[i].prev.unwrap_first()).unwrap(); + } else { + match inner.buckets[i].prev { + PrevPos::First(_) => { + let next_pos = inner.buckets[i].next; + inner.free_head = next_pos; + if next_pos != INVALID_POS { + inner.buckets[next_pos as usize].prev = PrevPos::First(INVALID_POS); + } + }, + PrevPos::Chained(j) => { + let next_pos = inner.buckets[i].next; + inner.buckets[j as usize].next = next_pos; + if next_pos != INVALID_POS { + inner.buckets[next_pos as usize].prev = PrevPos::Chained(j); + } } } } @@ -421,6 +417,5 @@ where inner.alloc_limit = INVALID_POS; Ok(()) - } - + } } diff --git a/libs/neon-shmem/src/hash/entry.rs b/libs/neon-shmem/src/hash/entry.rs index 64820b3d7b..7d3091c754 100644 --- a/libs/neon-shmem/src/hash/entry.rs +++ b/libs/neon-shmem/src/hash/entry.rs @@ -16,6 +16,15 @@ pub(crate) enum PrevPos { Chained(u32), } +impl PrevPos { + pub fn unwrap_first(&self) -> u32 { + match self { + Self::First(i) => *i, + _ => panic!("not first entry in chain") + } + } +} + pub struct OccupiedEntry<'a, 'b, K, V> { pub(crate) map: &'b mut CoreHashMap<'a, K, V>, pub(crate) _key: K, // The key of the occupied entry From 24e6c68772e60252b169da9747f72d3ee26d00cf Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Tue, 24 Jun 2025 13:34:22 -0700 Subject: [PATCH 12/24] Remove prev entry tracking, refactor HashMapInit into proper builder --- libs/neon-shmem/src/hash.rs | 233 ++++++++++++++++-------------- libs/neon-shmem/src/hash/core.rs | 37 ++--- libs/neon-shmem/src/hash/entry.rs | 27 ++-- libs/neon-shmem/src/hash/tests.rs | 91 +++++++----- 4 files changed, 200 insertions(+), 188 deletions(-) diff --git a/libs/neon-shmem/src/hash.rs b/libs/neon-shmem/src/hash.rs index efe7c96cb3..404a7ade50 100644 --- a/libs/neon-shmem/src/hash.rs +++ b/libs/neon-shmem/src/hash.rs @@ -22,16 +22,16 @@ pub mod entry; mod tests; use core::{CoreHashMap, INVALID_POS}; -use entry::{Entry, OccupiedEntry, PrevPos}; +use entry::{Entry, OccupiedEntry}; -#[derive(Debug)] -pub struct OutOfMemoryError(); pub struct HashMapInit<'a, K, V, S = rustc_hash::FxBuildHasher> { // Hash table can be allocated in a fixed memory area, or in a resizeable ShmemHandle. shmem_handle: Option, shared_ptr: *mut HashMapShared<'a, K, V>, + shared_size: usize, hasher: S, + num_buckets: u32, } pub struct HashMapAccess<'a, K, V, S = rustc_hash::FxBuildHasher> { @@ -43,8 +43,46 @@ pub struct HashMapAccess<'a, K, V, S = rustc_hash::FxBuildHasher> { unsafe impl<'a, K: Sync, V: Sync, S> Sync for HashMapAccess<'a, K, V, S> {} unsafe impl<'a, K: Send, V: Send, S> Send for HashMapAccess<'a, K, V, S> {} -impl<'a, K, V, S> HashMapInit<'a, K, V, S> { +impl<'a, K: Clone + Hash + Eq, V, S> HashMapInit<'a, K, V, S> { + pub fn with_hasher(self, hasher: S) -> HashMapInit<'a, K, V, S> { + Self { hasher, ..self } + } + + pub fn estimate_size(num_buckets: u32) -> usize { + // add some margin to cover alignment etc. + CoreHashMap::::estimate_size(num_buckets) + size_of::>() + 1000 + } + pub fn attach_writer(self) -> HashMapAccess<'a, K, V, S> { + // carve out the HashMapShared struct from the area. + let mut ptr: *mut u8 = self.shared_ptr.cast(); + let end_ptr: *mut u8 = unsafe { ptr.add(self.shared_size) }; + ptr = unsafe { ptr.add(ptr.align_offset(align_of::>())) }; + let shared_ptr: *mut HashMapShared = ptr.cast(); + ptr = unsafe { ptr.add(size_of::>()) }; + + // carve out the buckets + ptr = unsafe { ptr.byte_add(ptr.align_offset(align_of::>())) }; + let buckets_ptr = ptr; + ptr = unsafe { ptr.add(size_of::>() * self.num_buckets as usize) }; + + // use remaining space for the dictionary + ptr = unsafe { ptr.byte_add(ptr.align_offset(align_of::())) }; + assert!(ptr.addr() < end_ptr.addr()); + let dictionary_ptr = ptr; + let dictionary_size = unsafe { end_ptr.byte_offset_from(ptr) / size_of::() as isize }; + assert!(dictionary_size > 0); + + let buckets = + unsafe { std::slice::from_raw_parts_mut(buckets_ptr.cast(), self.num_buckets as usize) }; + let dictionary = unsafe { + std::slice::from_raw_parts_mut(dictionary_ptr.cast(), dictionary_size as usize) + }; + let hashmap = CoreHashMap::new(buckets, dictionary); + unsafe { + std::ptr::write(shared_ptr, HashMapShared { inner: hashmap }); + } + HashMapAccess { shmem_handle: self.shmem_handle, shared_ptr: self.shared_ptr, @@ -77,88 +115,56 @@ impl<'a, K, V> HashMapInit<'a, K, V, rustc_hash::FxBuildHasher> where K: Clone + Hash + Eq { - pub fn init_in_fixed_area( - num_buckets: u32, + pub fn with_fixed( + num_buckets: u32, area: &'a mut [MaybeUninit], ) -> HashMapInit<'a, K, V> { - Self::init_in_fixed_area_with_hasher(num_buckets, area, rustc_hash::FxBuildHasher::default()) + Self { + num_buckets, + shmem_handle: None, + shared_ptr: area.as_mut_ptr().cast(), + shared_size: area.len(), + hasher: rustc_hash::FxBuildHasher::default(), + } } /// Initialize a new hash map in the given shared memory area - pub fn init_in_shmem(num_buckets: u32, shmem: ShmemHandle) -> HashMapInit<'a, K, V> { - Self::init_in_shmem_with_hasher(num_buckets, shmem, rustc_hash::FxBuildHasher::default()) - } -} - -impl<'a, K, V, S: BuildHasher> HashMapInit<'a, K, V, S> -where - K: Clone + Hash + Eq -{ - pub fn estimate_size(num_buckets: u32) -> usize { - // add some margin to cover alignment etc. - CoreHashMap::::estimate_size(num_buckets) + size_of::>() + 1000 - } - - pub fn init_in_shmem_with_hasher(num_buckets: u32, mut shmem: ShmemHandle, hasher: S) -> HashMapInit<'a, K, V, S> { - let size = Self::estimate_size(num_buckets); - shmem + pub fn with_shmem(num_buckets: u32, shmem: ShmemHandle) -> HashMapInit<'a, K, V> { + let size = Self::estimate_size(num_buckets); + shmem .set_size(size) .expect("could not resize shared memory area"); - - let ptr = unsafe { shmem.data_ptr.as_mut() }; - Self::init_common(num_buckets, Some(shmem), ptr, size, hasher) + Self { + num_buckets, + shared_ptr: shmem.data_ptr.as_ptr().cast(), + shmem_handle: Some(shmem), + shared_size: size, + hasher: rustc_hash::FxBuildHasher::default() + } } - pub fn init_in_fixed_area_with_hasher( - num_buckets: u32, - area: &'a mut [MaybeUninit], - hasher: S, - ) -> HashMapInit<'a, K, V, S> { - Self::init_common(num_buckets, None, area.as_mut_ptr().cast(), area.len(), hasher) - } - - fn init_common( - num_buckets: u32, - shmem_handle: Option, - area_ptr: *mut u8, - area_len: usize, - hasher: S, - ) -> HashMapInit<'a, K, V, S> { - // carve out the HashMapShared struct from the area. - let mut ptr: *mut u8 = area_ptr; - let end_ptr: *mut u8 = unsafe { area_ptr.add(area_len) }; - ptr = unsafe { ptr.add(ptr.align_offset(align_of::>())) }; - let shared_ptr: *mut HashMapShared = ptr.cast(); - ptr = unsafe { ptr.add(size_of::>()) }; + pub fn new_resizeable_named(num_buckets: u32, max_buckets: u32, name: &str) -> HashMapInit<'a, K, V> { + let size = Self::estimate_size(num_buckets); + let max_size = Self::estimate_size(max_buckets); + let shmem = ShmemHandle::new(name, size, max_size) + .expect("failed to make shared memory area"); + + Self { + num_buckets, + shared_ptr: shmem.data_ptr.as_ptr().cast(), + shmem_handle: Some(shmem), + shared_size: size, + hasher: rustc_hash::FxBuildHasher::default() + } + } - // carve out the buckets - ptr = unsafe { ptr.byte_add(ptr.align_offset(align_of::>())) }; - let buckets_ptr = ptr; - ptr = unsafe { ptr.add(size_of::>() * num_buckets as usize) }; - - // use remaining space for the dictionary - ptr = unsafe { ptr.byte_add(ptr.align_offset(align_of::())) }; - assert!(ptr.addr() < end_ptr.addr()); - let dictionary_ptr = ptr; - let dictionary_size = unsafe { end_ptr.byte_offset_from(ptr) / size_of::() as isize }; - assert!(dictionary_size > 0); - - let buckets = - unsafe { std::slice::from_raw_parts_mut(buckets_ptr.cast(), num_buckets as usize) }; - let dictionary = unsafe { - std::slice::from_raw_parts_mut(dictionary_ptr.cast(), dictionary_size as usize) - }; - let hashmap = CoreHashMap::new(buckets, dictionary); - unsafe { - std::ptr::write(shared_ptr, HashMapShared { inner: hashmap }); - } - - HashMapInit { - shmem_handle, - shared_ptr, - hasher, - } - } + pub fn new_resizeable(num_buckets: u32, max_buckets: u32) -> HashMapInit<'a, K, V> { + use std::sync::atomic::{AtomicUsize, Ordering}; + const COUNTER: AtomicUsize = AtomicUsize::new(0); + let val = COUNTER.fetch_add(1, Ordering::Relaxed); + let name = format!("neon_shmem_hmap{}", val); + Self::new_resizeable_named(num_buckets, max_buckets, &name) + } } impl<'a, K, V, S: BuildHasher> HashMapAccess<'a, K, V, S> @@ -248,6 +254,8 @@ where num_buckets: u32, rehash_buckets: u32, ) { + inner.free_head = INVALID_POS; + // Recalculate the dictionary let buckets; let dictionary; @@ -268,7 +276,9 @@ where for i in 0..rehash_buckets as usize { if buckets[i].inner.is_none() { - continue; + buckets[i].next = inner.free_head; + inner.free_head = i as u32; + continue; } let hash = self.hasher.hash_one(&buckets[i].inner.as_ref().unwrap().0); @@ -286,19 +296,13 @@ where pub fn shuffle(&mut self) { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); let inner = &mut map.inner; - - let shmem_handle = self - .shmem_handle - .as_ref() - .expect("TODO(quantumish): make shuffle work w/ fixed-size table"); let num_buckets = inner.get_num_buckets() as u32; let size_bytes = HashMapInit::::estimate_size(num_buckets); - let end_ptr: *mut u8 = unsafe { shmem_handle.data_ptr.as_ptr().add(size_bytes) }; + let end_ptr: *mut u8 = unsafe { (self.shared_ptr as *mut u8).add(size_bytes) }; let buckets_ptr = inner.buckets.as_mut_ptr(); self.rehash_dict(inner, buckets_ptr, end_ptr, num_buckets, num_buckets); } - /// Grow /// /// 1. grow the underlying shared memory area @@ -336,11 +340,6 @@ where } else { inner.free_head }, - prev: if i > 0 { - PrevPos::Chained(i as u32 - 1) - } else { - PrevPos::First(INVALID_POS) - }, inner: None, }); } @@ -352,7 +351,7 @@ where Ok(()) } - /// Begin a shrink, limiting all new allocations to be in buckets with index less than `num_buckets`. + /// Begin a shrink, limiting all new allocations to be in buckets with index below `num_buckets`. pub fn begin_shrink(&mut self, num_buckets: u32) { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); if num_buckets > map.inner.get_num_buckets() as u32 { @@ -365,6 +364,26 @@ where map.inner.alloc_limit = num_buckets; } + /// Returns whether a shrink operation is currently in progress. + pub fn is_shrinking(&self) -> bool { + let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); + map.inner.is_shrinking() + } + + /// Returns how many entries need to be evicted before shrink can complete. + pub fn shrink_remaining(&self) -> usize { + let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); + let inner = &mut map.inner; + if !inner.is_shrinking() { + panic!("shrink_remaining called when no ongoing shrink") + } else { + inner.buckets_in_use + .checked_sub(inner.alloc_limit) + .unwrap_or(0) + as usize + } + } + /// Complete a shrink after caller has evicted entries, removing the unused buckets and rehashing. pub fn finish_shrink(&mut self) -> Result<(), crate::shmem::Error> { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); @@ -377,30 +396,24 @@ where if inner.get_num_buckets() == num_buckets as usize { return Ok(()); - } else if inner.get_num_buckets() > num_buckets as usize { + } else if inner.buckets_in_use > num_buckets { panic!("called finish_shrink before enough entries were removed"); } + + let mut open_spots = 0; + let mut curr = inner.free_head; + while curr != INVALID_POS { + if curr < num_buckets { + open_spots += 1; + } + curr = inner.buckets[curr as usize].next; + } for i in (num_buckets as usize)..inner.buckets.len() { - if let Some((k, v)) = inner.buckets[i].inner.take() { - inner.alloc_bucket(k, v, inner.buckets[i].prev.unwrap_first()).unwrap(); - } else { - match inner.buckets[i].prev { - PrevPos::First(_) => { - let next_pos = inner.buckets[i].next; - inner.free_head = next_pos; - if next_pos != INVALID_POS { - inner.buckets[next_pos as usize].prev = PrevPos::First(INVALID_POS); - } - }, - PrevPos::Chained(j) => { - let next_pos = inner.buckets[i].next; - inner.buckets[j as usize].next = next_pos; - if next_pos != INVALID_POS { - inner.buckets[next_pos as usize].prev = PrevPos::Chained(j); - } - } - } + if let Some((k, v)) = inner.buckets[i].inner.take() { + // alloc bucket increases buckets in use, so need to decrease since we're just moving + inner.buckets_in_use -= 1; + inner.alloc_bucket(k, v).unwrap(); } } diff --git a/libs/neon-shmem/src/hash/core.rs b/libs/neon-shmem/src/hash/core.rs index 049600e538..b27180a80a 100644 --- a/libs/neon-shmem/src/hash/core.rs +++ b/libs/neon-shmem/src/hash/core.rs @@ -12,14 +12,16 @@ pub(crate) const INVALID_POS: u32 = u32::MAX; // Bucket pub(crate) struct Bucket { - pub(crate) next: u32, - pub(crate) prev: PrevPos, + pub(crate) next: u32, pub(crate) inner: Option<(K, V)>, } pub(crate) struct CoreHashMap<'a, K, V> { + /// Dictionary used to map hashes to bucket indices. pub(crate) dictionary: &'a mut [u32], + /// Buckets containing key-value pairs. pub(crate) buckets: &'a mut [Bucket], + /// Head of the freelist. pub(crate) free_head: u32, pub(crate) _user_list_head: u32, @@ -63,12 +65,7 @@ where i as u32 + 1 } else { INVALID_POS - }, - prev: if i > 0 { - PrevPos::Chained(i as u32 - 1) - } else { - PrevPos::First(INVALID_POS) - }, + }, inner: None, }); } @@ -160,8 +157,8 @@ where self.alloc_limit != INVALID_POS } - - // TODO(quantumish): How does this interact with an ongoing shrink? + /// Clears all entries from the hashmap. + /// Does not reset any allocation limits, but does clear any entries beyond them. pub fn clear(&mut self) { for i in 0..self.buckets.len() { self.buckets[i] = Bucket { @@ -169,12 +166,7 @@ where i as u32 + 1 } else { INVALID_POS - }, - prev: if i > 0 { - PrevPos::Chained(i as u32 - 1) - } else { - PrevPos::First(INVALID_POS) - }, + }, inner: None, } } @@ -184,7 +176,6 @@ where } self.buckets_in_use = 0; - self.alloc_limit = INVALID_POS; } pub fn entry_at_bucket(&mut self, pos: usize) -> Option> { @@ -192,13 +183,12 @@ where return None; } - let prev = self.buckets[pos].prev; let entry = self.buckets[pos].inner.as_ref(); match entry { Some((key, _)) => Some(OccupiedEntry { _key: key.clone(), bucket_pos: pos as u32, - prev_pos: prev, + prev_pos: PrevPos::Unknown, map: self, }), _ => None, @@ -206,7 +196,7 @@ where } /// Find the position of an unused bucket via the freelist and initialize it. - pub(crate) fn alloc_bucket(&mut self, key: K, value: V, dict_pos: u32) -> Result { + pub(crate) fn alloc_bucket(&mut self, key: K, value: V) -> Result { let mut pos = self.free_head; // Find the first bucket we're *allowed* to use. @@ -225,17 +215,12 @@ where PrevPos::First(_) => { let next_pos = self.buckets[pos as usize].next; self.free_head = next_pos; - if next_pos != INVALID_POS { - self.buckets[next_pos as usize].prev = PrevPos::First(dict_pos); - } } PrevPos::Chained(p) => if p != INVALID_POS { let next_pos = self.buckets[pos as usize].next; self.buckets[p as usize].next = next_pos; - if next_pos != INVALID_POS { - self.buckets[next_pos as usize].prev = PrevPos::Chained(p); - } }, + PrevPos::Unknown => unreachable!() } // Initialize the bucket. diff --git a/libs/neon-shmem/src/hash/entry.rs b/libs/neon-shmem/src/hash/entry.rs index 7d3091c754..63af840c5d 100644 --- a/libs/neon-shmem/src/hash/entry.rs +++ b/libs/neon-shmem/src/hash/entry.rs @@ -10,13 +10,19 @@ pub enum Entry<'a, 'b, K, V> { Vacant(VacantEntry<'a, 'b, K, V>), } +/// Helper enum representing the previous position within a hashmap chain. #[derive(Clone, Copy)] pub(crate) enum PrevPos { + /// Starting index within the dictionary. First(u32), + /// Regular index within the buckets. Chained(u32), + /// Unknown - e.g. the associated entry was retrieved by index instead of chain. + Unknown, } impl PrevPos { + /// Unwrap an index from a `PrevPos::First`, panicking otherwise. pub fn unwrap_first(&self) -> u32 { match self { Self::First(i) => *i, @@ -26,10 +32,13 @@ impl PrevPos { } pub struct OccupiedEntry<'a, 'b, K, V> { - pub(crate) map: &'b mut CoreHashMap<'a, K, V>, - pub(crate) _key: K, // The key of the occupied entry + pub(crate) map: &'b mut CoreHashMap<'a, K, V>, + /// The key of the occupied entry + pub(crate) _key: K, + /// The index of the previous entry in the chain. pub(crate) prev_pos: PrevPos, - pub(crate) bucket_pos: u32, // The position of the bucket in the CoreHashMap's buckets array + /// The position of the bucket in the CoreHashMap's buckets array. + pub(crate) bucket_pos: u32, } impl<'a, 'b, K, V> OccupiedEntry<'a, 'b, K, V> { @@ -65,17 +74,14 @@ impl<'a, 'b, K, V> OccupiedEntry<'a, 'b, K, V> { PrevPos::First(dict_pos) => self.map.dictionary[dict_pos as usize] = bucket.next, PrevPos::Chained(bucket_pos) => { self.map.buckets[bucket_pos as usize].next = bucket.next - } + }, + PrevPos::Unknown => panic!("can't safely remove entry with unknown previous entry"), } // and add it to the freelist - if self.map.free_head != INVALID_POS { - self.map.buckets[self.map.free_head as usize].prev = PrevPos::Chained(self.bucket_pos); - } let bucket = &mut self.map.buckets[self.bucket_pos as usize]; let old_value = bucket.inner.take(); bucket.next = self.map.free_head; - bucket.prev = PrevPos::First(INVALID_POS); self.map.free_head = self.bucket_pos; self.map.buckets_in_use -= 1; @@ -91,14 +97,11 @@ pub struct VacantEntry<'a, 'b, K, V> { impl<'a, 'b, K: Clone + Hash + Eq, V> VacantEntry<'a, 'b, K, V> { pub fn insert(self, value: V) -> Result<&'b mut V, FullError> { - let pos = self.map.alloc_bucket(self.key, value, self.dict_pos)?; + let pos = self.map.alloc_bucket(self.key, value)?; if pos == INVALID_POS { return Err(FullError()); } let bucket = &mut self.map.buckets[pos as usize]; - if let PrevPos::First(INVALID_POS) = bucket.prev { - bucket.prev = PrevPos::First(self.dict_pos); - } bucket.next = self.map.dictionary[self.dict_pos as usize]; self.map.dictionary[self.dict_pos as usize] = pos; diff --git a/libs/neon-shmem/src/hash/tests.rs b/libs/neon-shmem/src/hash/tests.rs index ee6acfc144..a522423db1 100644 --- a/libs/neon-shmem/src/hash/tests.rs +++ b/libs/neon-shmem/src/hash/tests.rs @@ -38,11 +38,9 @@ impl<'a> From<&'a [u8]> for TestKey { } fn test_inserts + Copy>(keys: &[K]) { - const MAX_MEM_SIZE: usize = 10000000; - let shmem = ShmemHandle::new("test_inserts", 0, MAX_MEM_SIZE).unwrap(); - - let init_struct = HashMapInit::::init_in_shmem(100000, shmem); - let mut w = init_struct.attach_writer(); + let mut w = HashMapInit::::new_resizeable_named( + 100000, 120000, "test_inserts" + ).attach_writer(); for (idx, k) in keys.iter().enumerate() { let hash = w.get_hash_value(&(*k).into()); @@ -193,24 +191,23 @@ fn do_shrink( to: u32 ) { writer.begin_shrink(to); - for i in to..from { - if let Some(entry) = writer.entry_at_bucket(i as usize) { - shadow.remove(&entry._key); - entry.remove(); + while writer.get_num_buckets_in_use() > to as usize { + let (k, _) = shadow.pop_first().unwrap(); + let hash = writer.get_hash_value(&k); + let entry = writer.entry_with_hash(k, hash); + if let Entry::Occupied(mut e) = entry { + e.remove(); } } writer.finish_shrink().unwrap(); - } #[test] fn random_ops() { - let shmem = ShmemHandle::new("test_inserts", 0, 10000000).unwrap(); - - let init_struct = HashMapInit::::init_in_shmem(100000, shmem); - let mut writer = init_struct.attach_writer(); + let mut writer = HashMapInit::::new_resizeable_named( + 100000, 120000, "test_random" + ).attach_writer(); let mut shadow: std::collections::BTreeMap = BTreeMap::new(); - let distribution = Zipf::new(u128::MAX as f64, 1.1).unwrap(); let mut rng = rand::rng(); @@ -227,11 +224,25 @@ fn random_ops() { } } + +#[test] +fn test_shuffle() { + let mut writer = HashMapInit::::new_resizeable_named( + 1000, 1200, "test_shuf" + ).attach_writer(); + let mut shadow: std::collections::BTreeMap = BTreeMap::new(); + let mut rng = rand::rng(); + + do_random_ops(10000, 1000, 0.75, &mut writer, &mut shadow, &mut rng); + writer.shuffle(); + do_random_ops(10000, 1000, 0.75, &mut writer, &mut shadow, &mut rng); +} + #[test] fn test_grow() { - let shmem = ShmemHandle::new("test_grow", 0, 10000000).unwrap(); - let init_struct = HashMapInit::::init_in_shmem(1000, shmem); - let mut writer = init_struct.attach_writer(); + let mut writer = HashMapInit::::new_resizeable_named( + 1000, 2000, "test_grow" + ).attach_writer(); let mut shadow: std::collections::BTreeMap = BTreeMap::new(); let mut rng = rand::rng(); @@ -242,9 +253,9 @@ fn test_grow() { #[test] fn test_shrink() { - let shmem = ShmemHandle::new("test_shrink", 0, 10000000).unwrap(); - let init_struct = HashMapInit::::init_in_shmem(1500, shmem); - let mut writer = init_struct.attach_writer(); + let mut writer = HashMapInit::::new_resizeable_named( + 1500, 2000, "test_shrink" + ).attach_writer(); let mut shadow: std::collections::BTreeMap = BTreeMap::new(); let mut rng = rand::rng(); @@ -257,9 +268,9 @@ fn test_shrink() { #[test] fn test_shrink_grow_seq() { - let shmem = ShmemHandle::new("test_shrink_grow_seq", 0, 10000000).unwrap(); - let init_struct = HashMapInit::::init_in_shmem(1500, shmem); - let mut writer = init_struct.attach_writer(); + let mut writer = HashMapInit::::new_resizeable_named( + 1000, 20000, "test_grow_seq" + ).attach_writer(); let mut shadow: std::collections::BTreeMap = BTreeMap::new(); let mut rng = rand::rng(); @@ -281,9 +292,9 @@ fn test_shrink_grow_seq() { #[test] fn test_bucket_ops() { - let shmem = ShmemHandle::new("test_bucket_ops", 0, 10000000).unwrap(); - let init_struct = HashMapInit::::init_in_shmem(1000, shmem); - let mut writer = init_struct.attach_writer(); + let mut writer = HashMapInit::::new_resizeable_named( + 1000, 1200, "test_bucket_ops" + ).attach_writer(); let hash = writer.get_hash_value(&1.into()); match writer.entry_with_hash(1.into(), hash) { Entry::Occupied(mut e) => { e.insert(2); }, @@ -307,9 +318,9 @@ fn test_bucket_ops() { #[test] fn test_shrink_zero() { - let shmem = ShmemHandle::new("test_shrink_zero", 0, 10000000).unwrap(); - let init_struct = HashMapInit::::init_in_shmem(1500, shmem); - let mut writer = init_struct.attach_writer(); + let mut writer = HashMapInit::::new_resizeable_named( + 1500, 2000, "test_shrink_zero" + ).attach_writer(); writer.begin_shrink(0); for i in 0..1500 { writer.entry_at_bucket(i).map(|x| x.remove()); @@ -336,27 +347,27 @@ fn test_shrink_zero() { #[test] #[should_panic] fn test_grow_oom() { - let shmem = ShmemHandle::new("test_grow_oom", 0, 500).unwrap(); - let init_struct = HashMapInit::::init_in_shmem(5, shmem); - let mut writer = init_struct.attach_writer(); + let mut writer = HashMapInit::::new_resizeable_named( + 1500, 2000, "test_grow_oom" + ).attach_writer(); writer.grow(20000).unwrap(); } #[test] #[should_panic] fn test_shrink_bigger() { - let shmem = ShmemHandle::new("test_shrink_bigger", 0, 10000000).unwrap(); - let init_struct = HashMapInit::::init_in_shmem(1500, shmem); - let mut writer = init_struct.attach_writer(); + let mut writer = HashMapInit::::new_resizeable_named( + 1500, 2500, "test_shrink_bigger" + ).attach_writer(); writer.begin_shrink(2000); } #[test] #[should_panic] fn test_shrink_early_finish() { - let shmem = ShmemHandle::new("test_shrink_early_finish", 0, 10000000).unwrap(); - let init_struct = HashMapInit::::init_in_shmem(1500, shmem); - let mut writer = init_struct.attach_writer(); + let mut writer = HashMapInit::::new_resizeable_named( + 1500, 2500, "test_shrink_early_finish" + ).attach_writer(); writer.finish_shrink().unwrap(); } @@ -364,7 +375,7 @@ fn test_shrink_early_finish() { #[should_panic] fn test_shrink_fixed_size() { let mut area = [MaybeUninit::uninit(); 10000]; - let init_struct = HashMapInit::::init_in_fixed_area(3, &mut area); + let init_struct = HashMapInit::::with_fixed(3, &mut area); let mut writer = init_struct.attach_writer(); writer.begin_shrink(1); } From ae740ca1bb1a997e7768b940c384013ec3469d4b Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Tue, 24 Jun 2025 16:27:17 -0700 Subject: [PATCH 13/24] Document hashmap implementation, fix `get_bucket_for_value` Previously, `get_bucket_for_value` incorrectly divided by the size of `V` to get the bucket index. Now it divides by the size of `Bucket`. --- libs/neon-shmem/Cargo.toml | 4 -- libs/neon-shmem/src/hash.rs | 92 +++++++++++++++++-------------- libs/neon-shmem/src/hash/core.rs | 29 ++++++---- libs/neon-shmem/src/hash/entry.rs | 26 ++++----- libs/neon-shmem/src/hash/tests.rs | 7 ++- 5 files changed, 86 insertions(+), 72 deletions(-) diff --git a/libs/neon-shmem/Cargo.toml b/libs/neon-shmem/Cargo.toml index 284a19d55d..bf14eb2e83 100644 --- a/libs/neon-shmem/Cargo.toml +++ b/libs/neon-shmem/Cargo.toml @@ -23,7 +23,3 @@ tempfile = "3.14.0" [[bench]] name = "hmap_resize" harness = false - -[[bin]] -name = "hmap_test" -path = "main.rs" diff --git a/libs/neon-shmem/src/hash.rs b/libs/neon-shmem/src/hash.rs index 404a7ade50..032c1bd21f 100644 --- a/libs/neon-shmem/src/hash.rs +++ b/libs/neon-shmem/src/hash.rs @@ -1,18 +1,22 @@ -//! Hash table implementation on top of 'shmem' +//! Resizable hash table implementation on top of byte-level storage (either `shmem` or fixed byte array). //! -//! Features required in the long run by the communicator project: +//! This hash table has two major components: the bucket array and the dictionary. Each bucket within the +//! bucket array contains a Option<(K, V)> and an index of another bucket. In this way there is both an +//! implicit freelist within the bucket array (None buckets point to other None entries) and various hash +//! chains within the bucket array (a Some bucket will point to other Some buckets that had the same hash). //! -//! [X] Accessible from both Postgres processes and rust threads in the communicator process -//! [X] Low latency -//! [ ] Scalable to lots of concurrent accesses (currently relies on caller for locking) -//! [ ] Resizable +//! Buckets are never moved unless they are within a region that is being shrunk, and so the actual hash- +//! dependent component is done with the dictionary. When a new key is inserted into the map, a position +//! within the dictionary is decided based on its hash, the data is inserted into an empty bucket based +//! off of the freelist, and then the index of said bucket is placed in the dictionary. +//! +//! This map is resizable (if initialized on top of a `ShmemHandle`). Both growing and shrinking happen +//! in-place and are at a high level achieved by expanding/reducing the bucket array and rebuilding the +//! dictionary by rehashing all keys. -use std::fmt::Debug; -use std::hash::{Hash, Hasher, BuildHasher}; +use std::hash::{Hash, BuildHasher}; use std::mem::MaybeUninit; -use rustc_hash::FxBuildHasher; - use crate::shmem::ShmemHandle; mod core; @@ -21,12 +25,11 @@ pub mod entry; #[cfg(test)] mod tests; -use core::{CoreHashMap, INVALID_POS}; +use core::{Bucket, CoreHashMap, INVALID_POS}; use entry::{Entry, OccupiedEntry}; - +/// Builder for a `HashMapAccess`. pub struct HashMapInit<'a, K, V, S = rustc_hash::FxBuildHasher> { - // Hash table can be allocated in a fixed memory area, or in a resizeable ShmemHandle. shmem_handle: Option, shared_ptr: *mut HashMapShared<'a, K, V>, shared_size: usize, @@ -34,6 +37,7 @@ pub struct HashMapInit<'a, K, V, S = rustc_hash::FxBuildHasher> { num_buckets: u32, } +/// Accessor for a hash table. pub struct HashMapAccess<'a, K, V, S = rustc_hash::FxBuildHasher> { shmem_handle: Option, shared_ptr: *mut HashMapShared<'a, K, V>, @@ -43,16 +47,18 @@ pub struct HashMapAccess<'a, K, V, S = rustc_hash::FxBuildHasher> { unsafe impl<'a, K: Sync, V: Sync, S> Sync for HashMapAccess<'a, K, V, S> {} unsafe impl<'a, K: Send, V: Send, S> Send for HashMapAccess<'a, K, V, S> {} -impl<'a, K: Clone + Hash + Eq, V, S> HashMapInit<'a, K, V, S> { +impl<'a, K: Clone + Hash + Eq, V, S> HashMapInit<'a, K, V, S> { pub fn with_hasher(self, hasher: S) -> HashMapInit<'a, K, V, S> { Self { hasher, ..self } } - + + /// Loosely (over)estimate the size needed to store a hash table with `num_buckets` buckets. pub fn estimate_size(num_buckets: u32) -> usize { // add some margin to cover alignment etc. CoreHashMap::::estimate_size(num_buckets) + size_of::>() + 1000 } - + + /// Initialize a table for writing. pub fn attach_writer(self) -> HashMapAccess<'a, K, V, S> { // carve out the HashMapShared struct from the area. let mut ptr: *mut u8 = self.shared_ptr.cast(); @@ -90,13 +96,13 @@ impl<'a, K: Clone + Hash + Eq, V, S> HashMapInit<'a, K, V, S> { } } + /// Initialize a table for reading. Currently identical to `attach_writer`. pub fn attach_reader(self) -> HashMapAccess<'a, K, V, S> { - // no difference to attach_writer currently self.attach_writer() } } -/// This is stored in the shared memory area +/// Hash table data that is actually stored in the shared memory area. /// /// NOTE: We carve out the parts from a contiguous chunk. Growing and shrinking the hash table /// relies on the memory layout! The data structures are laid out in the contiguous shared memory @@ -115,6 +121,7 @@ impl<'a, K, V> HashMapInit<'a, K, V, rustc_hash::FxBuildHasher> where K: Clone + Hash + Eq { + /// Place the hash table within a user-supplied fixed memory area. pub fn with_fixed( num_buckets: u32, area: &'a mut [MaybeUninit], @@ -128,7 +135,7 @@ where } } - /// Initialize a new hash map in the given shared memory area + /// Place a new hash map in the given shared memory area pub fn with_shmem(num_buckets: u32, shmem: ShmemHandle) -> HashMapInit<'a, K, V> { let size = Self::estimate_size(num_buckets); shmem @@ -143,6 +150,7 @@ where } } + /// Make a resizable hash map within a new shared memory area with the given name. pub fn new_resizeable_named(num_buckets: u32, max_buckets: u32, name: &str) -> HashMapInit<'a, K, V> { let size = Self::estimate_size(num_buckets); let max_size = Self::estimate_size(max_buckets); @@ -158,6 +166,7 @@ where } } + /// Make a resizable hash map within a new anonymous shared memory area. pub fn new_resizeable(num_buckets: u32, max_buckets: u32) -> HashMapInit<'a, K, V> { use std::sync::atomic::{AtomicUsize, Ordering}; const COUNTER: AtomicUsize = AtomicUsize::new(0); @@ -171,22 +180,26 @@ impl<'a, K, V, S: BuildHasher> HashMapAccess<'a, K, V, S> where K: Clone + Hash + Eq, { + /// Hash a key using the map's hasher. pub fn get_hash_value(&self, key: &K) -> u64 { self.hasher.hash_one(key) } + /// Get a reference to the corresponding value for a key given its hash. pub fn get_with_hash<'e>(&'e self, key: &K, hash: u64) -> Option<&'e V> { let map = unsafe { self.shared_ptr.as_ref() }.unwrap(); map.inner.get_with_hash(key, hash) } + /// Get a reference to the entry containing a key given its hash. pub fn entry_with_hash(&mut self, key: K, hash: u64) -> Entry<'a, '_, K, V> { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); map.inner.entry_with_hash(key, hash) } + /// Remove a key given its hash. Does nothing if key is not present. pub fn remove_with_hash(&mut self, key: &K, hash: u64) { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); @@ -198,20 +211,23 @@ where }; } + /// Optionally return the entry for a bucket at a given index if it exists. pub fn entry_at_bucket(&mut self, pos: usize) -> Option> { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); map.inner.entry_at_bucket(pos) } + /// Returns the number of buckets in the table. pub fn get_num_buckets(&self) -> usize { let map = unsafe { self.shared_ptr.as_ref() }.unwrap(); map.inner.get_num_buckets() } /// Return the key and value stored in bucket with given index. This can be used to - /// iterate through the hash map. (An Iterator might be nicer. The communicator's - /// clock algorithm needs to _slowly_ iterate through all buckets with its clock hand, - /// without holding a lock. If we switch to an Iterator, it must not hold the lock.) + /// iterate through the hash map. + // TODO: An Iterator might be nicer. The communicator's clock algorithm needs to + // _slowly_ iterate through all buckets with its clock hand, without holding a lock. + // If we switch to an Iterator, it must not hold the lock. pub fn get_at_bucket(&self, pos: usize) -> Option<&(K, V)> { let map = unsafe { self.shared_ptr.as_ref() }.unwrap(); @@ -222,30 +238,33 @@ where bucket.inner.as_ref() } + /// Returns the index of the bucket a given value corresponds to. pub fn get_bucket_for_value(&self, val_ptr: *const V) -> usize { let map = unsafe { self.shared_ptr.as_ref() }.unwrap(); let origin = map.inner.buckets.as_ptr(); - let idx = (val_ptr as usize - origin as usize) / (size_of::() as usize); + let idx = (val_ptr as usize - origin as usize) / (size_of::>() as usize); assert!(idx < map.inner.buckets.len()); idx } - // for metrics + /// Returns the number of occupied buckets in the table. pub fn get_num_buckets_in_use(&self) -> usize { let map = unsafe { self.shared_ptr.as_ref() }.unwrap(); map.inner.buckets_in_use as usize } + /// Clears all entries in a table. Does not reset any shrinking operations. pub fn clear(&mut self) { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); let inner = &mut map.inner; inner.clear() } - /// Helper function that abstracts the common logic between growing and shrinking. - /// The only significant difference in the rehashing step is how many buckets to rehash. + /// Perform an in-place rehash of some region (0..`rehash_buckets`) of the table and reset + /// the `buckets` and `dictionary` slices to be as long as `num_buckets`. Resets the freelist + /// in the process. fn rehash_dict( &mut self, inner: &mut CoreHashMap<'a, K, V>, @@ -256,7 +275,6 @@ where ) { inner.free_head = INVALID_POS; - // Recalculate the dictionary let buckets; let dictionary; unsafe { @@ -287,12 +305,11 @@ where dictionary[pos] = i as u32; } - // Finally, update the CoreHashMap struct inner.dictionary = dictionary; inner.buckets = buckets; } - /// Rehash the map. Intended for benchmarking only. + /// Rehash the map without growing or shrinking. pub fn shuffle(&mut self) { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); let inner = &mut map.inner; @@ -303,11 +320,11 @@ where self.rehash_dict(inner, buckets_ptr, end_ptr, num_buckets, num_buckets); } - /// Grow + /// Grow the number of buckets within the table. /// - /// 1. grow the underlying shared memory area - /// 2. Initialize new buckets. This overwrites the current dictionary - /// 3. Recalculate the dictionary + /// 1. Grows the underlying shared memory area + /// 2. Initializes new buckets and overwrites the current dictionary + /// 3. Rehashes the dictionary pub fn grow(&mut self, num_buckets: u32) -> Result<(), crate::shmem::Error> { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); let inner = &mut map.inner; @@ -400,15 +417,6 @@ where panic!("called finish_shrink before enough entries were removed"); } - let mut open_spots = 0; - let mut curr = inner.free_head; - while curr != INVALID_POS { - if curr < num_buckets { - open_spots += 1; - } - curr = inner.buckets[curr as usize].next; - } - for i in (num_buckets as usize)..inner.buckets.len() { if let Some((k, v)) = inner.buckets[i].inner.take() { // alloc bucket increases buckets in use, so need to decrease since we're just moving diff --git a/libs/neon-shmem/src/hash/core.rs b/libs/neon-shmem/src/hash/core.rs index b27180a80a..22c44f20ac 100644 --- a/libs/neon-shmem/src/hash/core.rs +++ b/libs/neon-shmem/src/hash/core.rs @@ -1,7 +1,4 @@ -//! Simple hash table with chaining -//! -//! # Resizing -//! +//! Simple hash table with chaining. use std::hash::Hash; use std::mem::MaybeUninit; @@ -10,12 +7,16 @@ use crate::hash::entry::{Entry, OccupiedEntry, PrevPos, VacantEntry}; pub(crate) const INVALID_POS: u32 = u32::MAX; -// Bucket +/// Fundamental storage unit within the hash table. Either empty or contains a key-value pair. +/// Always part of a chain of some kind (either a freelist if empty or a hash chain if full). pub(crate) struct Bucket { + /// Index of next bucket in the chain. pub(crate) next: u32, + /// Key-value pair contained within bucket. pub(crate) inner: Option<(K, V)>, } +/// Core hash table implementation. pub(crate) struct CoreHashMap<'a, K, V> { /// Dictionary used to map hashes to bucket indices. pub(crate) dictionary: &'a mut [u32], @@ -23,15 +24,15 @@ pub(crate) struct CoreHashMap<'a, K, V> { pub(crate) buckets: &'a mut [Bucket], /// Head of the freelist. pub(crate) free_head: u32, - - pub(crate) _user_list_head: u32, /// Maximum index of a bucket allowed to be allocated. INVALID_POS if no limit. pub(crate) alloc_limit: u32, - - // metrics + /// The number of currently occupied buckets. pub(crate) buckets_in_use: u32, + // Unclear what the purpose of this is. + pub(crate) _user_list_head: u32, } +/// Error for when there are no empty buckets left but one is needed. #[derive(Debug)] pub struct FullError(); @@ -41,6 +42,7 @@ where { const FILL_FACTOR: f32 = 0.60; + /// Estimate the size of data contained within the the hash map. pub fn estimate_size(num_buckets: u32) -> usize { let mut size = 0; @@ -92,6 +94,7 @@ where } } + /// Get the value associated with a key (if it exists) given its hash. pub fn get_with_hash(&self, key: &K, hash: u64) -> Option<&V> { let mut next = self.dictionary[hash as usize % self.dictionary.len()]; loop { @@ -108,7 +111,7 @@ where } } - // all updates are done through Entry + /// Get the `Entry` associated with a key given hash. This should be used for updates/inserts. pub fn entry_with_hash(&mut self, key: K, hash: u64) -> Entry<'a, '_, K, V> { let dict_pos = hash as usize % self.dictionary.len(); let first = self.dictionary[dict_pos]; @@ -149,15 +152,18 @@ where } } + /// Get number of buckets in map. pub fn get_num_buckets(&self) -> usize { self.buckets.len() } + /// Returns whether there is an ongoing shrink operation. pub fn is_shrinking(&self) -> bool { self.alloc_limit != INVALID_POS } /// Clears all entries from the hashmap. + /// /// Does not reset any allocation limits, but does clear any entries beyond them. pub fn clear(&mut self) { for i in 0..self.buckets.len() { @@ -177,7 +183,8 @@ where self.buckets_in_use = 0; } - + + /// Optionally gets the entry at an index if it is occupied. pub fn entry_at_bucket(&mut self, pos: usize) -> Option> { if pos >= self.buckets.len() { return None; diff --git a/libs/neon-shmem/src/hash/entry.rs b/libs/neon-shmem/src/hash/entry.rs index 63af840c5d..24c124189b 100644 --- a/libs/neon-shmem/src/hash/entry.rs +++ b/libs/neon-shmem/src/hash/entry.rs @@ -5,12 +5,13 @@ use crate::hash::core::{CoreHashMap, FullError, INVALID_POS}; use std::hash::Hash; use std::mem; +/// View into an entry in the map (either vacant or occupied). pub enum Entry<'a, 'b, K, V> { - Occupied(OccupiedEntry<'a, 'b, K, V>), + Occupied(OccupiedEntry<'a, 'b, K, V>), Vacant(VacantEntry<'a, 'b, K, V>), } -/// Helper enum representing the previous position within a hashmap chain. +/// Enum representing the previous position within a chain. #[derive(Clone, Copy)] pub(crate) enum PrevPos { /// Starting index within the dictionary. @@ -21,17 +22,9 @@ pub(crate) enum PrevPos { Unknown, } -impl PrevPos { - /// Unwrap an index from a `PrevPos::First`, panicking otherwise. - pub fn unwrap_first(&self) -> u32 { - match self { - Self::First(i) => *i, - _ => panic!("not first entry in chain") - } - } -} - +/// View into an occupied entry within the map. pub struct OccupiedEntry<'a, 'b, K, V> { + /// Mutable reference to the map containing this entry. pub(crate) map: &'b mut CoreHashMap<'a, K, V>, /// The key of the occupied entry pub(crate) _key: K, @@ -58,6 +51,7 @@ impl<'a, 'b, K, V> OccupiedEntry<'a, 'b, K, V> { .1 } + /// Inserts a value into the entry, replacing (and returning) the existing value. pub fn insert(&mut self, value: V) -> V { let bucket = &mut self.map.buckets[self.bucket_pos as usize]; // This assumes inner is Some, which it must be for an OccupiedEntry @@ -65,6 +59,7 @@ impl<'a, 'b, K, V> OccupiedEntry<'a, 'b, K, V> { old_value } + /// Removes the entry from the hash map, returning the value originally stored within it. pub fn remove(self) -> V { // CoreHashMap::remove returns Option<(K, V)>. We know it's Some for an OccupiedEntry. let bucket = &mut self.map.buckets[self.bucket_pos as usize]; @@ -89,13 +84,18 @@ impl<'a, 'b, K, V> OccupiedEntry<'a, 'b, K, V> { } } +/// An abstract view into a vacant entry within the map. pub struct VacantEntry<'a, 'b, K, V> { + /// Mutable reference to the map containing this entry. pub(crate) map: &'b mut CoreHashMap<'a, K, V>, - pub(crate) key: K, // The key to insert + /// The key to be inserted into this entry. + pub(crate) key: K, + /// The position within the dictionary corresponding to the key's hash. pub(crate) dict_pos: u32, } impl<'a, 'b, K: Clone + Hash + Eq, V> VacantEntry<'a, 'b, K, V> { + /// Insert a value into the vacant entry, finding and populating an empty bucket in the process. pub fn insert(self, value: V) -> Result<&'b mut V, FullError> { let pos = self.map.alloc_bucket(self.key, value)?; if pos == INVALID_POS { diff --git a/libs/neon-shmem/src/hash/tests.rs b/libs/neon-shmem/src/hash/tests.rs index a522423db1..8a6e8a0a29 100644 --- a/libs/neon-shmem/src/hash/tests.rs +++ b/libs/neon-shmem/src/hash/tests.rs @@ -303,15 +303,18 @@ fn test_bucket_ops() { assert_eq!(writer.get_num_buckets_in_use(), 1); assert_eq!(writer.get_num_buckets(), 1000); assert_eq!(writer.get_with_hash(&1.into(), hash), Some(&2)); - match writer.entry_with_hash(1.into(), hash) { + let pos = match writer.entry_with_hash(1.into(), hash) { Entry::Occupied(e) => { assert_eq!(e._key, 1.into()); let pos = e.bucket_pos as usize; assert_eq!(writer.entry_at_bucket(pos).unwrap()._key, 1.into()); assert_eq!(writer.get_at_bucket(pos), Some(&(1.into(), 2))); + pos }, Entry::Vacant(_) => { panic!("Insert didn't affect entry"); }, - } + }; + let ptr: *const usize = writer.get_with_hash(&1.into(), hash).unwrap(); + assert_eq!(writer.get_bucket_for_value(ptr), pos); writer.remove_with_hash(&1.into(), hash); assert_eq!(writer.get_with_hash(&1.into(), hash), None); } From 00dfaa2eb4d819f0205f528d52c840917acf62ab Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Tue, 24 Jun 2025 16:30:59 -0700 Subject: [PATCH 14/24] Add Criterion microbenchmarks for rehashing and insertions --- Cargo.lock | 2982 ++++++++++++++---------- libs/neon-shmem/Cargo.toml | 7 +- libs/neon-shmem/benches/hmap_resize.rs | 287 +++ 3 files changed, 1984 insertions(+), 1292 deletions(-) create mode 100644 libs/neon-shmem/benches/hmap_resize.rs diff --git a/Cargo.lock b/Cargo.lock index 2f50f11cee..4fd5f5802b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,55 +17,49 @@ dependencies = [ "gimli", ] -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - [[package]] name = "adler2" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "ahash" -version = "0.8.11" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", "const-random", - "getrandom 0.2.11", + "getrandom 0.3.3", "once_cell", "version_check", - "zerocopy 0.7.31", + "zerocopy", ] [[package]] name = "aho-corasick" -version = "1.0.1" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] [[package]] name = "aligned-vec" -version = "0.6.1" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e0966165eaf052580bd70eb1b32cb3d6245774c0104d1b2793e9650bf83b52a" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" dependencies = [ "equator", ] [[package]] name = "allocator-api2" -version = "0.2.16" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android-tzdata" @@ -90,9 +84,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.15" +version = "0.6.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" dependencies = [ "anstyle", "anstyle-parse", @@ -105,43 +99,44 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.8" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" [[package]] name = "anstyle-parse" -version = "0.2.0" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.4" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "once_cell_polyfill", + "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.94" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" dependencies = [ "backtrace", ] @@ -190,9 +185,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.5" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc2d0cfb2a7388d34f590e76686704c494ed7aaceed62ee1ba35cbf363abc2a5" +checksum = "40f6024f3f856663b45fd0c9b6f2024034a702f453549449e0d84a305900dad4" dependencies = [ "flate2", "futures-core", @@ -205,20 +200,20 @@ dependencies = [ [[package]] name = "async-lock" -version = "3.2.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7125e42787d53db9dd54261812ef17e937c95a51e4d291373b670342fa44310c" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" dependencies = [ - "event-listener 4.0.0", + "event-listener 5.4.0", "event-listener-strategy", "pin-project-lite", ] [[package]] name = "async-stream" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ "async-stream-impl", "futures-core", @@ -227,24 +222,24 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] name = "async-trait" -version = "0.1.68" +version = "0.1.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -254,16 +249,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8ab6b55fe97976e46f91ddbed8d147d966475dc29b2032757ba47e02376fbc3" [[package]] -name = "autocfg" -version = "1.1.0" +name = "atomic-waker" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-config" -version = "1.5.10" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b49afaa341e8dd8577e1a2200468f98956d6eda50bcf4a53246cc00174ba924" +checksum = "455e9fb7743c6f6267eb2830ccc08686fbb3d13c9a689369562fd4d4ef9ea462" dependencies = [ "aws-credential-types", "aws-runtime", @@ -272,15 +273,15 @@ dependencies = [ "aws-sdk-sts", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json 0.60.7", + "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.2.0", + "fastrand 2.3.0", "hex", - "http 0.2.9", + "http 1.3.1", "ring", "time", "tokio", @@ -291,9 +292,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.1" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" +checksum = "687bc16bc431a8533fe0097c7f0182874767f920989d7260950172ae8e3c4465" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -302,10 +303,33 @@ dependencies = [ ] [[package]] -name = "aws-runtime" -version = "1.4.4" +name = "aws-lc-rs" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5ac934720fbb46206292d2c75b57e67acfc56fe7dfd34fb9a02334af08409ea" +checksum = "93fcc8f365936c834db5514fc45aee5b1202d677e6b40e48468aaaa8183ca8c7" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61b1d86e7705efe1be1b569bab41d4fa1e14e220b60a160f78de2db687add079" +dependencies = [ + "bindgen 0.69.5", + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "aws-runtime" +version = "1.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f6c68419d8ba16d9a7463671593c54f81ba58cab466e9b759418da606dcc2e2" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -317,10 +341,9 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.2.0", - "http 0.2.9", - "http-body 0.4.5", - "once_cell", + "fastrand 2.3.0", + "http 0.2.12", + "http-body 0.4.6", "percent-encoding", "pin-project-lite", "tracing", @@ -329,54 +352,54 @@ dependencies = [ [[package]] name = "aws-sdk-iam" -version = "1.53.0" +version = "1.77.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb8a6fea8d335cde419176b1f2c6d2d6e97997719e7df4b51e59064310f48e4a" +checksum = "2e273b8d4662875a313054584302defeba9c04e85dad7c662355e9fc3dba9ae1" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json 0.61.1", + "aws-smithy-json", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-smithy-xml", "aws-types", - "http 0.2.9", - "once_cell", + "fastrand 2.3.0", + "http 0.2.12", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-kms" -version = "1.51.0" +version = "1.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c30f6fd5646b99d9b45ec3a0c22e67112c175b2383100c960d7ee39d96c8d96" +checksum = "8565497721d9f18fa29a68bc5d8225b39e1cc7399d7fc6f1ad803ca934341804" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json 0.61.1", + "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", - "http 0.2.9", - "once_cell", + "fastrand 2.3.0", + "http 0.2.12", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-s3" -version = "1.65.0" +version = "1.93.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3ba2c5c0f2618937ce3d4a5ad574b86775576fa24006bcb3128c6e2cbf3c34e" +checksum = "16b9734dc8145b417a3c22eae8769a2879851690982dba718bdc52bd28ad04ce" dependencies = [ "aws-credential-types", "aws-runtime", @@ -385,20 +408,20 @@ dependencies = [ "aws-smithy-checksums", "aws-smithy-eventstream", "aws-smithy-http", - "aws-smithy-json 0.61.1", + "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-smithy-xml", "aws-types", "bytes", - "fastrand 2.2.0", + "fastrand 2.3.0", "hex", "hmac", - "http 0.2.9", - "http-body 0.4.5", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", "lru", - "once_cell", "percent-encoding", "regex-lite", "sha2", @@ -408,76 +431,76 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.50.0" +version = "1.73.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ca43a4ef210894f93096039ef1d6fa4ad3edfabb3be92b80908b9f2e4b4eab" +checksum = "b2ac1674cba7872061a29baaf02209fefe499ff034dfd91bd4cc59e4d7741489" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json 0.61.1", + "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", - "http 0.2.9", - "once_cell", + "fastrand 2.3.0", + "http 0.2.12", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-ssooidc" -version = "1.51.0" +version = "1.74.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abaf490c2e48eed0bb8e2da2fb08405647bd7f253996e0f93b981958ea0f73b0" +checksum = "3a6a22f077f5fd3e3c0270d4e1a110346cddf6769e9433eb9e6daceb4ca3b149" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json 0.61.1", + "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", - "http 0.2.9", - "once_cell", + "fastrand 2.3.0", + "http 0.2.12", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-sts" -version = "1.51.0" +version = "1.75.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b68fde0d69c8bfdc1060ea7da21df3e39f6014da316783336deff0a9ec28f4bf" +checksum = "e3258fa707f2f585ee3049d9550954b959002abd59176975150a01d5cf38ae3f" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json 0.61.1", + "aws-smithy-json", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-smithy-xml", "aws-types", - "http 0.2.9", - "once_cell", + "fastrand 2.3.0", + "http 0.2.12", "regex-lite", "tracing", ] [[package]] name = "aws-sigv4" -version = "1.2.6" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" +checksum = "ddfb9021f581b71870a17eac25b52335b82211cdc092e02b6876b2bcefa61666" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -489,9 +512,8 @@ dependencies = [ "form_urlencoded", "hex", "hmac", - "http 0.2.9", - "http 1.1.0", - "once_cell", + "http 0.2.12", + "http 1.3.1", "p256 0.11.1", "percent-encoding", "ring", @@ -504,9 +526,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.1" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" +checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" dependencies = [ "futures-util", "pin-project-lite", @@ -515,18 +537,17 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.60.13" +version = "0.63.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" +checksum = "d2f77a921dbd2c78ebe70726799787c1d110a2245dd65e39b20923dfdfb2deee" dependencies = [ "aws-smithy-http", "aws-smithy-types", "bytes", - "crc32c", - "crc32fast", + "crc-fast", "hex", - "http 0.2.9", - "http-body 0.4.5", + "http 0.2.12", + "http-body 0.4.6", "md-5", "pin-project-lite", "sha1", @@ -536,9 +557,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.60.5" +version = "0.60.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" +checksum = "338a3642c399c0a5d157648426110e199ca7fd1c689cc395676b81aa563700c4" dependencies = [ "aws-smithy-types", "bytes", @@ -547,9 +568,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.11" +version = "0.62.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" +checksum = "99335bec6cdc50a346fda1437f9fefe33abf8c99060739a546a16457f2862ca9" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -557,9 +578,9 @@ dependencies = [ "bytes", "bytes-utils", "futures-core", - "http 0.2.9", - "http-body 0.4.5", - "once_cell", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", "percent-encoding", "pin-project-lite", "pin-utils", @@ -567,21 +588,50 @@ dependencies = [ ] [[package]] -name = "aws-smithy-json" -version = "0.60.7" +name = "aws-smithy-http-client" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6" +checksum = "7f491388e741b7ca73b24130ff464c1478acc34d5b331b7dd0a2ee4643595a15" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.3.26", + "h2 0.4.10", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper 1.6.0", + "hyper-rustls 0.24.2", + "hyper-rustls 0.27.7", + "hyper-util", + "pin-project-lite", + "rustls 0.21.12", + "rustls 0.23.28", + "rustls-native-certs 0.8.1", + "rustls-pki-types", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.61.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a16e040799d29c17412943bdbf488fd75db04112d0c0d4b9290bacf5ae0014b9" dependencies = [ "aws-smithy-types", ] [[package]] -name = "aws-smithy-json" -version = "0.61.1" +name = "aws-smithy-observability" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" +checksum = "9364d5989ac4dd918e5cc4c4bdcc61c9be17dcd2586ea7f69e348fc7c6cab393" dependencies = [ - "aws-smithy-types", + "aws-smithy-runtime-api", ] [[package]] @@ -596,42 +646,39 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.4" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f20685047ca9d6f17b994a07f629c813f08b5bce65523e47124879e60103d45" +checksum = "14302f06d1d5b7d333fd819943075b13d27c7700b414f574c3c35859bfb55d5e" dependencies = [ "aws-smithy-async", "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "fastrand 2.2.0", - "h2 0.3.26", - "http 0.2.9", - "http-body 0.4.5", - "http-body 1.0.0", - "httparse", - "hyper 0.14.30", - "hyper-rustls 0.24.0", - "once_cell", + "fastrand 2.3.0", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "http-body 1.0.1", "pin-project-lite", "pin-utils", - "rustls 0.21.12", "tokio", "tracing", ] [[package]] name = "aws-smithy-runtime-api" -version = "1.7.3" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" +checksum = "bd8531b6d8882fd8f48f82a9754e682e29dd44cff27154af51fa3eb730f59efb" dependencies = [ "aws-smithy-async", "aws-smithy-types", "bytes", - "http 0.2.9", - "http 1.1.0", + "http 0.2.12", + "http 1.3.1", "pin-project-lite", "tokio", "tracing", @@ -640,18 +687,18 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.9" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fbd94a32b3a7d55d3806fe27d98d3ad393050439dd05eb53ece36ec5e3d3510" +checksum = "d498595448e43de7f4296b7b7a18a8a02c61ec9349128c80a368f7c3b4ab11a8" dependencies = [ "base64-simd", "bytes", "bytes-utils", "futures-core", - "http 0.2.9", - "http 1.1.0", - "http-body 0.4.5", - "http-body 1.0.0", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "http-body 1.0.1", "http-body-util", "itoa", "num-integer", @@ -666,18 +713,18 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.9" +version = "0.60.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" +checksum = "3db87b96cb1b16c024980f133968d52882ca0daaee3a086c6decc500f6c99728" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.3" +version = "1.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" +checksum = "8a322fec39e4df22777ed3ad8ea868ac2f94cd15e1a55f6ee8d8d6305057689a" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -689,19 +736,19 @@ dependencies = [ [[package]] name = "axum" -version = "0.8.1" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8" +checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5" dependencies = [ "axum-core", "base64 0.22.1", "bytes", "form_urlencoded", "futures-util", - "http 1.1.0", - "http-body 1.0.0", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.6.0", "hyper-util", "itoa", "matchit", @@ -715,10 +762,10 @@ dependencies = [ "serde_path_to_error", "serde_urlencoded", "sha1", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tokio", - "tokio-tungstenite 0.26.1", - "tower 0.5.2", + "tokio-tungstenite 0.26.2", + "tower", "tower-layer", "tower-service", "tracing", @@ -726,19 +773,19 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1362f362fd16024ae199c1970ce98f9661bf5ef94b9808fee734bc3698b733" +checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" dependencies = [ "bytes", - "futures-util", - "http 1.1.0", - "http-body 1.0.0", + "futures-core", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", "rustversion", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tower-layer", "tower-service", "tracing", @@ -746,22 +793,23 @@ dependencies = [ [[package]] name = "axum-extra" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fc6f625a1f7705c6cf62d0d070794e94668988b1c38111baeec177c715f7b" +checksum = "45bf463831f5131b7d3c756525b305d40f1185b688565648a92e1392ca35713d" dependencies = [ "axum", "axum-core", "bytes", "futures-util", "headers", - "http 1.1.0", - "http-body 1.0.0", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", + "rustversion", "serde", - "tower 0.5.2", + "tower", "tower-layer", "tower-service", ] @@ -776,7 +824,7 @@ dependencies = [ "bytes", "dyn-clone", "futures", - "getrandom 0.2.11", + "getrandom 0.2.16", "hmac", "http-types", "once_cell", @@ -869,14 +917,14 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.74" +version = "0.3.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide 0.8.0", + "miniz_oxide", "object", "rustc-demangle", "windows-targets 0.52.6", @@ -930,9 +978,9 @@ dependencies = [ [[package]] name = "base64ct" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" [[package]] name = "bb8" @@ -942,15 +990,15 @@ checksum = "d89aabfae550a5c44b43ab941844ffcd2e993cb6900b342debf59e9ea74acdb8" dependencies = [ "async-trait", "futures-util", - "parking_lot 0.12.1", + "parking_lot 0.12.4", "tokio", ] [[package]] name = "bcder" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c627747a6774aab38beb35990d88309481378558875a41da1a4b2e373c906ef0" +checksum = "89ffdaa8c6398acd07176317eb6c1f9082869dd1cc3fee7c72c6354866b928cc" dependencies = [ "bytes", "smallvec", @@ -965,16 +1013,39 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.9.1", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn", + "which", +] + [[package]] name = "bindgen" version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "cexpr", "clang-sys", - "itertools 0.12.1", + "itertools 0.13.0", "log", "prettyplease", "proc-macro2", @@ -982,7 +1053,7 @@ dependencies = [ "regex", "rustc-hash 2.1.1", "shlex", - "syn 2.0.100", + "syn", ] [[package]] @@ -999,9 +1070,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.8.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" [[package]] name = "block-buffer" @@ -1014,48 +1085,47 @@ dependencies = [ [[package]] name = "bstr" -version = "1.5.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a246e68bb43f6cd9db24bea052a53e40405417c5fb372e3d1a8a7f770a564ef5" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" dependencies = [ "memchr", - "once_cell", - "regex-automata 0.1.10", + "regex-automata 0.4.9", "serde", ] [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "bytemuck" -version = "1.16.3" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "102087e286b4677862ea56cf8fc58bb2cdfa8725c40ffb80fe3a008eb7f2fc83" +checksum = "5c76a5792e44e4abe34d3abf15636779261d45a7450612059293d1d2cfc63422" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" dependencies = [ "serde", ] [[package]] name = "bytes-utils" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" dependencies = [ "bytes", "either", @@ -1063,18 +1133,18 @@ dependencies = [ [[package]] name = "camino" -version = "1.1.6" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +checksum = "0da45bc31171d8d6960122e222a67740df867c1dd53b4d51caa297084c185cab" dependencies = [ "serde", ] [[package]] name = "camino-tempfile" -version = "1.0.2" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ab15a83d13f75dbd86f082bdefd160b628476ef58d3b900a0ef74e001bb097" +checksum = "64308c4c82a5c38679945ddf88738dc1483dcc563bbb5780755ae9f8497d2b20" dependencies = [ "camino", "tempfile", @@ -1100,16 +1170,16 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.100", + "syn", "tempfile", "toml", ] [[package]] name = "cc" -version = "1.2.16" +version = "1.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" +checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" dependencies = [ "jobserver", "libc", @@ -1127,9 +1197,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" [[package]] name = "cfg_aliases" @@ -1139,9 +1209,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "cgroups-rs" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb3af90c8d48ad5f432d8afb521b5b40c2a2fce46dd60e05912de51c47fba64" +checksum = "6db7c2f5545da4c12c5701455d9471da5f07db52e49b9cccb4f5512226dd0836" dependencies = [ "libc", "log", @@ -1152,9 +1222,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1194,9 +1264,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.6.1" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", @@ -1205,9 +1275,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.22" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69371e34337c4c984bbe322360c2547210bf632eb2814bbe78a6e87a2935bd2b" +checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" dependencies = [ "clap_builder", "clap_derive", @@ -1215,59 +1285,68 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.22" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e24c1b4099818523236a8ca881d2b45db98dadfb4625cf6608c12069fcbbde1" +checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim 0.11.1", + "strsim", ] [[package]] name = "clap_derive" -version = "4.5.18" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] name = "clap_lex" -version = "0.7.3" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "clashmap" -version = "1.0.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93bd59c81e2bd87a775ae2de75f070f7e2bfe97363a6ad652f46824564c23e4d" +checksum = "e8a055b1f1bf558eae4959f6dd77cf2d7d50ae1483928e60ef21ca5a24fd4321" dependencies = [ "crossbeam-utils", - "hashbrown 0.15.2", + "hashbrown 0.15.4 (registry+https://github.com/rust-lang/crates.io-index)", "lock_api", - "parking_lot_core 0.9.8", + "parking_lot_core 0.9.11", "polonius-the-crab", "replace_with", ] [[package]] -name = "colorchoice" -version = "1.0.0" +name = "cmake" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "combine" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" dependencies = [ "bytes", "futures-core", @@ -1279,13 +1358,12 @@ dependencies = [ [[package]] name = "comfy-table" -version = "7.1.1" +version = "7.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b34115915337defe99b2aff5c2ce6771e5fbc4079f4b506301f5cf394c8452f7" +checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a" dependencies = [ "crossterm", - "strum", - "strum_macros", + "unicode-segmentation", "unicode-width", ] @@ -1334,7 +1412,7 @@ dependencies = [ "fail", "flate2", "futures", - "http 1.1.0", + "http 1.3.1", "indexmap 2.9.0", "itertools 0.10.5", "jsonwebtoken", @@ -1364,7 +1442,7 @@ dependencies = [ "tokio-postgres", "tokio-stream", "tokio-util", - "tower 0.5.2", + "tower", "tower-http", "tower-otel", "tracing", @@ -1383,9 +1461,9 @@ dependencies = [ [[package]] name = "concurrent-queue" -version = "2.3.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" dependencies = [ "crossbeam-utils", ] @@ -1398,9 +1476,9 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "const-random" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaf16c9c2c612020bcfd042e170f6e32de9b9d75adb5277cdbbd2e2c8c8299a" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" dependencies = [ "const-random-macro", ] @@ -1411,25 +1489,25 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.11", + "getrandom 0.2.16", "once_cell", "tiny-keccak", ] [[package]] name = "const_format" -version = "0.2.30" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7309d9b4d3d2c0641e018d449232f2e28f1b22933c137f157d3dbc14228b8c0e" +checksum = "126f97965c8ad46d6d9163268ff28432e8f6a1196a55578867832e3049df63dd" dependencies = [ "const_format_proc_macros", ] [[package]] name = "const_format_proc_macros" -version = "0.2.29" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f47bf7270cf70d370f8f98c1abb6d2d4cf60a6845d30e05bfb90c6568650" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" dependencies = [ "proc-macro2", "quote", @@ -1460,7 +1538,7 @@ dependencies = [ "http-utils", "humantime", "humantime-serde", - "hyper 0.14.30", + "hyper 0.14.32", "jsonwebtoken", "nix 0.30.1", "once_cell", @@ -1493,9 +1571,19 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" dependencies = [ "core-foundation-sys", "libc", @@ -1503,9 +1591,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpp_demangle" @@ -1518,13 +1606,41 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc-fast" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf62af4cc77d8fe1c22dde4e721d87f2f54056139d8c412e1366b740305f56f" +dependencies = [ + "crc", + "digest", + "libc", + "rand 0.9.1", + "regex", +] + [[package]] name = "crc32c" version = "0.6.8" @@ -1536,9 +1652,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] @@ -1587,24 +1703,23 @@ checksum = "5877d3fbf742507b66bc2a1945106bd30dd8504019d596901ddd012a4dd01740" dependencies = [ "chrono", "once_cell", - "winnow", + "winnow 0.6.26", ] [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -1621,20 +1736,20 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crossterm" -version = "0.27.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "crossterm_winapi", - "libc", - "parking_lot 0.12.1", + "parking_lot 0.12.4", + "rustix 0.38.44", "winapi", ] @@ -1649,9 +1764,9 @@ dependencies = [ [[package]] name = "crunchy" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" [[package]] name = "crypto-bigint" @@ -1710,14 +1825,14 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] name = "darling" -version = "0.20.1" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0558d22a7b463ed0241e993f76f09f30b126687447751a8638587b864e4b3944" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ "darling_core", "darling_macro", @@ -1725,40 +1840,27 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.1" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab8bfa2e259f8ee1ce5e97824a3c55ec4404a0d772ca7fa96bf19f0752a046eb" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim 0.10.0", - "syn 2.0.100", + "strsim", + "syn", ] [[package]] name = "darling_macro" -version = "0.20.1" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn 2.0.100", -] - -[[package]] -name = "dashmap" -version = "5.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d" -dependencies = [ - "cfg-if", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core 0.9.8", + "syn", ] [[package]] @@ -1772,14 +1874,14 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core 0.9.8", + "parking_lot_core 0.9.11", ] [[package]] name = "data-encoding" -version = "2.4.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" [[package]] name = "debugid" @@ -1803,9 +1905,9 @@ dependencies = [ [[package]] name = "der" -version = "0.7.8" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ "const-oid", "der_derive", @@ -1822,7 +1924,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -1835,6 +1937,16 @@ dependencies = [ "serde", ] +[[package]] +name = "deranged" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +dependencies = [ + "powerfmt", + "serde", +] + [[package]] name = "desim" version = "0.1.0" @@ -1842,7 +1954,7 @@ dependencies = [ "anyhow", "bytes", "hex", - "parking_lot 0.12.1", + "parking_lot 0.12.4", "rand 0.8.5", "smallvec", "tracing", @@ -1857,11 +1969,11 @@ checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" [[package]] name = "diesel" -version = "2.2.6" +version = "2.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf1bedf64cdb9643204a36dd15b19a6ce8e7aa7f7b105868e9f1fad5ffa7d12" +checksum = "a917a9209950404d5be011c81d081a2692a822f73c3d6af586f0cab5ff50f614" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "byteorder", "chrono", "diesel_derives", @@ -1886,15 +1998,15 @@ dependencies = [ [[package]] name = "diesel_derives" -version = "2.2.1" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59de76a222c2b8059f789cbe07afbfd8deb8c31dd0bc2a21f85e256c1def8259" +checksum = "52841e97814f407b895d836fa0012091dff79c6268f39ad8155d384c21ae0d26" dependencies = [ "diesel_table_macro_syntax", "dsl_auto_type", "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -1914,7 +2026,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "209c735641a413bc68c4923a9d6ad4bcb3ca306b794edaa7eb0b3228a99ffb25" dependencies = [ - "syn 2.0.100", + "syn", ] [[package]] @@ -1937,13 +2049,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -1957,23 +2069,29 @@ dependencies = [ [[package]] name = "dsl_auto_type" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0892a17df262a24294c382f0d5997571006e7a4348b4327557c4ff1cd4a8bccc" +checksum = "139ae9aca7527f85f26dd76483eb38533fd84bd571065da1739656ef71c5ff5b" dependencies = [ "darling", "either", "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] -name = "dyn-clone" -version = "1.0.14" +name = "dunce" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "dyn-clone" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005" [[package]] name = "ecdsa" @@ -1993,7 +2111,7 @@ version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der 0.7.8", + "der 0.7.10", "digest", "elliptic-curve 0.13.8", "rfc6979 0.4.0", @@ -2025,9 +2143,9 @@ dependencies = [ [[package]] name = "either" -version = "1.8.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "elliptic-curve" @@ -2059,7 +2177,7 @@ dependencies = [ "base64ct", "crypto-bigint 0.5.5", "digest", - "ff 0.13.0", + "ff 0.13.1", "generic-array", "group 0.13.0", "pem-rfc7468", @@ -2093,7 +2211,7 @@ dependencies = [ "test-log", "tokio", "tokio-util", - "tower 0.5.2", + "tower", "tracing", "utils", "workspace_hack", @@ -2101,29 +2219,29 @@ dependencies = [ [[package]] name = "enum-map" -version = "2.5.0" +version = "2.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "988f0d17a0fa38291e5f41f71ea8d46a5d5497b9054d5a759fae2cbb819f2356" +checksum = "6866f3bfdf8207509a033af1a75a7b08abda06bbaaeae6669323fd5a097df2e9" dependencies = [ "enum-map-derive", ] [[package]] name = "enum-map-derive" -version = "0.11.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4da76b3b6116d758c7ba93f7ec6a35d2e2cf24feda76c6e38a375f4d5c59f2" +checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn", ] [[package]] name = "enumset" -version = "1.1.2" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e875f1719c16de097dee81ed675e2d9bb63096823ed3f0ca827b7dea3028bbbb" +checksum = "11a6b7c3d347de0a9f7bfd2f853be43fe32fa6fac30c70f6d6d67a1e936b87ee" dependencies = [ "enumset_derive", "serde", @@ -2131,14 +2249,14 @@ dependencies = [ [[package]] name = "enumset_derive" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08b6c6ab82d70f08844964ba10c7babb716de2ecaeab9be5717918a5177d3af" +checksum = "6da3ea9e1d1a3b1593e15781f930120e72aa7501610b2f82e5b6739c72e8eac5" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -2153,9 +2271,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.7" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3716d7a920fb4fac5d84e9d4bce8ceb321e9414b4409da61b07b75c1e3d0697" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" dependencies = [ "anstream", "anstyle", @@ -2166,38 +2284,38 @@ dependencies = [ [[package]] name = "equator" -version = "0.2.2" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c35da53b5a021d2484a7cc49b2ac7f2d840f8236a286f84202369bd338d761ea" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" dependencies = [ "equator-macro", ] [[package]] name = "equator-macro" -version = "0.2.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bf679796c0322556351f287a51b49e48f7c4986e727b5dd78c972d30e2e16cc" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] name = "equivalent" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -2208,9 +2326,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "4.0.0" +version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770d968249b5d99410d61f5bf89057f3199a077a04d087092f58e7d10692baae" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" dependencies = [ "concurrent-queue", "parking", @@ -2219,11 +2337,11 @@ dependencies = [ [[package]] name = "event-listener-strategy" -version = "0.4.0" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" dependencies = [ - "event-listener 4.0.0", + "event-listener 5.4.0", "pin-project-lite", ] @@ -2255,9 +2373,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "ff" @@ -2271,9 +2389,9 @@ dependencies = [ [[package]] name = "ff" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded41244b729663b1e574f1b4fb731469f69f79c17667b5d776b16cda0479449" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" dependencies = [ "rand_core 0.6.4", "subtle", @@ -2287,14 +2405,14 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "filetime" -version = "0.2.22" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", - "windows-sys 0.48.0", + "libredox", + "windows-sys 0.59.0", ] [[package]] @@ -2316,19 +2434,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] -name = "flagset" -version = "0.4.6" +name = "fixedbitset" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3ea1ec5f8307826a5b71094dd91fc04d4ae75d5709b20ad351c7fb4815c86ec" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flagset" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" [[package]] name = "flate2" -version = "1.0.26" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", - "miniz_oxide 0.7.1", + "miniz_oxide", ] [[package]] @@ -2337,6 +2461,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -2357,7 +2487,7 @@ dependencies = [ "futures-core", "futures-sink", "http-body-util", - "hyper 1.4.1", + "hyper 1.6.0", "hyper-util", "pin-project", "rand 0.8.5", @@ -2367,6 +2497,12 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -2378,9 +2514,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -2409,9 +2545,9 @@ checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -2447,7 +2583,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -2516,14 +2652,14 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -2534,9 +2670,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -2572,34 +2710,36 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "governor" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "842dc78579ce01e6a1576ad896edc92fca002dd60c9c3746b7fc2bec6fb429d0" +checksum = "be93b4ec2e4710b04d9264c0c7350cdd62a8c20e5e4ac732552ebb8f0debe8eb" dependencies = [ "cfg-if", - "dashmap 6.1.0", + "dashmap", "futures-sink", "futures-timer", "futures-util", + "getrandom 0.3.3", "no-std-compat", "nonzero_ext", - "parking_lot 0.12.1", + "parking_lot 0.12.4", "portable-atomic", "quanta", - "rand 0.8.5", + "rand 0.9.1", "smallvec", "spinning_top", + "web-time", ] [[package]] @@ -2619,7 +2759,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ - "ff 0.13.0", + "ff 0.13.1", "rand_core 0.6.4", "subtle", ] @@ -2635,7 +2775,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http 0.2.9", + "http 0.2.12", "indexmap 2.9.0", "slab", "tokio", @@ -2645,16 +2785,16 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.4" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" +checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5" dependencies = [ + "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "futures-util", - "http 1.1.0", + "http 1.3.1", "indexmap 2.9.0", "slab", "tokio", @@ -2664,9 +2804,9 @@ dependencies = [ [[package]] name = "half" -version = "2.4.1" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" dependencies = [ "cfg-if", "crunchy", @@ -2700,9 +2840,24 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.2" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.15.4" +source = "git+https://github.com/quantumish/hashbrown.git?rev=6610e6d#6610e6d2b1f288ef7b0709a3efefbc846395dc5e" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "hashlink" @@ -2729,14 +2884,14 @@ dependencies = [ [[package]] name = "headers" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322106e6bd0cba2d5ead589ddb8150a13d7c4217cf80d7c4f682ca994ccc6aa9" +checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "bytes", "headers-core", - "http 1.1.0", + "http 1.3.1", "httpdate", "mime", "sha1", @@ -2748,7 +2903,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http 1.1.0", + "http 1.3.1", ] [[package]] @@ -2765,9 +2920,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "hex" @@ -2803,21 +2958,30 @@ dependencies = [ ] [[package]] -name = "hostname" -version = "0.4.0" +name = "home" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9c7c7c8ac16c798734b8a24560c1362120597c40d5e1459f09498f8f6c8f2ba" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "hostname" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65" dependencies = [ "cfg-if", "libc", - "windows", + "windows-link", ] [[package]] name = "http" -version = "0.2.9" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -2826,9 +2990,9 @@ dependencies = [ [[package]] name = "http" -version = "1.1.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ "bytes", "fnv", @@ -2837,35 +3001,35 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http 0.2.9", + "http 0.2.12", "pin-project-lite", ] [[package]] name = "http-body" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.1.0", + "http 1.3.1", ] [[package]] name = "http-body-util" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", - "futures-util", - "http 1.1.0", - "http-body 1.0.0", + "futures-core", + "http 1.3.1", + "http-body 1.0.1", "pin-project-lite", ] @@ -2899,7 +3063,7 @@ dependencies = [ "camino", "fail", "futures", - "hyper 0.14.30", + "hyper 0.14.32", "itertools 0.10.5", "jemalloc_pprof", "jsonwebtoken", @@ -2908,8 +3072,8 @@ dependencies = [ "pprof", "regex", "routerify", - "rustls 0.23.27", - "rustls-pemfile 2.1.1", + "rustls 0.23.28", + "rustls-pemfile 2.2.0", "serde", "serde_json", "serde_path_to_error", @@ -2928,15 +3092,15 @@ dependencies = [ [[package]] name = "httparse" -version = "1.8.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" @@ -2956,17 +3120,17 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.30" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", "futures-core", "futures-util", "h2 0.3.26", - "http 0.2.9", - "http-body 0.4.5", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -2980,16 +3144,16 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.4", - "http 1.1.0", - "http-body 1.0.0", + "h2 0.4.10", + "http 1.3.1", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -3001,43 +3165,45 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.24.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ - "http 0.2.9", - "hyper 0.14.30", + "futures-util", + "http 0.2.12", + "hyper 0.14.32", "log", "rustls 0.21.12", - "rustls-native-certs 0.6.2", + "rustls-native-certs 0.6.3", "tokio", - "tokio-rustls 0.24.0", + "tokio-rustls 0.24.1", ] [[package]] name = "hyper-rustls" -version = "0.26.0" +version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "futures-util", - "http 1.1.0", - "hyper 1.4.1", + "http 1.3.1", + "hyper 1.6.0", "hyper-util", - "rustls 0.22.4", + "rustls 0.23.28", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls 0.26.2", "tower-service", + "webpki-roots 1.0.1", ] [[package]] name = "hyper-timeout" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.4.1", + "hyper 1.6.0", "hyper-util", "pin-project-lite", "tokio", @@ -3046,34 +3212,39 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.7" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" dependencies = [ + "base64 0.22.1", "bytes", "futures-channel", + "futures-core", "futures-util", - "http 1.1.0", - "http-body 1.0.0", - "hyper 1.4.1", + "http 1.3.1", + "http-body 1.0.1", + "hyper 1.6.0", + "ipnet", + "libc", + "percent-encoding", "pin-project-lite", "socket2", "tokio", - "tower 0.4.13", "tower-service", "tracing", ] [[package]] name = "iana-time-zone" -version = "0.1.61" +version = "0.1.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", + "log", "wasm-bindgen", "windows-core", ] @@ -3089,21 +3260,22 @@ dependencies = [ [[package]] name = "icu_collections" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" dependencies = [ "displaydoc", + "potential_utf", "yoke", "zerofrom", "zerovec", ] [[package]] -name = "icu_locid" -version = "1.5.0" +name = "icu_locale_core" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" dependencies = [ "displaydoc", "litemap", @@ -3112,31 +3284,11 @@ dependencies = [ "zerovec", ] -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" - [[package]] name = "icu_normalizer" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" dependencies = [ "displaydoc", "icu_collections", @@ -3144,67 +3296,54 @@ dependencies = [ "icu_properties", "icu_provider", "smallvec", - "utf16_iter", - "utf8_iter", - "write16", "zerovec", ] [[package]] name = "icu_normalizer_data" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" -version = "1.5.1" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" dependencies = [ "displaydoc", "icu_collections", - "icu_locid_transform", + "icu_locale_core", "icu_properties_data", "icu_provider", - "tinystr", + "potential_utf", + "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "1.5.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" [[package]] name = "icu_provider" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" dependencies = [ "displaydoc", - "icu_locid", - "icu_provider_macros", + "icu_locale_core", "stable_deref_trait", "tinystr", "writeable", "yoke", "zerofrom", + "zerotrie", "zerovec", ] -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - [[package]] name = "ident_case" version = "1.0.1" @@ -3224,9 +3363,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" dependencies = [ "icu_normalizer", "icu_properties", @@ -3250,15 +3389,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", - "hashbrown 0.15.2", + "hashbrown 0.15.4 (registry+https://github.com/rust-lang/crates.io-index)", "serde", ] [[package]] name = "indoc" -version = "2.0.5" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" [[package]] name = "infer" @@ -3286,22 +3425,22 @@ dependencies = [ [[package]] name = "inferno" -version = "0.12.0" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a5d75fee4d36809e6b021e4b96b686e763d365ffdb03af2bd00786353f84fe" +checksum = "2094aecddc672e902cd773bad7071542f63641e01e9187c3bba4b43005e837e9" dependencies = [ "ahash", "clap", "crossbeam-channel", "crossbeam-utils", - "dashmap 6.1.0", + "dashmap", "env_logger", "indexmap 2.9.0", "itoa", "log", "num-format", "once_cell", - "quick-xml 0.37.1", + "quick-xml 0.37.5", "rgb", "str_stack", ] @@ -3328,9 +3467,9 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", "js-sys", @@ -3346,9 +3485,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "io-uring" -version = "0.6.2" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460648e47a07a43110fbfa2e0b14afb2be920093c31e5dccc50e49568e099762" +checksum = "595a0399f411a508feb2ec1e970a4a30c249351e30208960d58298de8660b0e5" dependencies = [ "bitflags 1.3.2", "libc", @@ -3356,19 +3495,29 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] [[package]] name = "is-terminal" -version = "0.4.12" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3396,10 +3545,28 @@ dependencies = [ ] [[package]] -name = "itoa" -version = "1.0.11" +name = "itertools" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jemalloc_pprof" @@ -3420,9 +3587,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.4" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d699bc6dfc879fb1bf9bdff0d4c56f0884fc6f0d0eb0fba397a6d00cd9a6b85e" +checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49" dependencies = [ "jiff-static", "log", @@ -3433,21 +3600,22 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.4" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d16e75759ee0aa64c57a56acbf43916987b20c77373cb7e808979e02b93c9f9" +checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] name = "jobserver" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ + "getrandom 0.3.3", "libc", ] @@ -3489,10 +3657,11 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -3509,11 +3678,11 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "9.2.0" +version = "9.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7ea04a7c5c055c175f189b6dc6ba036fd62306b58c66c9f6389036c503a3f4" +checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "js-sys", "pem", "ring", @@ -3524,9 +3693,9 @@ dependencies = [ [[package]] name = "kqueue" -version = "1.0.8" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c" +checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" dependencies = [ "kqueue-sys", "libc", @@ -3544,12 +3713,12 @@ dependencies = [ [[package]] name = "lasso" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4644821e1c3d7a560fe13d842d13f587c07348a1a05d3a797152d41c90c56df2" +checksum = "6e14eda50a3494b3bf7b9ce51c52434a761e383d7238ce1dd5dcec2fbc13e9fb" dependencies = [ - "dashmap 5.5.0", - "hashbrown 0.13.2", + "dashmap", + "hashbrown 0.14.5", ] [[package]] @@ -3562,50 +3731,73 @@ dependencies = [ ] [[package]] -name = "libc" -version = "0.2.172" +name = "lazycell" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.174" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" [[package]] name = "libloading" -version = "0.7.4" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "winapi", + "windows-targets 0.53.2", ] [[package]] name = "libm" -version = "0.2.8" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.9.1", + "libc", + "redox_syscall 0.5.13", +] [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0b5399f6804fbab912acbd8878ed3532d506b7c951b8f9f164ef90fef39e3f4" +checksum = "2a385b1be4e5c3e362ad2ffa73c392e53f031eaa5b7d648e64cd87f27f6063d7" + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" [[package]] name = "litemap" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" dependencies = [ "autocfg", "scopeguard", @@ -3613,19 +3805,25 @@ dependencies = [ [[package]] name = "log" -version = "0.4.26" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "lru" -version = "0.12.3" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "mappings" version = "0.7.0" @@ -3656,10 +3854,11 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest", ] @@ -3682,7 +3881,7 @@ dependencies = [ "lasso", "measured-derive", "memchr", - "parking_lot 0.12.1", + "parking_lot 0.12.4", "rustc-hash 1.1.0", "ryu", ] @@ -3696,7 +3895,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -3712,15 +3911,15 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" [[package]] name = "memmap2" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45fd3a57831bf88bc63f8cebc0cf956116276e97fef3966103e96416209f7c92" +checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" dependencies = [ "libc", ] @@ -3736,9 +3935,9 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" dependencies = [ "autocfg", ] @@ -3756,7 +3955,7 @@ dependencies = [ "prometheus", "rand 0.8.5", "rand_distr 0.4.3", - "twox-hash", + "twox-hash 1.6.3", ] [[package]] @@ -3794,18 +3993,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" -dependencies = [ - "adler", -] - -[[package]] -name = "miniz_oxide" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", ] @@ -3818,39 +4008,45 @@ checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "log", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.48.0", ] [[package]] name = "mio" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.52.0", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", ] [[package]] name = "multimap" -version = "0.8.3" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" [[package]] name = "neon-shmem" version = "0.1.0" dependencies = [ + "ahash", "criterion", + "foldhash", + "hashbrown 0.15.4 (git+https://github.com/quantumish/hashbrown.git?rev=6610e6d)", "nix 0.30.1", "rand 0.9.1", "rand_distr 0.5.1", - "rustc-hash 1.1.0", + "rustc-hash 2.1.1", + "seahash", "tempfile", "thiserror 1.0.69", + "twox-hash 2.1.1", "workspace_hack", + "xxhash-rust", ] [[package]] @@ -3890,11 +4086,11 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "cfg-if", "cfg_aliases", "libc", - "memoffset 0.9.0", + "memoffset 0.9.1", ] [[package]] @@ -3925,7 +4121,7 @@ version = "6.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "crossbeam-channel", "filetime", "fsevent-sys", @@ -3959,9 +4155,9 @@ dependencies = [ [[package]] name = "num" -version = "0.4.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" dependencies = [ "num-bigint", "num-complex", @@ -4000,9 +4196,9 @@ dependencies = [ [[package]] name = "num-complex" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "num-traits", ] @@ -4034,9 +4230,9 @@ dependencies = [ [[package]] name = "num-iter" -version = "0.1.43" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" dependencies = [ "autocfg", "num-integer", @@ -4045,11 +4241,10 @@ dependencies = [ [[package]] name = "num-rational" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ - "autocfg", "num-bigint", "num-integer", "num-traits", @@ -4067,9 +4262,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ "hermit-abi", "libc", @@ -4083,8 +4278,8 @@ checksum = "c38841cdd844847e3e7c8d29cef9dcfed8877f8f56f9071f77843ecf3baf937f" dependencies = [ "base64 0.13.1", "chrono", - "getrandom 0.2.11", - "http 0.2.9", + "getrandom 0.2.16", + "http 0.2.12", "rand 0.8.5", "serde", "serde_json", @@ -4096,30 +4291,36 @@ dependencies = [ [[package]] name = "object" -version = "0.36.5" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.20.2" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" [[package]] name = "oorandom" -version = "11.1.3" +version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "openssl-probe" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "opentelemetry" @@ -4143,7 +4344,7 @@ checksum = "10a8a7f5f6ba7c1b286c2fbca0454eaba116f63bbe69ed250b642d36fbb04d80" dependencies = [ "async-trait", "bytes", - "http 1.1.0", + "http 1.3.1", "opentelemetry", "reqwest", ] @@ -4156,7 +4357,7 @@ checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" dependencies = [ "async-trait", "futures-core", - "http 1.1.0", + "http 1.3.1", "opentelemetry", "opentelemetry-http", "opentelemetry-proto", @@ -4226,20 +4427,21 @@ dependencies = [ [[package]] name = "os_info" -version = "3.7.0" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "006e42d5b888366f1880eda20371fedde764ed2213dc8496f49622fa0c99cd5e" +checksum = "d0e1ac5fde8d43c34139135df8ea9ee9465394b2d8d20f032d38998f64afffc3" dependencies = [ "log", + "plist", "serde", - "winapi", + "windows-sys 0.52.0", ] [[package]] name = "outref" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "overload" @@ -4272,9 +4474,9 @@ dependencies = [ [[package]] name = "p384" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70786f51bcc69f6a4c0360e063a4cac5419ef7c5cd5b3c99ad70f3be5ba79209" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" dependencies = [ "elliptic-curve 0.13.8", "primeorder", @@ -4359,11 +4561,11 @@ dependencies = [ "hashlink", "hex", "hex-literal", - "http 1.1.0", + "http 1.3.1", "http-utils", "humantime", "humantime-serde", - "hyper 0.14.30", + "hyper 0.14.32", "indoc", "itertools 0.10.5", "jsonwebtoken", @@ -4396,7 +4598,7 @@ dependencies = [ "reqwest", "rpds", "rstest", - "rustls 0.23.27", + "rustls 0.23.28", "scopeguard", "send-future", "serde", @@ -4422,10 +4624,10 @@ dependencies = [ "toml_edit", "tonic 0.13.1", "tonic-reflection", - "tower 0.5.2", + "tower", "tracing", "tracing-utils", - "twox-hash", + "twox-hash 1.6.3", "url", "utils", "uuid", @@ -4528,9 +4730,9 @@ dependencies = [ [[package]] name = "papaya" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6827e3fc394523c21d4464d02c0bb1c19966ea4a58a9844ad6d746214179d2bc" +checksum = "af228bb1296c9b044ee75e2a2325409c2d899bcfcc6150e5e41f148e0a87dd20" dependencies = [ "equivalent", "seize", @@ -4538,9 +4740,9 @@ dependencies = [ [[package]] name = "parking" -version = "2.1.1" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e52c774a4c39359c1d1c52e43f73dd91a75a614652c825408eec30c95a9b2067" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" @@ -4555,12 +4757,12 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", - "parking_lot_core 0.9.8", + "parking_lot_core 0.9.11", ] [[package]] @@ -4579,48 +4781,48 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall 0.5.13", "smallvec", - "windows-targets 0.48.0", + "windows-targets 0.52.6", ] [[package]] name = "parquet" -version = "53.0.0" +version = "53.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0fbf928021131daaa57d334ca8e3904fe9ae22f73c56244fc7db9b04eedc3d8" +checksum = "2f8cf58b29782a7add991f655ff42929e31a7859f5319e53db9e39a714cb113c" dependencies = [ "ahash", "bytes", "chrono", "half", - "hashbrown 0.14.5", + "hashbrown 0.15.4 (registry+https://github.com/rust-lang/crates.io-index)", "num", "num-bigint", "paste", "seq-macro", "thrift", - "twox-hash", + "twox-hash 1.6.3", "zstd", "zstd-sys", ] [[package]] name = "parquet_derive" -version = "53.0.0" +version = "53.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86e9fcfae007533a06b580429a3f7e07cb833ec8aa37c041c16563e7918f057e" +checksum = "a8a6a70e8ee504da797f192a2f7cf9ce61262a1026794f7cdf988daefa5d0814" dependencies = [ "parquet", "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -4636,9 +4838,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pbkdf2" @@ -4654,11 +4856,11 @@ dependencies = [ [[package]] name = "pem" -version = "3.0.3" +version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310" +checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "serde", ] @@ -4679,57 +4881,67 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ - "fixedbitset", - "indexmap 1.9.3", + "fixedbitset 0.4.2", + "indexmap 2.9.0", +] + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset 0.5.7", + "indexmap 2.9.0", ] [[package]] name = "phf" -version = "0.11.1" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ "phf_shared", ] [[package]] name = "phf_shared" -version = "0.11.1" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ "siphasher", ] [[package]] name = "pin-project" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfe2e71e1471fe07709406bf725f710b02927c9c54b2b5b2ec0e8087d97c327d" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6e859e6e5bd50440ab63c47e3ebabc90f26251f7c73c3d3e837b74a1cc3fa67" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -4743,7 +4955,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der 0.7.8", + "der 0.7.10", "pkcs8 0.10.2", "spki 0.7.3", ] @@ -4764,21 +4976,34 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der 0.7.8", + "der 0.7.10", "spki 0.7.3", ] [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "plist" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d77244ce2d584cd84f6a15f86195b8c9b2a0dfbfd817c09e0464244091a58ed" +dependencies = [ + "base64 0.22.1", + "indexmap 2.9.0", + "quick-xml 0.37.5", + "serde", + "time", +] [[package]] name = "plotters" -version = "0.3.4" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ "num-traits", "plotters-backend", @@ -4789,15 +5014,15 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.4" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" [[package]] name = "plotters-svg" -version = "0.3.3" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" dependencies = [ "plotters-backend", ] @@ -4814,9 +5039,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.10.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "portable-atomic-util" @@ -4830,7 +5055,7 @@ dependencies = [ [[package]] name = "postgres" version = "0.19.7" -source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#1f21e7959a96a34dcfbfce1b14b73286cdadffe9" +source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#f3cf448febde5fd298071d54d568a9c875a7a62b" dependencies = [ "bytes", "fallible-iterator", @@ -4902,8 +5127,8 @@ dependencies = [ "bytes", "once_cell", "pq_proto", - "rustls 0.23.27", - "rustls-pemfile 2.1.1", + "rustls 0.23.28", + "rustls-pemfile 2.2.0", "serde", "thiserror 1.0.69", "tokio", @@ -4930,13 +5155,13 @@ name = "postgres_ffi" version = "0.1.0" dependencies = [ "anyhow", - "bindgen", + "bindgen 0.71.1", "bytes", "crc32c", "criterion", "env_logger", "log", - "memoffset 0.9.0", + "memoffset 0.9.1", "once_cell", "postgres", "pprof", @@ -4976,6 +5201,15 @@ dependencies = [ "workspace_hack", ] +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -4998,7 +5232,7 @@ dependencies = [ "log", "nix 0.26.4", "once_cell", - "parking_lot 0.12.1", + "parking_lot 0.12.4", "prost 0.12.6", "prost-build 0.12.6", "prost-derive 0.12.6", @@ -5018,7 +5252,7 @@ dependencies = [ "anyhow", "backtrace", "flate2", - "inferno 0.12.0", + "inferno 0.12.2", "num", "paste", "prost 0.13.5", @@ -5026,9 +5260,12 @@ dependencies = [ [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] [[package]] name = "pq_proto" @@ -5046,12 +5283,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.17" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3928fb5db768cb86f891ff014f0144589297e3c6a1aba6ed7cecfdace270c7" +checksum = "061c1221631e079b26479d25bbf2275bfe5917ae8419cd7e34f13bfc2aa7539a" dependencies = [ "proc-macro2", - "syn 2.0.100", + "syn", ] [[package]] @@ -5065,9 +5302,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] @@ -5078,13 +5315,13 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "chrono", "flate2", "hex", "lazy_static", "procfs-core", - "rustix", + "rustix 0.38.44", ] [[package]] @@ -5093,7 +5330,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "chrono", "hex", ] @@ -5109,7 +5346,7 @@ dependencies = [ "lazy_static", "libc", "memchr", - "parking_lot 0.12.1", + "parking_lot 0.12.4", "procfs", "thiserror 1.0.69", ] @@ -5146,33 +5383,32 @@ dependencies = [ "log", "multimap", "once_cell", - "petgraph", + "petgraph 0.6.5", "prettyplease", "prost 0.12.6", "prost-types 0.12.6", "regex", - "syn 2.0.100", + "syn", "tempfile", ] [[package]] name = "prost-build" -version = "0.13.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1318b19085f08681016926435853bbf7858f9c082d0999b80550ff5d9abe15" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "bytes", "heck 0.5.0", - "itertools 0.12.1", + "itertools 0.14.0", "log", "multimap", "once_cell", - "petgraph", + "petgraph 0.7.1", "prettyplease", "prost 0.13.5", - "prost-types 0.13.3", + "prost-types 0.13.5", "regex", - "syn 2.0.100", + "syn", "tempfile", ] @@ -5186,7 +5422,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -5196,10 +5432,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -5213,9 +5449,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.13.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ "prost 0.13.5", ] @@ -5257,13 +5493,13 @@ dependencies = [ "hex", "hmac", "hostname", - "http 1.1.0", + "http 1.3.1", "http-body-util", "http-utils", "humantime", "humantime-serde", - "hyper 0.14.30", - "hyper 1.4.1", + "hyper 0.14.32", + "hyper 1.6.0", "hyper-util", "indexmap 2.9.0", "ipnet", @@ -5278,7 +5514,7 @@ dependencies = [ "opentelemetry", "p256 0.13.2", "papaya", - "parking_lot 0.12.1", + "parking_lot 0.12.4", "parquet", "parquet_derive", "pbkdf2", @@ -5299,9 +5535,9 @@ dependencies = [ "rsa", "rstest", "rustc-hash 1.1.0", - "rustls 0.23.27", - "rustls-native-certs 0.8.0", - "rustls-pemfile 2.1.1", + "rustls 0.23.28", + "rustls-native-certs 0.8.1", + "rustls-pemfile 2.2.0", "scopeguard", "serde", "serde_json", @@ -5336,20 +5572,20 @@ dependencies = [ "walkdir", "workspace_hack", "x509-cert", - "zerocopy 0.8.24", + "zerocopy", ] [[package]] name = "quanta" -version = "0.12.5" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bd1fe6824cea6538803de3ff1bc0cf3949024db3d43c9643024bfb33a807c0e" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" dependencies = [ "crossbeam-utils", "libc", "once_cell", "raw-cpuid", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi 0.11.1+wasi-snapshot-preview1", "web-sys", "winapi", ] @@ -5375,27 +5611,82 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.37.1" +version = "0.37.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f22f29bdff3987b4d8632ef95fd6424ec7e4e0a57e2f4fc63e489e75357f6a03" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" dependencies = [ "memchr", ] [[package]] -name = "quote" -version = "1.0.39" +name = "quinn" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1f1914ce909e1658d9907913b4b91947430c7d9be598b15a1912935b8c04801" +checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash 2.1.1", + "rustls 0.23.28", + "socket2", + "thiserror 2.0.12", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" +dependencies = [ + "bytes", + "getrandom 0.3.3", + "lru-slab", + "rand 0.9.1", + "ring", + "rustc-hash 2.1.1", + "rustls 0.23.28", + "rustls-pki-types", + "slab", + "thiserror 2.0.12", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.59.0", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] [[package]] name = "r-efi" -version = "5.2.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "rand" @@ -5476,7 +5767,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.11", + "getrandom 0.2.16", ] [[package]] @@ -5531,18 +5822,18 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.3.0" +version = "11.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6928fa44c097620b706542d428957635951bade7143269085389d42c8a4927e" +checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", ] [[package]] name = "rayon" -version = "1.7.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", @@ -5550,21 +5841,19 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] name = "rcgen" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54077e1872c46788540de1ea3d7f4ccb1983d12f9aa909b234468676c1a36779" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" dependencies = [ "pem", "ring", @@ -5575,9 +5864,9 @@ dependencies = [ [[package]] name = "redis" -version = "0.29.2" +version = "0.29.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b110459d6e323b7cda23980c46c77157601199c9da6241552b284cd565a7a133" +checksum = "1bc42f3a12fd4408ce64d8efef67048a924e543bd35c6591c0447fda9054695f" dependencies = [ "arc-swap", "bytes", @@ -5587,8 +5876,8 @@ dependencies = [ "num-bigint", "percent-encoding", "pin-project-lite", - "rustls 0.23.27", - "rustls-native-certs 0.8.0", + "rustls 0.23.28", + "rustls-native-certs 0.8.1", "ryu", "sha1_smol", "socket2", @@ -5618,23 +5907,23 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.1", ] [[package]] name = "regex" -version = "1.10.2" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.3", - "regex-syntax 0.8.2", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", ] [[package]] @@ -5648,20 +5937,20 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.2", + "regex-syntax 0.8.5", ] [[package]] name = "regex-lite" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" [[package]] name = "regex-syntax" @@ -5671,15 +5960,15 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "relative-path" -version = "1.9.0" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c707298afce11da2efef2f600116fa93ffa7a032b5d7b628aa17711ec81383ca" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "remote_storage" @@ -5704,7 +5993,7 @@ dependencies = [ "http-body-util", "http-types", "humantime-serde", - "hyper 1.4.1", + "hyper 1.6.0", "itertools 0.10.5", "metrics", "once_cell", @@ -5726,64 +6015,62 @@ dependencies = [ [[package]] name = "replace_with" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a8614ee435691de62bcffcf4a66d91b3594bf1428a5722e79103249a095690" +checksum = "51743d3e274e2b18df81c4dc6caf8a5b8e15dbe799e0dca05c7617380094e884" [[package]] name = "reqwest" -version = "0.12.4" +version = "0.12.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813" dependencies = [ "base64 0.22.1", "bytes", "futures-channel", "futures-core", "futures-util", - "http 1.1.0", - "http-body 1.0.0", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", - "hyper-rustls 0.26.0", + "hyper 1.6.0", + "hyper-rustls 0.27.7", "hyper-util", - "ipnet", "js-sys", "log", - "mime", - "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.22.4", - "rustls-native-certs 0.7.0", - "rustls-pemfile 2.1.1", + "quinn", + "rustls 0.23.28", + "rustls-native-certs 0.8.1", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.2", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls 0.26.2", "tokio-util", + "tower", + "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", - "winreg", + "webpki-roots 1.0.1", ] [[package]] name = "reqwest-middleware" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1ccd3b55e711f91a9885a2fa6fbbb2e39db1776420b062efc058c6410f7e5e3" +checksum = "57f17d28a6e6acfe1733fe24bcd30774d13bffa4b8a22535b4c8c98423088d4e" dependencies = [ "anyhow", "async-trait", - "http 1.1.0", + "http 1.3.1", "reqwest", "serde", "thiserror 1.0.69", @@ -5799,9 +6086,9 @@ dependencies = [ "anyhow", "async-trait", "futures", - "getrandom 0.2.11", - "http 1.1.0", - "hyper 1.4.1", + "getrandom 0.2.16", + "http 1.3.1", + "hyper 1.6.0", "parking_lot 0.11.2", "reqwest", "reqwest-middleware", @@ -5814,14 +6101,14 @@ dependencies = [ [[package]] name = "reqwest-tracing" -version = "0.5.5" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73e6153390585f6961341b50e5a1931d6be6dee4292283635903c26ef9d980d2" +checksum = "d70ea85f131b2ee9874f0b160ac5976f8af75f3c9badfe0d955880257d10bd83" dependencies = [ "anyhow", "async-trait", - "getrandom 0.2.11", - "http 1.1.0", + "getrandom 0.2.16", + "http 1.3.1", "matchit", "opentelemetry", "reqwest", @@ -5877,7 +6164,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.11", + "getrandom 0.2.16", "libc", "untrusted", "windows-sys 0.52.0", @@ -5885,9 +6172,9 @@ dependencies = [ [[package]] name = "rlimit" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3560f70f30a0f16d11d01ed078a07740fe6b489667abc7c7b029155d9f21c3d8" +checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a" dependencies = [ "libc", ] @@ -5898,8 +6185,8 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "496c1d3718081c45ba9c31fbfc07417900aa96f4070ff90dc29961836b7a9945" dependencies = [ - "http 0.2.9", - "hyper 0.14.30", + "http 0.2.12", + "hyper 0.14.32", "lazy_static", "percent-encoding", "regex", @@ -5916,9 +6203,9 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.6" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b" dependencies = [ "const-oid", "digest", @@ -5959,7 +6246,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.100", + "syn", "unicode-ident", ] @@ -5975,9 +6262,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" [[package]] name = "rustc-hash" @@ -5993,24 +6280,37 @@ checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] [[package]] name = "rustix" -version = "0.38.41" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "errno", "libc", - "linux-raw-sys 0.4.14", - "windows-sys 0.52.0", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +dependencies = [ + "bitflags 2.9.1", + "errno", + "libc", + "linux-raw-sys 0.9.4", + "windows-sys 0.59.0", ] [[package]] @@ -6027,24 +6327,11 @@ dependencies = [ [[package]] name = "rustls" -version = "0.22.4" +version = "0.23.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" -dependencies = [ - "log", - "ring", - "rustls-pki-types", - "rustls-webpki 0.102.8", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls" -version = "0.23.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" +checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -6056,66 +6343,55 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ "openssl-probe", - "rustls-pemfile 1.0.2", + "rustls-pemfile 1.0.4", "schannel", - "security-framework", + "security-framework 2.11.1", ] [[package]] name = "rustls-native-certs" -version = "0.7.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" dependencies = [ "openssl-probe", - "rustls-pemfile 2.1.1", "rustls-pki-types", "schannel", - "security-framework", -] - -[[package]] -name = "rustls-native-certs" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" -dependencies = [ - "openssl-probe", - "rustls-pemfile 2.1.1", - "rustls-pki-types", - "schannel", - "security-framework", + "security-framework 3.2.0", ] [[package]] name = "rustls-pemfile" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ "base64 0.21.7", ] [[package]] name = "rustls-pemfile" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f48172685e6ff52a556baa527774f61fcaa884f59daf3375c62a3f1cd2549dab" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.21.7", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +dependencies = [ + "web-time", + "zeroize", +] [[package]] name = "rustls-webpki" @@ -6127,23 +6403,13 @@ dependencies = [ "untrusted", ] -[[package]] -name = "rustls-webpki" -version = "0.102.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted", -] - [[package]] name = "rustls-webpki" version = "0.103.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -6151,15 +6417,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.12" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "safekeeper" @@ -6180,16 +6446,16 @@ dependencies = [ "fail", "futures", "hex", - "http 1.1.0", + "http 1.3.1", "http-utils", "humantime", - "hyper 0.14.30", + "hyper 0.14.32", "itertools 0.10.5", "jsonwebtoken", "metrics", "once_cell", "pageserver_api", - "parking_lot 0.12.1", + "parking_lot 0.12.4", "pem", "postgres-protocol", "postgres_backend", @@ -6200,7 +6466,7 @@ dependencies = [ "regex", "remote_storage", "reqwest", - "rustls 0.23.27", + "rustls 0.23.28", "safekeeper_api", "safekeeper_client", "scopeguard", @@ -6269,11 +6535,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6287,9 +6553,9 @@ dependencies = [ [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" @@ -6303,9 +6569,18 @@ dependencies = [ [[package]] name = "sd-notify" -version = "0.4.1" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "621e3680f3e07db4c9c2c3fb07c6223ab2fab2e54bd3c04c3ae037990f428c32" +checksum = "b943eadf71d8b69e661330cb0e2656e31040acf21ee7708e2c238a0ec6af2bf4" +dependencies = [ + "libc", +] + +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "sec1" @@ -6328,7 +6603,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct 0.2.0", - "der 0.7.8", + "der 0.7.10", "generic-array", "pkcs8 0.10.2", "serdect", @@ -6338,12 +6613,25 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.9.1" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 1.3.2", - "core-foundation", + "bitflags 2.9.1", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +dependencies = [ + "bitflags 2.9.1", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -6351,9 +6639,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -6371,9 +6659,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.17" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" [[package]] name = "send-future" @@ -6389,7 +6677,7 @@ checksum = "255914a8e53822abd946e2ce8baa41d4cded6b8e938913b7f7b9da5b7ab44335" dependencies = [ "httpdate", "reqwest", - "rustls 0.23.27", + "rustls 0.23.28", "sentry-backtrace", "sentry-contexts", "sentry-core", @@ -6397,7 +6685,7 @@ dependencies = [ "sentry-tracing", "tokio", "ureq", - "webpki-roots", + "webpki-roots 0.26.11", ] [[package]] @@ -6480,15 +6768,15 @@ dependencies = [ [[package]] name = "seq-macro" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.215" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" dependencies = [ "serde_derive", ] @@ -6505,20 +6793,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] name = "serde_json" -version = "1.0.125" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" dependencies = [ "itoa", "memchr", @@ -6528,9 +6816,9 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.14" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" +checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a" dependencies = [ "itoa", "serde", @@ -6549,9 +6837,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.6" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" dependencies = [ "serde", ] @@ -6593,7 +6881,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -6619,15 +6907,15 @@ dependencies = [ [[package]] name = "sha1_smol" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -6637,18 +6925,18 @@ dependencies = [ [[package]] name = "sha2-asm" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27ba7066011e3fb30d808b51affff34f0a66d3a03a58edd787c6e420e40e44e" +checksum = "b845214d6175804686b2bd482bcffe96651bb2d1200742b712003504a2dac1ab" dependencies = [ "cc", ] [[package]] name = "sharded-slab" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] @@ -6661,9 +6949,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook" -version = "0.3.15" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" dependencies = [ "libc", "signal-hook-registry", @@ -6671,9 +6959,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" dependencies = [ "libc", ] @@ -6700,54 +6988,51 @@ dependencies = [ [[package]] name = "simple_asn1" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" +checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" dependencies = [ "num-bigint", "num-traits", - "thiserror 1.0.69", + "thiserror 2.0.12", "time", ] [[package]] name = "siphasher" -version = "0.3.10" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" -version = "0.4.8" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" -dependencies = [ - "autocfg", -] +checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" [[package]] name = "smallvec" -version = "1.13.2" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "smol_str" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6845563ada680337a52d43bb0b29f396f2d911616f6573012645b9e3d048a49" +checksum = "dd538fb6910ac1099850255cf94a94df6551fbdd602454387d0adb2d1ca6dead" dependencies = [ "serde", ] [[package]] name = "socket2" -version = "0.5.5" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -6782,7 +7067,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der 0.7.8", + "der 0.7.10", ] [[package]] @@ -6813,13 +7098,13 @@ dependencies = [ "http-body-util", "http-utils", "humantime", - "hyper 1.4.1", + "hyper 1.6.0", "hyper-util", "metrics", "once_cell", - "parking_lot 0.12.1", + "parking_lot 0.12.4", "prost 0.13.5", - "rustls 0.23.27", + "rustls 0.23.28", "tokio", "tokio-rustls 0.26.2", "tonic 0.13.1", @@ -6850,7 +7135,7 @@ dependencies = [ "hex", "http-utils", "humantime", - "hyper 0.14.30", + "hyper 0.14.32", "itertools 0.10.5", "json-structural-diff", "lasso", @@ -6864,8 +7149,8 @@ dependencies = [ "regex", "reqwest", "routerify", - "rustls 0.23.27", - "rustls-native-certs 0.8.0", + "rustls 0.23.28", + "rustls-native-certs 0.8.1", "safekeeper_api", "safekeeper_client", "scoped-futures", @@ -6917,8 +7202,8 @@ dependencies = [ "postgres_ffi", "remote_storage", "reqwest", - "rustls 0.23.27", - "rustls-native-certs 0.8.0", + "rustls 0.23.28", + "rustls-native-certs 0.8.1", "serde", "serde_json", "storage_controller_client", @@ -6962,20 +7247,15 @@ checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" [[package]] name = "stringprep" -version = "0.1.2" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" dependencies = [ "unicode-bidi", "unicode-normalization", + "unicode-properties", ] -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - [[package]] name = "strsim" version = "0.11.1" @@ -6998,26 +7278,26 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.100", + "syn", ] [[package]] name = "subtle" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "svg_fmt" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20e16a0f46cf5fd675563ef54f26e83e20f2366bcf027bcb3cc3ed2b98aaf2ca" +checksum = "0193cc4331cfd2f3d2011ef287590868599a2f33c3e69bc22c1a3d3acf9e02fb" [[package]] name = "symbolic-common" -version = "12.12.0" +version = "12.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "366f1b4c6baf6cfefc234bbd4899535fca0b06c74443039a73f6dfb2fad88d77" +checksum = "6a1150bdda9314f6cfeeea801c23f5593c6e6a6c72e64f67e48d723a12b8efdb" dependencies = [ "debugid", "memmap2", @@ -7027,9 +7307,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "12.12.0" +version = "12.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aba05ba5b9962ea5617baf556293720a8b2d0a282aa14ee4bf10e22efc7da8c8" +checksum = "9f66537def48fbc704a92e4fdaab7833bc7cb2255faca8182592fb5fa617eb82" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -7038,20 +7318,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.109" +version = "2.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" dependencies = [ "proc-macro2", "quote", @@ -7069,26 +7338,29 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] [[package]] name = "synstructure" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] name = "sysinfo" -version = "0.29.7" +version = "0.29.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "165d6d8539689e3d3bc8b98ac59541e1f21c7de7c85d60dc80e43ae0ed2113db" +checksum = "cd727fc423c2060f6c92d9534cef765c65a6ed3f428a03d7def74a8c4348e666" dependencies = [ "cfg-if", "core-foundation-sys", @@ -7101,9 +7373,9 @@ dependencies = [ [[package]] name = "tar" -version = "0.4.40" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" dependencies = [ "filetime", "libc", @@ -7112,14 +7384,14 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.14.0" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" dependencies = [ - "cfg-if", - "fastrand 2.2.0", + "fastrand 2.3.0", + "getrandom 0.3.3", "once_cell", - "rustix", + "rustix 1.0.7", "windows-sys 0.59.0", ] @@ -7150,7 +7422,7 @@ checksum = "78ea17a2dc368aeca6f554343ced1b1e31f76d63683fa8016e5844bd7a5144a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -7172,7 +7444,7 @@ checksum = "888d0c3c6db53c0fdab160d2ed5e12ba745383d3e85813f2ea0f2b1475ab553f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -7186,11 +7458,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" dependencies = [ - "thiserror-impl 2.0.11", + "thiserror-impl 2.0.12", ] [[package]] @@ -7201,28 +7473,27 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] name = "thiserror-impl" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] name = "thread_local" -version = "1.1.7" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ "cfg-if", - "once_cell", ] [[package]] @@ -7269,11 +7540,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" dependencies = [ - "deranged", + "deranged 0.4.0", "itoa", "js-sys", "num-conv", @@ -7285,15 +7556,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" dependencies = [ "num-conv", "time-core", @@ -7310,9 +7581,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.7.6" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" dependencies = [ "displaydoc", "zerovec", @@ -7330,9 +7601,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" dependencies = [ "tinyvec_macros", ] @@ -7361,20 +7632,20 @@ checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] name = "tokio" -version = "1.43.1" +version = "1.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "492a604e2fd7f814268a378409e6c92b5525d747d10db9a229723f55a417958c" +checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" dependencies = [ "backtrace", "bytes", "libc", - "mio 1.0.3", - "parking_lot 0.12.1", + "mio 1.0.4", + "parking_lot 0.12.4", "pin-project-lite", "signal-hook-registry", "socket2", @@ -7416,7 +7687,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -7431,7 +7702,7 @@ dependencies = [ "futures-channel", "futures-util", "log", - "parking_lot 0.12.1", + "parking_lot 0.12.4", "percent-encoding", "phf", "pin-project-lite", @@ -7451,7 +7722,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04fb792ccd6bbcd4bba408eb8a292f70fc4a3589e5d793626f45190e6454b6ab" dependencies = [ "ring", - "rustls 0.23.27", + "rustls 0.23.28", "tokio", "tokio-postgres", "tokio-rustls 0.26.2", @@ -7465,7 +7736,7 @@ dependencies = [ "bytes", "fallible-iterator", "futures-util", - "parking_lot 0.12.1", + "parking_lot 0.12.4", "pin-project-lite", "postgres-protocol2", "postgres-types2", @@ -7477,40 +7748,29 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ "rustls 0.21.12", "tokio", ] -[[package]] -name = "tokio-rustls" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" -dependencies = [ - "rustls 0.22.4", - "rustls-pki-types", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.27", + "rustls 0.23.28", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -7546,38 +7806,37 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4bf6fecd69fcdede0ec680aaf474cdab988f9de6bc73d3758f0160e3b7025a" +checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" dependencies = [ "futures-util", "log", "tokio", - "tungstenite 0.26.1", + "tungstenite 0.26.2", ] [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", "futures-io", "futures-sink", "futures-util", - "hashbrown 0.14.5", + "hashbrown 0.15.4 (registry+https://github.com/rust-lang/crates.io-index)", "pin-project-lite", "tokio", - "tracing", ] [[package]] name = "toml" -version = "0.8.14" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", "serde_spanned", @@ -7587,26 +7846,33 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.6" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.22.14" +version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap 2.9.0", "serde", "serde_spanned", "toml_datetime", - "winnow", + "toml_write", + "winnow 0.7.11", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "tonic" version = "0.12.3" @@ -7616,8 +7882,8 @@ dependencies = [ "async-trait", "base64 0.22.1", "bytes", - "http 1.1.0", - "http-body 1.0.0", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", "percent-encoding", "pin-project", @@ -7638,22 +7904,22 @@ dependencies = [ "axum", "base64 0.22.1", "bytes", - "h2 0.4.4", - "http 1.1.0", - "http-body 1.0.0", + "h2 0.4.10", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.6.0", "hyper-timeout", "hyper-util", "percent-encoding", "pin-project", "prost 0.13.5", - "rustls-native-certs 0.8.0", + "rustls-native-certs 0.8.1", "socket2", "tokio", "tokio-rustls 0.26.2", "tokio-stream", - "tower 0.5.2", + "tower", "tower-layer", "tower-service", "tracing", @@ -7667,10 +7933,10 @@ checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847" dependencies = [ "prettyplease", "proc-macro2", - "prost-build 0.13.3", - "prost-types 0.13.3", + "prost-build 0.13.5", + "prost-types 0.13.5", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -7680,27 +7946,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9687bd5bfeafebdded2356950f278bba8226f0b32109537c4253406e09aafe1" dependencies = [ "prost 0.13.5", - "prost-types 0.13.3", + "prost-types 0.13.5", "tokio", "tokio-stream", "tonic 0.13.1", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", -] - [[package]] name = "tower" version = "0.5.2" @@ -7712,7 +7963,7 @@ dependencies = [ "indexmap 2.9.0", "pin-project-lite", "slab", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tokio", "tokio-util", "tower-layer", @@ -7722,17 +7973,20 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.2" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ "base64 0.22.1", - "bitflags 2.8.0", + "bitflags 2.9.1", "bytes", - "http 1.1.0", - "http-body 1.0.0", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "iri-string", "mime", "pin-project-lite", + "tower", "tower-layer", "tower-service", "tracing", @@ -7750,7 +8004,7 @@ name = "tower-otel" version = "0.2.0" source = "git+https://github.com/mattiapenati/tower-otel?rev=56a7321053bcb72443888257b622ba0d43a11fcd#56a7321053bcb72443888257b622ba0d43a11fcd" dependencies = [ - "http 1.1.0", + "http 1.3.1", "opentelemetry", "pin-project", "tower-layer", @@ -7779,31 +8033,32 @@ dependencies = [ [[package]] name = "tracing-appender" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e" +checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf" dependencies = [ "crossbeam-channel", + "thiserror 1.0.69", "time", "tracing-subscriber", ] [[package]] name = "tracing-attributes" -version = "0.1.28" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] name = "tracing-core" -version = "0.1.33" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" dependencies = [ "once_cell", "valuable", @@ -7811,9 +8066,9 @@ dependencies = [ [[package]] name = "tracing-error" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" +checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db" dependencies = [ "tracing", "tracing-subscriber", @@ -7897,14 +8152,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" dependencies = [ "quote", - "syn 2.0.100", + "syn", ] [[package]] name = "tracing-utils" version = "0.1.0" dependencies = [ - "hyper 0.14.30", + "hyper 0.14.32", "opentelemetry", "opentelemetry-otlp", "opentelemetry-semantic-conventions", @@ -7931,7 +8186,7 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http 1.1.0", + "http 1.3.1", "httparse", "log", "rand 0.8.5", @@ -7943,19 +8198,18 @@ dependencies = [ [[package]] name = "tungstenite" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413083a99c579593656008130e29255e54dcaae495be556cc26888f211648c24" +checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" dependencies = [ - "byteorder", "bytes", "data-encoding", - "http 1.1.0", + "http 1.3.1", "httparse", "log", - "rand 0.8.5", + "rand 0.9.1", "sha1", - "thiserror 2.0.11", + "thiserror 2.0.12", "utf-8", ] @@ -7969,6 +8223,15 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "twox-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56" +dependencies = [ + "rand 0.9.1", +] + [[package]] name = "typed-json" version = "0.1.1" @@ -7981,9 +8244,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.16.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "uname" @@ -7996,15 +8259,15 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.14" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] name = "unicode-normalization" @@ -8016,16 +8279,28 @@ dependencies = [ ] [[package]] -name = "unicode-width" -version = "0.1.10" +name = "unicode-properties" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" [[package]] name = "unicode-xid" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "untrusted" @@ -8035,17 +8310,17 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.10.1" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" dependencies = [ "base64 0.22.1", "log", "once_cell", - "rustls 0.23.27", + "rustls 0.23.28", "rustls-pki-types", "url", - "webpki-roots", + "webpki-roots 0.26.11", ] [[package]] @@ -8056,7 +8331,7 @@ dependencies = [ "bytes", "io-uring", "libc", - "linux-raw-sys 0.6.4", + "linux-raw-sys 0.6.5", ] [[package]] @@ -8073,9 +8348,9 @@ dependencies = [ [[package]] name = "urlencoding" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" [[package]] name = "utf-8" @@ -8083,12 +8358,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -8097,9 +8366,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "utf8parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "utils" @@ -8157,25 +8426,27 @@ dependencies = [ [[package]] name = "uuid" -version = "1.10.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" dependencies = [ - "getrandom 0.2.11", + "getrandom 0.3.3", + "js-sys", "serde", + "wasm-bindgen", ] [[package]] name = "valuable" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vm_monitor" @@ -8204,9 +8475,9 @@ checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" [[package]] name = "waker-fn" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" +checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" [[package]] name = "wal_craft" @@ -8266,18 +8537,17 @@ name = "walproposer" version = "0.1.0" dependencies = [ "anyhow", - "bindgen", + "bindgen 0.71.1", "postgres_ffi", "utils", ] [[package]] name = "want" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "log", "try-lock", ] @@ -8289,9 +8559,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" @@ -8310,46 +8580,48 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", + "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.100", + "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -8357,28 +8629,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "wasm-streams" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" dependencies = [ "futures-util", "js-sys", @@ -8404,9 +8679,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -8424,20 +8699,41 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.1" +version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.1", +] + +[[package]] +name = "webpki-roots" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8782dd5a41a24eed3a4f40b606249b3e236ca61adf1f25ea4d45c73de122b502" dependencies = [ "rustls-pki-types", ] [[package]] -name = "whoami" -version = "1.5.1" +name = "which" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ - "redox_syscall 0.4.1", + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + +[[package]] +name = "whoami" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" +dependencies = [ + "redox_syscall 0.5.13", "wasite", "web-sys", ] @@ -8460,11 +8756,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "winapi", + "windows-sys 0.59.0", ] [[package]] @@ -8474,22 +8770,62 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.52.0" +name = "windows-core" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ - "windows-core", - "windows-targets 0.52.6", + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", ] [[package]] -name = "windows-core" -version = "0.52.0" +name = "windows-implement" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ - "windows-targets 0.52.6", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link", ] [[package]] @@ -8498,7 +8834,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets 0.48.5", ] [[package]] @@ -8520,18 +8856,27 @@ dependencies = [ ] [[package]] -name = "windows-targets" -version = "0.48.0" +name = "windows-sys" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows-targets 0.53.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -8543,7 +8888,7 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", @@ -8551,10 +8896,26 @@ dependencies = [ ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.0" +name = "windows-targets" +version = "0.53.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" @@ -8563,10 +8924,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] -name = "windows_aarch64_msvc" -version = "0.48.0" +name = "windows_aarch64_gnullvm" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" @@ -8575,10 +8942,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] -name = "windows_i686_gnu" -version = "0.48.0" +name = "windows_aarch64_msvc" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" @@ -8586,6 +8959,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" @@ -8593,10 +8972,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] -name = "windows_i686_msvc" -version = "0.48.0" +name = "windows_i686_gnullvm" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" @@ -8605,10 +8990,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] -name = "windows_x86_64_gnu" -version = "0.48.0" +name = "windows_i686_msvc" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" @@ -8617,10 +9008,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.0" +name = "windows_x86_64_gnu" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" @@ -8629,10 +9026,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] -name = "windows_x86_64_msvc" -version = "0.48.0" +name = "windows_x86_64_gnullvm" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" @@ -8640,6 +9043,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "winnow" version = "0.6.26" @@ -8650,13 +9059,12 @@ dependencies = [ ] [[package]] -name = "winreg" -version = "0.52.0" +name = "winnow" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd" dependencies = [ - "cfg-if", - "windows-sys 0.48.0", + "memchr", ] [[package]] @@ -8665,7 +9073,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", ] [[package]] @@ -8688,8 +9096,8 @@ dependencies = [ "clap_builder", "const-oid", "crypto-bigint 0.5.5", - "der 0.7.8", - "deranged", + "der 0.7.10", + "deranged 0.3.11", "digest", "ecdsa 0.16.9", "either", @@ -8703,13 +9111,13 @@ dependencies = [ "futures-io", "futures-util", "generic-array", - "getrandom 0.2.11", + "getrandom 0.2.16", "half", "hashbrown 0.14.5", "hex", "hmac", - "hyper 0.14.30", - "hyper 1.4.1", + "hyper 0.14.32", + "hyper 1.6.0", "hyper-util", "indexmap 2.9.0", "itertools 0.12.1", @@ -8736,10 +9144,10 @@ dependencies = [ "quote", "rand 0.8.5", "regex", - "regex-automata 0.4.3", - "regex-syntax 0.8.2", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", "reqwest", - "rustls 0.23.27", + "rustls 0.23.28", "rustls-pki-types", "rustls-webpki 0.103.3", "scopeguard", @@ -8752,7 +9160,7 @@ dependencies = [ "spki 0.7.3", "stable_deref_trait", "subtle", - "syn 2.0.100", + "syn", "sync_wrapper 0.1.2", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", @@ -8763,7 +9171,7 @@ dependencies = [ "tokio-stream", "tokio-util", "toml_edit", - "tower 0.5.2", + "tower", "tracing", "tracing-core", "tracing-log", @@ -8776,17 +9184,11 @@ dependencies = [ "zstd-sys", ] -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - [[package]] name = "writeable" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" [[package]] name = "x509-cert" @@ -8795,7 +9197,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" dependencies = [ "const-oid", - "der 0.7.8", + "der 0.7.10", "spki 0.7.3", "tls_codec", ] @@ -8809,7 +9211,7 @@ dependencies = [ "bcder", "bytes", "chrono", - "der 0.7.8", + "der 0.7.10", "hex", "pem", "ring", @@ -8821,18 +9223,25 @@ dependencies = [ [[package]] name = "xattr" -version = "1.0.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea263437ca03c1522846a4ddafbca2542d0ad5ed9b784909d4b27b76f62bc34a" +checksum = "0d65cbf2f12c15564212d48f4e3dfb87923d25d611f2aed18f4cb23f0413d89e" dependencies = [ "libc", + "rustix 1.0.7", ] [[package]] name = "xmlparser" -version = "0.13.5" +version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" [[package]] name = "yasna" @@ -8845,9 +9254,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" dependencies = [ "serde", "stable_deref_trait", @@ -8857,74 +9266,54 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", "synstructure", ] [[package]] name = "zerocopy" -version = "0.7.31" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" dependencies = [ - "zerocopy-derive 0.7.31", -] - -[[package]] -name = "zerocopy" -version = "0.8.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" -dependencies = [ - "zerocopy-derive 0.8.24", + "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.31" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", + "syn", ] [[package]] name = "zerofrom" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", "synstructure", ] @@ -8946,14 +9335,25 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", +] + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", ] [[package]] name = "zerovec" -version = "0.10.4" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" dependencies = [ "yoke", "zerofrom", @@ -8962,38 +9362,38 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.10.3" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] name = "zstd" -version = "0.13.0" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.0.0" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.9+zstd.1.5.5" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", diff --git a/libs/neon-shmem/Cargo.toml b/libs/neon-shmem/Cargo.toml index bf14eb2e83..1c47ffed37 100644 --- a/libs/neon-shmem/Cargo.toml +++ b/libs/neon-shmem/Cargo.toml @@ -9,13 +9,17 @@ thiserror.workspace = true nix.workspace = true workspace_hack = { version = "0.1", path = "../../workspace_hack" } rustc-hash = { version = "2.1.1" } +rand = "0.9.1" [dev-dependencies] criterion = { workspace = true, features = ["html_reports"] } -rand = "0.9.1" rand_distr = "0.5.1" xxhash-rust = { version = "0.8.15", features = ["xxh3"] } ahash.workspace = true +twox-hash = { version = "2.1.1" } +seahash = "4.1.0" +hashbrown = { git = "https://github.com/quantumish/hashbrown.git", rev = "6610e6d" } +foldhash = "0.1.5" [target.'cfg(target_os = "macos")'.dependencies] tempfile = "3.14.0" @@ -23,3 +27,4 @@ tempfile = "3.14.0" [[bench]] name = "hmap_resize" harness = false + diff --git a/libs/neon-shmem/benches/hmap_resize.rs b/libs/neon-shmem/benches/hmap_resize.rs new file mode 100644 index 0000000000..fa4743a895 --- /dev/null +++ b/libs/neon-shmem/benches/hmap_resize.rs @@ -0,0 +1,287 @@ +use std::hint::black_box; +use criterion::{criterion_group, criterion_main, BatchSize, Criterion, BenchmarkId}; +use neon_shmem::hash::HashMapAccess; +use neon_shmem::hash::HashMapInit; +use neon_shmem::hash::entry::Entry; +use neon_shmem::shmem::ShmemHandle; +use rand::prelude::*; +use rand::distr::{Distribution, StandardUniform}; +use std::hash::BuildHasher; +use std::default::Default; + +// Taken from bindings to C code + +#[derive(Clone, Debug, Hash, Eq, PartialEq)] +#[repr(C)] +pub struct FileCacheKey { + pub _spc_id: u32, + pub _db_id: u32, + pub _rel_number: u32, + pub _fork_num: u32, + pub _block_num: u32, +} + +impl Distribution for StandardUniform { + // questionable, but doesn't need to be good randomness + fn sample(&self, rng: &mut R) -> FileCacheKey { + FileCacheKey { + _spc_id: rng.random(), + _db_id: rng.random(), + _rel_number: rng.random(), + _fork_num: rng.random(), + _block_num: rng.random() + } + } +} + +#[derive(Clone, Debug)] +#[repr(C)] +pub struct FileCacheEntry { + pub _offset: u32, + pub _access_count: u32, + pub _prev: *mut FileCacheEntry, + pub _next: *mut FileCacheEntry, + pub _state: [u32; 8], +} + +impl FileCacheEntry { + fn dummy() -> Self { + Self { + _offset: 0, + _access_count: 0, + _prev: std::ptr::null_mut(), + _next: std::ptr::null_mut(), + _state: [0; 8] + } + } +} + +// Utilities for applying operations. + +#[derive(Clone, Debug)] +struct TestOp(K, Option); + +fn apply_op( + op: TestOp, + map: &mut HashMapAccess, +) { + let hash = map.get_hash_value(&op.0); + let entry = map.entry_with_hash(op.0, hash); + + match op.1 { + Some(new) => { + match entry { + Entry::Occupied(mut e) => Some(e.insert(new)), + Entry::Vacant(e) => { e.insert(new).unwrap(); None }, + } + }, + None => { + match entry { + Entry::Occupied(e) => Some(e.remove()), + Entry::Vacant(_) => None, + } + }, + }; +} + +// Hash utilities + +struct SeaRandomState { + k1: u64, + k2: u64, + k3: u64, + k4: u64 +} + +impl std::hash::BuildHasher for SeaRandomState { + type Hasher = seahash::SeaHasher; + + fn build_hasher(&self) -> Self::Hasher { + seahash::SeaHasher::with_seeds(self.k1, self.k2, self.k3, self.k4) + } +} + +impl SeaRandomState { + fn new() -> Self { + let mut rng = rand::rng(); + Self { k1: rng.random(), k2: rng.random(), k3: rng.random(), k4: rng.random() } + } +} + +fn small_benchs(c: &mut Criterion) { + let mut group = c.benchmark_group("Small maps"); + group.sample_size(10); + + group.bench_function("small_rehash", |b| { + let ideal_filled = 4_000_000; + let size = 5_000_000; + let mut writer = HashMapInit::new_resizeable(size, size * 2).attach_writer(); + let mut rng = rand::rng(); + while writer.get_num_buckets_in_use() < ideal_filled as usize { + let key: FileCacheKey = rng.random(); + let val = FileCacheEntry::dummy(); + apply_op(TestOp(key, Some(val)), &mut writer); + } + b.iter(|| writer.shuffle()); + }); + + + group.bench_function("small_rehash_xxhash", |b| { + let ideal_filled = 4_000_000; + let size = 5_000_000; + let shmem = ShmemHandle::new("bench", 0, 1073741824 * 2).unwrap(); + let init_struct = HashMapInit::::init_in_shmem_with_hasher( + size, shmem, twox_hash::xxhash64::RandomState::default(), + ); + let mut writer = init_struct.attach_writer(); + let mut rng = rand::rng(); + while writer.get_num_buckets_in_use() < ideal_filled as usize { + let key: FileCacheKey = rng.random(); + let val = FileCacheEntry::dummy(); + apply_op(TestOp(key, Some(val)), &mut writer); + } + b.iter(|| writer.shuffle()); + }); + + + group.bench_function("small_rehash_ahash", |b| { + let ideal_filled = 4_000_000; + let size = 5_000_000; + let shmem = ShmemHandle::new("bench", 0, 1073741824 * 2).unwrap(); + let init_struct = HashMapInit::::init_in_shmem_with_hasher( + size, shmem, ahash::RandomState::default() + ); + let mut writer = init_struct.attach_writer(); + let mut rng = rand::rng(); + while writer.get_num_buckets_in_use() < ideal_filled as usize { + let key: FileCacheKey = rng.random(); + let val = FileCacheEntry::dummy(); + apply_op(TestOp(key, Some(val)), &mut writer); + } + b.iter(|| writer.shuffle()); + }); + + group.bench_function("small_rehash_seahash", |b| { + let ideal_filled = 4_000_000; + let size = 5_000_000; + let shmem = ShmemHandle::new("bench", 0, 1073741824 * 2).unwrap(); + let init_struct = HashMapInit::::init_in_shmem_with_hasher( + size, shmem, SeaRandomState::new() + ); + let mut writer = init_struct.attach_writer(); + let mut rng = rand::rng(); + while writer.get_num_buckets_in_use() < ideal_filled as usize { + let key: FileCacheKey = rng.random(); + let val = FileCacheEntry::dummy(); + apply_op(TestOp(key, Some(val)), &mut writer); + } + b.iter(|| writer.shuffle()); + }); + + group.finish(); +} + +fn real_benchs(c: &mut Criterion) { + let mut group = c.benchmark_group("Realistic workloads"); + group.sample_size(10); + group.bench_function("real_bulk_insert", |b| { + let size = 125_000_000; + let ideal_filled = 100_000_000; + let mut rng = rand::rng(); + b.iter_batched( + || HashMapInit::new_resizeable(size, size * 2).attach_writer(), + |mut writer| { + for i in 0..ideal_filled { + let key: FileCacheKey = rng.random(); + let val = FileCacheEntry::dummy(); + let hash = writer.get_hash_value(&key); + let entry = writer.entry_with_hash(key, hash); + std::hint::black_box(match entry { + Entry::Occupied(mut e) => { e.insert(val); }, + Entry::Vacant(e) => { e.insert(val).unwrap(); }, + }) + } + }, + BatchSize::SmallInput, + ) + }); + + group.bench_function("real_rehash", |b| { + let size = 125_000_000; + let ideal_filled = 100_000_000; + let mut writer = HashMapInit::new_resizeable(size, size).attach_writer(); + let mut rng = rand::rng(); + while writer.get_num_buckets_in_use() < ideal_filled { + let key: FileCacheKey = rng.random(); + let val = FileCacheEntry::dummy(); + apply_op(TestOp(key, Some(val)), &mut writer); + } + b.iter(|| writer.shuffle()); + }); + + group.bench_function("real_rehash_hashbrown", |b| { + let size = 125_000_000; + let ideal_filled = 100_000_000; + let mut writer = hashbrown::raw::RawTable::new(); + let mut rng = rand::rng(); + let hasher = rustc_hash::FxBuildHasher::default(); + while writer.len() < ideal_filled as usize { + let key: FileCacheKey = rng.random(); + let val = FileCacheEntry::dummy(); + writer.insert(hasher.hash_one(&key), (key, val), |(k,_)| hasher.hash_one(&k)); + } + b.iter(|| unsafe { writer.table.rehash_in_place( + &|table, index| hasher.hash_one(&table.bucket::<(FileCacheKey, FileCacheEntry)>(index).as_ref().0), + std::mem::size_of::<(FileCacheKey, FileCacheEntry)>(), + if std::mem::needs_drop::<(FileCacheKey, FileCacheEntry)>() { + Some(|ptr| std::ptr::drop_in_place(ptr as *mut (FileCacheKey, FileCacheEntry))) + } else { + None + }, + ) }); + }); + + for elems in [2, 4, 8, 16, 32, 64, 96, 112] { + group.bench_with_input(BenchmarkId::new("real_rehash_varied", elems), &elems, |b, &size| { + let ideal_filled = size * 1_000_000; + let size = 125_000_000; + let mut writer = HashMapInit::new_resizeable(size, size).attach_writer(); + let mut rng = rand::rng(); + while writer.get_num_buckets_in_use() < ideal_filled as usize { + let key: FileCacheKey = rng.random(); + let val = FileCacheEntry::dummy(); + apply_op(TestOp(key, Some(val)), &mut writer); + } + b.iter(|| writer.shuffle()); + }); + group.bench_with_input(BenchmarkId::new("real_rehash_varied_hashbrown", elems), &elems, |b, &size| { + let ideal_filled = size * 1_000_000; + let size = 125_000_000; + let mut writer = hashbrown::raw::RawTable::new(); + let mut rng = rand::rng(); + let hasher = rustc_hash::FxBuildHasher::default(); + unsafe { + writer.resize(size, |(k,_)| hasher.hash_one(&k), hashbrown::raw::Fallibility::Infallible).unwrap(); + } + while writer.len() < ideal_filled as usize { + let key: FileCacheKey = rng.random(); + let val = FileCacheEntry::dummy(); + writer.insert(hasher.hash_one(&key), (key, val), |(k,_)| hasher.hash_one(&k)); + } + b.iter(|| unsafe { writer.table.rehash_in_place( + &|table, index| hasher.hash_one(&table.bucket::<(FileCacheKey, FileCacheEntry)>(index).as_ref().0), + std::mem::size_of::<(FileCacheKey, FileCacheEntry)>(), + if std::mem::needs_drop::<(FileCacheKey, FileCacheEntry)>() { + Some(|ptr| std::ptr::drop_in_place(ptr as *mut (FileCacheKey, FileCacheEntry))) + } else { + None + }, + ) }); + }); + } + + group.finish(); +} + +criterion_group!(benches, small_benchs, real_benchs); +criterion_main!(benches); From 1fb363917038e6828871a5c6ed8ec24de8bd34e4 Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Wed, 25 Jun 2025 03:03:19 -0700 Subject: [PATCH 15/24] Properly change type of HashMapInit in .with_hasher() --- libs/neon-shmem/benches/hmap_resize.rs | 33 +++++++++++++------------- libs/neon-shmem/src/hash.rs | 15 ++++++++---- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/libs/neon-shmem/benches/hmap_resize.rs b/libs/neon-shmem/benches/hmap_resize.rs index fa4743a895..30a3dca296 100644 --- a/libs/neon-shmem/benches/hmap_resize.rs +++ b/libs/neon-shmem/benches/hmap_resize.rs @@ -129,11 +129,9 @@ fn small_benchs(c: &mut Criterion) { group.bench_function("small_rehash_xxhash", |b| { let ideal_filled = 4_000_000; let size = 5_000_000; - let shmem = ShmemHandle::new("bench", 0, 1073741824 * 2).unwrap(); - let init_struct = HashMapInit::::init_in_shmem_with_hasher( - size, shmem, twox_hash::xxhash64::RandomState::default(), - ); - let mut writer = init_struct.attach_writer(); + let mut writer = HashMapInit::new_resizeable(size, size * 2) + .with_hasher(twox_hash::xxhash64::RandomState::default()) + .attach_writer(); let mut rng = rand::rng(); while writer.get_num_buckets_in_use() < ideal_filled as usize { let key: FileCacheKey = rng.random(); @@ -147,11 +145,9 @@ fn small_benchs(c: &mut Criterion) { group.bench_function("small_rehash_ahash", |b| { let ideal_filled = 4_000_000; let size = 5_000_000; - let shmem = ShmemHandle::new("bench", 0, 1073741824 * 2).unwrap(); - let init_struct = HashMapInit::::init_in_shmem_with_hasher( - size, shmem, ahash::RandomState::default() - ); - let mut writer = init_struct.attach_writer(); + let mut writer = HashMapInit::new_resizeable(size, size * 2) + .with_hasher(ahash::RandomState::default()) + .attach_writer(); let mut rng = rand::rng(); while writer.get_num_buckets_in_use() < ideal_filled as usize { let key: FileCacheKey = rng.random(); @@ -164,12 +160,10 @@ fn small_benchs(c: &mut Criterion) { group.bench_function("small_rehash_seahash", |b| { let ideal_filled = 4_000_000; let size = 5_000_000; - let shmem = ShmemHandle::new("bench", 0, 1073741824 * 2).unwrap(); - let init_struct = HashMapInit::::init_in_shmem_with_hasher( - size, shmem, SeaRandomState::new() - ); - let mut writer = init_struct.attach_writer(); - let mut rng = rand::rng(); + let mut writer = HashMapInit::new_resizeable(size, size * 2) + .with_hasher(SeaRandomState::new()) + .attach_writer(); + let mut rng = rand::rng(); while writer.get_num_buckets_in_use() < ideal_filled as usize { let key: FileCacheKey = rng.random(); let val = FileCacheEntry::dummy(); @@ -225,6 +219,10 @@ fn real_benchs(c: &mut Criterion) { let mut writer = hashbrown::raw::RawTable::new(); let mut rng = rand::rng(); let hasher = rustc_hash::FxBuildHasher::default(); + unsafe { + writer.resize(size, |(k,_)| hasher.hash_one(&k), + hashbrown::raw::Fallibility::Infallible).unwrap(); + } while writer.len() < ideal_filled as usize { let key: FileCacheKey = rng.random(); let val = FileCacheEntry::dummy(); @@ -261,7 +259,8 @@ fn real_benchs(c: &mut Criterion) { let mut rng = rand::rng(); let hasher = rustc_hash::FxBuildHasher::default(); unsafe { - writer.resize(size, |(k,_)| hasher.hash_one(&k), hashbrown::raw::Fallibility::Infallible).unwrap(); + writer.resize(size, |(k,_)| hasher.hash_one(&k), + hashbrown::raw::Fallibility::Infallible).unwrap(); } while writer.len() < ideal_filled as usize { let key: FileCacheKey = rng.random(); diff --git a/libs/neon-shmem/src/hash.rs b/libs/neon-shmem/src/hash.rs index 032c1bd21f..a3d465db93 100644 --- a/libs/neon-shmem/src/hash.rs +++ b/libs/neon-shmem/src/hash.rs @@ -48,8 +48,14 @@ unsafe impl<'a, K: Sync, V: Sync, S> Sync for HashMapAccess<'a, K, V, S> {} unsafe impl<'a, K: Send, V: Send, S> Send for HashMapAccess<'a, K, V, S> {} impl<'a, K: Clone + Hash + Eq, V, S> HashMapInit<'a, K, V, S> { - pub fn with_hasher(self, hasher: S) -> HashMapInit<'a, K, V, S> { - Self { hasher, ..self } + pub fn with_hasher(self, hasher: T) -> HashMapInit<'a, K, V, T> { + HashMapInit { + hasher, + shmem_handle: self.shmem_handle, + shared_ptr: self.shared_ptr, + shared_size: self.shared_size, + num_buckets: self.num_buckets, + } } /// Loosely (over)estimate the size needed to store a hash table with `num_buckets` buckets. @@ -287,11 +293,12 @@ where buckets = std::slice::from_raw_parts_mut(buckets_ptr, num_buckets as usize); dictionary = std::slice::from_raw_parts_mut(dictionary_ptr, dictionary_size); - } + (dictionary_ptr, dictionary_size) + } for i in 0..dictionary.len() { dictionary[i] = INVALID_POS; } - + for i in 0..rehash_buckets as usize { if buckets[i].inner.is_none() { buckets[i].next = inner.free_head; From b1e3161d4e0f602e0f56efa854a171312c00d6a1 Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Thu, 26 Jun 2025 14:32:32 -0700 Subject: [PATCH 16/24] Satisfy `cargo clippy` lints, simplify shrinking API --- libs/neon-shmem/src/hash.rs | 193 ++++++++++++++++++------------ libs/neon-shmem/src/hash/core.rs | 24 ++-- libs/neon-shmem/src/hash/entry.rs | 28 +++-- libs/neon-shmem/src/hash/tests.rs | 40 +------ libs/neon-shmem/src/shmem.rs | 92 +++++++------- 5 files changed, 200 insertions(+), 177 deletions(-) diff --git a/libs/neon-shmem/src/hash.rs b/libs/neon-shmem/src/hash.rs index a3d465db93..36fbb1112c 100644 --- a/libs/neon-shmem/src/hash.rs +++ b/libs/neon-shmem/src/hash.rs @@ -1,8 +1,8 @@ -//! Resizable hash table implementation on top of byte-level storage (either `shmem` or fixed byte array). +//! Resizable hash table implementation on top of byte-level storage (either a [`ShmemHandle`] or a fixed byte array). //! //! This hash table has two major components: the bucket array and the dictionary. Each bucket within the -//! bucket array contains a Option<(K, V)> and an index of another bucket. In this way there is both an -//! implicit freelist within the bucket array (None buckets point to other None entries) and various hash +//! bucket array contains a `Option<(K, V)>` and an index of another bucket. In this way there is both an +//! implicit freelist within the bucket array (`None` buckets point to other `None` entries) and various hash //! chains within the bucket array (a Some bucket will point to other Some buckets that had the same hash). //! //! Buckets are never moved unless they are within a region that is being shrunk, and so the actual hash- @@ -10,14 +10,15 @@ //! within the dictionary is decided based on its hash, the data is inserted into an empty bucket based //! off of the freelist, and then the index of said bucket is placed in the dictionary. //! -//! This map is resizable (if initialized on top of a `ShmemHandle`). Both growing and shrinking happen +//! This map is resizable (if initialized on top of a [`ShmemHandle`]). Both growing and shrinking happen //! in-place and are at a high level achieved by expanding/reducing the bucket array and rebuilding the //! dictionary by rehashing all keys. use std::hash::{Hash, BuildHasher}; use std::mem::MaybeUninit; +use std::default::Default; -use crate::shmem::ShmemHandle; +use crate::{shmem, shmem::ShmemHandle}; mod core; pub mod entry; @@ -28,11 +29,13 @@ mod tests; use core::{Bucket, CoreHashMap, INVALID_POS}; use entry::{Entry, OccupiedEntry}; -/// Builder for a `HashMapAccess`. +/// Builder for a [`HashMapAccess`]. +#[must_use] pub struct HashMapInit<'a, K, V, S = rustc_hash::FxBuildHasher> { shmem_handle: Option, shared_ptr: *mut HashMapShared<'a, K, V>, shared_size: usize, + shrink_mode: HashMapShrinkMode, hasher: S, num_buckets: u32, } @@ -42,12 +45,34 @@ pub struct HashMapAccess<'a, K, V, S = rustc_hash::FxBuildHasher> { shmem_handle: Option, shared_ptr: *mut HashMapShared<'a, K, V>, hasher: S, + shrink_mode: HashMapShrinkMode, } -unsafe impl<'a, K: Sync, V: Sync, S> Sync for HashMapAccess<'a, K, V, S> {} -unsafe impl<'a, K: Send, V: Send, S> Send for HashMapAccess<'a, K, V, S> {} +/// Enum specifying what behavior to have surrounding occupied entries in what is +/// about-to-be-shrinked space during a call to [`HashMapAccess::finish_shrink`]. +#[derive(PartialEq, Eq)] +pub enum HashMapShrinkMode { + /// Remap entry to the range of buckets that will remain after shrinking. + /// + /// Requires that caller has left enough room within the map such that this is possible. + Remap, + /// Remove any entries remaining in soon to be deallocated space. + /// + /// Only really useful if you legitimately do not care what entries are removed. + /// Should primarily be used for testing. + Remove, +} -impl<'a, K: Clone + Hash + Eq, V, S> HashMapInit<'a, K, V, S> { +impl Default for HashMapShrinkMode { + fn default() -> Self { + Self::Remap + } +} + +unsafe impl Sync for HashMapAccess<'_, K, V, S> {} +unsafe impl Send for HashMapAccess<'_, K, V, S> {} + +impl<'a, K: Clone + Hash + Eq, V, S> HashMapInit<'a, K, V, S> { pub fn with_hasher(self, hasher: T) -> HashMapInit<'a, K, V, T> { HashMapInit { hasher, @@ -55,9 +80,14 @@ impl<'a, K: Clone + Hash + Eq, V, S> HashMapInit<'a, K, V, S> { shared_ptr: self.shared_ptr, shared_size: self.shared_size, num_buckets: self.num_buckets, + shrink_mode: self.shrink_mode, } } + pub fn with_shrink_mode(self, mode: HashMapShrinkMode) -> Self { + Self { shrink_mode: mode, ..self } + } + /// Loosely (over)estimate the size needed to store a hash table with `num_buckets` buckets. pub fn estimate_size(num_buckets: u32) -> usize { // add some margin to cover alignment etc. @@ -98,11 +128,12 @@ impl<'a, K: Clone + Hash + Eq, V, S> HashMapInit<'a, K, V, S> { HashMapAccess { shmem_handle: self.shmem_handle, shared_ptr: self.shared_ptr, + shrink_mode: self.shrink_mode, hasher: self.hasher, } } - /// Initialize a table for reading. Currently identical to `attach_writer`. + /// Initialize a table for reading. Currently identical to [`HashMapInit::attach_writer`]. pub fn attach_reader(self) -> HashMapAccess<'a, K, V, S> { self.attach_writer() } @@ -114,7 +145,7 @@ impl<'a, K: Clone + Hash + Eq, V, S> HashMapInit<'a, K, V, S> { /// relies on the memory layout! The data structures are laid out in the contiguous shared memory /// area as follows: /// -/// HashMapShared +/// [`HashMapShared`] /// [buckets] /// [dictionary] /// @@ -131,18 +162,22 @@ where pub fn with_fixed( num_buckets: u32, area: &'a mut [MaybeUninit], - ) -> HashMapInit<'a, K, V> { + ) -> Self { Self { num_buckets, shmem_handle: None, shared_ptr: area.as_mut_ptr().cast(), shared_size: area.len(), - hasher: rustc_hash::FxBuildHasher::default(), + shrink_mode: HashMapShrinkMode::default(), + hasher: rustc_hash::FxBuildHasher, } } /// Place a new hash map in the given shared memory area - pub fn with_shmem(num_buckets: u32, shmem: ShmemHandle) -> HashMapInit<'a, K, V> { + /// + /// # Panics + /// Will panic on failure to resize area to expected map size. + pub fn with_shmem(num_buckets: u32, shmem: ShmemHandle) -> Self { let size = Self::estimate_size(num_buckets); shmem .set_size(size) @@ -152,12 +187,13 @@ where shared_ptr: shmem.data_ptr.as_ptr().cast(), shmem_handle: Some(shmem), shared_size: size, - hasher: rustc_hash::FxBuildHasher::default() + shrink_mode: HashMapShrinkMode::default(), + hasher: rustc_hash::FxBuildHasher } } /// Make a resizable hash map within a new shared memory area with the given name. - pub fn new_resizeable_named(num_buckets: u32, max_buckets: u32, name: &str) -> HashMapInit<'a, K, V> { + pub fn new_resizeable_named(num_buckets: u32, max_buckets: u32, name: &str) -> Self { let size = Self::estimate_size(num_buckets); let max_size = Self::estimate_size(max_buckets); let shmem = ShmemHandle::new(name, size, max_size) @@ -168,16 +204,17 @@ where shared_ptr: shmem.data_ptr.as_ptr().cast(), shmem_handle: Some(shmem), shared_size: size, - hasher: rustc_hash::FxBuildHasher::default() + shrink_mode: HashMapShrinkMode::default(), + hasher: rustc_hash::FxBuildHasher } } /// Make a resizable hash map within a new anonymous shared memory area. - pub fn new_resizeable(num_buckets: u32, max_buckets: u32) -> HashMapInit<'a, K, V> { + pub fn new_resizeable(num_buckets: u32, max_buckets: u32) -> Self { use std::sync::atomic::{AtomicUsize, Ordering}; - const COUNTER: AtomicUsize = AtomicUsize::new(0); + static COUNTER: AtomicUsize = AtomicUsize::new(0); let val = COUNTER.fetch_add(1, Ordering::Relaxed); - let name = format!("neon_shmem_hmap{}", val); + let name = format!("neon_shmem_hmap{val}"); Self::new_resizeable_named(num_buckets, max_buckets, &name) } } @@ -214,7 +251,7 @@ where e.remove(); } Entry::Vacant(_) => {} - }; + } } /// Optionally return the entry for a bucket at a given index if it exists. @@ -249,7 +286,7 @@ where let map = unsafe { self.shared_ptr.as_ref() }.unwrap(); let origin = map.inner.buckets.as_ptr(); - let idx = (val_ptr as usize - origin as usize) / (size_of::>() as usize); + let idx = (val_ptr as usize - origin as usize) / size_of::>(); assert!(idx < map.inner.buckets.len()); idx @@ -265,14 +302,14 @@ where pub fn clear(&mut self) { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); let inner = &mut map.inner; - inner.clear() + inner.clear(); } /// Perform an in-place rehash of some region (0..`rehash_buckets`) of the table and reset /// the `buckets` and `dictionary` slices to be as long as `num_buckets`. Resets the freelist /// in the process. fn rehash_dict( - &mut self, + &self, inner: &mut CoreHashMap<'a, K, V>, buckets_ptr: *mut core::Bucket, end_ptr: *mut u8, @@ -293,22 +330,21 @@ where buckets = std::slice::from_raw_parts_mut(buckets_ptr, num_buckets as usize); dictionary = std::slice::from_raw_parts_mut(dictionary_ptr, dictionary_size); - (dictionary_ptr, dictionary_size) } - for i in 0..dictionary.len() { - dictionary[i] = INVALID_POS; + for e in dictionary.iter_mut() { + *e = INVALID_POS; } - for i in 0..rehash_buckets as usize { - if buckets[i].inner.is_none() { - buckets[i].next = inner.free_head; + for (i, bucket) in buckets.iter_mut().enumerate().take(rehash_buckets as usize) { + if bucket.inner.is_none() { + bucket.next = inner.free_head; inner.free_head = i as u32; continue; } - let hash = self.hasher.hash_one(&buckets[i].inner.as_ref().unwrap().0); + let hash = self.hasher.hash_one(&bucket.inner.as_ref().unwrap().0); let pos: usize = (hash % dictionary.len() as u64) as usize; - buckets[i].next = dictionary[pos]; + bucket.next = dictionary[pos]; dictionary[pos] = i as u32; } @@ -322,7 +358,7 @@ where let inner = &mut map.inner; let num_buckets = inner.get_num_buckets() as u32; let size_bytes = HashMapInit::::estimate_size(num_buckets); - let end_ptr: *mut u8 = unsafe { (self.shared_ptr as *mut u8).add(size_bytes) }; + let end_ptr: *mut u8 = unsafe { self.shared_ptr.byte_add(size_bytes).cast() }; let buckets_ptr = inner.buckets.as_mut_ptr(); self.rehash_dict(inner, buckets_ptr, end_ptr, num_buckets, num_buckets); } @@ -332,14 +368,18 @@ where /// 1. Grows the underlying shared memory area /// 2. Initializes new buckets and overwrites the current dictionary /// 3. Rehashes the dictionary - pub fn grow(&mut self, num_buckets: u32) -> Result<(), crate::shmem::Error> { + /// + /// # Panics + /// Panics if called on a map initialized with [`HashMapInit::with_fixed`]. + /// + /// # Errors + /// Returns an [`shmem::Error`] if any errors occur resizing the memory region. + pub fn grow(&mut self, num_buckets: u32) -> Result<(), shmem::Error> { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); let inner = &mut map.inner; let old_num_buckets = inner.buckets.len() as u32; - if num_buckets < old_num_buckets { - panic!("grow called with a smaller number of buckets"); - } + assert!(num_buckets >= old_num_buckets, "grow called with a smaller number of buckets"); if num_buckets == old_num_buckets { return Ok(()); } @@ -352,15 +392,15 @@ where shmem_handle.set_size(size_bytes)?; let end_ptr: *mut u8 = unsafe { shmem_handle.data_ptr.as_ptr().add(size_bytes) }; - // Initialize new buckets. The new buckets are linked to the free list. NB: This overwrites - // the dictionary! + // Initialize new buckets. The new buckets are linked to the free list. + // NB: This overwrites the dictionary! let buckets_ptr = inner.buckets.as_mut_ptr(); unsafe { for i in old_num_buckets..num_buckets { - let bucket_ptr = buckets_ptr.add(i as usize); - bucket_ptr.write(core::Bucket { + let bucket = buckets_ptr.add(i as usize); + bucket.write(core::Bucket { next: if i < num_buckets-1 { - i as u32 + 1 + i + 1 } else { inner.free_head }, @@ -376,11 +416,16 @@ where } /// Begin a shrink, limiting all new allocations to be in buckets with index below `num_buckets`. + /// + /// # Panics + /// Panics if called on a map initialized with [`HashMapInit::with_fixed`] or if `num_buckets` is + /// greater than the number of buckets in the map. pub fn begin_shrink(&mut self, num_buckets: u32) { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); - if num_buckets > map.inner.get_num_buckets() as u32 { - panic!("shrink called with a larger number of buckets"); - } + assert!( + num_buckets <= map.inner.get_num_buckets() as u32, + "shrink called with a larger number of buckets" + ); _ = self .shmem_handle .as_ref() @@ -388,47 +433,47 @@ where map.inner.alloc_limit = num_buckets; } - /// Returns whether a shrink operation is currently in progress. - pub fn is_shrinking(&self) -> bool { + /// If a shrink operation is underway, returns the target size of the map. Otherwise, returns None. + pub fn shrink_goal(&self) -> Option { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); - map.inner.is_shrinking() + let goal = map.inner.alloc_limit; + if goal == INVALID_POS { None } else { Some(goal as usize) } } - /// Returns how many entries need to be evicted before shrink can complete. - pub fn shrink_remaining(&self) -> usize { - let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); - let inner = &mut map.inner; - if !inner.is_shrinking() { - panic!("shrink_remaining called when no ongoing shrink") - } else { - inner.buckets_in_use - .checked_sub(inner.alloc_limit) - .unwrap_or(0) - as usize - } - } - /// Complete a shrink after caller has evicted entries, removing the unused buckets and rehashing. - pub fn finish_shrink(&mut self) -> Result<(), crate::shmem::Error> { + /// + /// # Panics + /// The following cases result in a panic: + /// - Calling this function on a map initialized with [`HashMapInit::with_fixed`]. + /// - Calling this function on a map when no shrink operation is in progress. + /// - Calling this function on a map with `shrink_mode` set to [`HashMapShrinkMode::Remap`] and + /// [`HashMapAccess::get_num_buckets_in_use`] returns a value higher than [`HashMapAccess::shrink_goal`]. + /// + /// # Errors + /// Returns an [`shmem::Error`] if any errors occur resizing the memory region. + pub fn finish_shrink(&mut self) -> Result<(), shmem::Error> { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); let inner = &mut map.inner; - if !inner.is_shrinking() { - panic!("called finish_shrink when no shrink is in progress"); - } + assert!(inner.is_shrinking(), "called finish_shrink when no shrink is in progress"); let num_buckets = inner.alloc_limit; if inner.get_num_buckets() == num_buckets as usize { return Ok(()); - } else if inner.buckets_in_use > num_buckets { - panic!("called finish_shrink before enough entries were removed"); - } + } - for i in (num_buckets as usize)..inner.buckets.len() { - if let Some((k, v)) = inner.buckets[i].inner.take() { - // alloc bucket increases buckets in use, so need to decrease since we're just moving - inner.buckets_in_use -= 1; - inner.alloc_bucket(k, v).unwrap(); + if self.shrink_mode == HashMapShrinkMode::Remap { + assert!( + inner.buckets_in_use <= num_buckets, + "called finish_shrink before enough entries were removed" + ); + + for i in (num_buckets as usize)..inner.buckets.len() { + if let Some((k, v)) = inner.buckets[i].inner.take() { + // alloc bucket increases buckets in use, so need to decrease since we're just moving + inner.buckets_in_use -= 1; + inner.alloc_bucket(k, v).unwrap(); + } } } diff --git a/libs/neon-shmem/src/hash/core.rs b/libs/neon-shmem/src/hash/core.rs index 22c44f20ac..b2cf788d21 100644 --- a/libs/neon-shmem/src/hash/core.rs +++ b/libs/neon-shmem/src/hash/core.rs @@ -5,6 +5,7 @@ use std::mem::MaybeUninit; use crate::hash::entry::{Entry, OccupiedEntry, PrevPos, VacantEntry}; +/// Invalid position within the map (either within the dictionary or bucket array). pub(crate) const INVALID_POS: u32 = u32::MAX; /// Fundamental storage unit within the hash table. Either empty or contains a key-value pair. @@ -18,13 +19,13 @@ pub(crate) struct Bucket { /// Core hash table implementation. pub(crate) struct CoreHashMap<'a, K, V> { - /// Dictionary used to map hashes to bucket indices. + /// Dictionary used to map hashes to bucket indices. pub(crate) dictionary: &'a mut [u32], /// Buckets containing key-value pairs. pub(crate) buckets: &'a mut [Bucket], /// Head of the freelist. pub(crate) free_head: u32, - /// Maximum index of a bucket allowed to be allocated. INVALID_POS if no limit. + /// Maximum index of a bucket allowed to be allocated. [`INVALID_POS`] if no limit. pub(crate) alloc_limit: u32, /// The number of currently occupied buckets. pub(crate) buckets_in_use: u32, @@ -36,10 +37,7 @@ pub(crate) struct CoreHashMap<'a, K, V> { #[derive(Debug)] pub struct FullError(); -impl<'a, K: Hash + Eq, V> CoreHashMap<'a, K, V> -where - K: Clone + Hash + Eq, -{ +impl<'a, K: Clone + Hash + Eq, V> CoreHashMap<'a, K, V> { const FILL_FACTOR: f32 = 0.60; /// Estimate the size of data contained within the the hash map. @@ -59,7 +57,7 @@ where pub fn new( buckets: &'a mut [MaybeUninit>], dictionary: &'a mut [MaybeUninit], - ) -> CoreHashMap<'a, K, V> { + ) -> Self { // Initialize the buckets for i in 0..buckets.len() { buckets[i].write(Bucket { @@ -73,8 +71,8 @@ where } // Initialize the dictionary - for i in 0..dictionary.len() { - dictionary[i].write(INVALID_POS); + for e in dictionary.iter_mut() { + e.write(INVALID_POS); } // TODO: use std::slice::assume_init_mut() once it stabilizes @@ -84,7 +82,7 @@ where std::slice::from_raw_parts_mut(dictionary.as_mut_ptr().cast(), dictionary.len()) }; - CoreHashMap { + Self { dictionary, buckets, free_head: 0, @@ -105,13 +103,13 @@ where let bucket = &self.buckets[next as usize]; let (bucket_key, bucket_value) = bucket.inner.as_ref().expect("entry is in use"); if bucket_key == key { - return Some(&bucket_value); + return Some(bucket_value); } next = bucket.next; } } - /// Get the `Entry` associated with a key given hash. This should be used for updates/inserts. + /// Get the [`Entry`] associated with a key given hash. This should be used for updates/inserts. pub fn entry_with_hash(&mut self, key: K, hash: u64) -> Entry<'a, '_, K, V> { let dict_pos = hash as usize % self.dictionary.len(); let first = self.dictionary[dict_pos]; @@ -236,7 +234,7 @@ where bucket.next = INVALID_POS; bucket.inner = Some((key, value)); - return Ok(pos); + Ok(pos) } } diff --git a/libs/neon-shmem/src/hash/entry.rs b/libs/neon-shmem/src/hash/entry.rs index 24c124189b..5231061b8e 100644 --- a/libs/neon-shmem/src/hash/entry.rs +++ b/libs/neon-shmem/src/hash/entry.rs @@ -1,4 +1,4 @@ -//! Like std::collections::hash_map::Entry; +//! Equivalent of [`std::collections::hash_map::Entry`] for this hashmap. use crate::hash::core::{CoreHashMap, FullError, INVALID_POS}; @@ -30,11 +30,11 @@ pub struct OccupiedEntry<'a, 'b, K, V> { pub(crate) _key: K, /// The index of the previous entry in the chain. pub(crate) prev_pos: PrevPos, - /// The position of the bucket in the CoreHashMap's buckets array. + /// The position of the bucket in the [`CoreHashMap`] bucket array. pub(crate) bucket_pos: u32, } -impl<'a, 'b, K, V> OccupiedEntry<'a, 'b, K, V> { +impl OccupiedEntry<'_, '_, K, V> { pub fn get(&self) -> &V { &self.map.buckets[self.bucket_pos as usize] .inner @@ -55,20 +55,25 @@ impl<'a, 'b, K, V> OccupiedEntry<'a, 'b, K, V> { pub fn insert(&mut self, value: V) -> V { let bucket = &mut self.map.buckets[self.bucket_pos as usize]; // This assumes inner is Some, which it must be for an OccupiedEntry - let old_value = mem::replace(&mut bucket.inner.as_mut().unwrap().1, value); - old_value + mem::replace(&mut bucket.inner.as_mut().unwrap().1, value) } /// Removes the entry from the hash map, returning the value originally stored within it. + /// + /// # Panics + /// Panics if the `prev_pos` field is equal to [`PrevPos::Unknown`]. In practice, this means + /// the entry was obtained via calling something like [`CoreHashMap::entry_at_bucket`]. pub fn remove(self) -> V { // CoreHashMap::remove returns Option<(K, V)>. We know it's Some for an OccupiedEntry. let bucket = &mut self.map.buckets[self.bucket_pos as usize]; // unlink it from the chain match self.prev_pos { - PrevPos::First(dict_pos) => self.map.dictionary[dict_pos as usize] = bucket.next, + PrevPos::First(dict_pos) => { + self.map.dictionary[dict_pos as usize] = bucket.next; + }, PrevPos::Chained(bucket_pos) => { - self.map.buckets[bucket_pos as usize].next = bucket.next + self.map.buckets[bucket_pos as usize].next = bucket.next; }, PrevPos::Unknown => panic!("can't safely remove entry with unknown previous entry"), } @@ -80,7 +85,7 @@ impl<'a, 'b, K, V> OccupiedEntry<'a, 'b, K, V> { self.map.free_head = self.bucket_pos; self.map.buckets_in_use -= 1; - return old_value.unwrap().1; + old_value.unwrap().1 } } @@ -94,8 +99,11 @@ pub struct VacantEntry<'a, 'b, K, V> { pub(crate) dict_pos: u32, } -impl<'a, 'b, K: Clone + Hash + Eq, V> VacantEntry<'a, 'b, K, V> { +impl<'b, K: Clone + Hash + Eq, V> VacantEntry<'_, 'b, K, V> { /// Insert a value into the vacant entry, finding and populating an empty bucket in the process. + /// + /// # Errors + /// Will return [`FullError`] if there are no unoccupied buckets in the map. pub fn insert(self, value: V) -> Result<&'b mut V, FullError> { let pos = self.map.alloc_bucket(self.key, value)?; if pos == INVALID_POS { @@ -106,6 +114,6 @@ impl<'a, 'b, K: Clone + Hash + Eq, V> VacantEntry<'a, 'b, K, V> { self.map.dictionary[self.dict_pos as usize] = pos; let result = &mut self.map.buckets[pos as usize].inner.as_mut().unwrap().1; - return Ok(result); + Ok(result) } } diff --git a/libs/neon-shmem/src/hash/tests.rs b/libs/neon-shmem/src/hash/tests.rs index 8a6e8a0a29..209db599b5 100644 --- a/libs/neon-shmem/src/hash/tests.rs +++ b/libs/neon-shmem/src/hash/tests.rs @@ -1,14 +1,11 @@ use std::collections::BTreeMap; use std::collections::HashSet; -use std::fmt::{Debug, Formatter}; -use std::mem::uninitialized; +use std::fmt::Debug; use std::mem::MaybeUninit; -use std::sync::atomic::{AtomicUsize, Ordering}; use crate::hash::HashMapAccess; use crate::hash::HashMapInit; use crate::hash::Entry; -use crate::shmem::ShmemHandle; use rand::seq::SliceRandom; use rand::{Rng, RngCore}; @@ -98,30 +95,6 @@ fn sparse() { test_inserts(&keys); } -struct TestValue(AtomicUsize); - -impl TestValue { - fn new(val: usize) -> TestValue { - TestValue(AtomicUsize::new(val)) - } - - fn load(&self) -> usize { - self.0.load(Ordering::Relaxed) - } -} - -impl Clone for TestValue { - fn clone(&self) -> TestValue { - TestValue::new(self.load()) - } -} - -impl Debug for TestValue { - fn fmt(&self, fmt: &mut Formatter<'_>) -> Result<(), std::fmt::Error> { - write!(fmt, "{:?}", self.load()) - } -} - #[derive(Clone, Debug)] struct TestOp(TestKey, Option); @@ -177,7 +150,7 @@ fn do_deletes( writer: &mut HashMapAccess, shadow: &mut BTreeMap, ) { - for i in 0..num_ops { + for _ in 0..num_ops { let (k, _) = shadow.pop_first().unwrap(); let hash = writer.get_hash_value(&k); writer.remove_with_hash(&k, hash); @@ -187,7 +160,6 @@ fn do_deletes( fn do_shrink( writer: &mut HashMapAccess, shadow: &mut BTreeMap, - from: u32, to: u32 ) { writer.begin_shrink(to); @@ -195,7 +167,7 @@ fn do_shrink( let (k, _) = shadow.pop_first().unwrap(); let hash = writer.get_hash_value(&k); let entry = writer.entry_with_hash(k, hash); - if let Entry::Occupied(mut e) = entry { + if let Entry::Occupied(e) = entry { e.remove(); } } @@ -260,7 +232,7 @@ fn test_shrink() { let mut rng = rand::rng(); do_random_ops(10000, 1500, 0.75, &mut writer, &mut shadow, &mut rng); - do_shrink(&mut writer, &mut shadow, 1500, 1000); + do_shrink(&mut writer, &mut shadow, 1000); do_deletes(500, &mut writer, &mut shadow); do_random_ops(10000, 500, 0.75, &mut writer, &mut shadow, &mut rng); assert!(writer.get_num_buckets_in_use() <= 1000); @@ -276,13 +248,13 @@ fn test_shrink_grow_seq() { do_random_ops(500, 1000, 0.1, &mut writer, &mut shadow, &mut rng); eprintln!("Shrinking to 750"); - do_shrink(&mut writer, &mut shadow, 1000, 750); + do_shrink(&mut writer, &mut shadow, 750); do_random_ops(200, 1000, 0.5, &mut writer, &mut shadow, &mut rng); eprintln!("Growing to 1500"); writer.grow(1500).unwrap(); do_random_ops(600, 1500, 0.1, &mut writer, &mut shadow, &mut rng); eprintln!("Shrinking to 200"); - do_shrink(&mut writer, &mut shadow, 1500, 200); + do_shrink(&mut writer, &mut shadow, 200); do_deletes(100, &mut writer, &mut shadow); do_random_ops(50, 1500, 0.25, &mut writer, &mut shadow, &mut rng); eprintln!("Growing to 10k"); diff --git a/libs/neon-shmem/src/shmem.rs b/libs/neon-shmem/src/shmem.rs index 21b1454b10..7c7285f67e 100644 --- a/libs/neon-shmem/src/shmem.rs +++ b/libs/neon-shmem/src/shmem.rs @@ -12,14 +12,14 @@ use nix::sys::mman::mmap as nix_mmap; use nix::sys::mman::munmap as nix_munmap; use nix::unistd::ftruncate as nix_ftruncate; -/// ShmemHandle represents a shared memory area that can be shared by processes over fork(). -/// Unlike shared memory allocated by Postgres, this area is resizable, up to 'max_size' that's +/// `ShmemHandle` represents a shared memory area that can be shared by processes over `fork()`. +/// Unlike shared memory allocated by Postgres, this area is resizable, up to `max_size` that's /// specified at creation. /// -/// The area is backed by an anonymous file created with memfd_create(). The full address space for -/// 'max_size' is reserved up-front with mmap(), but whenever you call [`ShmemHandle::set_size`], +/// The area is backed by an anonymous file created with `memfd_create()`. The full address space for +/// `max_size` is reserved up-front with `mmap()`, but whenever you call [`ShmemHandle::set_size`], /// the underlying file is resized. Do not access the area beyond the current size. Currently, that -/// will cause the file to be expanded, but we might use mprotect() etc. to enforce that in the +/// will cause the file to be expanded, but we might use `mprotect()` etc. to enforce that in the /// future. pub struct ShmemHandle { /// memfd file descriptor @@ -38,7 +38,7 @@ pub struct ShmemHandle { struct SharedStruct { max_size: usize, - /// Current size of the backing file. The high-order bit is used for the RESIZE_IN_PROGRESS flag + /// Current size of the backing file. The high-order bit is used for the [`RESIZE_IN_PROGRESS`] flag. current_size: AtomicUsize, } @@ -46,7 +46,7 @@ const RESIZE_IN_PROGRESS: usize = 1 << 63; const HEADER_SIZE: usize = std::mem::size_of::(); -/// Error type returned by the ShmemHandle functions. +/// Error type returned by the [`ShmemHandle`] functions. #[derive(thiserror::Error, Debug)] #[error("{msg}: {errno}")] pub struct Error { @@ -55,8 +55,8 @@ pub struct Error { } impl Error { - fn new(msg: &str, errno: Errno) -> Error { - Error { + fn new(msg: &str, errno: Errno) -> Self { + Self { msg: msg.to_string(), errno, } @@ -65,11 +65,11 @@ impl Error { impl ShmemHandle { /// Create a new shared memory area. To communicate between processes, the processes need to be - /// fork()'d after calling this, so that the ShmemHandle is inherited by all processes. + /// `fork()`'d after calling this, so that the `ShmemHandle` is inherited by all processes. /// - /// If the ShmemHandle is dropped, the memory is unmapped from the current process. Other + /// If the `ShmemHandle` is dropped, the memory is unmapped from the current process. Other /// processes can continue using it, however. - pub fn new(name: &str, initial_size: usize, max_size: usize) -> Result { + pub fn new(name: &str, initial_size: usize, max_size: usize) -> Result { // create the backing anonymous file. let fd = create_backing_file(name)?; @@ -80,17 +80,17 @@ impl ShmemHandle { fd: OwnedFd, initial_size: usize, max_size: usize, - ) -> Result { - // We reserve the high-order bit for the RESIZE_IN_PROGRESS flag, and the actual size + ) -> Result { + // We reserve the high-order bit for the `RESIZE_IN_PROGRESS` flag, and the actual size // is a little larger than this because of the SharedStruct header. Make the upper limit // somewhat smaller than that, because with anything close to that, you'll run out of // memory anyway. - if max_size >= 1 << 48 { - panic!("max size {} too large", max_size); - } - if initial_size > max_size { - panic!("initial size {initial_size} larger than max size {max_size}"); - } + assert!(max_size < 1 << 48, "max size {max_size} too large"); + + assert!( + initial_size <= max_size, + "initial size {initial_size} larger than max size {max_size}" + ); // The actual initial / max size is the one given by the caller, plus the size of // 'SharedStruct'. @@ -110,7 +110,7 @@ impl ShmemHandle { 0, ) } - .map_err(|e| Error::new("mmap failed: {e}", e))?; + .map_err(|e| Error::new("mmap failed", e))?; // Reserve space for the initial size enlarge_file(fd.as_fd(), initial_size as u64)?; @@ -121,13 +121,13 @@ impl ShmemHandle { shared.write(SharedStruct { max_size: max_size.into(), current_size: AtomicUsize::new(initial_size), - }) - }; + }); + } // The user data begins after the header let data_ptr = unsafe { start_ptr.cast().add(HEADER_SIZE) }; - Ok(ShmemHandle { + Ok(Self { fd, max_size: max_size.into(), shared_ptr: shared, @@ -140,28 +140,28 @@ impl ShmemHandle { unsafe { self.shared_ptr.as_ref() } } - /// Resize the shared memory area. 'new_size' must not be larger than the 'max_size' specified + /// Resize the shared memory area. `new_size` must not be larger than the `max_size` specified /// when creating the area. /// /// This may only be called from one process/thread concurrently. We detect that case - /// and return an Error. + /// and return an [`shmem::Error`](Error). pub fn set_size(&self, new_size: usize) -> Result<(), Error> { let new_size = new_size + HEADER_SIZE; let shared = self.shared(); - if new_size > self.max_size { - panic!( - "new size ({} is greater than max size ({})", - new_size, self.max_size - ); - } - assert_eq!(self.max_size, shared.max_size); + assert!( + new_size <= self.max_size, + "new size ({new_size}) is greater than max size ({})", + self.max_size + ); - // Lock the area by setting the bit in 'current_size' + assert_eq!(self.max_size, shared.max_size); + + // Lock the area by setting the bit in `current_size` // // Ordering::Relaxed would probably be sufficient here, as we don't access any other memory - // and the posix_fallocate/ftruncate call is surely a synchronization point anyway. But - // since this is not performance-critical, better safe than sorry . + // and the `posix_fallocate`/`ftruncate` call is surely a synchronization point anyway. But + // since this is not performance-critical, better safe than sorry. let mut old_size = shared.current_size.load(Ordering::Acquire); loop { if (old_size & RESIZE_IN_PROGRESS) != 0 { @@ -188,7 +188,7 @@ impl ShmemHandle { use std::cmp::Ordering::{Equal, Greater, Less}; match new_size.cmp(&old_size) { Less => nix_ftruncate(&self.fd, new_size as i64).map_err(|e| { - Error::new("could not shrink shmem segment, ftruncate failed: {e}", e) + Error::new("could not shrink shmem segment, ftruncate failed", e) }), Equal => Ok(()), Greater => enlarge_file(self.fd.as_fd(), new_size as u64), @@ -206,8 +206,8 @@ impl ShmemHandle { /// Returns the current user-visible size of the shared memory segment. /// - /// NOTE: a concurrent set_size() call can change the size at any time. It is the caller's - /// responsibility not to access the area beyond the current size. + /// NOTE: a concurrent [`ShmemHandle::set_size()`] call can change the size at any time. + /// It is the caller's responsibility not to access the area beyond the current size. pub fn current_size(&self) -> usize { let total_current_size = self.shared().current_size.load(Ordering::Relaxed) & !RESIZE_IN_PROGRESS; @@ -224,23 +224,23 @@ impl Drop for ShmemHandle { } } -/// Create a "backing file" for the shared memory area. On Linux, use memfd_create(), to create an +/// Create a "backing file" for the shared memory area. On Linux, use `memfd_create()`, to create an /// anonymous in-memory file. One macos, fall back to a regular file. That's good enough for /// development and testing, but in production we want the file to stay in memory. /// -/// disable 'unused_variables' warnings, because in the macos path, 'name' is unused. +/// Disable unused variables warnings because `name` is unused in the macos path. #[allow(unused_variables)] fn create_backing_file(name: &str) -> Result { #[cfg(not(target_os = "macos"))] { nix::sys::memfd::memfd_create(name, nix::sys::memfd::MFdFlags::empty()) - .map_err(|e| Error::new("memfd_create failed: {e}", e)) + .map_err(|e| Error::new("memfd_create failed", e)) } #[cfg(target_os = "macos")] { let file = tempfile::tempfile().map_err(|e| { Error::new( - "could not create temporary file to back shmem area: {e}", + "could not create temporary file to back shmem area", nix::errno::Errno::from_raw(e.raw_os_error().unwrap_or(0)), ) })?; @@ -255,7 +255,7 @@ fn enlarge_file(fd: BorrowedFd, size: u64) -> Result<(), Error> { { nix::fcntl::posix_fallocate(fd, 0, size as i64).map_err(|e| { Error::new( - "could not grow shmem segment, posix_fallocate failed: {e}", + "could not grow shmem segment, posix_fallocate failed", e, ) }) @@ -264,7 +264,7 @@ fn enlarge_file(fd: BorrowedFd, size: u64) -> Result<(), Error> { #[cfg(target_os = "macos")] { nix::unistd::ftruncate(fd, size as i64) - .map_err(|e| Error::new("could not grow shmem segment, ftruncate failed: {e}", e)) + .map_err(|e| Error::new("could not grow shmem segment, ftruncate failed", e)) } } @@ -330,7 +330,7 @@ mod tests { Ok(()) } - /// This is used in tests to coordinate between test processes. It's like std::sync::Barrier, + /// This is used in tests to coordinate between test processes. It's like `std::sync::Barrier`, /// but is stored in the shared memory area and works across processes. It's implemented by /// polling, because e.g. standard rust mutexes are not guaranteed to work across processes. struct SimpleBarrier { From 47664e40d452a4e3f771e6c6b4905f02d6c710a2 Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Thu, 26 Jun 2025 16:00:33 -0700 Subject: [PATCH 17/24] Initial work in visualizing properties of hashmap --- Cargo.lock | 299 ++++++++++++++++++++++++++++++ libs/neon-shmem/Cargo.toml | 8 + libs/neon-shmem/src/hash.rs | 26 ++- libs/neon-shmem/src/hmap_stats.rs | 139 ++++++++++++++ 4 files changed, 471 insertions(+), 1 deletion(-) create mode 100644 libs/neon-shmem/src/hmap_stats.rs diff --git a/Cargo.lock b/Cargo.lock index 4fd5f5802b..c407806b3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1336,6 +1336,12 @@ dependencies = [ "cc", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "colorchoice" version = "1.0.4" @@ -1595,6 +1601,42 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core-graphics" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "core-graphics-types", + "foreign-types", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "libc", +] + +[[package]] +name = "core-text" +version = "20.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9d2790b5c08465d49f8dc05c8bcae9fea467855947db39b0f8145c091aaced5" +dependencies = [ + "core-foundation 0.9.4", + "core-graphics", + "foreign-types", + "libc", +] + [[package]] name = "cpp_demangle" version = "0.4.4" @@ -2047,6 +2089,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.59.0", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -2058,6 +2121,15 @@ dependencies = [ "syn", ] +[[package]] +name = "dlib" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330c60081dcc4c72131f8eb70510f1ac07223e5d4163db481a04a0befcffa412" +dependencies = [ + "libloading", +] + [[package]] name = "dlv-list" version = "0.5.2" @@ -2087,6 +2159,18 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "dwrote" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfe1f192fcce01590bd8d839aca53ce0d11d803bf291b2a6c4ad925a8f0024be" +dependencies = [ + "lazy_static", + "libc", + "winapi", + "wio", +] + [[package]] name = "dyn-clone" version = "1.0.19" @@ -2377,6 +2461,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "ff" version = "0.12.1" @@ -2455,6 +2548,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-ord" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" + [[package]] name = "fnv" version = "1.0.7" @@ -2467,6 +2566,58 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "font-kit" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c7e611d49285d4c4b2e1727b72cf05353558885cc5252f93707b845dfcaf3d3" +dependencies = [ + "bitflags 2.9.1", + "byteorder", + "core-foundation 0.9.4", + "core-graphics", + "core-text", + "dirs", + "dwrote", + "float-ord", + "freetype-sys", + "lazy_static", + "libc", + "log", + "pathfinder_geometry", + "pathfinder_simd", + "walkdir", + "winapi", + "yeslogic-fontconfig-sys", +] + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -2497,6 +2648,17 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "freetype-sys" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7edc5b9669349acfda99533e9e0bcf26a51862ab43b08ee7745c55d28eb134" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -2687,6 +2849,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "gif" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80792593675e051cf94a4b111980da2ba60d4a83e43e0048c5693baab3977045" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.31.1" @@ -3371,6 +3543,20 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.24.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "jpeg-decoder", + "num-traits", + "png", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -3655,6 +3841,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "jpeg-decoder" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07" + [[package]] name = "js-sys" version = "0.3.77" @@ -3998,6 +4190,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -4038,6 +4231,7 @@ dependencies = [ "foldhash", "hashbrown 0.15.4 (git+https://github.com/quantumish/hashbrown.git?rev=6610e6d)", "nix 0.30.1", + "plotters", "rand 0.9.1", "rand_distr 0.5.1", "rustc-hash 2.1.1", @@ -4406,6 +4600,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "ordered-float" version = "2.10.1" @@ -4842,6 +5042,25 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pathfinder_geometry" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b7e7b4ea703700ce73ebf128e1450eb69c3a8329199ffbfb9b2a0418e5ad3" +dependencies = [ + "log", + "pathfinder_simd", +] + +[[package]] +name = "pathfinder_simd" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf9027960355bf3afff9841918474a81a5f972ac6d226d518060bba758b5ad57" +dependencies = [ + "rustc_version", +] + [[package]] name = "pbkdf2" version = "0.12.2" @@ -5005,9 +5224,16 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ + "chrono", + "font-kit", + "image", + "lazy_static", "num-traits", + "pathfinder_geometry", "plotters-backend", + "plotters-bitmap", "plotters-svg", + "ttf-parser", "wasm-bindgen", "web-sys", ] @@ -5018,6 +5244,17 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" +[[package]] +name = "plotters-bitmap" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ce181e3f6bf82d6c1dc569103ca7b1bd964c60ba03d7e6cdfbb3e3eb7f7405" +dependencies = [ + "gif", + "image", + "plotters-backend", +] + [[package]] name = "plotters-svg" version = "0.3.7" @@ -5027,6 +5264,19 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "png" +version = "0.17.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "polonius-the-crab" version = "0.4.2" @@ -5914,6 +6164,17 @@ dependencies = [ "bitflags 2.9.1", ] +[[package]] +name = "redox_users" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b" +dependencies = [ + "getrandom 0.2.16", + "libredox", + "thiserror 2.0.12", +] + [[package]] name = "regex" version = "1.11.1" @@ -6986,6 +7247,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "simple_asn1" version = "0.6.3" @@ -8177,6 +8444,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "ttf-parser" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17f77d76d837a7830fe1d4f12b7b4ba4192c1888001c7164257e4bc6d21d96b4" + [[package]] name = "tungstenite" version = "0.21.0" @@ -8715,6 +8988,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "weezl" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a751b3277700db47d3e574514de2eced5e54dc8a5436a3bf7a0b248b2cee16f3" + [[package]] name = "which" version = "4.4.2" @@ -9067,6 +9346,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "wio" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d129932f4644ac2396cb456385cbf9e63b5b30c6e8dc4820bdca4eb082037a5" +dependencies = [ + "winapi", +] + [[package]] name = "wit-bindgen-rt" version = "0.39.0" @@ -9252,6 +9540,17 @@ dependencies = [ "time", ] +[[package]] +name = "yeslogic-fontconfig-sys" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503a066b4c037c440169d995b869046827dbc71263f6e8f3be6d77d4f3229dbd" +dependencies = [ + "dlib", + "once_cell", + "pkg-config", +] + [[package]] name = "yoke" version = "0.8.0" diff --git a/libs/neon-shmem/Cargo.toml b/libs/neon-shmem/Cargo.toml index 1c47ffed37..3a22ce6c8b 100644 --- a/libs/neon-shmem/Cargo.toml +++ b/libs/neon-shmem/Cargo.toml @@ -10,6 +10,7 @@ nix.workspace = true workspace_hack = { version = "0.1", path = "../../workspace_hack" } rustc-hash = { version = "2.1.1" } rand = "0.9.1" +plotters = { version = "0.3.7", optional = true } [dev-dependencies] criterion = { workspace = true, features = ["html_reports"] } @@ -21,6 +22,7 @@ seahash = "4.1.0" hashbrown = { git = "https://github.com/quantumish/hashbrown.git", rev = "6610e6d" } foldhash = "0.1.5" + [target.'cfg(target_os = "macos")'.dependencies] tempfile = "3.14.0" @@ -28,3 +30,9 @@ tempfile = "3.14.0" name = "hmap_resize" harness = false +[[bin]] +name = "hmap_stats" +path = "src/hmap_stats.rs" + +[features] +stats = ["dep:plotters"] diff --git a/libs/neon-shmem/src/hash.rs b/libs/neon-shmem/src/hash.rs index 36fbb1112c..6cc641814a 100644 --- a/libs/neon-shmem/src/hash.rs +++ b/libs/neon-shmem/src/hash.rs @@ -490,5 +490,29 @@ where inner.alloc_limit = INVALID_POS; Ok(()) - } + } + + #[cfg(feature = "stats")] + pub fn dict_len(&self) -> usize { + let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); + map.inner.dictionary.len() + } + + #[cfg(feature = "stats")] + pub fn chain_distribution(&self) -> (Vec<(usize, usize)>, usize) { + let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); + let mut out = Vec::new(); + let mut max = 0; + for (i, d) in map.inner.dictionary.iter().enumerate() { + let mut curr = *d; + let mut len = 0; + while curr != INVALID_POS { + curr = map.inner.buckets[curr as usize].next; + len += 1; + } + out.push((i, len)); + max = max.max(len); + } + (out, max) + } } diff --git a/libs/neon-shmem/src/hmap_stats.rs b/libs/neon-shmem/src/hmap_stats.rs new file mode 100644 index 0000000000..9e55ad1f05 --- /dev/null +++ b/libs/neon-shmem/src/hmap_stats.rs @@ -0,0 +1,139 @@ +use neon_shmem::hash::HashMapAccess; +use neon_shmem::hash::HashMapInit; +use neon_shmem::hash::entry::Entry; +use rand::prelude::*; +use rand::distr::{Distribution, StandardUniform}; +use plotters::prelude::*; + +#[derive(Clone, Debug, Hash, Eq, PartialEq)] +#[repr(C)] +pub struct FileCacheKey { + pub _spc_id: u32, + pub _db_id: u32, + pub _rel_number: u32, + pub _fork_num: u32, + pub _block_num: u32, +} + +impl Distribution for StandardUniform { + // questionable, but doesn't need to be good randomness + fn sample(&self, rng: &mut R) -> FileCacheKey { + FileCacheKey { + _spc_id: rng.random(), + _db_id: rng.random(), + _rel_number: rng.random(), + _fork_num: rng.random(), + _block_num: rng.random() + } + } +} + +#[derive(Clone, Debug)] +#[repr(C)] +pub struct FileCacheEntry { + pub _offset: u32, + pub _access_count: u32, + pub _prev: *mut FileCacheEntry, + pub _next: *mut FileCacheEntry, + pub _state: [u32; 8], +} + +impl FileCacheEntry { + fn dummy() -> Self { + Self { + _offset: 0, + _access_count: 0, + _prev: std::ptr::null_mut(), + _next: std::ptr::null_mut(), + _state: [0; 8] + } + } +} + +// Utilities for applying operations. + +#[derive(Clone, Debug)] +struct TestOp(K, Option); + +fn apply_op( + op: TestOp, + map: &mut HashMapAccess, +) { + let hash = map.get_hash_value(&op.0); + let entry = map.entry_with_hash(op.0, hash); + + match op.1 { + Some(new) => { + match entry { + Entry::Occupied(mut e) => Some(e.insert(new)), + Entry::Vacant(e) => { e.insert(new).unwrap(); None }, + } + }, + None => { + match entry { + Entry::Occupied(e) => Some(e.remove()), + Entry::Vacant(_) => None, + } + }, + }; +} + +#[cfg(feature = "stats")] +fn main() { + let ideal_filled = 16_000_000; + let size = 20_000_000; + let mut writer = HashMapInit::new_resizeable(size, size).attach_writer(); + let mut rng = rand::rng(); + while writer.get_num_buckets_in_use() < ideal_filled as usize { + let key: FileCacheKey = rng.random(); + let val = FileCacheEntry::dummy(); + apply_op(TestOp(key, Some(val)), &mut writer); + } + println!("Inserted {ideal_filled} entries into a map with capacity {size}."); + let (distr, max) = writer.chain_distribution(); + + let root_area = BitMapBackend::new("chain_distr.png", (800, 400)) + .into_drawing_area(); + root_area.fill(&WHITE).unwrap(); + + let mut ctx = ChartBuilder::on(&root_area) + .set_label_area_size(LabelAreaPosition::Left, 40) + .set_label_area_size(LabelAreaPosition::Bottom, 40) + .build_cartesian_2d((0..max).into_segmented(), (0..ideal_filled * 2).log_scale()) + .unwrap(); + + ctx.configure_mesh() + .y_label_formatter(&|y| format!("{:e}", y)) + .draw().unwrap(); + + ctx.draw_series( + Histogram::vertical(&ctx) + .margin(10) + .data(distr.iter().map(|x| (x.1, 1))) + ).unwrap(); + + // let root_area = BitMapBackend::new("dict_distr.png", (2000, 400)) + // .into_drawing_area(); + // root_area.fill(&WHITE).unwrap(); + + // let mut ctx = ChartBuilder::on(&root_area) + // .set_label_area_size(LabelAreaPosition::Left, 40) + // .set_label_area_size(LabelAreaPosition::Bottom, 40) + // .build_cartesian_2d((0..writer.dict_len()), (0..(max as f32 * 1.5) as usize)) + // .unwrap(); + + // ctx.configure_mesh().draw().unwrap(); + + // ctx.draw_series(LineSeries::new( + // distr.iter().map(|(bin, count)| (*bin, *count)), + // &RED, + // )).unwrap(); + + // println!("Longest chain: {}", writer.longest_chain()); +} + +#[cfg(not(feature = "stats"))] +fn main() { + println!("Enable the `stats` feature to use this binary!"); +} + From 78b6da270b9f6f6fb71300b30ca3766c23402470 Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Thu, 26 Jun 2025 16:45:48 -0700 Subject: [PATCH 18/24] Sketchily integrate hashmap rewrite with `integrated_cache` --- libs/neon-shmem/src/hash.rs | 12 +- .../neon/communicator/src/integrated_cache.rs | 212 +++++++++--------- 2 files changed, 110 insertions(+), 114 deletions(-) diff --git a/libs/neon-shmem/src/hash.rs b/libs/neon-shmem/src/hash.rs index ca358ed4e5..e97ad51b0d 100644 --- a/libs/neon-shmem/src/hash.rs +++ b/libs/neon-shmem/src/hash.rs @@ -199,14 +199,14 @@ where } /// Get a reference to the entry containing a key given its hash. - pub fn entry_with_hash(&mut self, key: K, hash: u64) -> Entry<'a, '_, K, V> { + pub fn entry_with_hash(&self, key: K, hash: u64) -> Entry<'a, '_, K, V> { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); map.inner.entry_with_hash(key, hash) } /// Remove a key given its hash. Does nothing if key is not present. - pub fn remove_with_hash(&mut self, key: &K, hash: u64) { + pub fn remove_with_hash(&self, key: &K, hash: u64) { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); match map.inner.entry_with_hash(key.clone(), hash) { @@ -218,7 +218,7 @@ where } /// Optionally return the entry for a bucket at a given index if it exists. - pub fn entry_at_bucket(&mut self, pos: usize) -> Option> { + pub fn entry_at_bucket(&self, pos: usize) -> Option> { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); map.inner.entry_at_bucket(pos) } @@ -272,7 +272,7 @@ where /// the `buckets` and `dictionary` slices to be as long as `num_buckets`. Resets the freelist /// in the process. fn rehash_dict( - &mut self, + &self, inner: &mut CoreHashMap<'a, K, V>, buckets_ptr: *mut core::Bucket, end_ptr: *mut u8, @@ -331,7 +331,7 @@ where /// 1. Grows the underlying shared memory area /// 2. Initializes new buckets and overwrites the current dictionary /// 3. Rehashes the dictionary - pub fn grow(&mut self, num_buckets: u32) -> Result<(), crate::shmem::Error> { + pub fn grow(&self, num_buckets: u32) -> Result<(), crate::shmem::Error> { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); let inner = &mut map.inner; let old_num_buckets = inner.buckets.len() as u32; @@ -408,7 +408,7 @@ where } /// Complete a shrink after caller has evicted entries, removing the unused buckets and rehashing. - pub fn finish_shrink(&mut self) -> Result<(), crate::shmem::Error> { + pub fn finish_shrink(&self) -> Result<(), crate::shmem::Error> { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); let inner = &mut map.inner; if !inner.is_shrinking() { diff --git a/pgxn/neon/communicator/src/integrated_cache.rs b/pgxn/neon/communicator/src/integrated_cache.rs index 4c65af6bdb..d710f0e35b 100644 --- a/pgxn/neon/communicator/src/integrated_cache.rs +++ b/pgxn/neon/communicator/src/integrated_cache.rs @@ -33,8 +33,7 @@ use pageserver_page_api::RelTag; use metrics::{IntCounter, IntGauge}; -use neon_shmem::hash::HashMapInit; -use neon_shmem::hash::UpdateAction; +use neon_shmem::hash::{HashMapInit, entry::Entry}; use neon_shmem::shmem::ShmemHandle; // in # of entries @@ -95,7 +94,7 @@ impl<'t> IntegratedCacheInitStruct<'t> { ) -> IntegratedCacheInitStruct<'t> { // Initialize the relsize cache in the fixed-size area let relsize_cache_handle = - neon_shmem::hash::HashMapInit::init_in_fixed_area(RELSIZE_CACHE_SIZE, shmem_area); + neon_shmem::hash::HashMapInit::with_fixed(RELSIZE_CACHE_SIZE, shmem_area); let max_bytes = HashMapInit::::estimate_size(max_file_cache_size as u32); @@ -103,7 +102,7 @@ impl<'t> IntegratedCacheInitStruct<'t> { // Initialize the block map in a separate resizable shared memory area let shmem_handle = ShmemHandle::new("block mapping", 0, max_bytes).unwrap(); - let block_map_handle = neon_shmem::hash::HashMapInit::init_in_shmem( + let block_map_handle = neon_shmem::hash::HashMapInit::with_shmem( initial_file_cache_size as u32, shmem_handle, ); @@ -268,7 +267,8 @@ impl<'t> IntegratedCacheWriteAccess<'t> { block_number: u32, dst: impl uring_common::buf::IoBufMut + Send + Sync, ) -> Result, std::io::Error> { - let x = if let Some(block_entry) = self.block_map.get(&BlockKey::from((rel, block_number))) + let hash = self.block_map.get_hash_value(&BlockKey::from((rel, block_number))); + let x = if let Some(block_entry) = self.block_map.get_with_hash(&BlockKey::from((rel, block_number)), hash) { block_entry.referenced.store(true, Ordering::Relaxed); @@ -302,7 +302,8 @@ impl<'t> IntegratedCacheWriteAccess<'t> { rel: &RelTag, block_number: u32, ) -> Result, std::io::Error> { - if let Some(block_entry) = self.block_map.get(&BlockKey::from((rel, block_number))) { + let hash = self.block_map.get_hash_value(&BlockKey::from((rel, block_number))); + if let Some(block_entry) = self.block_map.get_with_hash(&BlockKey::from((rel, block_number)), hash) { // This is used for prefetch requests. Treat the probe as an 'access', to keep it // in cache. block_entry.referenced.store(true, Ordering::Relaxed); @@ -324,7 +325,8 @@ impl<'t> IntegratedCacheWriteAccess<'t> { /// information, i.e. we don't know if the relation exists or not. pub fn get_rel_exists(&'t self, rel: &RelTag) -> CacheResult { // we don't currently cache negative entries, so if the relation is in the cache, it exists - if let Some(_rel_entry) = self.relsize_cache.get(&RelKey::from(rel)) { + let hash = self.relsize_cache.get_hash_value(&RelKey::from(rel)); + if let Some(_rel_entry) = self.relsize_cache.get_with_hash(&RelKey::from(rel), hash) { CacheResult::Found(true) } else { let lsn = Lsn(self.global_lw_lsn.load(Ordering::Relaxed)); @@ -343,24 +345,20 @@ impl<'t> IntegratedCacheWriteAccess<'t> { } pub fn remember_rel_size(&'t self, rel: &RelTag, nblocks: u32) { - let result = - self.relsize_cache - .update_with_fn(&RelKey::from(rel), |existing| match existing { - None => { - tracing::info!("inserting rel entry for {rel:?}, {nblocks} blocks"); - UpdateAction::Insert(RelEntry { - nblocks: AtomicU32::new(nblocks), - }) - } - Some(e) => { - tracing::info!("updating rel entry for {rel:?}, {nblocks} blocks"); - e.nblocks.store(nblocks, Ordering::Relaxed); - UpdateAction::Nothing - } - }); - - // FIXME: what to do if we run out of memory? Evict other relation entries? - result.expect("out of memory"); + let hash = self.relsize_cache.get_hash_value(&RelKey::from(rel)); + match self.relsize_cache.entry_with_hash(RelKey::from(rel), hash) { + Entry::Vacant(e) => { + tracing::info!("inserting rel entry for {rel:?}, {nblocks} blocks"); + // FIXME: what to do if we run out of memory? Evict other relation entries? + e.insert(RelEntry { + nblocks: AtomicU32::new(nblocks), + }).expect("out of memory"); + }, + Entry::Occupied(e) => { + tracing::info!("updating rel entry for {rel:?}, {nblocks} blocks"); + e.get().nblocks.store(nblocks, Ordering::Relaxed); + } + }; } /// Remember the given page contents in the cache. @@ -386,34 +384,28 @@ impl<'t> IntegratedCacheWriteAccess<'t> { let mut old_cache_block = None; let mut found_existing = false; - let res = self.block_map.update_with_fn(&key, |existing| { - if let Some(block_entry) = existing { - found_existing = true; - - // Prevent this entry from being evicted - let pin_count = block_entry.pinned.fetch_add(1, Ordering::Relaxed); - if pin_count > 0 { - // this is unexpected, because the caller has obtained the io-in-progress lock, - // so no one else should try to modify the page at the same time. - // XXX: and I think a read should not be happening either, because the postgres - // buffer is held locked. TODO: check these conditions and tidy this up a little. Seems fragile to just panic. - panic!("block entry was unexpectedly pinned"); - } - - let cache_block = block_entry.cache_block.load(Ordering::Relaxed); - old_cache_block = if cache_block != INVALID_CACHE_BLOCK { - Some(cache_block) - } else { - None - }; + let hash = self.block_map.get_hash_value(&key); + if let Entry::Occupied(e) = self.block_map.entry_with_hash(key.clone(), hash) { + let block_entry = e.get(); + found_existing = true; + + // Prevent this entry from being evicted + let pin_count = block_entry.pinned.fetch_add(1, Ordering::Relaxed); + if pin_count > 0 { + // this is unexpected, because the caller has obtained the io-in-progress lock, + // so no one else should try to modify the page at the same time. + // XXX: and I think a read should not be happening either, because the postgres + // buffer is held locked. TODO: check these conditions and tidy this up a little. Seems fragile to just panic. + panic!("block entry was unexpectedly pinned"); } - // if there was no existing entry, we will insert one, but not yet - UpdateAction::Nothing - }); - - // FIXME: what to do if we run out of memory? Evict other relation entries? Remove - // block entries first? - res.expect("out of memory"); + + let cache_block = block_entry.cache_block.load(Ordering::Relaxed); + old_cache_block = if cache_block != INVALID_CACHE_BLOCK { + Some(cache_block) + } else { + None + }; + } // Allocate a new block if required let cache_block = old_cache_block.unwrap_or_else(|| { @@ -436,9 +428,12 @@ impl<'t> IntegratedCacheWriteAccess<'t> { // FIXME: unpin the block entry on error // Update the block entry - let res = self.block_map.update_with_fn(&key, |existing| { - assert_eq!(found_existing, existing.is_some()); - if let Some(block_entry) = existing { + let hash = self.block_map.get_hash_value(&key); + let entry = self.block_map.entry_with_hash(key, hash); + assert_eq!(found_existing, matches!(entry, Entry::Occupied(_))); + match entry { + Entry::Occupied(e) => { + let block_entry = e.get(); // Update the cache block let old_blk = block_entry.cache_block.compare_exchange( INVALID_CACHE_BLOCK, @@ -454,20 +449,18 @@ impl<'t> IntegratedCacheWriteAccess<'t> { let pin_count = block_entry.pinned.fetch_sub(1, Ordering::Relaxed); assert!(pin_count > 0); - UpdateAction::Nothing - } else { - UpdateAction::Insert(BlockEntry { + } + Entry::Vacant(e) => { + // FIXME: what to do if we run out of memory? Evict other relation entries? Remove + // block entries first? + e.insert(BlockEntry { lw_lsn: AtomicLsn::new(lw_lsn.0), cache_block: AtomicU64::new(cache_block), pinned: AtomicU64::new(0), referenced: AtomicBool::new(true), - }) + }).expect("out of memory"); } - }); - - // FIXME: what to do if we run out of memory? Evict other relation entries? Remove - // block entries first? - res.expect("out of memory"); + } } else { // !is_write // @@ -494,36 +487,37 @@ impl<'t> IntegratedCacheWriteAccess<'t> { .expect("error writing to cache"); // FIXME: handle errors gracefully. - let res = self.block_map.update_with_fn(&key, |existing| { - if let Some(block_entry) = existing { - // FIXME: could there be concurrent readers? + let hash = self.block_map.get_hash_value(&key); + match self.block_map.entry_with_hash(key, hash) { + Entry::Occupied(e) => { + let block_entry = e.get(); + // FIXME: could there be concurrent readers? assert!(block_entry.pinned.load(Ordering::Relaxed) == 0); let old_cache_block = block_entry.cache_block.swap(cache_block, Ordering::Relaxed); if old_cache_block != INVALID_CACHE_BLOCK { panic!("remember_page called in !is_write mode, but page is already cached at blk {}", old_cache_block); } - UpdateAction::Nothing - } else { - UpdateAction::Insert(BlockEntry { + }, + Entry::Vacant(e) => { + // FIXME: what to do if we run out of memory? Evict other relation entries? Remove + // block entries first? + e.insert(BlockEntry { lw_lsn: AtomicLsn::new(lw_lsn.0), cache_block: AtomicU64::new(cache_block), pinned: AtomicU64::new(0), referenced: AtomicBool::new(true), - }) + }).expect("out of memory"); } - }); - - // FIXME: what to do if we run out of memory? Evict other relation entries? Remove - // block entries first? - res.expect("out of memory"); + } } } /// Forget information about given relation in the cache. (For DROP TABLE and such) pub fn forget_rel(&'t self, rel: &RelTag) { tracing::info!("forgetting rel entry for {rel:?}"); - self.relsize_cache.remove(&RelKey::from(rel)); + let hash = self.relsize_cache.get_hash_value(&RelKey::from(rel)); + self.relsize_cache.remove_with_hash(&RelKey::from(rel), hash); // also forget all cached blocks for the relation // FIXME @@ -585,13 +579,13 @@ impl<'t> IntegratedCacheWriteAccess<'t> { let num_buckets = self.block_map.get_num_buckets(); match self .block_map - .get_bucket((*clock_hand) % num_buckets) + .get_at_bucket((*clock_hand) % num_buckets) .as_deref() { None => { // This bucket was unused } - Some(blk_entry) => { + Some((_, blk_entry)) => { if !blk_entry.referenced.swap(false, Ordering::Relaxed) { // Evict this. Maybe. evict_this = true; @@ -602,37 +596,37 @@ impl<'t> IntegratedCacheWriteAccess<'t> { if evict_this { // grab the write lock let mut evicted_cache_block = None; - let res = - self.block_map - .update_with_fn_at_bucket(*clock_hand % num_buckets, |old| { - match old { - None => UpdateAction::Nothing, - Some(old) => { - // note: all the accesses to 'pinned' currently happen - // within update_with_fn(), or while holding ValueReadGuard, which protects from concurrent - // updates. Otherwise, another thread could set the 'pinned' - // flag just after we have checked it here. - if old.pinned.load(Ordering::Relaxed) != 0 { - return UpdateAction::Nothing; - } + todo!("quantumish: re-add support for point removal without demolishing performance"); + // self.block_map + // .update_with_fn_at_bucket(*clock_hand % num_buckets, |old| { + // match old { + // None => UpdateAction::Nothing, + // Some(old) => { + // // note: all the accesses to 'pinned' currently happen + // // within update_with_fn(), or while holding ValueReadGuard, which protects from concurrent + // // updates. Otherwise, another thread could set the 'pinned' + // // flag just after we have checked it here. + // if old.pinned.load(Ordering::Relaxed) != 0 { + // return UpdateAction::Nothing; + // } - let _ = self - .global_lw_lsn - .fetch_max(old.lw_lsn.load().0, Ordering::Relaxed); - let cache_block = old - .cache_block - .swap(INVALID_CACHE_BLOCK, Ordering::Relaxed); - if cache_block != INVALID_CACHE_BLOCK { - evicted_cache_block = Some(cache_block); - } - UpdateAction::Remove - } - } - }); + // let _ = self + // .global_lw_lsn + // .fetch_max(old.lw_lsn.load().0, Ordering::Relaxed); + // let cache_block = old + // .cache_block + // .swap(INVALID_CACHE_BLOCK, Ordering::Relaxed); + // if cache_block != INVALID_CACHE_BLOCK { + // evicted_cache_block = Some(cache_block); + // } + // UpdateAction::Remove + // } + // } + // }); // Out of memory should not happen here, as we're only updating existing values, // not inserting new entries to the map. - res.expect("out of memory"); + // res.expect("out of memory"); if evicted_cache_block.is_some() { self.page_evictions_counter.inc(); @@ -711,7 +705,8 @@ fn get_rel_size<'t>( r: &neon_shmem::hash::HashMapAccess, rel: &RelTag, ) -> Option { - if let Some(rel_entry) = r.get(&RelKey::from(rel)) { + let hash = r.get_hash_value(&RelKey::from(rel)); + if let Some(rel_entry) = r.get_with_hash(&RelKey::from(rel), hash) { let nblocks = rel_entry.nblocks.load(Ordering::Relaxed); if nblocks != u32::MAX { Some(nblocks) @@ -755,10 +750,11 @@ impl<'e> BackendCacheReadOp<'e> { /// After you have completed the read, call BackendCacheReadResult::finish() to check if the /// read was in fact valid or not. If it was concurrently invalidated, you need to retry. pub fn get_page(&mut self, rel: &RelTag, block_number: u32) -> Option { + let hash = self.map_access.block_map.get_hash_value(&BlockKey::from((rel, block_number))); if let Some(block_entry) = self .map_access .block_map - .get(&BlockKey::from((rel, block_number))) + .get_with_hash(&BlockKey::from((rel, block_number)), hash) { block_entry.referenced.store(true, Ordering::Relaxed); From c3c136ef3a7b9dfceba63fc837f4a1cb6287e4e9 Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Fri, 27 Jun 2025 17:10:52 -0700 Subject: [PATCH 19/24] Remove statistics utilities from neon_shmem crate --- Cargo.lock | 299 ------------------------------ libs/neon-shmem/Cargo.toml | 8 - libs/neon-shmem/src/hmap_stats.rs | 139 -------------- 3 files changed, 446 deletions(-) delete mode 100644 libs/neon-shmem/src/hmap_stats.rs diff --git a/Cargo.lock b/Cargo.lock index c407806b3d..4fd5f5802b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1336,12 +1336,6 @@ dependencies = [ "cc", ] -[[package]] -name = "color_quant" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" - [[package]] name = "colorchoice" version = "1.0.4" @@ -1601,42 +1595,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core-graphics" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081" -dependencies = [ - "bitflags 1.3.2", - "core-foundation 0.9.4", - "core-graphics-types", - "foreign-types", - "libc", -] - -[[package]] -name = "core-graphics-types" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" -dependencies = [ - "bitflags 1.3.2", - "core-foundation 0.9.4", - "libc", -] - -[[package]] -name = "core-text" -version = "20.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d2790b5c08465d49f8dc05c8bcae9fea467855947db39b0f8145c091aaced5" -dependencies = [ - "core-foundation 0.9.4", - "core-graphics", - "foreign-types", - "libc", -] - [[package]] name = "cpp_demangle" version = "0.4.4" @@ -2089,27 +2047,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "dirs" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.59.0", -] - [[package]] name = "displaydoc" version = "0.2.5" @@ -2121,15 +2058,6 @@ dependencies = [ "syn", ] -[[package]] -name = "dlib" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "330c60081dcc4c72131f8eb70510f1ac07223e5d4163db481a04a0befcffa412" -dependencies = [ - "libloading", -] - [[package]] name = "dlv-list" version = "0.5.2" @@ -2159,18 +2087,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" -[[package]] -name = "dwrote" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe1f192fcce01590bd8d839aca53ce0d11d803bf291b2a6c4ad925a8f0024be" -dependencies = [ - "lazy_static", - "libc", - "winapi", - "wio", -] - [[package]] name = "dyn-clone" version = "1.0.19" @@ -2461,15 +2377,6 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" -[[package]] -name = "fdeflate" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" -dependencies = [ - "simd-adler32", -] - [[package]] name = "ff" version = "0.12.1" @@ -2548,12 +2455,6 @@ dependencies = [ "miniz_oxide", ] -[[package]] -name = "float-ord" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" - [[package]] name = "fnv" version = "1.0.7" @@ -2566,58 +2467,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" -[[package]] -name = "font-kit" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c7e611d49285d4c4b2e1727b72cf05353558885cc5252f93707b845dfcaf3d3" -dependencies = [ - "bitflags 2.9.1", - "byteorder", - "core-foundation 0.9.4", - "core-graphics", - "core-text", - "dirs", - "dwrote", - "float-ord", - "freetype-sys", - "lazy_static", - "libc", - "log", - "pathfinder_geometry", - "pathfinder_simd", - "walkdir", - "winapi", - "yeslogic-fontconfig-sys", -] - -[[package]] -name = "foreign-types" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" -dependencies = [ - "foreign-types-macros", - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-macros" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "foreign-types-shared" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -2648,17 +2497,6 @@ dependencies = [ "tokio-util", ] -[[package]] -name = "freetype-sys" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7edc5b9669349acfda99533e9e0bcf26a51862ab43b08ee7745c55d28eb134" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "fs_extra" version = "1.3.0" @@ -2849,16 +2687,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "gif" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80792593675e051cf94a4b111980da2ba60d4a83e43e0048c5693baab3977045" -dependencies = [ - "color_quant", - "weezl", -] - [[package]] name = "gimli" version = "0.31.1" @@ -3543,20 +3371,6 @@ dependencies = [ "icu_properties", ] -[[package]] -name = "image" -version = "0.24.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d" -dependencies = [ - "bytemuck", - "byteorder", - "color_quant", - "jpeg-decoder", - "num-traits", - "png", -] - [[package]] name = "indexmap" version = "1.9.3" @@ -3841,12 +3655,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "jpeg-decoder" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07" - [[package]] name = "js-sys" version = "0.3.77" @@ -4190,7 +3998,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", - "simd-adler32", ] [[package]] @@ -4231,7 +4038,6 @@ dependencies = [ "foldhash", "hashbrown 0.15.4 (git+https://github.com/quantumish/hashbrown.git?rev=6610e6d)", "nix 0.30.1", - "plotters", "rand 0.9.1", "rand_distr 0.5.1", "rustc-hash 2.1.1", @@ -4600,12 +4406,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - [[package]] name = "ordered-float" version = "2.10.1" @@ -5042,25 +4842,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" -[[package]] -name = "pathfinder_geometry" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b7e7b4ea703700ce73ebf128e1450eb69c3a8329199ffbfb9b2a0418e5ad3" -dependencies = [ - "log", - "pathfinder_simd", -] - -[[package]] -name = "pathfinder_simd" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf9027960355bf3afff9841918474a81a5f972ac6d226d518060bba758b5ad57" -dependencies = [ - "rustc_version", -] - [[package]] name = "pbkdf2" version = "0.12.2" @@ -5224,16 +5005,9 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ - "chrono", - "font-kit", - "image", - "lazy_static", "num-traits", - "pathfinder_geometry", "plotters-backend", - "plotters-bitmap", "plotters-svg", - "ttf-parser", "wasm-bindgen", "web-sys", ] @@ -5244,17 +5018,6 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" -[[package]] -name = "plotters-bitmap" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72ce181e3f6bf82d6c1dc569103ca7b1bd964c60ba03d7e6cdfbb3e3eb7f7405" -dependencies = [ - "gif", - "image", - "plotters-backend", -] - [[package]] name = "plotters-svg" version = "0.3.7" @@ -5264,19 +5027,6 @@ dependencies = [ "plotters-backend", ] -[[package]] -name = "png" -version = "0.17.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" -dependencies = [ - "bitflags 1.3.2", - "crc32fast", - "fdeflate", - "flate2", - "miniz_oxide", -] - [[package]] name = "polonius-the-crab" version = "0.4.2" @@ -6164,17 +5914,6 @@ dependencies = [ "bitflags 2.9.1", ] -[[package]] -name = "redox_users" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b" -dependencies = [ - "getrandom 0.2.16", - "libredox", - "thiserror 2.0.12", -] - [[package]] name = "regex" version = "1.11.1" @@ -7247,12 +6986,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "simd-adler32" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" - [[package]] name = "simple_asn1" version = "0.6.3" @@ -8444,12 +8177,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "ttf-parser" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f77d76d837a7830fe1d4f12b7b4ba4192c1888001c7164257e4bc6d21d96b4" - [[package]] name = "tungstenite" version = "0.21.0" @@ -8988,12 +8715,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "weezl" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a751b3277700db47d3e574514de2eced5e54dc8a5436a3bf7a0b248b2cee16f3" - [[package]] name = "which" version = "4.4.2" @@ -9346,15 +9067,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "wio" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d129932f4644ac2396cb456385cbf9e63b5b30c6e8dc4820bdca4eb082037a5" -dependencies = [ - "winapi", -] - [[package]] name = "wit-bindgen-rt" version = "0.39.0" @@ -9540,17 +9252,6 @@ dependencies = [ "time", ] -[[package]] -name = "yeslogic-fontconfig-sys" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503a066b4c037c440169d995b869046827dbc71263f6e8f3be6d77d4f3229dbd" -dependencies = [ - "dlib", - "once_cell", - "pkg-config", -] - [[package]] name = "yoke" version = "0.8.0" diff --git a/libs/neon-shmem/Cargo.toml b/libs/neon-shmem/Cargo.toml index 3a22ce6c8b..08b5ac2138 100644 --- a/libs/neon-shmem/Cargo.toml +++ b/libs/neon-shmem/Cargo.toml @@ -10,7 +10,6 @@ nix.workspace = true workspace_hack = { version = "0.1", path = "../../workspace_hack" } rustc-hash = { version = "2.1.1" } rand = "0.9.1" -plotters = { version = "0.3.7", optional = true } [dev-dependencies] criterion = { workspace = true, features = ["html_reports"] } @@ -29,10 +28,3 @@ tempfile = "3.14.0" [[bench]] name = "hmap_resize" harness = false - -[[bin]] -name = "hmap_stats" -path = "src/hmap_stats.rs" - -[features] -stats = ["dep:plotters"] diff --git a/libs/neon-shmem/src/hmap_stats.rs b/libs/neon-shmem/src/hmap_stats.rs deleted file mode 100644 index 9e55ad1f05..0000000000 --- a/libs/neon-shmem/src/hmap_stats.rs +++ /dev/null @@ -1,139 +0,0 @@ -use neon_shmem::hash::HashMapAccess; -use neon_shmem::hash::HashMapInit; -use neon_shmem::hash::entry::Entry; -use rand::prelude::*; -use rand::distr::{Distribution, StandardUniform}; -use plotters::prelude::*; - -#[derive(Clone, Debug, Hash, Eq, PartialEq)] -#[repr(C)] -pub struct FileCacheKey { - pub _spc_id: u32, - pub _db_id: u32, - pub _rel_number: u32, - pub _fork_num: u32, - pub _block_num: u32, -} - -impl Distribution for StandardUniform { - // questionable, but doesn't need to be good randomness - fn sample(&self, rng: &mut R) -> FileCacheKey { - FileCacheKey { - _spc_id: rng.random(), - _db_id: rng.random(), - _rel_number: rng.random(), - _fork_num: rng.random(), - _block_num: rng.random() - } - } -} - -#[derive(Clone, Debug)] -#[repr(C)] -pub struct FileCacheEntry { - pub _offset: u32, - pub _access_count: u32, - pub _prev: *mut FileCacheEntry, - pub _next: *mut FileCacheEntry, - pub _state: [u32; 8], -} - -impl FileCacheEntry { - fn dummy() -> Self { - Self { - _offset: 0, - _access_count: 0, - _prev: std::ptr::null_mut(), - _next: std::ptr::null_mut(), - _state: [0; 8] - } - } -} - -// Utilities for applying operations. - -#[derive(Clone, Debug)] -struct TestOp(K, Option); - -fn apply_op( - op: TestOp, - map: &mut HashMapAccess, -) { - let hash = map.get_hash_value(&op.0); - let entry = map.entry_with_hash(op.0, hash); - - match op.1 { - Some(new) => { - match entry { - Entry::Occupied(mut e) => Some(e.insert(new)), - Entry::Vacant(e) => { e.insert(new).unwrap(); None }, - } - }, - None => { - match entry { - Entry::Occupied(e) => Some(e.remove()), - Entry::Vacant(_) => None, - } - }, - }; -} - -#[cfg(feature = "stats")] -fn main() { - let ideal_filled = 16_000_000; - let size = 20_000_000; - let mut writer = HashMapInit::new_resizeable(size, size).attach_writer(); - let mut rng = rand::rng(); - while writer.get_num_buckets_in_use() < ideal_filled as usize { - let key: FileCacheKey = rng.random(); - let val = FileCacheEntry::dummy(); - apply_op(TestOp(key, Some(val)), &mut writer); - } - println!("Inserted {ideal_filled} entries into a map with capacity {size}."); - let (distr, max) = writer.chain_distribution(); - - let root_area = BitMapBackend::new("chain_distr.png", (800, 400)) - .into_drawing_area(); - root_area.fill(&WHITE).unwrap(); - - let mut ctx = ChartBuilder::on(&root_area) - .set_label_area_size(LabelAreaPosition::Left, 40) - .set_label_area_size(LabelAreaPosition::Bottom, 40) - .build_cartesian_2d((0..max).into_segmented(), (0..ideal_filled * 2).log_scale()) - .unwrap(); - - ctx.configure_mesh() - .y_label_formatter(&|y| format!("{:e}", y)) - .draw().unwrap(); - - ctx.draw_series( - Histogram::vertical(&ctx) - .margin(10) - .data(distr.iter().map(|x| (x.1, 1))) - ).unwrap(); - - // let root_area = BitMapBackend::new("dict_distr.png", (2000, 400)) - // .into_drawing_area(); - // root_area.fill(&WHITE).unwrap(); - - // let mut ctx = ChartBuilder::on(&root_area) - // .set_label_area_size(LabelAreaPosition::Left, 40) - // .set_label_area_size(LabelAreaPosition::Bottom, 40) - // .build_cartesian_2d((0..writer.dict_len()), (0..(max as f32 * 1.5) as usize)) - // .unwrap(); - - // ctx.configure_mesh().draw().unwrap(); - - // ctx.draw_series(LineSeries::new( - // distr.iter().map(|(bin, count)| (*bin, *count)), - // &RED, - // )).unwrap(); - - // println!("Longest chain: {}", writer.longest_chain()); -} - -#[cfg(not(feature = "stats"))] -fn main() { - println!("Enable the `stats` feature to use this binary!"); -} - From 74330920ee6ced0099010b8ecf0974fbbb539ad6 Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Fri, 27 Jun 2025 17:11:22 -0700 Subject: [PATCH 20/24] Simplify API, squash bugs, and expand hashmap test suite --- libs/neon-shmem/src/hash.rs | 108 ++++++++++++++++------------ libs/neon-shmem/src/hash/core.rs | 29 +------- libs/neon-shmem/src/hash/entry.rs | 27 +++++-- libs/neon-shmem/src/hash/tests.rs | 114 ++++++++++++++++++++++++------ 4 files changed, 178 insertions(+), 100 deletions(-) diff --git a/libs/neon-shmem/src/hash.rs b/libs/neon-shmem/src/hash.rs index 6cc641814a..7d47a4f5e5 100644 --- a/libs/neon-shmem/src/hash.rs +++ b/libs/neon-shmem/src/hash.rs @@ -224,40 +224,75 @@ where K: Clone + Hash + Eq, { /// Hash a key using the map's hasher. - pub fn get_hash_value(&self, key: &K) -> u64 { + #[inline] + fn get_hash_value(&self, key: &K) -> u64 { self.hasher.hash_one(key) } - /// Get a reference to the corresponding value for a key given its hash. - pub fn get_with_hash<'e>(&'e self, key: &K, hash: u64) -> Option<&'e V> { + /// Get a reference to the corresponding value for a key. + pub fn get<'e>(&'e self, key: &K) -> Option<&'e V> { let map = unsafe { self.shared_ptr.as_ref() }.unwrap(); - + let hash = self.get_hash_value(key); map.inner.get_with_hash(key, hash) } - /// Get a reference to the entry containing a key given its hash. - pub fn entry_with_hash(&mut self, key: K, hash: u64) -> Entry<'a, '_, K, V> { + /// Get a reference to the entry containing a key. + pub fn entry(&self, key: K) -> Entry<'a, '_, K, V> { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); - + let hash = self.get_hash_value(&key); map.inner.entry_with_hash(key, hash) } - /// Remove a key given its hash. Does nothing if key is not present. - pub fn remove_with_hash(&mut self, key: &K, hash: u64) { + /// Remove a key given its hash. Returns the associated value if it existed. + pub fn remove(&self, key: &K) -> Option { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); - + let hash = self.get_hash_value(&key); match map.inner.entry_with_hash(key.clone(), hash) { - Entry::Occupied(e) => { - e.remove(); - } - Entry::Vacant(_) => {} + Entry::Occupied(e) => Some(e.remove()), + Entry::Vacant(_) => None } } - /// Optionally return the entry for a bucket at a given index if it exists. - pub fn entry_at_bucket(&mut self, pos: usize) -> Option> { + /// Insert/update a key. Returns the previous associated value if it existed. + /// + /// # Errors + /// Will return [`core::FullError`] if there is no more space left in the map. + pub fn insert(&self, key: K, value: V) -> Result, core::FullError> { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); - map.inner.entry_at_bucket(pos) + let hash = self.get_hash_value(&key); + match map.inner.entry_with_hash(key.clone(), hash) { + Entry::Occupied(mut e) => Ok(Some(e.insert(value))), + Entry::Vacant(e) => { + e.insert(value)?; + Ok(None) + } + } + } + + /// Optionally return the entry for a bucket at a given index if it exists. + /// + /// Has more overhead than one would intuitively expect: performs both a clone of the key + /// due to the [`OccupiedEntry`] type owning the key and also a hash of the key in order + /// to enable repairing the hash chain if the entry is removed. + pub fn entry_at_bucket(&self, pos: usize) -> Option> { + let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); + let inner = &mut map.inner; + if pos >= inner.buckets.len() { + return None; + } + + let entry = inner.buckets[pos].inner.as_ref(); + match entry { + Some((key, _)) => Some(OccupiedEntry { + _key: key.clone(), + bucket_pos: pos as u32, + prev_pos: entry::PrevPos::Unknown( + self.get_hash_value(&key) + ), + map: inner, + }), + _ => None, + } } /// Returns the number of buckets in the table. @@ -299,7 +334,7 @@ where } /// Clears all entries in a table. Does not reset any shrinking operations. - pub fn clear(&mut self) { + pub fn clear(&self) { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); let inner = &mut map.inner; inner.clear(); @@ -353,7 +388,7 @@ where } /// Rehash the map without growing or shrinking. - pub fn shuffle(&mut self) { + pub fn shuffle(&self) { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); let inner = &mut map.inner; let num_buckets = inner.get_num_buckets() as u32; @@ -447,14 +482,17 @@ where /// - Calling this function on a map initialized with [`HashMapInit::with_fixed`]. /// - Calling this function on a map when no shrink operation is in progress. /// - Calling this function on a map with `shrink_mode` set to [`HashMapShrinkMode::Remap`] and - /// [`HashMapAccess::get_num_buckets_in_use`] returns a value higher than [`HashMapAccess::shrink_goal`]. + /// there are more buckets in use than the value returned by [`HashMapAccess::shrink_goal`]. /// /// # Errors /// Returns an [`shmem::Error`] if any errors occur resizing the memory region. - pub fn finish_shrink(&mut self) -> Result<(), shmem::Error> { + pub fn finish_shrink(&self) -> Result<(), shmem::Error> { let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); let inner = &mut map.inner; - assert!(inner.is_shrinking(), "called finish_shrink when no shrink is in progress"); + assert!( + inner.alloc_limit != INVALID_POS, + "called finish_shrink when no shrink is in progress" + ); let num_buckets = inner.alloc_limit; @@ -470,7 +508,7 @@ where for i in (num_buckets as usize)..inner.buckets.len() { if let Some((k, v)) = inner.buckets[i].inner.take() { - // alloc bucket increases buckets in use, so need to decrease since we're just moving + // alloc_bucket increases count, so need to decrease since we're just moving inner.buckets_in_use -= 1; inner.alloc_bucket(k, v).unwrap(); } @@ -491,28 +529,4 @@ where Ok(()) } - - #[cfg(feature = "stats")] - pub fn dict_len(&self) -> usize { - let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); - map.inner.dictionary.len() - } - - #[cfg(feature = "stats")] - pub fn chain_distribution(&self) -> (Vec<(usize, usize)>, usize) { - let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); - let mut out = Vec::new(); - let mut max = 0; - for (i, d) in map.inner.dictionary.iter().enumerate() { - let mut curr = *d; - let mut len = 0; - while curr != INVALID_POS { - curr = map.inner.buckets[curr as usize].next; - len += 1; - } - out.push((i, len)); - max = max.max(len); - } - (out, max) - } } diff --git a/libs/neon-shmem/src/hash/core.rs b/libs/neon-shmem/src/hash/core.rs index b2cf788d21..28c58e851e 100644 --- a/libs/neon-shmem/src/hash/core.rs +++ b/libs/neon-shmem/src/hash/core.rs @@ -34,7 +34,7 @@ pub(crate) struct CoreHashMap<'a, K, V> { } /// Error for when there are no empty buckets left but one is needed. -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub struct FullError(); impl<'a, K: Clone + Hash + Eq, V> CoreHashMap<'a, K, V> { @@ -155,11 +155,6 @@ impl<'a, K: Clone + Hash + Eq, V> CoreHashMap<'a, K, V> { self.buckets.len() } - /// Returns whether there is an ongoing shrink operation. - pub fn is_shrinking(&self) -> bool { - self.alloc_limit != INVALID_POS - } - /// Clears all entries from the hashmap. /// /// Does not reset any allocation limits, but does clear any entries beyond them. @@ -174,32 +169,14 @@ impl<'a, K: Clone + Hash + Eq, V> CoreHashMap<'a, K, V> { inner: None, } } - for i in 0..self.dictionary.len() { self.dictionary[i] = INVALID_POS; } + self.free_head = 0; self.buckets_in_use = 0; } - /// Optionally gets the entry at an index if it is occupied. - pub fn entry_at_bucket(&mut self, pos: usize) -> Option> { - if pos >= self.buckets.len() { - return None; - } - - let entry = self.buckets[pos].inner.as_ref(); - match entry { - Some((key, _)) => Some(OccupiedEntry { - _key: key.clone(), - bucket_pos: pos as u32, - prev_pos: PrevPos::Unknown, - map: self, - }), - _ => None, - } - } - /// Find the position of an unused bucket via the freelist and initialize it. pub(crate) fn alloc_bucket(&mut self, key: K, value: V) -> Result { let mut pos = self.free_head; @@ -225,7 +202,7 @@ impl<'a, K: Clone + Hash + Eq, V> CoreHashMap<'a, K, V> { let next_pos = self.buckets[pos as usize].next; self.buckets[p as usize].next = next_pos; }, - PrevPos::Unknown => unreachable!() + _ => unreachable!() } // Initialize the bucket. diff --git a/libs/neon-shmem/src/hash/entry.rs b/libs/neon-shmem/src/hash/entry.rs index 5231061b8e..b4c973d9f5 100644 --- a/libs/neon-shmem/src/hash/entry.rs +++ b/libs/neon-shmem/src/hash/entry.rs @@ -19,7 +19,7 @@ pub(crate) enum PrevPos { /// Regular index within the buckets. Chained(u32), /// Unknown - e.g. the associated entry was retrieved by index instead of chain. - Unknown, + Unknown(u64), } /// View into an occupied entry within the map. @@ -31,7 +31,7 @@ pub struct OccupiedEntry<'a, 'b, K, V> { /// The index of the previous entry in the chain. pub(crate) prev_pos: PrevPos, /// The position of the bucket in the [`CoreHashMap`] bucket array. - pub(crate) bucket_pos: u32, + pub(crate) bucket_pos: u32, } impl OccupiedEntry<'_, '_, K, V> { @@ -60,22 +60,39 @@ impl OccupiedEntry<'_, '_, K, V> { /// Removes the entry from the hash map, returning the value originally stored within it. /// + /// This may result in multiple bucket accesses if the entry was obtained by index as the + /// previous chain entry needs to be discovered in this case. + /// /// # Panics /// Panics if the `prev_pos` field is equal to [`PrevPos::Unknown`]. In practice, this means /// the entry was obtained via calling something like [`CoreHashMap::entry_at_bucket`]. pub fn remove(self) -> V { + // If this bucket was queried by index, go ahead and follow its chain from the start. + let prev = if let PrevPos::Unknown(hash) = self.prev_pos { + let dict_idx = hash as usize % self.map.dictionary.len(); + let mut prev = PrevPos::First(dict_idx as u32); + let mut curr = self.map.dictionary[dict_idx]; + while curr != self.bucket_pos { + curr = self.map.buckets[curr as usize].next; + prev = PrevPos::Chained(curr); + } + prev + } else { + self.prev_pos + }; + // CoreHashMap::remove returns Option<(K, V)>. We know it's Some for an OccupiedEntry. let bucket = &mut self.map.buckets[self.bucket_pos as usize]; - + // unlink it from the chain - match self.prev_pos { + match prev { PrevPos::First(dict_pos) => { self.map.dictionary[dict_pos as usize] = bucket.next; }, PrevPos::Chained(bucket_pos) => { self.map.buckets[bucket_pos as usize].next = bucket.next; }, - PrevPos::Unknown => panic!("can't safely remove entry with unknown previous entry"), + _ => unreachable!(), } // and add it to the freelist diff --git a/libs/neon-shmem/src/hash/tests.rs b/libs/neon-shmem/src/hash/tests.rs index 209db599b5..d838aa0b86 100644 --- a/libs/neon-shmem/src/hash/tests.rs +++ b/libs/neon-shmem/src/hash/tests.rs @@ -6,6 +6,7 @@ use std::mem::MaybeUninit; use crate::hash::HashMapAccess; use crate::hash::HashMapInit; use crate::hash::Entry; +use crate::hash::core::FullError; use rand::seq::SliceRandom; use rand::{Rng, RngCore}; @@ -40,8 +41,7 @@ fn test_inserts + Copy>(keys: &[K]) { ).attach_writer(); for (idx, k) in keys.iter().enumerate() { - let hash = w.get_hash_value(&(*k).into()); - let res = w.entry_with_hash((*k).into(), hash); + let res = w.entry((*k).into()); match res { Entry::Occupied(mut e) => { e.insert(idx); } Entry::Vacant(e) => { @@ -52,8 +52,7 @@ fn test_inserts + Copy>(keys: &[K]) { } for (idx, k) in keys.iter().enumerate() { - let hash = w.get_hash_value(&(*k).into()); - let x = w.get_with_hash(&(*k).into(), hash); + let x = w.get(&(*k).into()); let value = x.as_deref().copied(); assert_eq!(value, Some(idx)); } @@ -110,8 +109,7 @@ fn apply_op( shadow.remove(&op.0) }; - let hash = map.get_hash_value(&op.0); - let entry = map.entry_with_hash(op.0, hash); + let entry = map.entry(op.0); let hash_existing = match op.1 { Some(new) => { match entry { @@ -152,8 +150,7 @@ fn do_deletes( ) { for _ in 0..num_ops { let (k, _) = shadow.pop_first().unwrap(); - let hash = writer.get_hash_value(&k); - writer.remove_with_hash(&k, hash); + writer.remove(&k); } } @@ -162,16 +159,20 @@ fn do_shrink( shadow: &mut BTreeMap, to: u32 ) { + assert!(writer.shrink_goal().is_none()); writer.begin_shrink(to); + assert_eq!(writer.shrink_goal(), Some(to as usize)); while writer.get_num_buckets_in_use() > to as usize { let (k, _) = shadow.pop_first().unwrap(); - let hash = writer.get_hash_value(&k); - let entry = writer.entry_with_hash(k, hash); + let entry = writer.entry(k); if let Entry::Occupied(e) = entry { e.remove(); } } + let old_usage = writer.get_num_buckets_in_use(); writer.finish_shrink().unwrap(); + assert!(writer.shrink_goal().is_none()); + assert_eq!(writer.get_num_buckets_in_use(), old_usage); } #[test] @@ -219,10 +220,80 @@ fn test_grow() { let mut rng = rand::rng(); do_random_ops(10000, 1000, 0.75, &mut writer, &mut shadow, &mut rng); + let old_usage = writer.get_num_buckets_in_use(); writer.grow(1500).unwrap(); + assert_eq!(writer.get_num_buckets_in_use(), old_usage); + assert_eq!(writer.get_num_buckets(), 1500); do_random_ops(10000, 1500, 0.75, &mut writer, &mut shadow, &mut rng); } +#[test] +fn test_clear() { + let mut writer = HashMapInit::::new_resizeable_named( + 1500, 2000, "test_clear" + ).attach_writer(); + let mut shadow: std::collections::BTreeMap = BTreeMap::new(); + let mut rng = rand::rng(); + do_random_ops(2000, 1500, 0.75, &mut writer, &mut shadow, &mut rng); + writer.clear(); + assert_eq!(writer.get_num_buckets_in_use(), 0); + assert_eq!(writer.get_num_buckets(), 1500); + while let Some((key, _)) = shadow.pop_first() { + assert!(writer.get(&key).is_none()); + } + do_random_ops(2000, 1500, 0.75, &mut writer, &mut shadow, &mut rng); + for i in 0..(1500 - writer.get_num_buckets_in_use()) { + writer.insert((1500 + i as u128).into(), 0).unwrap(); + } + assert_eq!(writer.insert(5000.into(), 0), Err(FullError {})); + writer.clear(); + assert!(writer.insert(5000.into(), 0).is_ok()); +} + +#[test] +fn test_idx_remove() { + let mut writer = HashMapInit::::new_resizeable_named( + 1500, 2000, "test_clear" + ).attach_writer(); + let mut shadow: std::collections::BTreeMap = BTreeMap::new(); + let mut rng = rand::rng(); + do_random_ops(2000, 1500, 0.25, &mut writer, &mut shadow, &mut rng); + for _ in 0..100 { + let idx = (rng.next_u32() % 1500) as usize; + if let Some(e) = writer.entry_at_bucket(idx) { + shadow.remove(&e._key); + e.remove(); + } + + } + while let Some((key, val)) = shadow.pop_first() { + assert_eq!(writer.get(&key), Some(&val)); + } +} + +#[test] +fn test_idx_get() { + let mut writer = HashMapInit::::new_resizeable_named( + 1500, 2000, "test_clear" + ).attach_writer(); + let mut shadow: std::collections::BTreeMap = BTreeMap::new(); + let mut rng = rand::rng(); + do_random_ops(2000, 1500, 0.25, &mut writer, &mut shadow, &mut rng); + for _ in 0..100 { + let idx = (rng.next_u32() % 1500) as usize; + if let Some(mut e) = writer.entry_at_bucket(idx) { + { + let v: *const usize = e.get(); + assert_eq!(writer.get_bucket_for_value(v), idx); + } + { + let v: *const usize = e.get_mut(); + assert_eq!(writer.get_bucket_for_value(v), idx); + } + } + } +} + #[test] fn test_shrink() { let mut writer = HashMapInit::::new_resizeable_named( @@ -231,8 +302,9 @@ fn test_shrink() { let mut shadow: std::collections::BTreeMap = BTreeMap::new(); let mut rng = rand::rng(); - do_random_ops(10000, 1500, 0.75, &mut writer, &mut shadow, &mut rng); - do_shrink(&mut writer, &mut shadow, 1000); + do_random_ops(10000, 1500, 0.75, &mut writer, &mut shadow, &mut rng); + do_shrink(&mut writer, &mut shadow, 1000); + assert_eq!(writer.get_num_buckets(), 1000); do_deletes(500, &mut writer, &mut shadow); do_random_ops(10000, 500, 0.75, &mut writer, &mut shadow, &mut rng); assert!(writer.get_num_buckets_in_use() <= 1000); @@ -267,15 +339,14 @@ fn test_bucket_ops() { let mut writer = HashMapInit::::new_resizeable_named( 1000, 1200, "test_bucket_ops" ).attach_writer(); - let hash = writer.get_hash_value(&1.into()); - match writer.entry_with_hash(1.into(), hash) { + match writer.entry(1.into()) { Entry::Occupied(mut e) => { e.insert(2); }, Entry::Vacant(e) => { e.insert(2).unwrap(); }, } assert_eq!(writer.get_num_buckets_in_use(), 1); assert_eq!(writer.get_num_buckets(), 1000); - assert_eq!(writer.get_with_hash(&1.into(), hash), Some(&2)); - let pos = match writer.entry_with_hash(1.into(), hash) { + assert_eq!(writer.get(&1.into()), Some(&2)); + let pos = match writer.entry(1.into()) { Entry::Occupied(e) => { assert_eq!(e._key, 1.into()); let pos = e.bucket_pos as usize; @@ -285,10 +356,10 @@ fn test_bucket_ops() { }, Entry::Vacant(_) => { panic!("Insert didn't affect entry"); }, }; - let ptr: *const usize = writer.get_with_hash(&1.into(), hash).unwrap(); + let ptr: *const usize = writer.get(&1.into()).unwrap(); assert_eq!(writer.get_bucket_for_value(ptr), pos); - writer.remove_with_hash(&1.into(), hash); - assert_eq!(writer.get_with_hash(&1.into(), hash), None); + writer.remove(&1.into()); + assert_eq!(writer.get(&1.into()), None); } #[test] @@ -302,15 +373,14 @@ fn test_shrink_zero() { } writer.finish_shrink().unwrap(); assert_eq!(writer.get_num_buckets_in_use(), 0); - let hash = writer.get_hash_value(&1.into()); - let entry = writer.entry_with_hash(1.into(), hash); + let entry = writer.entry(1.into()); if let Entry::Vacant(v) = entry { assert!(v.insert(2).is_err()); } else { panic!("Somehow got non-vacant entry in empty map.") } writer.grow(50).unwrap(); - let entry = writer.entry_with_hash(1.into(), hash); + let entry = writer.entry(1.into()); if let Entry::Vacant(v) = entry { assert!(v.insert(2).is_ok()); } else { From 9d3e07ef2cebb53ba439df6d7e39195d1cf22d5d Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Mon, 30 Jun 2025 17:05:32 -0700 Subject: [PATCH 21/24] Add initial prototype of shmem sync primitives --- Cargo.lock | 1 + libs/neon-shmem/Cargo.toml | 1 + libs/neon-shmem/src/lib.rs | 1 + libs/neon-shmem/src/sync.rs | 255 ++++++++++++++++++++++++++++++++++++ 4 files changed, 258 insertions(+) create mode 100644 libs/neon-shmem/src/sync.rs diff --git a/Cargo.lock b/Cargo.lock index 4fd5f5802b..4bb6d2d474 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4037,6 +4037,7 @@ dependencies = [ "criterion", "foldhash", "hashbrown 0.15.4 (git+https://github.com/quantumish/hashbrown.git?rev=6610e6d)", + "libc", "nix 0.30.1", "rand 0.9.1", "rand_distr 0.5.1", diff --git a/libs/neon-shmem/Cargo.toml b/libs/neon-shmem/Cargo.toml index 08b5ac2138..8306bbf778 100644 --- a/libs/neon-shmem/Cargo.toml +++ b/libs/neon-shmem/Cargo.toml @@ -10,6 +10,7 @@ nix.workspace = true workspace_hack = { version = "0.1", path = "../../workspace_hack" } rustc-hash = { version = "2.1.1" } rand = "0.9.1" +libc.workspace = true [dev-dependencies] criterion = { workspace = true, features = ["html_reports"] } diff --git a/libs/neon-shmem/src/lib.rs b/libs/neon-shmem/src/lib.rs index f601010122..61ca168073 100644 --- a/libs/neon-shmem/src/lib.rs +++ b/libs/neon-shmem/src/lib.rs @@ -2,3 +2,4 @@ pub mod hash; pub mod shmem; +pub mod sync; diff --git a/libs/neon-shmem/src/sync.rs b/libs/neon-shmem/src/sync.rs new file mode 100644 index 0000000000..68ef7b904d --- /dev/null +++ b/libs/neon-shmem/src/sync.rs @@ -0,0 +1,255 @@ +//! Simple utilities akin to what's in [`std::sync`] but designed to work with shared memory. + +use std::mem::MaybeUninit; +use std::ptr::NonNull; +use std::cell::UnsafeCell; +use std::ops::{Deref, DerefMut}; +use thiserror::Error; + +/// Shared memory read-write lock. +struct RwLock<'a, T: ?Sized> { + inner: &'a mut libc::pthread_rwlock_t, + data: UnsafeCell, +} + +/// RAII guard for a read lock. +struct RwLockReadGuard<'a, 'b, T: ?Sized> { + data: NonNull, + lock: &'a RwLock<'b, T>, +} + +/// RAII guard for a write lock. +struct RwLockWriteGuard<'a, 'b, T: ?Sized> { + lock: &'a RwLock<'b, T>, +} + +// TODO(quantumish): Support poisoning errors? +#[derive(Error, Debug)] +enum RwLockError { + #[error("deadlock detected")] + Deadlock, + #[error("max number of read locks exceeded")] + MaxReadLocks, + #[error("nonblocking operation would block")] + WouldBlock, +} + +unsafe impl Send for RwLock<'_, T> {} +unsafe impl Sync for RwLock<'_, T> {} + +impl<'a, T> RwLock<'a, T> { + fn new(lock: &'a mut MaybeUninit, data: T) -> Self { + unsafe { + let mut attrs = MaybeUninit::uninit(); + // Ignoring return value here - only possible error is OOM. + libc::pthread_rwlockattr_init(attrs.as_mut_ptr()); + libc::pthread_rwlockattr_setpshared( + attrs.as_mut_ptr(), + libc::PTHREAD_PROCESS_SHARED + ); + // TODO(quantumish): worth making this function return Result? + libc::pthread_rwlock_init(lock.as_mut_ptr(), attrs.as_mut_ptr()); + // Safety: POSIX specifies that "any function affecting the attributes + // object (including destruction) shall not affect any previously + // initialized read-write locks". + libc::pthread_rwlockattr_destroy(attrs.as_mut_ptr()); + Self { + inner: lock.assume_init_mut(), + data: data.into(), + } + } + } + + fn read(&self) -> Result, RwLockError> { + unsafe { + let res = libc::pthread_rwlock_rdlock(self.inner as *const _ as *mut _); + match res { + 0 => (), + libc::EINVAL => panic!("failed to properly initialize lock"), + libc::EDEADLK => return Err(RwLockError::Deadlock), + libc::EAGAIN => return Err(RwLockError::MaxReadLocks), + e => panic!("unknown error code returned: {e}") + } + Ok(RwLockReadGuard { + data: NonNull::new_unchecked(self.data.get()), + lock: self + }) + } + } + + fn try_read(&self) -> Result, RwLockError> { + unsafe { + let res = libc::pthread_rwlock_tryrdlock(self.inner as *const _ as *mut _); + match res { + 0 => (), + libc::EINVAL => panic!("failed to properly initialize lock"), + libc::EDEADLK => return Err(RwLockError::Deadlock), + libc::EAGAIN => return Err(RwLockError::MaxReadLocks), + libc::EBUSY => return Err(RwLockError::WouldBlock), + e => panic!("unknown error code returned: {e}") + } + Ok(RwLockReadGuard { + data: NonNull::new_unchecked(self.data.get()), + lock: self + }) + } + } + + fn write(&self) -> Result, RwLockError> { + unsafe { + let res = libc::pthread_rwlock_wrlock(self.inner as *const _ as *mut _); + match res { + 0 => (), + libc::EINVAL => panic!("failed to properly initialize lock"), + libc::EDEADLK => return Err(RwLockError::Deadlock), + e => panic!("unknown error code returned: {e}") + } + } + Ok(RwLockWriteGuard { lock: self }) + } + + fn try_write(&self) -> Result, RwLockError> { + unsafe { + let res = libc::pthread_rwlock_trywrlock(self.inner as *const _ as *mut _); + match res { + 0 => (), + libc::EINVAL => panic!("failed to properly initialize lock"), + libc::EDEADLK => return Err(RwLockError::Deadlock), + libc::EBUSY => return Err(RwLockError::WouldBlock), + e => panic!("unknown error code returned: {e}") + } + } + Ok(RwLockWriteGuard { lock: self }) + } +} + +unsafe impl Sync for RwLockReadGuard<'_, '_, T> {} +unsafe impl Sync for RwLockWriteGuard<'_, '_, T> {} + +impl Deref for RwLockReadGuard<'_, '_, T> { + type Target = T; + + fn deref(&self) -> &T { + unsafe { self.data.as_ref() } + } +} + +impl Deref for RwLockWriteGuard<'_, '_, T> { + type Target = T; + + fn deref(&self) -> &T { + unsafe { &*self.lock.data.get() } + } +} + +impl DerefMut for RwLockWriteGuard<'_, '_, T> { + fn deref_mut(&mut self) -> &mut T { + unsafe { &mut *self.lock.data.get() } + } +} + +impl Drop for RwLockReadGuard<'_, '_, T> { + fn drop(&mut self) -> () { + let res = unsafe { libc::pthread_rwlock_unlock( + self.lock.inner as *const _ as *mut _ + ) }; + debug_assert!(res == 0); + } +} + +impl Drop for RwLockWriteGuard<'_, '_, T> { + fn drop(&mut self) -> () { + let res = unsafe { libc::pthread_rwlock_unlock( + self.lock.inner as *const _ as *mut _ + ) }; + debug_assert!(res == 0); + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; +use RwLockError::*; + + #[test] + fn test_single_process() { + let mut lock = MaybeUninit::uninit(); + let wrapper = RwLock::new(&mut lock, 0); + let mut writer = wrapper.write().unwrap(); + assert!(matches!(wrapper.try_write(), Err(Deadlock | WouldBlock))); + assert!(matches!(wrapper.try_read(), Err(Deadlock | WouldBlock))); + *writer = 5; + drop(writer); + let reader = wrapper.read().unwrap(); + assert!(matches!(wrapper.try_write(), Err(Deadlock | WouldBlock))); + assert!(matches!(wrapper.read(), Ok(_))); + assert_eq!(*reader, 5); + drop(reader); + assert!(matches!(wrapper.try_write(), Ok(_))); + } + + #[test] + fn test_multi_thread() { + let lock = Box::new(MaybeUninit::uninit()); + let wrapper = Arc::new(RwLock::new(Box::leak(lock), 0)); + let mut writer = wrapper.write().unwrap(); + let t1 = { + let wrapper = wrapper.clone(); + std::thread::spawn(move || { + let mut writer = wrapper.write().unwrap(); + *writer = 20; + }) + }; + assert_eq!(*writer, 0); + *writer = 10; + assert_eq!(*writer, 10); + drop(writer); + t1.join().unwrap(); + let mut writer = wrapper.write().unwrap(); + assert_eq!(*writer, 20); + drop(writer); + let mut handles = vec![]; + for _ in 0..5 { + handles.push({ + let wrapper = wrapper.clone(); + std::thread::spawn(move || { + let reader = wrapper.read().unwrap(); + assert_eq!(*reader, 20); + }) + }); + } + for h in handles { + h.join().unwrap(); + } + let writer = wrapper.write().unwrap(); + assert_eq!(*writer, 20); + } + + // // TODO(quantumish): Terrible time-based synchronization, fix me. + // #[test] + // fn test_multi_process() { + // let max_size = 100; + // let init_struct = crate::shmem::ShmemHandle::new("test_multi_process", 0, max_size).unwrap(); + // let ptr = init_struct.data_ptr.as_ptr(); + // let lock: &mut _ = unsafe { ptr.add( + // ptr.align_offset(std::mem::align_of::>()) + // ).cast::>().as_mut().unwrap() } ; + // let wrapper = RwLock::new(lock, 0); + + // let fork_result = unsafe { nix::unistd::fork().unwrap() }; + + // if !fork_result.is_parent() { + // let mut writer = wrapper.write().unwrap(); + // std::thread::sleep(std::time::Duration::from_secs(5)); + // *writer = 2; + // } else { + // std::thread::sleep(std::time::Duration::from_secs(1)); + // assert!(matches!(wrapper.try_write(), Err(WouldBlock))); + // std::thread::sleep(std::time::Duration::from_secs(10)); + // let writer = wrapper.try_write().unwrap(); + // assert_eq!(*writer, 2); + // } + // } +} From 19b5618578ceab75b6ead3bf84d9ae607203624c Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Wed, 2 Jul 2025 11:44:38 -0700 Subject: [PATCH 22/24] Switch to neon_shmem::sync lock_api and integrate into hashmap --- Cargo.lock | 1 + libs/neon-shmem/Cargo.toml | 1 + libs/neon-shmem/src/hash.rs | 233 ++++++++++++------------- libs/neon-shmem/src/hash/core.rs | 44 +---- libs/neon-shmem/src/hash/entry.rs | 27 +-- libs/neon-shmem/src/hash/tests.rs | 22 ++- libs/neon-shmem/src/sync.rs | 280 +++++++----------------------- 7 files changed, 210 insertions(+), 398 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4bb6d2d474..f798a1bdda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4038,6 +4038,7 @@ dependencies = [ "foldhash", "hashbrown 0.15.4 (git+https://github.com/quantumish/hashbrown.git?rev=6610e6d)", "libc", + "lock_api", "nix 0.30.1", "rand 0.9.1", "rand_distr 0.5.1", diff --git a/libs/neon-shmem/Cargo.toml b/libs/neon-shmem/Cargo.toml index 8306bbf778..8ce5b52deb 100644 --- a/libs/neon-shmem/Cargo.toml +++ b/libs/neon-shmem/Cargo.toml @@ -11,6 +11,7 @@ workspace_hack = { version = "0.1", path = "../../workspace_hack" } rustc-hash = { version = "2.1.1" } rand = "0.9.1" libc.workspace = true +lock_api = "0.4.13" [dev-dependencies] criterion = { workspace = true, features = ["html_reports"] } diff --git a/libs/neon-shmem/src/hash.rs b/libs/neon-shmem/src/hash.rs index 7d47a4f5e5..733e4b6f33 100644 --- a/libs/neon-shmem/src/hash.rs +++ b/libs/neon-shmem/src/hash.rs @@ -16,9 +16,9 @@ use std::hash::{Hash, BuildHasher}; use std::mem::MaybeUninit; -use std::default::Default; -use crate::{shmem, shmem::ShmemHandle}; +use crate::{shmem, sync::*}; +use crate::shmem::ShmemHandle; mod core; pub mod entry; @@ -27,15 +27,14 @@ pub mod entry; mod tests; use core::{Bucket, CoreHashMap, INVALID_POS}; -use entry::{Entry, OccupiedEntry}; +use entry::{Entry, OccupiedEntry, VacantEntry, PrevPos}; /// Builder for a [`HashMapAccess`]. #[must_use] pub struct HashMapInit<'a, K, V, S = rustc_hash::FxBuildHasher> { shmem_handle: Option, - shared_ptr: *mut HashMapShared<'a, K, V>, + shared_ptr: *mut RwLock>, shared_size: usize, - shrink_mode: HashMapShrinkMode, hasher: S, num_buckets: u32, } @@ -45,28 +44,6 @@ pub struct HashMapAccess<'a, K, V, S = rustc_hash::FxBuildHasher> { shmem_handle: Option, shared_ptr: *mut HashMapShared<'a, K, V>, hasher: S, - shrink_mode: HashMapShrinkMode, -} - -/// Enum specifying what behavior to have surrounding occupied entries in what is -/// about-to-be-shrinked space during a call to [`HashMapAccess::finish_shrink`]. -#[derive(PartialEq, Eq)] -pub enum HashMapShrinkMode { - /// Remap entry to the range of buckets that will remain after shrinking. - /// - /// Requires that caller has left enough room within the map such that this is possible. - Remap, - /// Remove any entries remaining in soon to be deallocated space. - /// - /// Only really useful if you legitimately do not care what entries are removed. - /// Should primarily be used for testing. - Remove, -} - -impl Default for HashMapShrinkMode { - fn default() -> Self { - Self::Remap - } } unsafe impl Sync for HashMapAccess<'_, K, V, S> {} @@ -80,14 +57,9 @@ impl<'a, K: Clone + Hash + Eq, V, S> HashMapInit<'a, K, V, S> { shared_ptr: self.shared_ptr, shared_size: self.shared_size, num_buckets: self.num_buckets, - shrink_mode: self.shrink_mode, } } - pub fn with_shrink_mode(self, mode: HashMapShrinkMode) -> Self { - Self { shrink_mode: mode, ..self } - } - /// Loosely (over)estimate the size needed to store a hash table with `num_buckets` buckets. pub fn estimate_size(num_buckets: u32) -> usize { // add some margin to cover alignment etc. @@ -96,13 +68,17 @@ impl<'a, K: Clone + Hash + Eq, V, S> HashMapInit<'a, K, V, S> { /// Initialize a table for writing. pub fn attach_writer(self) -> HashMapAccess<'a, K, V, S> { - // carve out the HashMapShared struct from the area. let mut ptr: *mut u8 = self.shared_ptr.cast(); let end_ptr: *mut u8 = unsafe { ptr.add(self.shared_size) }; - ptr = unsafe { ptr.add(ptr.align_offset(align_of::>())) }; - let shared_ptr: *mut HashMapShared = ptr.cast(); - ptr = unsafe { ptr.add(size_of::>()) }; + // carve out area for the One Big Lock (TM) and the HashMapShared. + ptr = unsafe { ptr.add(ptr.align_offset(align_of::())) }; + let raw_lock_ptr = ptr; + ptr = unsafe { ptr.add(size_of::()) }; + ptr = unsafe { ptr.add(ptr.align_offset(align_of::>())) }; + let shared_ptr: *mut HashMapShared = ptr.cast(); + ptr = unsafe { ptr.add(size_of::>()) }; + // carve out the buckets ptr = unsafe { ptr.byte_add(ptr.align_offset(align_of::>())) }; let buckets_ptr = ptr; @@ -121,14 +97,14 @@ impl<'a, K: Clone + Hash + Eq, V, S> HashMapInit<'a, K, V, S> { std::slice::from_raw_parts_mut(dictionary_ptr.cast(), dictionary_size as usize) }; let hashmap = CoreHashMap::new(buckets, dictionary); - unsafe { - std::ptr::write(shared_ptr, HashMapShared { inner: hashmap }); - } + let lock = RwLock::from_raw(PthreadRwLock::new(raw_lock_ptr.cast()), hashmap); + unsafe { + std::ptr::write(shared_ptr, lock); + } HashMapAccess { shmem_handle: self.shmem_handle, - shared_ptr: self.shared_ptr, - shrink_mode: self.shrink_mode, + shared_ptr, hasher: self.hasher, } } @@ -145,14 +121,13 @@ impl<'a, K: Clone + Hash + Eq, V, S> HashMapInit<'a, K, V, S> { /// relies on the memory layout! The data structures are laid out in the contiguous shared memory /// area as follows: /// +/// [`libc::pthread_rwlock_t`] /// [`HashMapShared`] /// [buckets] /// [dictionary] /// /// In between the above parts, there can be padding bytes to align the parts correctly. -struct HashMapShared<'a, K, V> { - inner: CoreHashMap<'a, K, V> -} +type HashMapShared<'a, K, V> = RwLock>; impl<'a, K, V> HashMapInit<'a, K, V, rustc_hash::FxBuildHasher> where @@ -168,7 +143,6 @@ where shmem_handle: None, shared_ptr: area.as_mut_ptr().cast(), shared_size: area.len(), - shrink_mode: HashMapShrinkMode::default(), hasher: rustc_hash::FxBuildHasher, } } @@ -187,7 +161,6 @@ where shared_ptr: shmem.data_ptr.as_ptr().cast(), shmem_handle: Some(shmem), shared_size: size, - shrink_mode: HashMapShrinkMode::default(), hasher: rustc_hash::FxBuildHasher } } @@ -204,7 +177,6 @@ where shared_ptr: shmem.data_ptr.as_ptr().cast(), shmem_handle: Some(shmem), shared_size: size, - shrink_mode: HashMapShrinkMode::default(), hasher: rustc_hash::FxBuildHasher } } @@ -229,25 +201,64 @@ where self.hasher.hash_one(key) } + fn entry_with_hash(&self, key: K, hash: u64) -> Entry<'a, '_, K, V> { + let mut map = unsafe { self.shared_ptr.as_ref() }.unwrap().write(); + let dict_pos = hash as usize % map.dictionary.len(); + let first = map.dictionary[dict_pos]; + if first == INVALID_POS { + // no existing entry + return Entry::Vacant(VacantEntry { + map, + key, + dict_pos: dict_pos as u32, + }); + } + + let mut prev_pos = PrevPos::First(dict_pos as u32); + let mut next = first; + loop { + let bucket = &mut map.buckets[next as usize]; + let (bucket_key, _bucket_value) = bucket.inner.as_mut().expect("entry is in use"); + if *bucket_key == key { + // found existing entry + return Entry::Occupied(OccupiedEntry { + map, + _key: key, + prev_pos, + bucket_pos: next, + }); + } + + if bucket.next == INVALID_POS { + // No existing entry + return Entry::Vacant(VacantEntry { + map, + key, + dict_pos: dict_pos as u32, + }); + } + prev_pos = PrevPos::Chained(next); + next = bucket.next; + } + } + /// Get a reference to the corresponding value for a key. - pub fn get<'e>(&'e self, key: &K) -> Option<&'e V> { - let map = unsafe { self.shared_ptr.as_ref() }.unwrap(); + pub fn get<'e>(&'e self, key: &K) -> Option> { let hash = self.get_hash_value(key); - map.inner.get_with_hash(key, hash) + let map = unsafe { self.shared_ptr.as_ref() }.unwrap().read(); + RwLockReadGuard::try_map(map, |m| m.get_with_hash(key, hash)).ok() } /// Get a reference to the entry containing a key. pub fn entry(&self, key: K) -> Entry<'a, '_, K, V> { - let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); let hash = self.get_hash_value(&key); - map.inner.entry_with_hash(key, hash) + self.entry_with_hash(key, hash) } /// Remove a key given its hash. Returns the associated value if it existed. pub fn remove(&self, key: &K) -> Option { - let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); let hash = self.get_hash_value(&key); - match map.inner.entry_with_hash(key.clone(), hash) { + match self.entry_with_hash(key.clone(), hash) { Entry::Occupied(e) => Some(e.remove()), Entry::Vacant(_) => None } @@ -258,12 +269,11 @@ where /// # Errors /// Will return [`core::FullError`] if there is no more space left in the map. pub fn insert(&self, key: K, value: V) -> Result, core::FullError> { - let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); let hash = self.get_hash_value(&key); - match map.inner.entry_with_hash(key.clone(), hash) { + match self.entry_with_hash(key.clone(), hash) { Entry::Occupied(mut e) => Ok(Some(e.insert(value))), Entry::Vacant(e) => { - e.insert(value)?; + _ = e.insert(value)?; Ok(None) } } @@ -275,13 +285,12 @@ where /// due to the [`OccupiedEntry`] type owning the key and also a hash of the key in order /// to enable repairing the hash chain if the entry is removed. pub fn entry_at_bucket(&self, pos: usize) -> Option> { - let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); - let inner = &mut map.inner; - if pos >= inner.buckets.len() { + let map = unsafe { self.shared_ptr.as_mut() }.unwrap().write(); + if pos >= map.buckets.len() { return None; } - let entry = inner.buckets[pos].inner.as_ref(); + let entry = map.buckets[pos].inner.as_ref(); match entry { Some((key, _)) => Some(OccupiedEntry { _key: key.clone(), @@ -289,7 +298,7 @@ where prev_pos: entry::PrevPos::Unknown( self.get_hash_value(&key) ), - map: inner, + map, }), _ => None, } @@ -297,8 +306,8 @@ where /// Returns the number of buckets in the table. pub fn get_num_buckets(&self) -> usize { - let map = unsafe { self.shared_ptr.as_ref() }.unwrap(); - map.inner.get_num_buckets() + let map = unsafe { self.shared_ptr.as_ref() }.unwrap().read(); + map.get_num_buckets() } /// Return the key and value stored in bucket with given index. This can be used to @@ -306,38 +315,35 @@ where // TODO: An Iterator might be nicer. The communicator's clock algorithm needs to // _slowly_ iterate through all buckets with its clock hand, without holding a lock. // If we switch to an Iterator, it must not hold the lock. - pub fn get_at_bucket(&self, pos: usize) -> Option<&(K, V)> { - let map = unsafe { self.shared_ptr.as_ref() }.unwrap(); - - if pos >= map.inner.buckets.len() { + pub fn get_at_bucket(&self, pos: usize) -> Option> { + let map = unsafe { self.shared_ptr.as_ref() }.unwrap().read(); + if pos >= map.buckets.len() { return None; } - let bucket = &map.inner.buckets[pos]; - bucket.inner.as_ref() + RwLockReadGuard::try_map(map, |m| m.buckets[pos].inner.as_ref()).ok() } /// Returns the index of the bucket a given value corresponds to. pub fn get_bucket_for_value(&self, val_ptr: *const V) -> usize { - let map = unsafe { self.shared_ptr.as_ref() }.unwrap(); + let map = unsafe { self.shared_ptr.as_ref() }.unwrap().read(); - let origin = map.inner.buckets.as_ptr(); + let origin = map.buckets.as_ptr(); let idx = (val_ptr as usize - origin as usize) / size_of::>(); - assert!(idx < map.inner.buckets.len()); + assert!(idx < map.buckets.len()); idx } /// Returns the number of occupied buckets in the table. pub fn get_num_buckets_in_use(&self) -> usize { - let map = unsafe { self.shared_ptr.as_ref() }.unwrap(); - map.inner.buckets_in_use as usize + let map = unsafe { self.shared_ptr.as_ref() }.unwrap().read(); + map.buckets_in_use as usize } /// Clears all entries in a table. Does not reset any shrinking operations. pub fn clear(&self) { - let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); - let inner = &mut map.inner; - inner.clear(); + let mut map = unsafe { self.shared_ptr.as_mut() }.unwrap().write(); + map.clear(); } /// Perform an in-place rehash of some region (0..`rehash_buckets`) of the table and reset @@ -389,13 +395,12 @@ where /// Rehash the map without growing or shrinking. pub fn shuffle(&self) { - let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); - let inner = &mut map.inner; - let num_buckets = inner.get_num_buckets() as u32; + let mut map = unsafe { self.shared_ptr.as_mut() }.unwrap().write(); + let num_buckets = map.get_num_buckets() as u32; let size_bytes = HashMapInit::::estimate_size(num_buckets); let end_ptr: *mut u8 = unsafe { self.shared_ptr.byte_add(size_bytes).cast() }; - let buckets_ptr = inner.buckets.as_mut_ptr(); - self.rehash_dict(inner, buckets_ptr, end_ptr, num_buckets, num_buckets); + let buckets_ptr = map.buckets.as_mut_ptr(); + self.rehash_dict(&mut map, buckets_ptr, end_ptr, num_buckets, num_buckets); } /// Grow the number of buckets within the table. @@ -409,10 +414,9 @@ where /// /// # Errors /// Returns an [`shmem::Error`] if any errors occur resizing the memory region. - pub fn grow(&mut self, num_buckets: u32) -> Result<(), shmem::Error> { - let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); - let inner = &mut map.inner; - let old_num_buckets = inner.buckets.len() as u32; + pub fn grow(&self, num_buckets: u32) -> Result<(), shmem::Error> { + let mut map = unsafe { self.shared_ptr.as_mut() }.unwrap().write(); + let old_num_buckets = map.buckets.len() as u32; assert!(num_buckets >= old_num_buckets, "grow called with a smaller number of buckets"); if num_buckets == old_num_buckets { @@ -429,7 +433,7 @@ where // Initialize new buckets. The new buckets are linked to the free list. // NB: This overwrites the dictionary! - let buckets_ptr = inner.buckets.as_mut_ptr(); + let buckets_ptr = map.buckets.as_mut_ptr(); unsafe { for i in old_num_buckets..num_buckets { let bucket = buckets_ptr.add(i as usize); @@ -437,15 +441,15 @@ where next: if i < num_buckets-1 { i + 1 } else { - inner.free_head + map.free_head }, inner: None, }); } } - self.rehash_dict(inner, buckets_ptr, end_ptr, num_buckets, old_num_buckets); - inner.free_head = old_num_buckets; + self.rehash_dict(&mut map, buckets_ptr, end_ptr, num_buckets, old_num_buckets); + map.free_head = old_num_buckets; Ok(()) } @@ -456,22 +460,22 @@ where /// Panics if called on a map initialized with [`HashMapInit::with_fixed`] or if `num_buckets` is /// greater than the number of buckets in the map. pub fn begin_shrink(&mut self, num_buckets: u32) { - let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); + let mut map = unsafe { self.shared_ptr.as_mut() }.unwrap().write(); assert!( - num_buckets <= map.inner.get_num_buckets() as u32, + num_buckets <= map.get_num_buckets() as u32, "shrink called with a larger number of buckets" ); _ = self .shmem_handle .as_ref() .expect("shrink called on a fixed-size hash table"); - map.inner.alloc_limit = num_buckets; + map.alloc_limit = num_buckets; } /// If a shrink operation is underway, returns the target size of the map. Otherwise, returns None. pub fn shrink_goal(&self) -> Option { - let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); - let goal = map.inner.alloc_limit; + let map = unsafe { self.shared_ptr.as_mut() }.unwrap().read(); + let goal = map.alloc_limit; if goal == INVALID_POS { None } else { Some(goal as usize) } } @@ -487,31 +491,28 @@ where /// # Errors /// Returns an [`shmem::Error`] if any errors occur resizing the memory region. pub fn finish_shrink(&self) -> Result<(), shmem::Error> { - let map = unsafe { self.shared_ptr.as_mut() }.unwrap(); - let inner = &mut map.inner; + let mut map = unsafe { self.shared_ptr.as_mut() }.unwrap().write(); assert!( - inner.alloc_limit != INVALID_POS, + map.alloc_limit != INVALID_POS, "called finish_shrink when no shrink is in progress" ); - let num_buckets = inner.alloc_limit; + let num_buckets = map.alloc_limit; - if inner.get_num_buckets() == num_buckets as usize { + if map.get_num_buckets() == num_buckets as usize { return Ok(()); } - if self.shrink_mode == HashMapShrinkMode::Remap { - assert!( - inner.buckets_in_use <= num_buckets, - "called finish_shrink before enough entries were removed" - ); - - for i in (num_buckets as usize)..inner.buckets.len() { - if let Some((k, v)) = inner.buckets[i].inner.take() { - // alloc_bucket increases count, so need to decrease since we're just moving - inner.buckets_in_use -= 1; - inner.alloc_bucket(k, v).unwrap(); - } + assert!( + map.buckets_in_use <= num_buckets, + "called finish_shrink before enough entries were removed" + ); + + for i in (num_buckets as usize)..map.buckets.len() { + if let Some((k, v)) = map.buckets[i].inner.take() { + // alloc_bucket increases count, so need to decrease since we're just moving + map.buckets_in_use -= 1; + map.alloc_bucket(k, v).unwrap(); } } @@ -523,9 +524,9 @@ where let size_bytes = HashMapInit::::estimate_size(num_buckets); shmem_handle.set_size(size_bytes)?; let end_ptr: *mut u8 = unsafe { shmem_handle.data_ptr.as_ptr().add(size_bytes) }; - let buckets_ptr = inner.buckets.as_mut_ptr(); - self.rehash_dict(inner, buckets_ptr, end_ptr, num_buckets, num_buckets); - inner.alloc_limit = INVALID_POS; + let buckets_ptr = map.buckets.as_mut_ptr(); + self.rehash_dict(&mut map, buckets_ptr, end_ptr, num_buckets, num_buckets); + map.alloc_limit = INVALID_POS; Ok(()) } diff --git a/libs/neon-shmem/src/hash/core.rs b/libs/neon-shmem/src/hash/core.rs index 28c58e851e..473c417ece 100644 --- a/libs/neon-shmem/src/hash/core.rs +++ b/libs/neon-shmem/src/hash/core.rs @@ -3,7 +3,7 @@ use std::hash::Hash; use std::mem::MaybeUninit; -use crate::hash::entry::{Entry, OccupiedEntry, PrevPos, VacantEntry}; +use crate::hash::entry::*; /// Invalid position within the map (either within the dictionary or bucket array). pub(crate) const INVALID_POS: u32 = u32::MAX; @@ -29,6 +29,7 @@ pub(crate) struct CoreHashMap<'a, K, V> { pub(crate) alloc_limit: u32, /// The number of currently occupied buckets. pub(crate) buckets_in_use: u32, + // pub(crate) lock: libc::pthread_mutex_t, // Unclear what the purpose of this is. pub(crate) _user_list_head: u32, } @@ -109,47 +110,6 @@ impl<'a, K: Clone + Hash + Eq, V> CoreHashMap<'a, K, V> { } } - /// Get the [`Entry`] associated with a key given hash. This should be used for updates/inserts. - pub fn entry_with_hash(&mut self, key: K, hash: u64) -> Entry<'a, '_, K, V> { - let dict_pos = hash as usize % self.dictionary.len(); - let first = self.dictionary[dict_pos]; - if first == INVALID_POS { - // no existing entry - return Entry::Vacant(VacantEntry { - map: self, - key, - dict_pos: dict_pos as u32, - }); - } - - let mut prev_pos = PrevPos::First(dict_pos as u32); - let mut next = first; - loop { - let bucket = &mut self.buckets[next as usize]; - let (bucket_key, _bucket_value) = bucket.inner.as_mut().expect("entry is in use"); - if *bucket_key == key { - // found existing entry - return Entry::Occupied(OccupiedEntry { - map: self, - _key: key, - prev_pos, - bucket_pos: next, - }); - } - - if bucket.next == INVALID_POS { - // No existing entry - return Entry::Vacant(VacantEntry { - map: self, - key, - dict_pos: dict_pos as u32, - }); - } - prev_pos = PrevPos::Chained(next); - next = bucket.next; - } - } - /// Get number of buckets in map. pub fn get_num_buckets(&self) -> usize { self.buckets.len() diff --git a/libs/neon-shmem/src/hash/entry.rs b/libs/neon-shmem/src/hash/entry.rs index b4c973d9f5..a5832665aa 100644 --- a/libs/neon-shmem/src/hash/entry.rs +++ b/libs/neon-shmem/src/hash/entry.rs @@ -1,11 +1,12 @@ //! Equivalent of [`std::collections::hash_map::Entry`] for this hashmap. use crate::hash::core::{CoreHashMap, FullError, INVALID_POS}; +use crate::sync::{RwLockWriteGuard, ValueWriteGuard}; use std::hash::Hash; use std::mem; -/// View into an entry in the map (either vacant or occupied). + pub enum Entry<'a, 'b, K, V> { Occupied(OccupiedEntry<'a, 'b, K, V>), Vacant(VacantEntry<'a, 'b, K, V>), @@ -22,10 +23,9 @@ pub(crate) enum PrevPos { Unknown(u64), } -/// View into an occupied entry within the map. pub struct OccupiedEntry<'a, 'b, K, V> { /// Mutable reference to the map containing this entry. - pub(crate) map: &'b mut CoreHashMap<'a, K, V>, + pub(crate) map: RwLockWriteGuard<'b, CoreHashMap<'a, K, V>>, /// The key of the occupied entry pub(crate) _key: K, /// The index of the previous entry in the chain. @@ -66,7 +66,7 @@ impl OccupiedEntry<'_, '_, K, V> { /// # Panics /// Panics if the `prev_pos` field is equal to [`PrevPos::Unknown`]. In practice, this means /// the entry was obtained via calling something like [`CoreHashMap::entry_at_bucket`]. - pub fn remove(self) -> V { + pub fn remove(mut self) -> V { // If this bucket was queried by index, go ahead and follow its chain from the start. let prev = if let PrevPos::Unknown(hash) = self.prev_pos { let dict_idx = hash as usize % self.map.dictionary.len(); @@ -90,15 +90,17 @@ impl OccupiedEntry<'_, '_, K, V> { self.map.dictionary[dict_pos as usize] = bucket.next; }, PrevPos::Chained(bucket_pos) => { + // println!("we think prev of {} is {bucket_pos}", self.bucket_pos); self.map.buckets[bucket_pos as usize].next = bucket.next; }, _ => unreachable!(), } - // and add it to the freelist + // and add it to the freelist + let free = self.map.free_head; let bucket = &mut self.map.buckets[self.bucket_pos as usize]; let old_value = bucket.inner.take(); - bucket.next = self.map.free_head; + bucket.next = free; self.map.free_head = self.bucket_pos; self.map.buckets_in_use -= 1; @@ -109,7 +111,7 @@ impl OccupiedEntry<'_, '_, K, V> { /// An abstract view into a vacant entry within the map. pub struct VacantEntry<'a, 'b, K, V> { /// Mutable reference to the map containing this entry. - pub(crate) map: &'b mut CoreHashMap<'a, K, V>, + pub(crate) map: RwLockWriteGuard<'b, CoreHashMap<'a, K, V>>, /// The key to be inserted into this entry. pub(crate) key: K, /// The position within the dictionary corresponding to the key's hash. @@ -121,16 +123,17 @@ impl<'b, K: Clone + Hash + Eq, V> VacantEntry<'_, 'b, K, V> { /// /// # Errors /// Will return [`FullError`] if there are no unoccupied buckets in the map. - pub fn insert(self, value: V) -> Result<&'b mut V, FullError> { + pub fn insert(mut self, value: V) -> Result, FullError> { let pos = self.map.alloc_bucket(self.key, value)?; if pos == INVALID_POS { return Err(FullError()); } - let bucket = &mut self.map.buckets[pos as usize]; - bucket.next = self.map.dictionary[self.dict_pos as usize]; + self.map.buckets[pos as usize].next = self.map.dictionary[self.dict_pos as usize]; self.map.dictionary[self.dict_pos as usize] = pos; - let result = &mut self.map.buckets[pos as usize].inner.as_mut().unwrap().1; - Ok(result) + Ok(RwLockWriteGuard::map( + self.map, + |m| &mut m.buckets[pos as usize].inner.as_mut().unwrap().1 + )) } } diff --git a/libs/neon-shmem/src/hash/tests.rs b/libs/neon-shmem/src/hash/tests.rs index d838aa0b86..0c760c56b7 100644 --- a/libs/neon-shmem/src/hash/tests.rs +++ b/libs/neon-shmem/src/hash/tests.rs @@ -36,7 +36,7 @@ impl<'a> From<&'a [u8]> for TestKey { } fn test_inserts + Copy>(keys: &[K]) { - let mut w = HashMapInit::::new_resizeable_named( + let w = HashMapInit::::new_resizeable_named( 100000, 120000, "test_inserts" ).attach_writer(); @@ -190,10 +190,6 @@ fn random_ops() { let op = TestOp(key, if rng.random_bool(0.75) { Some(i) } else { None }); apply_op(&op, &mut writer, &mut shadow); - - if i % 1000 == 0 { - eprintln!("{i} ops processed"); - } } } @@ -267,7 +263,7 @@ fn test_idx_remove() { } while let Some((key, val)) = shadow.pop_first() { - assert_eq!(writer.get(&key), Some(&val)); + assert_eq!(*writer.get(&key).unwrap(), val); } } @@ -326,8 +322,10 @@ fn test_shrink_grow_seq() { writer.grow(1500).unwrap(); do_random_ops(600, 1500, 0.1, &mut writer, &mut shadow, &mut rng); eprintln!("Shrinking to 200"); + while shadow.len() > 100 { + do_deletes(1, &mut writer, &mut shadow); + } do_shrink(&mut writer, &mut shadow, 200); - do_deletes(100, &mut writer, &mut shadow); do_random_ops(50, 1500, 0.25, &mut writer, &mut shadow, &mut rng); eprintln!("Growing to 10k"); writer.grow(10000).unwrap(); @@ -336,7 +334,7 @@ fn test_shrink_grow_seq() { #[test] fn test_bucket_ops() { - let mut writer = HashMapInit::::new_resizeable_named( + let writer = HashMapInit::::new_resizeable_named( 1000, 1200, "test_bucket_ops" ).attach_writer(); match writer.entry(1.into()) { @@ -345,21 +343,21 @@ fn test_bucket_ops() { } assert_eq!(writer.get_num_buckets_in_use(), 1); assert_eq!(writer.get_num_buckets(), 1000); - assert_eq!(writer.get(&1.into()), Some(&2)); + assert_eq!(*writer.get(&1.into()).unwrap(), 2); let pos = match writer.entry(1.into()) { Entry::Occupied(e) => { assert_eq!(e._key, 1.into()); let pos = e.bucket_pos as usize; assert_eq!(writer.entry_at_bucket(pos).unwrap()._key, 1.into()); - assert_eq!(writer.get_at_bucket(pos), Some(&(1.into(), 2))); + assert_eq!(*writer.get_at_bucket(pos).unwrap(), (1.into(), 2)); pos }, Entry::Vacant(_) => { panic!("Insert didn't affect entry"); }, }; - let ptr: *const usize = writer.get(&1.into()).unwrap(); + let ptr: *const usize = &*writer.get(&1.into()).unwrap(); assert_eq!(writer.get_bucket_for_value(ptr), pos); writer.remove(&1.into()); - assert_eq!(writer.get(&1.into()), None); + assert!(writer.get(&1.into()).is_none()); } #[test] diff --git a/libs/neon-shmem/src/sync.rs b/libs/neon-shmem/src/sync.rs index 68ef7b904d..8887299a92 100644 --- a/libs/neon-shmem/src/sync.rs +++ b/libs/neon-shmem/src/sync.rs @@ -2,43 +2,18 @@ use std::mem::MaybeUninit; use std::ptr::NonNull; -use std::cell::UnsafeCell; -use std::ops::{Deref, DerefMut}; -use thiserror::Error; + +pub type RwLock = lock_api::RwLock; +pub(crate) type RwLockReadGuard<'a, T> = lock_api::RwLockReadGuard<'a, PthreadRwLock, T>; +pub type RwLockWriteGuard<'a, T> = lock_api::RwLockWriteGuard<'a, PthreadRwLock, T>; +pub type ValueReadGuard<'a, T> = lock_api::MappedRwLockReadGuard<'a, PthreadRwLock, T>; +pub type ValueWriteGuard<'a, T> = lock_api::MappedRwLockWriteGuard<'a, PthreadRwLock, T>; /// Shared memory read-write lock. -struct RwLock<'a, T: ?Sized> { - inner: &'a mut libc::pthread_rwlock_t, - data: UnsafeCell, -} +pub struct PthreadRwLock(Option>); -/// RAII guard for a read lock. -struct RwLockReadGuard<'a, 'b, T: ?Sized> { - data: NonNull, - lock: &'a RwLock<'b, T>, -} - -/// RAII guard for a write lock. -struct RwLockWriteGuard<'a, 'b, T: ?Sized> { - lock: &'a RwLock<'b, T>, -} - -// TODO(quantumish): Support poisoning errors? -#[derive(Error, Debug)] -enum RwLockError { - #[error("deadlock detected")] - Deadlock, - #[error("max number of read locks exceeded")] - MaxReadLocks, - #[error("nonblocking operation would block")] - WouldBlock, -} - -unsafe impl Send for RwLock<'_, T> {} -unsafe impl Sync for RwLock<'_, T> {} - -impl<'a, T> RwLock<'a, T> { - fn new(lock: &'a mut MaybeUninit, data: T) -> Self { +impl PthreadRwLock { + pub fn new(lock: *mut libc::pthread_rwlock_t) -> Self { unsafe { let mut attrs = MaybeUninit::uninit(); // Ignoring return value here - only possible error is OOM. @@ -48,208 +23,81 @@ impl<'a, T> RwLock<'a, T> { libc::PTHREAD_PROCESS_SHARED ); // TODO(quantumish): worth making this function return Result? - libc::pthread_rwlock_init(lock.as_mut_ptr(), attrs.as_mut_ptr()); + libc::pthread_rwlock_init(lock, attrs.as_mut_ptr()); // Safety: POSIX specifies that "any function affecting the attributes // object (including destruction) shall not affect any previously // initialized read-write locks". libc::pthread_rwlockattr_destroy(attrs.as_mut_ptr()); - Self { - inner: lock.assume_init_mut(), - data: data.into(), - } - } - } - - fn read(&self) -> Result, RwLockError> { - unsafe { - let res = libc::pthread_rwlock_rdlock(self.inner as *const _ as *mut _); - match res { - 0 => (), - libc::EINVAL => panic!("failed to properly initialize lock"), - libc::EDEADLK => return Err(RwLockError::Deadlock), - libc::EAGAIN => return Err(RwLockError::MaxReadLocks), - e => panic!("unknown error code returned: {e}") - } - Ok(RwLockReadGuard { - data: NonNull::new_unchecked(self.data.get()), - lock: self - }) - } - } - - fn try_read(&self) -> Result, RwLockError> { - unsafe { - let res = libc::pthread_rwlock_tryrdlock(self.inner as *const _ as *mut _); - match res { - 0 => (), - libc::EINVAL => panic!("failed to properly initialize lock"), - libc::EDEADLK => return Err(RwLockError::Deadlock), - libc::EAGAIN => return Err(RwLockError::MaxReadLocks), - libc::EBUSY => return Err(RwLockError::WouldBlock), - e => panic!("unknown error code returned: {e}") - } - Ok(RwLockReadGuard { - data: NonNull::new_unchecked(self.data.get()), - lock: self - }) + Self(Some(NonNull::new_unchecked(lock))) } } - fn write(&self) -> Result, RwLockError> { + fn inner(&self) -> NonNull { + match self.0 { + None => panic!("PthreadRwLock constructed badly - something likely used RawMutex::INIT"), + Some(x) => x, + } + } +} + +unsafe impl lock_api::RawRwLock for PthreadRwLock { + type GuardMarker = lock_api::GuardSend; + const INIT: Self = Self(None); + + fn lock_shared(&self) { unsafe { - let res = libc::pthread_rwlock_wrlock(self.inner as *const _ as *mut _); - match res { - 0 => (), - libc::EINVAL => panic!("failed to properly initialize lock"), - libc::EDEADLK => return Err(RwLockError::Deadlock), - e => panic!("unknown error code returned: {e}") + let res = libc::pthread_rwlock_rdlock(self.inner().as_ptr()); + if res != 0 { + panic!("rdlock failed with {res}"); } } - Ok(RwLockWriteGuard { lock: self }) } - fn try_write(&self) -> Result, RwLockError> { + fn try_lock_shared(&self) -> bool { unsafe { - let res = libc::pthread_rwlock_trywrlock(self.inner as *const _ as *mut _); + let res = libc::pthread_rwlock_tryrdlock(self.inner().as_ptr()); match res { - 0 => (), - libc::EINVAL => panic!("failed to properly initialize lock"), - libc::EDEADLK => return Err(RwLockError::Deadlock), - libc::EBUSY => return Err(RwLockError::WouldBlock), - e => panic!("unknown error code returned: {e}") + 0 => true, + libc::EAGAIN => false, + o => panic!("try_rdlock failed with {o}") } } - Ok(RwLockWriteGuard { lock: self }) - } -} - -unsafe impl Sync for RwLockReadGuard<'_, '_, T> {} -unsafe impl Sync for RwLockWriteGuard<'_, '_, T> {} - -impl Deref for RwLockReadGuard<'_, '_, T> { - type Target = T; - - fn deref(&self) -> &T { - unsafe { self.data.as_ref() } - } -} - -impl Deref for RwLockWriteGuard<'_, '_, T> { - type Target = T; - - fn deref(&self) -> &T { - unsafe { &*self.lock.data.get() } - } -} - -impl DerefMut for RwLockWriteGuard<'_, '_, T> { - fn deref_mut(&mut self) -> &mut T { - unsafe { &mut *self.lock.data.get() } - } -} - -impl Drop for RwLockReadGuard<'_, '_, T> { - fn drop(&mut self) -> () { - let res = unsafe { libc::pthread_rwlock_unlock( - self.lock.inner as *const _ as *mut _ - ) }; - debug_assert!(res == 0); - } -} - -impl Drop for RwLockWriteGuard<'_, '_, T> { - fn drop(&mut self) -> () { - let res = unsafe { libc::pthread_rwlock_unlock( - self.lock.inner as *const _ as *mut _ - ) }; - debug_assert!(res == 0); - } -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use super::*; -use RwLockError::*; - - #[test] - fn test_single_process() { - let mut lock = MaybeUninit::uninit(); - let wrapper = RwLock::new(&mut lock, 0); - let mut writer = wrapper.write().unwrap(); - assert!(matches!(wrapper.try_write(), Err(Deadlock | WouldBlock))); - assert!(matches!(wrapper.try_read(), Err(Deadlock | WouldBlock))); - *writer = 5; - drop(writer); - let reader = wrapper.read().unwrap(); - assert!(matches!(wrapper.try_write(), Err(Deadlock | WouldBlock))); - assert!(matches!(wrapper.read(), Ok(_))); - assert_eq!(*reader, 5); - drop(reader); - assert!(matches!(wrapper.try_write(), Ok(_))); } - #[test] - fn test_multi_thread() { - let lock = Box::new(MaybeUninit::uninit()); - let wrapper = Arc::new(RwLock::new(Box::leak(lock), 0)); - let mut writer = wrapper.write().unwrap(); - let t1 = { - let wrapper = wrapper.clone(); - std::thread::spawn(move || { - let mut writer = wrapper.write().unwrap(); - *writer = 20; - }) - }; - assert_eq!(*writer, 0); - *writer = 10; - assert_eq!(*writer, 10); - drop(writer); - t1.join().unwrap(); - let mut writer = wrapper.write().unwrap(); - assert_eq!(*writer, 20); - drop(writer); - let mut handles = vec![]; - for _ in 0..5 { - handles.push({ - let wrapper = wrapper.clone(); - std::thread::spawn(move || { - let reader = wrapper.read().unwrap(); - assert_eq!(*reader, 20); - }) - }); + fn lock_exclusive(&self) { + unsafe { + let res = libc::pthread_rwlock_wrlock(self.inner().as_ptr()); + if res != 0 { + panic!("wrlock failed with {res}"); + } } - for h in handles { - h.join().unwrap(); - } - let writer = wrapper.write().unwrap(); - assert_eq!(*writer, 20); } - // // TODO(quantumish): Terrible time-based synchronization, fix me. - // #[test] - // fn test_multi_process() { - // let max_size = 100; - // let init_struct = crate::shmem::ShmemHandle::new("test_multi_process", 0, max_size).unwrap(); - // let ptr = init_struct.data_ptr.as_ptr(); - // let lock: &mut _ = unsafe { ptr.add( - // ptr.align_offset(std::mem::align_of::>()) - // ).cast::>().as_mut().unwrap() } ; - // let wrapper = RwLock::new(lock, 0); + fn try_lock_exclusive(&self) -> bool { + unsafe { + let res = libc::pthread_rwlock_trywrlock(self.inner().as_ptr()); + match res { + 0 => true, + libc::EAGAIN => false, + o => panic!("try_wrlock failed with {o}") + } + } + } - // let fork_result = unsafe { nix::unistd::fork().unwrap() }; - - // if !fork_result.is_parent() { - // let mut writer = wrapper.write().unwrap(); - // std::thread::sleep(std::time::Duration::from_secs(5)); - // *writer = 2; - // } else { - // std::thread::sleep(std::time::Duration::from_secs(1)); - // assert!(matches!(wrapper.try_write(), Err(WouldBlock))); - // std::thread::sleep(std::time::Duration::from_secs(10)); - // let writer = wrapper.try_write().unwrap(); - // assert_eq!(*writer, 2); - // } - // } + unsafe fn unlock_exclusive(&self) { + unsafe { + let res = libc::pthread_rwlock_unlock(self.inner().as_ptr()); + if res != 0 { + panic!("unlock failed with {res}"); + } + } + } + unsafe fn unlock_shared(&self) { + unsafe { + let res = libc::pthread_rwlock_unlock(self.inner().as_ptr()); + if res != 0 { + panic!("unlock failed with {res}"); + } + } + } } From 2fe27f510df465ddd663086d0420f01e121886b4 Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Wed, 2 Jul 2025 11:57:34 -0700 Subject: [PATCH 23/24] Make neon-shmem tests thread-safe and report errno in panics --- libs/neon-shmem/benches/hmap_resize.rs | 16 ++++++---------- libs/neon-shmem/src/hash/tests.rs | 24 +++++++++++++----------- libs/neon-shmem/src/sync.rs | 14 ++++++++------ 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/libs/neon-shmem/benches/hmap_resize.rs b/libs/neon-shmem/benches/hmap_resize.rs index 30a3dca296..6b86e7ed27 100644 --- a/libs/neon-shmem/benches/hmap_resize.rs +++ b/libs/neon-shmem/benches/hmap_resize.rs @@ -1,9 +1,7 @@ -use std::hint::black_box; use criterion::{criterion_group, criterion_main, BatchSize, Criterion, BenchmarkId}; use neon_shmem::hash::HashMapAccess; use neon_shmem::hash::HashMapInit; use neon_shmem::hash::entry::Entry; -use neon_shmem::shmem::ShmemHandle; use rand::prelude::*; use rand::distr::{Distribution, StandardUniform}; use std::hash::BuildHasher; @@ -65,14 +63,13 @@ fn apply_op( op: TestOp, map: &mut HashMapAccess, ) { - let hash = map.get_hash_value(&op.0); - let entry = map.entry_with_hash(op.0, hash); + let entry = map.entry(op.0); match op.1 { Some(new) => { match entry { Entry::Occupied(mut e) => Some(e.insert(new)), - Entry::Vacant(e) => { e.insert(new).unwrap(); None }, + Entry::Vacant(e) => { _ = e.insert(new).unwrap(); None }, } }, None => { @@ -184,15 +181,14 @@ fn real_benchs(c: &mut Criterion) { let mut rng = rand::rng(); b.iter_batched( || HashMapInit::new_resizeable(size, size * 2).attach_writer(), - |mut writer| { - for i in 0..ideal_filled { + |writer| { + for _ in 0..ideal_filled { let key: FileCacheKey = rng.random(); let val = FileCacheEntry::dummy(); - let hash = writer.get_hash_value(&key); - let entry = writer.entry_with_hash(key, hash); + let entry = writer.entry(key); std::hint::black_box(match entry { Entry::Occupied(mut e) => { e.insert(val); }, - Entry::Vacant(e) => { e.insert(val).unwrap(); }, + Entry::Vacant(e) => { _ = e.insert(val).unwrap(); }, }) } }, diff --git a/libs/neon-shmem/src/hash/tests.rs b/libs/neon-shmem/src/hash/tests.rs index 0c760c56b7..d7b8d19050 100644 --- a/libs/neon-shmem/src/hash/tests.rs +++ b/libs/neon-shmem/src/hash/tests.rs @@ -114,7 +114,7 @@ fn apply_op( Some(new) => { match entry { Entry::Occupied(mut e) => Some(e.insert(new)), - Entry::Vacant(e) => { e.insert(new).unwrap(); None }, + Entry::Vacant(e) => { _ = e.insert(new).unwrap(); None }, } }, None => { @@ -277,13 +277,13 @@ fn test_idx_get() { do_random_ops(2000, 1500, 0.25, &mut writer, &mut shadow, &mut rng); for _ in 0..100 { let idx = (rng.next_u32() % 1500) as usize; - if let Some(mut e) = writer.entry_at_bucket(idx) { + if let Some(pair) = writer.get_at_bucket(idx) { { - let v: *const usize = e.get(); + let v: *const usize = &pair.1; assert_eq!(writer.get_bucket_for_value(v), idx); } { - let v: *const usize = e.get_mut(); + let v: *const usize = &pair.1; assert_eq!(writer.get_bucket_for_value(v), idx); } } @@ -339,7 +339,7 @@ fn test_bucket_ops() { ).attach_writer(); match writer.entry(1.into()) { Entry::Occupied(mut e) => { e.insert(2); }, - Entry::Vacant(e) => { e.insert(2).unwrap(); }, + Entry::Vacant(e) => { _ = e.insert(2).unwrap(); }, } assert_eq!(writer.get_num_buckets_in_use(), 1); assert_eq!(writer.get_num_buckets(), 1000); @@ -348,14 +348,16 @@ fn test_bucket_ops() { Entry::Occupied(e) => { assert_eq!(e._key, 1.into()); let pos = e.bucket_pos as usize; - assert_eq!(writer.entry_at_bucket(pos).unwrap()._key, 1.into()); - assert_eq!(*writer.get_at_bucket(pos).unwrap(), (1.into(), 2)); pos }, Entry::Vacant(_) => { panic!("Insert didn't affect entry"); }, }; - let ptr: *const usize = &*writer.get(&1.into()).unwrap(); - assert_eq!(writer.get_bucket_for_value(ptr), pos); + assert_eq!(writer.entry_at_bucket(pos).unwrap()._key, 1.into()); + assert_eq!(*writer.get_at_bucket(pos).unwrap(), (1.into(), 2)); + { + let ptr: *const usize = &*writer.get(&1.into()).unwrap(); + assert_eq!(writer.get_bucket_for_value(ptr), pos); + } writer.remove(&1.into()); assert!(writer.get(&1.into()).is_none()); } @@ -390,7 +392,7 @@ fn test_shrink_zero() { #[test] #[should_panic] fn test_grow_oom() { - let mut writer = HashMapInit::::new_resizeable_named( + let writer = HashMapInit::::new_resizeable_named( 1500, 2000, "test_grow_oom" ).attach_writer(); writer.grow(20000).unwrap(); @@ -408,7 +410,7 @@ fn test_shrink_bigger() { #[test] #[should_panic] fn test_shrink_early_finish() { - let mut writer = HashMapInit::::new_resizeable_named( + let writer = HashMapInit::::new_resizeable_named( 1500, 2500, "test_shrink_early_finish" ).attach_writer(); writer.finish_shrink().unwrap(); diff --git a/libs/neon-shmem/src/sync.rs b/libs/neon-shmem/src/sync.rs index 8887299a92..fc39df9100 100644 --- a/libs/neon-shmem/src/sync.rs +++ b/libs/neon-shmem/src/sync.rs @@ -3,6 +3,8 @@ use std::mem::MaybeUninit; use std::ptr::NonNull; +use nix::errno::Errno; + pub type RwLock = lock_api::RwLock; pub(crate) type RwLockReadGuard<'a, T> = lock_api::RwLockReadGuard<'a, PthreadRwLock, T>; pub type RwLockWriteGuard<'a, T> = lock_api::RwLockWriteGuard<'a, PthreadRwLock, T>; @@ -48,7 +50,7 @@ unsafe impl lock_api::RawRwLock for PthreadRwLock { unsafe { let res = libc::pthread_rwlock_rdlock(self.inner().as_ptr()); if res != 0 { - panic!("rdlock failed with {res}"); + panic!("rdlock failed with {}", Errno::from_raw(res)); } } } @@ -59,7 +61,7 @@ unsafe impl lock_api::RawRwLock for PthreadRwLock { match res { 0 => true, libc::EAGAIN => false, - o => panic!("try_rdlock failed with {o}") + o => panic!("try_rdlock failed with {}", Errno::from_raw(res)), } } } @@ -68,7 +70,7 @@ unsafe impl lock_api::RawRwLock for PthreadRwLock { unsafe { let res = libc::pthread_rwlock_wrlock(self.inner().as_ptr()); if res != 0 { - panic!("wrlock failed with {res}"); + panic!("wrlock failed with {}", Errno::from_raw(res)); } } } @@ -79,7 +81,7 @@ unsafe impl lock_api::RawRwLock for PthreadRwLock { match res { 0 => true, libc::EAGAIN => false, - o => panic!("try_wrlock failed with {o}") + o => panic!("try_wrlock failed with {}", Errno::from_raw(res)), } } } @@ -88,7 +90,7 @@ unsafe impl lock_api::RawRwLock for PthreadRwLock { unsafe { let res = libc::pthread_rwlock_unlock(self.inner().as_ptr()); if res != 0 { - panic!("unlock failed with {res}"); + panic!("unlock failed with {}", Errno::from_raw(res)); } } } @@ -96,7 +98,7 @@ unsafe impl lock_api::RawRwLock for PthreadRwLock { unsafe { let res = libc::pthread_rwlock_unlock(self.inner().as_ptr()); if res != 0 { - panic!("unlock failed with {res}"); + panic!("unlock failed with {}", Errno::from_raw(res)); } } } From 86fb7b966a920ee97739f054e76b1ac9c364fd73 Mon Sep 17 00:00:00 2001 From: David Freifeld Date: Wed, 2 Jul 2025 12:18:37 -0700 Subject: [PATCH 24/24] Update `integrated_cache.rs` to use new hashmap API --- libs/neon-shmem/src/hash.rs | 1 - .../neon/communicator/src/integrated_cache.rs | 88 +++++++------------ 2 files changed, 34 insertions(+), 55 deletions(-) diff --git a/libs/neon-shmem/src/hash.rs b/libs/neon-shmem/src/hash.rs index b46a58faaf..733e4b6f33 100644 --- a/libs/neon-shmem/src/hash.rs +++ b/libs/neon-shmem/src/hash.rs @@ -302,7 +302,6 @@ where }), _ => None, } ->>>>>>> quantumish/lfc-resizable-map } /// Returns the number of buckets in the table. diff --git a/pgxn/neon/communicator/src/integrated_cache.rs b/pgxn/neon/communicator/src/integrated_cache.rs index d710f0e35b..acd73b3b40 100644 --- a/pgxn/neon/communicator/src/integrated_cache.rs +++ b/pgxn/neon/communicator/src/integrated_cache.rs @@ -267,8 +267,7 @@ impl<'t> IntegratedCacheWriteAccess<'t> { block_number: u32, dst: impl uring_common::buf::IoBufMut + Send + Sync, ) -> Result, std::io::Error> { - let hash = self.block_map.get_hash_value(&BlockKey::from((rel, block_number))); - let x = if let Some(block_entry) = self.block_map.get_with_hash(&BlockKey::from((rel, block_number)), hash) + let x = if let Some(block_entry) = self.block_map.get(&BlockKey::from((rel, block_number))) { block_entry.referenced.store(true, Ordering::Relaxed); @@ -302,8 +301,7 @@ impl<'t> IntegratedCacheWriteAccess<'t> { rel: &RelTag, block_number: u32, ) -> Result, std::io::Error> { - let hash = self.block_map.get_hash_value(&BlockKey::from((rel, block_number))); - if let Some(block_entry) = self.block_map.get_with_hash(&BlockKey::from((rel, block_number)), hash) { + if let Some(block_entry) = self.block_map.get(&BlockKey::from((rel, block_number))) { // This is used for prefetch requests. Treat the probe as an 'access', to keep it // in cache. block_entry.referenced.store(true, Ordering::Relaxed); @@ -325,8 +323,7 @@ impl<'t> IntegratedCacheWriteAccess<'t> { /// information, i.e. we don't know if the relation exists or not. pub fn get_rel_exists(&'t self, rel: &RelTag) -> CacheResult { // we don't currently cache negative entries, so if the relation is in the cache, it exists - let hash = self.relsize_cache.get_hash_value(&RelKey::from(rel)); - if let Some(_rel_entry) = self.relsize_cache.get_with_hash(&RelKey::from(rel), hash) { + if let Some(_rel_entry) = self.relsize_cache.get(&RelKey::from(rel)) { CacheResult::Found(true) } else { let lsn = Lsn(self.global_lw_lsn.load(Ordering::Relaxed)); @@ -345,12 +342,11 @@ impl<'t> IntegratedCacheWriteAccess<'t> { } pub fn remember_rel_size(&'t self, rel: &RelTag, nblocks: u32) { - let hash = self.relsize_cache.get_hash_value(&RelKey::from(rel)); - match self.relsize_cache.entry_with_hash(RelKey::from(rel), hash) { + match self.relsize_cache.entry(RelKey::from(rel)) { Entry::Vacant(e) => { tracing::info!("inserting rel entry for {rel:?}, {nblocks} blocks"); // FIXME: what to do if we run out of memory? Evict other relation entries? - e.insert(RelEntry { + _ = e.insert(RelEntry { nblocks: AtomicU32::new(nblocks), }).expect("out of memory"); }, @@ -384,8 +380,9 @@ impl<'t> IntegratedCacheWriteAccess<'t> { let mut old_cache_block = None; let mut found_existing = false; - let hash = self.block_map.get_hash_value(&key); - if let Entry::Occupied(e) = self.block_map.entry_with_hash(key.clone(), hash) { + // NOTE(quantumish): honoring original semantics here (used to be update_with_fn) + // but I don't see any reason why this has to take a write lock. + if let Entry::Occupied(e) = self.block_map.entry(key.clone()) { let block_entry = e.get(); found_existing = true; @@ -428,8 +425,7 @@ impl<'t> IntegratedCacheWriteAccess<'t> { // FIXME: unpin the block entry on error // Update the block entry - let hash = self.block_map.get_hash_value(&key); - let entry = self.block_map.entry_with_hash(key, hash); + let entry = self.block_map.entry(key); assert_eq!(found_existing, matches!(entry, Entry::Occupied(_))); match entry { Entry::Occupied(e) => { @@ -453,7 +449,7 @@ impl<'t> IntegratedCacheWriteAccess<'t> { Entry::Vacant(e) => { // FIXME: what to do if we run out of memory? Evict other relation entries? Remove // block entries first? - e.insert(BlockEntry { + _ = e.insert(BlockEntry { lw_lsn: AtomicLsn::new(lw_lsn.0), cache_block: AtomicU64::new(cache_block), pinned: AtomicU64::new(0), @@ -487,8 +483,7 @@ impl<'t> IntegratedCacheWriteAccess<'t> { .expect("error writing to cache"); // FIXME: handle errors gracefully. - let hash = self.block_map.get_hash_value(&key); - match self.block_map.entry_with_hash(key, hash) { + match self.block_map.entry(key) { Entry::Occupied(e) => { let block_entry = e.get(); // FIXME: could there be concurrent readers? @@ -502,7 +497,7 @@ impl<'t> IntegratedCacheWriteAccess<'t> { Entry::Vacant(e) => { // FIXME: what to do if we run out of memory? Evict other relation entries? Remove // block entries first? - e.insert(BlockEntry { + _ = e.insert(BlockEntry { lw_lsn: AtomicLsn::new(lw_lsn.0), cache_block: AtomicU64::new(cache_block), pinned: AtomicU64::new(0), @@ -516,8 +511,7 @@ impl<'t> IntegratedCacheWriteAccess<'t> { /// Forget information about given relation in the cache. (For DROP TABLE and such) pub fn forget_rel(&'t self, rel: &RelTag) { tracing::info!("forgetting rel entry for {rel:?}"); - let hash = self.relsize_cache.get_hash_value(&RelKey::from(rel)); - self.relsize_cache.remove_with_hash(&RelKey::from(rel), hash); + self.relsize_cache.remove(&RelKey::from(rel)); // also forget all cached blocks for the relation // FIXME @@ -596,37 +590,25 @@ impl<'t> IntegratedCacheWriteAccess<'t> { if evict_this { // grab the write lock let mut evicted_cache_block = None; - todo!("quantumish: re-add support for point removal without demolishing performance"); - // self.block_map - // .update_with_fn_at_bucket(*clock_hand % num_buckets, |old| { - // match old { - // None => UpdateAction::Nothing, - // Some(old) => { - // // note: all the accesses to 'pinned' currently happen - // // within update_with_fn(), or while holding ValueReadGuard, which protects from concurrent - // // updates. Otherwise, another thread could set the 'pinned' - // // flag just after we have checked it here. - // if old.pinned.load(Ordering::Relaxed) != 0 { - // return UpdateAction::Nothing; - // } - - // let _ = self - // .global_lw_lsn - // .fetch_max(old.lw_lsn.load().0, Ordering::Relaxed); - // let cache_block = old - // .cache_block - // .swap(INVALID_CACHE_BLOCK, Ordering::Relaxed); - // if cache_block != INVALID_CACHE_BLOCK { - // evicted_cache_block = Some(cache_block); - // } - // UpdateAction::Remove - // } - // } - // }); - - // Out of memory should not happen here, as we're only updating existing values, - // not inserting new entries to the map. - // res.expect("out of memory"); + if let Some(e) = self.block_map.entry_at_bucket(*clock_hand % num_buckets) { + let old = e.get(); + // note: all the accesses to 'pinned' currently happen + // within update_with_fn(), or while holding ValueReadGuard, which protects from concurrent + // updates. Otherwise, another thread could set the 'pinned' + // flag just after we have checked it here. + if old.pinned.load(Ordering::Relaxed) == 0 { + let _ = self + .global_lw_lsn + .fetch_max(old.lw_lsn.load().0, Ordering::Relaxed); + let cache_block = old + .cache_block + .swap(INVALID_CACHE_BLOCK, Ordering::Relaxed); + if cache_block != INVALID_CACHE_BLOCK { + evicted_cache_block = Some(cache_block); + } + e.remove(); + } + } if evicted_cache_block.is_some() { self.page_evictions_counter.inc(); @@ -705,8 +687,7 @@ fn get_rel_size<'t>( r: &neon_shmem::hash::HashMapAccess, rel: &RelTag, ) -> Option { - let hash = r.get_hash_value(&RelKey::from(rel)); - if let Some(rel_entry) = r.get_with_hash(&RelKey::from(rel), hash) { + if let Some(rel_entry) = r.get(&RelKey::from(rel)) { let nblocks = rel_entry.nblocks.load(Ordering::Relaxed); if nblocks != u32::MAX { Some(nblocks) @@ -750,11 +731,10 @@ impl<'e> BackendCacheReadOp<'e> { /// After you have completed the read, call BackendCacheReadResult::finish() to check if the /// read was in fact valid or not. If it was concurrently invalidated, you need to retry. pub fn get_page(&mut self, rel: &RelTag, block_number: u32) -> Option { - let hash = self.map_access.block_map.get_hash_value(&BlockKey::from((rel, block_number))); if let Some(block_entry) = self .map_access .block_map - .get_with_hash(&BlockKey::from((rel, block_number)), hash) + .get(&BlockKey::from((rel, block_number))) { block_entry.referenced.store(true, Ordering::Relaxed);