mirror of
https://github.com/neondatabase/neon.git
synced 2025-12-22 21:59:59 +00:00
a7ce323949
This PR commits the benchmarks I ran to qualify concurrent IO before we
released it.
Changes:
- Add `l0stack` fixture; a reusable abstraction for creating a stack of
L0 deltas
each of which has 1 Value::Delta per page.
- Such a stack of L0 deltas is a good and understandable demo for
concurrent IO
because to reconstruct any page, $layer_stack_height` Values need to be
read.
Before concurrent IO, the reads were sequential.
With concurrent IO, they are executed concurrently.
- So, switch `test_latency` to use the l0stack.
- Teach `pagebench`, which is used by `test_latency`, to limit itself to
the blocks of the relation created by the l0stack abstraction.
- Additional parametrization of `test_latency` over dimensions
`ps_io_concurrency,l0_stack_height,queue_depth`
- Use better names for the tests to reflect what they do, leave
interpretation of the (now quite high-dimensional) results to the reader
- `test_{throughput => postgres_seqscan}`
- `test_{latency => random_reads}`
- Cut down on permutations to those we use in production. Runtime is
about 2min.
Refs
- concurrent IO epic https://github.com/neondatabase/neon/issues/9378
- batching task: fixes https://github.com/neondatabase/neon/issues/9837
---------
Co-authored-by: Peter Bendel <peterbendel@neon.tech>
60 lines
2.3 KiB
Python
60 lines
2.3 KiB
Python
"""
|
|
Script to creates a stack of L0 deltas each of which should have 1 Value::Delta per page in `data`,
|
|
in your running neon_local setup.
|
|
|
|
Use this bash setup to reset your neon_local environment.
|
|
The last line of this bash snippet will run this file here.
|
|
```
|
|
export NEON_REPO_DIR=$PWD/.neon
|
|
export NEON_BIN_DIR=$PWD/target/release
|
|
$NEON_BIN_DIR/neon_local stop
|
|
rm -rf $NEON_REPO_DIR
|
|
$NEON_BIN_DIR/neon_local init
|
|
cat >> $NEON_REPO_DIR/pageserver_1/pageserver.toml <<"EOF"
|
|
# customizations
|
|
virtual_file_io_mode = "direct-rw"
|
|
page_service_pipelining={mode="pipelined", max_batch_size=32, execution="concurrent-futures"}
|
|
get_vectored_concurrent_io={mode="sidecar-task"}
|
|
EOF
|
|
$NEON_BIN_DIR/neon_local start
|
|
|
|
psql 'postgresql://localhost:1235/storage_controller' -c 'DELETE FROM tenant_shards'
|
|
sed 's/.*get_vectored_concurrent_io.*/get_vectored_concurrent_io={mode="sidecar-task"}/' -i $NEON_REPO_DIR/pageserver_1/pageserver.toml
|
|
$NEON_BIN_DIR/neon_local pageserver restart
|
|
sleep 2
|
|
$NEON_BIN_DIR/neon_local tenant create --set-default
|
|
./target/debug/neon_local endpoint stop foo
|
|
rm -rf $NEON_REPO_DIR/endpoints/foo
|
|
./target/debug/neon_local endpoint create foo
|
|
echo 'full_page_writes=off' >> $NEON_REPO_DIR/endpoints/foo/postgresql.conf
|
|
./target/debug/neon_local endpoint start foo
|
|
|
|
pushd test_runner; poetry run python3 -m bin.neon_local_create_deep_l0_stack 10; popd
|
|
```
|
|
"""
|
|
|
|
import sys
|
|
|
|
import psycopg2
|
|
from fixtures.common_types import TenantShardId, TimelineId
|
|
from fixtures.pageserver.http import PageserverHttpClient
|
|
from fixtures.pageserver.makelayers.l0stack import L0StackShape, make_l0_stack_standalone
|
|
|
|
ps_http = PageserverHttpClient(port=9898, is_testing_enabled_or_skip=lambda: None)
|
|
vps_http = PageserverHttpClient(port=1234, is_testing_enabled_or_skip=lambda: None)
|
|
|
|
tenants = ps_http.tenant_list()
|
|
assert len(tenants) == 1
|
|
tenant_shard_id = TenantShardId.parse(tenants[0]["id"])
|
|
|
|
timlines = ps_http.timeline_list(tenant_shard_id)
|
|
assert len(timlines) == 1
|
|
timeline_id = TimelineId(timlines[0]["timeline_id"])
|
|
|
|
connstr = "postgresql://cloud_admin@localhost:55432/postgres"
|
|
conn = psycopg2.connect(connstr)
|
|
|
|
shape = L0StackShape(logical_table_size_mib=50, delta_stack_height=int(sys.argv[1]))
|
|
|
|
make_l0_stack_standalone(vps_http, ps_http, tenant_shard_id, timeline_id, conn, shape)
|