diff --git a/test_runner/deep_layers_with_delta.py b/test_runner/deep_layers_with_delta.py index 3945098ff6..d9db58f8de 100644 --- a/test_runner/deep_layers_with_delta.py +++ b/test_runner/deep_layers_with_delta.py @@ -21,31 +21,49 @@ Some commands from shell history used in combination with this script: starting from scrach: psql 'postgresql://localhost:1235/storage_controller' -c 'DELETE FROM tenant_shards' - ./target/release/neon_local pageserver restart + NEON_PAGESERVER_VALUE_RECONSTRUCT_IO_CONCURRENCY=... ./target/release/neon_local pageserver restart ./target/debug/neon_local endpoint stop foo rm -rf .neon/endpoints/foo ./target/debug/neon_local endpoint create foo + echo 'full_page_writes=off' >> .neon/endpoints/foo/postgresql.conf ./target/debug/neon_local endpoint start foo non-package-mode-py3.10christian@neon-hetzner-dev-christian:[~/src/neon/test_runner]: poetry run python3 deep_layers_with_delta.py + Need 6400 pages, 38400 rows + 0 + modified rows 6400 + 1 + modified rows 6400 + ... + 19 + modified rows 6400 - Layer files created: -christian@neon-hetzner-dev-christian:[~/src/neon]: ls -lrt .neon/pageserver_1/tenants/1cef6c5e3f94e02ac61d41d837fd6be9/timelines/0e1551891a0418657128d1b5ed2c16fb -total 64172 --rw-r--r-- 1 christian 23265280 Dec 12 20:16 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014F1378-00000000014F13F1-v1-00000001 --rw-r--r-- 1 christian 9805824 Dec 12 20:16 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014F13F1-0000000001C63B59-v1-00000002 --rw-r--r-- 1 christian 3334144 Dec 12 20:16 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001C63B59-0000000001EEFA19-v1-00000002 --rw-r--r-- 1 christian 4734976 Dec 12 20:16 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001EEFA19-000000000229BE69-v1-00000002 --rw-r--r-- 1 christian 6127616 Dec 12 20:16 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__000000000229BE69-000000000275BF99-v1-00000002 --rw-r--r-- 1 christian 7520256 Dec 12 20:16 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__000000000275BF99-0000000002D3C661-v1-00000002 --rw-r--r-- 1 christian 8863744 Dec 12 20:16 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000002D3C661-000000000343D2A1-v1-00000002 --rw-r--r-- 1 christian 802816 Dec 12 20:16 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__000000000343D2A1-00000000034D3589-v1-00000002 --rw-r--r-- 1 christian 24576 Dec 12 20:16 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000034D3589-00000000034D39E1-v1-00000002 --rw-r--r-- 1 christian 630784 Dec 12 20:17 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000034D39E1-0000000003507881-v1-00000002 --rw-r--r-- 1 christian 598016 Dec 12 20:17 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000003507881-000000000353A0C9-v1-00000002 --rw-r--r-- 1 christian 49152 Dec 12 20:17 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__000000000353A0C9-0000000003541269-v1-00000002 --rw-r--r-- 1 christian 24576 Dec 12 20:17 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000003541269-00000000035412A1-v1-00000002 +christian@neon-hetzner-dev-christian:[~/src/neon]: ls -hlrt .neon/pageserver_1/tenants/*/timelines/* +total 67M +-rw-r--r-- 1 christian 23M Dec 13 12:38 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014F1378-00000000014F13F1-v1-00000001 +-rw-r--r-- 1 christian 9.3M Dec 13 12:41 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014F13F1-0000000001C4D021-v1-00000001 +-rw-r--r-- 1 christian 1.9M Dec 13 12:41 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001C4D021-0000000001DC51F1-v1-00000001 +-rw-r--r-- 1 christian 1.9M Dec 13 12:41 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001DC51F1-0000000001F3D3C1-v1-00000001 +-rw-r--r-- 1 christian 1.9M Dec 13 12:41 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001F3D3C1-00000000020B55A1-v1-00000001 +-rw-r--r-- 1 christian 1.9M Dec 13 12:41 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000020B55A1-000000000222D771-v1-00000001 +-rw-r--r-- 1 christian 1.9M Dec 13 12:41 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__000000000222D771-00000000023A5941-v1-00000001 +-rw-r--r-- 1 christian 1.9M Dec 13 12:42 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000023A5941-000000000251DB11-v1-00000001 +-rw-r--r-- 1 christian 1.9M Dec 13 12:42 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__000000000251DB11-0000000002695D19-v1-00000001 +-rw-r--r-- 1 christian 1.9M Dec 13 12:42 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000002695D19-000000000280DEE9-v1-00000001 +-rw-r--r-- 1 christian 1.9M Dec 13 12:42 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__000000000280DEE9-00000000029860D1-v1-00000001 +-rw-r--r-- 1 christian 1.9M Dec 13 12:42 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000029860D1-0000000002AFE2A1-v1-00000001 +-rw-r--r-- 1 christian 1.9M Dec 13 12:42 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000002AFE2A1-0000000002C76471-v1-00000001 +-rw-r--r-- 1 christian 1.9M Dec 13 12:42 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000002C76471-0000000002DEE679-v1-00000001 +-rw-r--r-- 1 christian 1.9M Dec 13 12:42 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000002DEE679-0000000002F66849-v1-00000001 +-rw-r--r-- 1 christian 1.9M Dec 13 12:42 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000002F66849-00000000030DEA29-v1-00000001 +-rw-r--r-- 1 christian 1.9M Dec 13 12:42 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000030DEA29-0000000003256BF9-v1-00000001 +-rw-r--r-- 1 christian 2.0M Dec 13 12:42 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000003256BF9-00000000033CEE39-v1-00000001 +-rw-r--r-- 1 christian 1.7M Dec 13 12:42 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000033CEE39-0000000003512B91-v1-00000001 +-rw-r--r-- 1 christian 1.4M Dec 13 12:42 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000003512B91-0000000003633D91-v1-00000001 +-rw-r--r-- 1 christian 1.9M Dec 13 12:42 000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000003633D91-00000000037AE2E9-v1-00000001 +-rw-r--r-- 1 christian 0 Dec 13 12:42 ephemeral-22 + """ @@ -102,17 +120,18 @@ cur = conn.cursor() # (8k-24bytes) / 123 bytes = 63 tuples per page # set fillfactor to 10 to have 6 tuples per page cur.execute("DROP TABLE IF EXISTS data") -cur.execute("CREATE TABLE data(row char(100)) with (fillfactor=10)") +cur.execute("CREATE TABLE data(id bigint, row char(92)) with (fillfactor=10)") desired_size = 50 * 1024 * 1024 # 50MiB need_pages = desired_size // 8192 need_rows = need_pages * 6 print(f"Need {need_pages} pages, {need_rows} rows") -cur.execute(f"INSERT INTO data SELECT i % 6 FROM generate_series(1, {need_rows}) as i") +cur.execute(f"INSERT INTO data SELECT i,'row'||i FROM generate_series(1, {need_rows}) as i") # every iteration updates one tuple in each page for i in range( 0, 20): print(i) - cur.execute(f"UPDATE data set row = ((row::bigint + 1) % 6) where row::bigint % 6 = {i}") + cur.execute(f"UPDATE data set row = row||',u' where id % 6 = {i%6}") + print("modified rows", cur.rowcount) cur.execute("SELECT pg_current_wal_flush_lsn()") flush_lsn = Lsn(cur.fetchall()[0][0])