mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-07 05:22:56 +00:00
6dec85b19d
Previously, the /v1/tenant/:tenant_id/timeline/:timeline_id/do_gc API call performed a flush and compaction on the timeline before GC. Change it not to do that, and change all the tests that used that API to perform compaction explicitly. The compaction happens at a slightly different point now. Previously, the code performed the `refresh_gc_info_internal` step first, and only then did compaction on all the timelines. I don't think that was what was originally intended here. Presumably the idea with compaction was to make some old layer files available for GC. But if we're going to flush the current in-memory layer to disk, surely you would want to include the newly-written layer in the compaction too. I guess this didn't make any difference to the tests in practice, but in any case, the tests now perform the flush and compaction before any of the GC steps. Some of the tests might not need the compaction at all, but I didn't try hard to determine which ones might need it. I left it out from a few tests that intentionally tested calling do_gc with an invalid tenant or timeline ID, though.
178 lines
7.3 KiB
Python
178 lines
7.3 KiB
Python
import threading
|
|
import time
|
|
|
|
import pytest
|
|
from fixtures.log_helper import log
|
|
from fixtures.neon_fixtures import NeonEnv
|
|
from fixtures.types import Lsn
|
|
from fixtures.utils import query_scalar
|
|
|
|
|
|
# Test the GC implementation when running with branching.
|
|
# This test reproduces the issue https://github.com/neondatabase/neon/issues/707.
|
|
#
|
|
# Consider two LSNs `lsn1` and `lsn2` with some delta files as follows:
|
|
# ...
|
|
# p -> has an image layer xx_p with p < lsn1
|
|
# ...
|
|
# lsn1
|
|
# ...
|
|
# q -> has an image layer yy_q with lsn1 < q < lsn2
|
|
# ...
|
|
# lsn2
|
|
#
|
|
# Consider running a GC iteration such that the GC horizon is between p and lsn1
|
|
# ...
|
|
# p -> has an image layer xx_p with p < lsn1
|
|
# D_start -> is a delta layer D's start (e.g D = '...-...-D_start-D_end')
|
|
# ...
|
|
# GC_h -> is a gc horizon such that p < GC_h < lsn1
|
|
# ...
|
|
# lsn1
|
|
# ...
|
|
# D_end -> is a delta layer D's end
|
|
# ...
|
|
# q -> has an image layer yy_q with lsn1 < q < lsn2
|
|
# ...
|
|
# lsn2
|
|
#
|
|
# As described in the issue #707, the image layer xx_p will be deleted as
|
|
# its range is below the GC horizon and there exists a newer image layer yy_q (q > p).
|
|
# However, removing xx_p will corrupt any delta layers that depend on xx_p that
|
|
# are not deleted by GC. For example, the delta layer D is corrupted in the
|
|
# above example because D depends on the image layer xx_p for value reconstruction.
|
|
#
|
|
# Because the delta layer D covering lsn1 is corrupted, creating a branch
|
|
# starting from lsn1 should return an error as follows:
|
|
# could not find data for key ... at LSN ..., for request at LSN ...
|
|
def test_branch_and_gc(neon_simple_env: NeonEnv):
|
|
env = neon_simple_env
|
|
pageserver_http_client = env.pageserver.http_client()
|
|
|
|
tenant, _ = env.neon_cli.create_tenant(
|
|
conf={
|
|
# disable background GC
|
|
"gc_period": "0s",
|
|
# small checkpoint distance to create more delta layer files
|
|
"checkpoint_distance": f"{1024 ** 2}",
|
|
# set the target size to be large to allow the image layer to cover the whole key space
|
|
"compaction_target_size": f"{1024 ** 3}",
|
|
# tweak the default settings to allow quickly create image layers and L1 layers
|
|
"compaction_period": "1 s",
|
|
"compaction_threshold": "2",
|
|
"image_creation_threshold": "1",
|
|
# set PITR interval to be small, so we can do GC
|
|
"pitr_interval": "1 s",
|
|
}
|
|
)
|
|
|
|
timeline_main = env.neon_cli.create_timeline("test_main", tenant_id=tenant)
|
|
pg_main = env.postgres.create_start("test_main", tenant_id=tenant)
|
|
|
|
main_cur = pg_main.connect().cursor()
|
|
|
|
main_cur.execute(
|
|
"CREATE TABLE foo(key serial primary key, t text default 'foooooooooooooooooooooooooooooooooooooooooooooooooooo')"
|
|
)
|
|
main_cur.execute("INSERT INTO foo SELECT FROM generate_series(1, 100000)")
|
|
lsn1 = Lsn(query_scalar(main_cur, "SELECT pg_current_wal_insert_lsn()"))
|
|
log.info(f"LSN1: {lsn1}")
|
|
|
|
main_cur.execute("INSERT INTO foo SELECT FROM generate_series(1, 100000)")
|
|
lsn2 = Lsn(query_scalar(main_cur, "SELECT pg_current_wal_insert_lsn()"))
|
|
log.info(f"LSN2: {lsn2}")
|
|
|
|
# Set the GC horizon so that lsn1 is inside the horizon, which means
|
|
# we can create a new branch starting from lsn1.
|
|
pageserver_http_client.timeline_checkpoint(tenant, timeline_main)
|
|
pageserver_http_client.timeline_gc(tenant, timeline_main, lsn2 - lsn1 + 1024)
|
|
|
|
env.neon_cli.create_branch(
|
|
"test_branch", "test_main", tenant_id=tenant, ancestor_start_lsn=lsn1
|
|
)
|
|
pg_branch = env.postgres.create_start("test_branch", tenant_id=tenant)
|
|
|
|
branch_cur = pg_branch.connect().cursor()
|
|
branch_cur.execute("INSERT INTO foo SELECT FROM generate_series(1, 100000)")
|
|
|
|
assert query_scalar(branch_cur, "SELECT count(*) FROM foo") == 200000
|
|
|
|
|
|
# This test simulates a race condition happening when branch creation and GC are performed concurrently.
|
|
#
|
|
# Suppose we want to create a new timeline 't' from a source timeline 's' starting
|
|
# from a lsn 'lsn'. Upon creating 't', if we don't hold the GC lock and compare 'lsn' with
|
|
# the latest GC information carefully, it's possible for GC to accidentally remove data
|
|
# needed by the new timeline.
|
|
#
|
|
# In this test, GC is requested before the branch creation but is delayed to happen after branch creation.
|
|
# As a result, when doing GC for the source timeline, we don't have any information about
|
|
# the upcoming new branches, so it's possible to remove data that may be needed by the new branches.
|
|
# It's the branch creation task's job to make sure the starting 'lsn' is not out of scope
|
|
# and prevent creating branches with invalid starting LSNs.
|
|
#
|
|
# For more details, see discussion in https://github.com/neondatabase/neon/pull/2101#issuecomment-1185273447.
|
|
def test_branch_creation_before_gc(neon_simple_env: NeonEnv):
|
|
env = neon_simple_env
|
|
pageserver_http_client = env.pageserver.http_client()
|
|
|
|
env.pageserver.allowed_errors.extend(
|
|
[
|
|
".*invalid branch start lsn: less than latest GC cutoff.*",
|
|
".*invalid branch start lsn: less than planned GC cutoff.*",
|
|
]
|
|
)
|
|
|
|
# Disable background GC but set the `pitr_interval` to be small, so GC can delete something
|
|
tenant, _ = env.neon_cli.create_tenant(
|
|
conf={
|
|
# disable background GC
|
|
"gc_period": "0s",
|
|
# small checkpoint distance to create more delta layer files
|
|
"checkpoint_distance": f"{1024 ** 2}",
|
|
# set the target size to be large to allow the image layer to cover the whole key space
|
|
"compaction_target_size": f"{1024 ** 3}",
|
|
# tweak the default settings to allow quickly create image layers and L1 layers
|
|
"compaction_period": "1 s",
|
|
"compaction_threshold": "2",
|
|
"image_creation_threshold": "1",
|
|
# set PITR interval to be small, so we can do GC
|
|
"pitr_interval": "0 s",
|
|
}
|
|
)
|
|
|
|
b0 = env.neon_cli.create_branch("b0", tenant_id=tenant)
|
|
pg0 = env.postgres.create_start("b0", tenant_id=tenant)
|
|
res = pg0.safe_psql_many(
|
|
queries=[
|
|
"CREATE TABLE t(key serial primary key)",
|
|
"INSERT INTO t SELECT FROM generate_series(1, 100000)",
|
|
"SELECT pg_current_wal_insert_lsn()",
|
|
"INSERT INTO t SELECT FROM generate_series(1, 100000)",
|
|
]
|
|
)
|
|
lsn = Lsn(res[2][0][0])
|
|
|
|
# Use `failpoint=sleep` and `threading` to make the GC iteration triggers *before* the
|
|
# branch creation task but the individual timeline GC iteration happens *after*
|
|
# the branch creation task.
|
|
pageserver_http_client.configure_failpoints(("before-timeline-gc", "sleep(2000)"))
|
|
pageserver_http_client.timeline_checkpoint(tenant, b0)
|
|
|
|
def do_gc():
|
|
pageserver_http_client.timeline_gc(tenant, b0, 0)
|
|
|
|
thread = threading.Thread(target=do_gc, daemon=True)
|
|
thread.start()
|
|
|
|
# because of network latency and other factors, GC iteration might be processed
|
|
# after the `create_branch` request. Add a sleep here to make sure that GC is
|
|
# always processed before.
|
|
time.sleep(1.0)
|
|
|
|
# The starting LSN is invalid as the corresponding record is scheduled to be removed by in-queue GC.
|
|
with pytest.raises(Exception, match="invalid branch start lsn: .*"):
|
|
env.neon_cli.create_branch("b1", "b0", tenant_id=tenant, ancestor_start_lsn=lsn)
|
|
|
|
thread.join()
|