From a89d6dc76e8406ad15e45b190bc687b8b208c3e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Tue, 19 Dec 2023 11:29:16 +0100 Subject: [PATCH] Always send a json response for timeline_get_lsn_by_timestamp (#6178) As part of the transition laid out in [this](https://github.com/neondatabase/cloud/pull/7553#discussion_r1370473911) comment, don't read the `version` query parameter in `timeline_get_lsn_by_timestamp`, but always return the structured json response. Follow-up of https://github.com/neondatabase/neon/pull/5608 --- pageserver/src/http/routes.rs | 37 ++++--------- test_runner/fixtures/pageserver/http.py | 12 ++++- test_runner/regress/test_lsn_mapping.py | 71 ++----------------------- 3 files changed, 24 insertions(+), 96 deletions(-) diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 601fad5bde..bc8b677f77 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -592,8 +592,6 @@ async fn get_lsn_by_timestamp_handler( ))); } - let version: Option = parse_query_param(&request, "version")?; - let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; let timestamp_raw = must_get_query_param(&request, "timestamp")?; let timestamp = humantime::parse_rfc3339(×tamp_raw) @@ -606,31 +604,18 @@ async fn get_lsn_by_timestamp_handler( let result = timeline .find_lsn_for_timestamp(timestamp_pg, &cancel, &ctx) .await?; - - if version.unwrap_or(0) > 1 { - #[derive(serde::Serialize)] - struct Result { - lsn: Lsn, - kind: &'static str, - } - let (lsn, kind) = match result { - LsnForTimestamp::Present(lsn) => (lsn, "present"), - LsnForTimestamp::Future(lsn) => (lsn, "future"), - LsnForTimestamp::Past(lsn) => (lsn, "past"), - LsnForTimestamp::NoData(lsn) => (lsn, "nodata"), - }; - json_response(StatusCode::OK, Result { lsn, kind }) - } else { - // FIXME: this is a temporary crutch not to break backwards compatibility - // See https://github.com/neondatabase/neon/pull/5608 - let result = match result { - LsnForTimestamp::Present(lsn) => format!("{lsn}"), - LsnForTimestamp::Future(_lsn) => "future".into(), - LsnForTimestamp::Past(_lsn) => "past".into(), - LsnForTimestamp::NoData(_lsn) => "nodata".into(), - }; - json_response(StatusCode::OK, result) + #[derive(serde::Serialize)] + struct Result { + lsn: Lsn, + kind: &'static str, } + let (lsn, kind) = match result { + LsnForTimestamp::Present(lsn) => (lsn, "present"), + LsnForTimestamp::Future(lsn) => (lsn, "future"), + LsnForTimestamp::Past(lsn) => (lsn, "past"), + LsnForTimestamp::NoData(lsn) => (lsn, "nodata"), + }; + json_response(StatusCode::OK, Result { lsn, kind }) } async fn get_timestamp_of_lsn_handler( diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index eda8813c36..add6c4288a 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -510,13 +510,21 @@ class PageserverHttpClient(requests.Session): assert res_json is None def timeline_get_lsn_by_timestamp( - self, tenant_id: TenantId, timeline_id: TimelineId, timestamp, version: int + self, + tenant_id: TenantId, + timeline_id: TimelineId, + timestamp, + version: Optional[int] = None, ): log.info( f"Requesting lsn by timestamp {timestamp}, tenant {tenant_id}, timeline {timeline_id}" ) + if version is None: + version_str = "" + else: + version_str = f"&version={version}" res = self.get( - f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/get_lsn_by_timestamp?timestamp={timestamp}&version={version}", + f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/get_lsn_by_timestamp?timestamp={timestamp}{version_str}", ) self.verbose_error(res) res_json = res.json() diff --git a/test_runner/regress/test_lsn_mapping.py b/test_runner/regress/test_lsn_mapping.py index f79c1c347c..65d6d7a9fd 100644 --- a/test_runner/regress/test_lsn_mapping.py +++ b/test_runner/regress/test_lsn_mapping.py @@ -8,71 +8,6 @@ from fixtures.types import Lsn from fixtures.utils import query_scalar -# -# Test pageserver get_lsn_by_timestamp API -# -def test_lsn_mapping_old(neon_env_builder: NeonEnvBuilder): - env = neon_env_builder.init_start() - - new_timeline_id = env.neon_cli.create_branch("test_lsn_mapping") - endpoint_main = env.endpoints.create_start("test_lsn_mapping") - log.info("postgres is running on 'test_lsn_mapping' branch") - - cur = endpoint_main.connect().cursor() - # Create table, and insert rows, each in a separate transaction - # Disable synchronous_commit to make this initialization go faster. - # - # Each row contains current insert LSN and the current timestamp, when - # the row was inserted. - cur.execute("SET synchronous_commit=off") - cur.execute("CREATE TABLE foo (x integer)") - tbl = [] - for i in range(1000): - cur.execute("INSERT INTO foo VALUES(%s)", (i,)) - # Get the timestamp at UTC - after_timestamp = query_scalar(cur, "SELECT clock_timestamp()").replace(tzinfo=None) - tbl.append([i, after_timestamp]) - - # Execute one more transaction with synchronous_commit enabled, to flush - # all the previous transactions - cur.execute("SET synchronous_commit=on") - cur.execute("INSERT INTO foo VALUES (-1)") - - # Wait until WAL is received by pageserver - wait_for_last_flush_lsn(env, endpoint_main, env.initial_tenant, new_timeline_id) - - with env.pageserver.http_client() as client: - # Check edge cases: timestamp in the future - probe_timestamp = tbl[-1][1] + timedelta(hours=1) - result = client.timeline_get_lsn_by_timestamp( - env.initial_tenant, new_timeline_id, f"{probe_timestamp.isoformat()}Z", 1 - ) - assert result == "future" - - # timestamp too the far history - probe_timestamp = tbl[0][1] - timedelta(hours=10) - result = client.timeline_get_lsn_by_timestamp( - env.initial_tenant, new_timeline_id, f"{probe_timestamp.isoformat()}Z", 1 - ) - assert result == "past" - - # Probe a bunch of timestamps in the valid range - for i in range(1, len(tbl), 100): - probe_timestamp = tbl[i][1] - lsn = client.timeline_get_lsn_by_timestamp( - env.initial_tenant, new_timeline_id, f"{probe_timestamp.isoformat()}Z", 1 - ) - # Call get_lsn_by_timestamp to get the LSN - # Launch a new read-only node at that LSN, and check that only the rows - # that were supposed to be committed at that point in time are visible. - endpoint_here = env.endpoints.create_start( - branch_name="test_lsn_mapping", endpoint_id="ep-lsn_mapping_read", lsn=lsn - ) - assert endpoint_here.safe_psql("SELECT max(x) FROM foo")[0][0] == i - - endpoint_here.stop_and_destroy() - - # # Test pageserver get_lsn_by_timestamp API # @@ -130,7 +65,7 @@ def test_lsn_mapping(neon_env_builder: NeonEnvBuilder): # Timestamp is in the future probe_timestamp = tbl[-1][1] + timedelta(hours=1) result = client.timeline_get_lsn_by_timestamp( - tenant_id, timeline_id, f"{probe_timestamp.isoformat()}Z", 2 + tenant_id, timeline_id, f"{probe_timestamp.isoformat()}Z" ) assert result["kind"] == "future" # make sure that we return a well advanced lsn here @@ -139,7 +74,7 @@ def test_lsn_mapping(neon_env_builder: NeonEnvBuilder): # Timestamp is in the unreachable past probe_timestamp = tbl[0][1] - timedelta(hours=10) result = client.timeline_get_lsn_by_timestamp( - tenant_id, timeline_id, f"{probe_timestamp.isoformat()}Z", 2 + tenant_id, timeline_id, f"{probe_timestamp.isoformat()}Z" ) assert result["kind"] == "past" # make sure that we return the minimum lsn here at the start of the range @@ -149,7 +84,7 @@ def test_lsn_mapping(neon_env_builder: NeonEnvBuilder): for i in range(1, len(tbl), 100): probe_timestamp = tbl[i][1] result = client.timeline_get_lsn_by_timestamp( - tenant_id, timeline_id, f"{probe_timestamp.isoformat()}Z", 2 + tenant_id, timeline_id, f"{probe_timestamp.isoformat()}Z" ) assert result["kind"] not in ["past", "nodata"] lsn = result["lsn"]