Use fsync+rename for atomic downloads from remote storage

Use failpoint in test_remote_storage to check the behavior

test_runner/batch_others/test_remote_storage.py

@@ -4,10 +4,11 @@
 import shutil, os
 from contextlib import closing
 from pathlib import Path
+import time
 from uuid import UUID
 from fixtures.zenith_fixtures import ZenithEnvBuilder, assert_local, wait_for, wait_for_last_record_lsn, wait_for_upload
 from fixtures.log_helper import log
-from fixtures.utils import lsn_from_hex
+from fixtures.utils import lsn_from_hex, lsn_to_hex
 import pytest
 
 
@@ -23,14 +24,14 @@ import pytest
 #
 # 2. Second pageserver
 #   * starts another pageserver, connected to the same remote storage
-#   * same timeline id is queried for status, triggering timeline's download
+#   * timeline_attach is called for the same timeline id
 #   * timeline status is polled until it's downloaded
 #   * queries the specific data, ensuring that it matches the one stored before
 #
 # The tests are done for all types of remote storage pageserver supports.
 @pytest.mark.parametrize('storage_type', ['local_fs', 'mock_s3'])
 def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder, storage_type: str):
-    zenith_env_builder.rust_log_override = 'debug'
+    # zenith_env_builder.rust_log_override = 'debug'
     zenith_env_builder.num_safekeepers = 1
     if storage_type == 'local_fs':
         zenith_env_builder.enable_local_fs_remote_storage()
@@ -67,9 +68,7 @@ def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder,
         wait_for_last_record_lsn(client, UUID(tenant_id), UUID(timeline_id), current_lsn)
 
         # run checkpoint manually to be sure that data landed in remote storage
-        with closing(env.pageserver.connect()) as psconn:
-            with psconn.cursor() as pscur:
-                pscur.execute(f"checkpoint {tenant_id} {timeline_id}")
+        env.pageserver.safe_psql(f"checkpoint {tenant_id} {timeline_id}")
 
         log.info(f'waiting for checkpoint {checkpoint_number} upload')
         # wait until pageserver successfully uploaded a checkpoint to remote storage
@@ -87,6 +86,27 @@ def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder,
     ##### Second start, restore the data and ensure it's the same
     env.pageserver.start()
 
+    # Introduce failpoint in download
+    env.pageserver.safe_psql(f"failpoints remote-storage-download-pre-rename=return")
+
+    client.timeline_attach(UUID(tenant_id), UUID(timeline_id))
+
+    # is there a better way to assert that fafilpoint triggered?
+    time.sleep(10)
+
+    # assert cannot attach timeline that is scheduled for download
+    with pytest.raises(Exception, match="Timeline download is already in progress"):
+        client.timeline_attach(UUID(tenant_id), UUID(timeline_id))
+
+    detail = client.timeline_detail(UUID(tenant_id), UUID(timeline_id))
+    log.info("Timeline detail with active failpoint: %s", detail)
+    assert detail['local'] is None
+    assert detail['remote']['awaits_download']
+
+    # trigger temporary download files removal
+    env.pageserver.stop()
+    env.pageserver.start()
+
     client.timeline_attach(UUID(tenant_id), UUID(timeline_id))
 
     log.info("waiting for timeline redownload")
@@ -94,6 +114,12 @@ def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder,
              interval=1,
              func=lambda: assert_local(client, UUID(tenant_id), UUID(timeline_id)))
 
+    detail = client.timeline_detail(UUID(tenant_id), UUID(timeline_id))
+    assert detail['local'] is not None
+    log.info("Timeline detail after attach completed: %s", detail)
+    assert lsn_from_hex(detail['local']['last_record_lsn']) == current_lsn
+    assert not detail['remote']['awaits_download']
+
     pg = env.postgres.create_start('main')
     with closing(pg.connect()) as conn:
         with conn.cursor() as cur: