diff --git a/pgxn/neon/relsize_cache.c b/pgxn/neon/relsize_cache.c
index 0cfd57d832..a1847b6047 100644
--- a/pgxn/neon/relsize_cache.c
+++ b/pgxn/neon/relsize_cache.c
@@ -506,7 +506,8 @@ stop_unlogged_build(NRelFileInfo rinfo, ForkNumber forknum)
 		{
 			relsize_ctl->misses += 1;
 		}
-		LWLockRelease(relsize_lock);
+		if (!unlogged)
+			LWLockRelease(relsize_lock);
 	}
 	return unlogged;
 }