WIP

docs/storage_broker.md

@@ -1,6 +1,6 @@
 # Storage broker
 
-Storage broker targets two issues:
+Storage broker targets two issues
 - Allowing safekeepers and pageservers learn which nodes also hold their
   timelines, and timeline statuses there.
 - Avoiding O(n^2) connections between storage nodes while doing so.
@@ -19,7 +19,7 @@ Currently, the only message is `SafekeeperTimelineInfo`. Each safekeeper, for
 each active timeline, once in a while pushes timeline status to the broker.
 Other nodes subscribe and receive this info, using it per above.
 
-Broker serves /metrics on the same port as grpc service. 
+Broker serves /metrics on the same port as grpc service.
 
 grpcurl can be used to check which values are currently being pushed:
 ```

safekeeper/src/timeline_manager.rs

@@ -495,6 +495,7 @@ impl Manager {
     }
 
     /// Update is_active flag and returns its value.
+    // Timelines marked active are pushed to the broker by the `push_loop` task.
     fn update_is_active(
         &mut self,
         is_wal_backup_required: bool,

safekeeper/src/wal_backup.rs

@@ -61,7 +61,9 @@ pub(crate) fn is_wal_backup_required(
     state: &StateSnapshot,
 ) -> bool {
     num_computes > 0 ||
-    // Currently only the whole segment is offloaded, so compare segment numbers.
+    // This task backups completed segments only.
+    // The current partial segment is backed up by a separate task/code module (wal_backup_partial).
+    // So, need for completed segment backup <=> last backup was at at older segment.
     (state.commit_lsn.segment_number(wal_seg_size) > state.backup_lsn.segment_number(wal_seg_size))
 }
 
@@ -69,6 +71,11 @@ pub(crate) fn is_wal_backup_required(
 /// is me, run (per timeline) task, if not yet. OTOH, if it is not me and task
 /// is running, kill it.
 pub(crate) async fn update_task(mgr: &mut Manager, need_backup: bool, state: &StateSnapshot) {
+    // Based on the peer information received from broker, each safekeeper figures out
+    // whether it, or one of the peers, is the offloader.
+    // The algorithm is deterministic, so, if all peers have the same information,
+    // the system converges. In unconverged state, multiple peers upload the same
+    // segments, which is inefficient but safe.
     let (offloader, election_dbg_str) =
         determine_offloader(&state.peers, state.backup_lsn, mgr.tli.ttid, &mgr.conf);
     let elected_me = Some(mgr.conf.my_id) == offloader;

storage_broker/spec/modelcheck.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# Usage: ./modelcheck.sh <config_file> <spec_file>, e.g.
+# ./modelcheck.sh models/MCProposerAcceptorStatic_p2_a3_t3_l3.cfg MCProposerAcceptorStatic.tla
+# ./modelcheck.sh models/MCProposerAcceptorReconfig_p2_a3_t3_l3_c3.cfg MCProposerAcceptorReconfig.tla
+CONFIG=$1
+SPEC=$2
+
+MEM=4G
+TOOLSPATH=/Applications/TLA+\ Toolbox.app/Contents/Eclipse/tla2tools.jar
+
+mkdir -p "tlc-results"
+CONFIG_FILE=$(basename -- "$CONFIG")
+outfilename="$SPEC-${CONFIG_FILE}-$(date --utc +%Y-%m-%d--%H-%M-%S)".log
+outfile="tlc-results/$outfilename"
+echo "saving results to $outfile"
+touch $outfile
+
+# Save some info about the run.
+GIT_REV=`git rev-parse --short HEAD`
+INFO=`uname -a`
+
+# First for Linux, second for Mac.
+CPUNAMELinux=$(lscpu | grep 'Model name' | cut -f 2 -d ":" | awk '{$1=$1}1')
+CPUCORESLinux=`nproc`
+CPUNAMEMac=`sysctl -n machdep.cpu.brand_string`
+CPUCORESMac=`sysctl -n machdep.cpu.thread_count`
+
+echo "git revision: $GIT_REV" >> $outfile
+echo "Platform: $INFO" >> $outfile
+echo "CPU Info Linux: $CPUNAMELinux" >> $outfile
+echo "CPU Cores Linux: $CPUCORESLinux" >> $outfile
+echo "CPU Info Mac: $CPUNAMEMac" >> $outfile
+echo "CPU Cores Mac: $CPUCORESMac" >> $outfile
+echo "Spec: $SPEC" >> $outfile
+echo "Config: $CONFIG" >> $outfile
+echo "----" >> $outfile
+cat $CONFIG >> $outfile
+echo "" >> $outfile
+echo "----" >> $outfile
+echo "" >> $outfile
+
+# see
+# https://lamport.azurewebsites.net/tla/current-tools.pdf
+# for TLC options.
+# OffHeapDiskFPSet is the optimal fingerprint set implementation
+# https://docs.tlapl.us/codebase:architecture#fingerprint_sets_fpsets
+#
+# Add -simulate to run in infinite simulation mode.
+# -coverage 1 is useful for profiling (check how many times actions are taken).
+java -Xmx$MEM -XX:MaxDirectMemorySize=$MEM -XX:+UseParallelGC -Dtlc2.tool.fp.FPSet.impl=tlc2.tool.fp.OffHeapDiskFPSet \
+  -cp "${TOOLSPATH}" tlc2.TLC $SPEC -config $CONFIG -workers auto -gzip | tee -a $outfile

storage_broker/spec/modelcheck.sh~

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# Usage: ./modelcheck.sh <config_file> <spec_file>, e.g.
+# ./modelcheck.sh models/MCProposerAcceptorStatic_p2_a3_t3_l3.cfg MCProposerAcceptorStatic.tla
+# ./modelcheck.sh models/MCProposerAcceptorReconfig_p2_a3_t3_l3_c3.cfg MCProposerAcceptorReconfig.tla
+CONFIG=$1
+SPEC=$2
+
+MEM=7G
+TOOLSPATH="/opt/TLA+Toolbox/tla2tools.jar"
+
+mkdir -p "tlc-results"
+CONFIG_FILE=$(basename -- "$CONFIG")
+outfilename="$SPEC-${CONFIG_FILE}-$(date --utc +%Y-%m-%d--%H-%M-%S)".log
+outfile="tlc-results/$outfilename"
+echo "saving results to $outfile"
+touch $outfile
+
+# Save some info about the run.
+GIT_REV=`git rev-parse --short HEAD`
+INFO=`uname -a`
+
+# First for Linux, second for Mac.
+CPUNAMELinux=$(lscpu | grep 'Model name' | cut -f 2 -d ":" | awk '{$1=$1}1')
+CPUCORESLinux=`nproc`
+CPUNAMEMac=`sysctl -n machdep.cpu.brand_string`
+CPUCORESMac=`sysctl -n machdep.cpu.thread_count`
+
+echo "git revision: $GIT_REV" >> $outfile
+echo "Platform: $INFO" >> $outfile
+echo "CPU Info Linux: $CPUNAMELinux" >> $outfile
+echo "CPU Cores Linux: $CPUCORESLinux" >> $outfile
+echo "CPU Info Mac: $CPUNAMEMac" >> $outfile
+echo "CPU Cores Mac: $CPUCORESMac" >> $outfile
+echo "Spec: $SPEC" >> $outfile
+echo "Config: $CONFIG" >> $outfile
+echo "----" >> $outfile
+cat $CONFIG >> $outfile
+echo "" >> $outfile
+echo "----" >> $outfile
+echo "" >> $outfile
+
+# see
+# https://lamport.azurewebsites.net/tla/current-tools.pdf
+# for TLC options.
+# OffHeapDiskFPSet is the optimal fingerprint set implementation
+# https://docs.tlapl.us/codebase:architecture#fingerprint_sets_fpsets
+#
+# Add -simulate to run in infinite simulation mode.
+# -coverage 1 is useful for profiling (check how many times actions are taken).
+java -Xmx$MEM -XX:MaxDirectMemorySize=$MEM -XX:+UseParallelGC -Dtlc2.tool.fp.FPSet.impl=tlc2.tool.fp.OffHeapDiskFPSet \
+  -cp "${TOOLSPATH}" tlc2.TLC $SPEC -config $CONFIG -workers auto -gzip | tee -a $outfile

storage_broker/spec/replicated/replicated.tla

@@ -0,0 +1,175 @@
+---- MODULE replicated ----
+
+EXTENDS Integers, FiniteSets
+
+VARIABLES broker_state, safekeeper_state, pageserver_state, online
+
+
+CONSTANT
+    brokers,
+    safekeepers,
+    pageservers,
+    azs,
+    az_mapping
+
+CONSTANT
+    NULL
+    
+CONSTANT
+    max_commit_lsn
+    
+\* HELPERS
+
+Max(a,b) == IF a > b THEN a ELSE b
+
+(* The minimum of a non-empty set of integers *)
+MinOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x <= y
+  
+(* The minimum of a non-empty set of integers *)
+MaxOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x >= y
+
+\* END HELPERS
+
+StateConstraint ==
+    /\ \A s \in safekeepers: 
+        /\ safekeeper_state[s].commit_lsn <= max_commit_lsn
+    /\ \A b \in brokers:
+        /\ \A s \in DOMAIN broker_state[b].sk:
+            /\ broker_state[b].sk[s].commit_lsn <= max_commit_lsn
+
+
+ 
+InitSafekeeper == [prune_lsn |-> 0, commit_lsn |-> 0, rx |-> [b \in brokers |-> NULL] ]
+InitBroker == [sk |-> [s \in safekeepers |-> [prune_lsn |-> 0, commit_lsn |-> 0]]]
+InitPageserver == [last_record_lsn |-> 0, preferred_sk |-> NULL, sk |-> [s \in safekeepers |-> [commit_lsn |-> 0]]]
+InitOnline == safekeepers \cup brokers \cup pageservers
+
+Init ==
+    /\ broker_state = [b \in brokers |-> InitBroker]
+    /\ safekeeper_state = [s \in safekeepers |-> InitSafekeeper]
+    /\ pageserver_state = [p \in pageservers |-> InitPageserver]
+    /\ online = InitOnline
+  
+NodeOnlineOffline ==
+    /\ online' = CHOOSE ss \in SUBSET InitOnline: 
+        /\ Cardinality(ss \cap safekeepers) >= 2
+        /\ Cardinality(ss \cap brokers) >= 2
+        /\ ss \cap pageservers = pageservers \* assume no PS failures for now
+    /\ UNCHANGED <<safekeeper_state,broker_state,pageserver_state>>
+   
+SkCommit(s1, s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ s1 # s2
+    /\ safekeeper_state[s1].commit_lsn = safekeeper_state[s2].commit_lsn
+    /\ LET
+            new_commit_lsn == safekeeper_state[s1].commit_lsn + 1
+       IN
+            safekeeper_state' = [safekeeper_state EXCEPT
+                ![s1].commit_lsn = new_commit_lsn,
+                ![s2].commit_lsn = new_commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPeerRecovery(s1,s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ safekeeper_state[s1].commit_lsn < safekeeper_state[s2].commit_lsn \* s2 has more WAL than s1
+    /\ safekeeper_state[s2].prune_lsn < safekeeper_state[s1].commit_lsn \* s2 has not yet trimmed the WAL the WAL
+    /\ safekeeper_state' = [safekeeper_state EXCEPT![s1].commit_lsn = safekeeper_state[s2].commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPushToBroker(s,b) ==
+    /\ {s, b} \subseteq online 
+    /\ broker_state' = LET
+           broker_side == broker_state[b].sk[s]
+           sk_side == safekeeper_state[s]
+           merged == [broker_side EXCEPT
+                !["commit_lsn"] = Max(broker_side.commit_lsn, sk_side.commit_lsn),
+                !["prune_lsn"] = Max(broker_side.prune_lsn, sk_side.prune_lsn)]
+           IN
+           [broker_state EXCEPT ![b].sk[s] = merged]
+    /\ UNCHANGED <<safekeeper_state, pageserver_state,online>> 
+
+PsRecvBroker(b,p,s) ==
+    /\ {b,p,s} \subseteq online
+    /\ LET
+            bsk == broker_state[b].sk[s]
+            psk == pageserver_state[p].sk[s]
+            updpsk == [psk EXCEPT !["commit_lsn"] = bsk.commit_lsn]
+       IN
+            pageserver_state' = IF bsk.commit_lsn > psk.commit_lsn
+                THEN [pageserver_state EXCEPT ![p].sk[s] = updpsk]
+                ELSE pageserver_state
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>
+    
+    
+SkRecvBroker(b,s) ==
+    /\ {b,s} \subseteq online
+    /\ safekeeper_state' = [safekeeper_state EXCEPT ![s].rx[b] = broker_state[b]] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+
+SkPrune(s) ==
+    /\ {s} \subseteq online
+    /\ LET
+        sk == safekeeper_state[s]
+        skis == {<<b,DOMAIN bi.sk>>: <<b,bi>> \in {<<b,bi>> \in { <<b,sk.rx[b]>>: b \in sk.rx }: bi # NULL} }
+        
+       IN
+        safekeeper_state' = safekeeper_state   
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+    
+
+SksWithNewerWal(p) ==
+    LET
+        ps == pageserver_state[p]
+    IN
+    {s \in DOMAIN ps.sk: ps.sk[s].commit_lsn > ps.last_record_lsn}
+
+PsChooseSk(p) ==
+    /\ {p} \subseteq online
+    /\ SksWithNewerWal(p) # {}
+    /\ pageserver_state' = [pageserver_state EXCEPT![p].preferred_sk = CHOOSE s \in SksWithNewerWal(p): TRUE]
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>    
+                 
+    
+Next ==
+    \/ NodeOnlineOffline
+    \/ \E s1 \in safekeepers: \E s2 \in safekeepers:
+        \/ SkCommit(s1, s2)
+        \/ SkPeerRecovery(s1, s2)
+    \/ \E s \in safekeepers: \E b \in brokers:
+        \/ SkPushToBroker(s, b)
+        \/ SkRecvBroker(b, s)
+    \/ \E s \in safekeepers:
+        \/ SkPrune(s)  
+    \/ \E s \in safekeepers: \E b \in brokers: \E p \in pageservers: PsRecvBroker(b,p,s)
+    \/ \E p \in pageservers: PsChooseSk(p)
+    
+
+Spec == Init /\ [][Next]_<< broker_state, safekeeper_state, pageserver_state,online>>
+
+
+\* invariants
+
+PsLagsSk == \A p \in pageservers: \A s \in DOMAIN pageserver_state[p].sk: 
+        /\ pageserver_state[p].sk[s].commit_lsn <= safekeeper_state[s].commit_lsn
+
+PeerRecoveryIsPossible ==
+    \A laggard \in (safekeepers \cap online): \E donor \in (safekeepers \cap online):
+        /\ (safekeeper_state[laggard].commit_lsn < safekeeper_state[donor].commit_lsn)
+            => safekeeper_state[donor].prune_lsn <= safekeeper_state[laggard].commit_lsn
+        
+
+EventuallyLaggingSkIsNotPreferredSk == <>(
+        LET
+            sks == safekeeper_state
+            lagging_sks == { s \in safekeepers: \A s2 \in safekeepers: sks[s].commit_lsn <= sks[s2].commit_lsn }
+            preferred_sks == {pageserver_state[p].preferred_sk: p \in pageservers}
+        IN
+            preferred_sks \cap lagging_sks = {}
+            
+    )
+
+    
+
+====

storage_broker/spec/replicated/replicated.toolbox/.project

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>replicated</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>toolbox.builder.TLAParserBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>toolbox.natures.TLANature</nature>
+	</natures>
+	<linkedResources>
+		<link>
+			<name>replicated.tla</name>
+			<type>1</type>
+			<locationURI>PARENT-1-PROJECT_LOC/replicated.tla</locationURI>
+		</link>
+	</linkedResources>
+</projectDescription>

storage_broker/spec/replicated/replicated.toolbox/.settings/org.lamport.tla.toolbox.prefs

@@ -0,0 +1,2 @@
+ProjectRootFile=PARENT-1-PROJECT_LOC/replicated.tla
+eclipse.preferences.version=1

storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.cfg

@@ -0,0 +1,54 @@
+\* MV CONSTANT declarations
+CONSTANTS
+b1 = b1
+b2 = b2
+b3 = b3
+\* MV CONSTANT declarations
+CONSTANTS
+s1 = s1
+s2 = s2
+s3 = s3
+\* MV CONSTANT declarations
+CONSTANTS
+p1 = p1
+\* MV CONSTANT declarations
+CONSTANTS
+az1 = az1
+az2 = az2
+az3 = az3
+\* MV CONSTANT definitions
+CONSTANT
+brokers <- const_1737327160682382000
+\* CONSTANT declarations
+CONSTANT NULL = NULL
+\* MV CONSTANT definitions
+CONSTANT
+safekeepers <- const_1737327160682383000
+\* MV CONSTANT definitions
+CONSTANT
+pageservers <- const_1737327160682384000
+\* MV CONSTANT definitions
+CONSTANT
+azs <- const_1737327160682385000
+\* SYMMETRY definition
+SYMMETRY symm_1737327160682386000
+\* CONSTANT definitions
+CONSTANT
+max_commit_lsn <- const_1737327160682387000
+\* CONSTANT definitions
+CONSTANT
+az_mapping <- const_1737327160682388000
+\* CONSTRAINT definition
+CONSTRAINT
+constr_1737327160682389000
+\* SPECIFICATION definition
+SPECIFICATION
+Spec
+\* INVARIANT definition
+INVARIANT
+PsLagsSk
+PeerRecoveryIsPossible
+\* PROPERTY definition
+PROPERTY
+EventuallyLaggingSkIsNotPreferredSk
+\* Generated on Sun Jan 19 23:52:40 CET 2025

storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.out

@@ -0,0 +1,53 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 121 and seed -1130553646976177579 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 25330] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 23:52:41)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 23:52:43.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 2192:0 @!@!@
+Checking temporal properties for the current state space with 1627 total distinct states at (2025-01-19 23:52:46)
+@!@!@ENDMSG 2192 @!@!@
+@!@!@STARTMSG 2267:0 @!@!@
+Finished checking temporal properties in 00s at 2025-01-19 23:52:46
+@!@!@ENDMSG 2267 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(9) at 2025-01-19 23:52:46: 45.678 states generated (45.678 s/min), 7.117 distinct states found (7.117 ds/min), 5.486 states left on queue.
+@!@!@ENDMSG 2200 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla

@@ -0,0 +1,65 @@
+---- MODULE MC ----
+EXTENDS replicated, TLC
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+b1, b2, b3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+s1, s2, s3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+p1
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+az1, az2, az3
+----
+
+\* MV CONSTANT definitions brokers
+const_1737327160682382000 == 
+{b1, b2, b3}
+----
+
+\* MV CONSTANT definitions safekeepers
+const_1737327160682383000 == 
+{s1, s2, s3}
+----
+
+\* MV CONSTANT definitions pageservers
+const_1737327160682384000 == 
+{p1}
+----
+
+\* MV CONSTANT definitions azs
+const_1737327160682385000 == 
+{az1, az2, az3}
+----
+
+\* SYMMETRY definition
+symm_1737327160682386000 == 
+Permutations(const_1737327160682382000) \union Permutations(const_1737327160682383000) \union Permutations(const_1737327160682384000) \union Permutations(const_1737327160682385000)
+----
+
+\* CONSTANT definitions @modelParameterConstants:4max_commit_lsn
+const_1737327160682387000 == 
+2
+----
+
+\* CONSTANT definitions @modelParameterConstants:5az_mapping
+const_1737327160682388000 == 
+[ az1 |-> {b1,s1,p1} , az2 |-> {b2,s2} , az3 |-> {b3,s3}]
+----
+
+\* CONSTRAINT definition @modelParameterContraint:0
+constr_1737327160682389000 ==
+StateConstraint
+----
+=============================================================================
+\* Modification History
+\* Created Sun Jan 19 23:52:40 CET 2025 by cs

storage_broker/spec/replicated/replicated.toolbox/Model_1/MC_TE.out

@@ -0,0 +1,53 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 121 and seed -1130553646976177579 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 25330] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 23:52:41)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 23:52:43.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 2192:0 @!@!@
+Checking temporal properties for the current state space with 1627 total distinct states at (2025-01-19 23:52:46)
+@!@!@ENDMSG 2192 @!@!@
+@!@!@STARTMSG 2267:0 @!@!@
+Finished checking temporal properties in 00s at 2025-01-19 23:52:46
+@!@!@ENDMSG 2267 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(9) at 2025-01-19 23:52:46: 45.678 states generated (45.678 s/min), 7.117 distinct states found (7.117 ds/min), 5.486 states left on queue.
+@!@!@ENDMSG 2200 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla

@@ -0,0 +1,174 @@
+---- MODULE replicated ----
+
+EXTENDS Integers, FiniteSets
+
+VARIABLES broker_state, safekeeper_state, pageserver_state, online
+
+
+CONSTANT
+    brokers,
+    safekeepers,
+    pageservers,
+    azs,
+    az_mapping
+
+CONSTANT
+    NULL
+    
+CONSTANT
+    max_commit_lsn
+    
+\* HELPERS
+
+Max(a,b) == IF a > b THEN a ELSE b
+
+(* The minimum of a non-empty set of integers *)
+MinOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x <= y
+  
+(* The minimum of a non-empty set of integers *)
+MaxOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x >= y
+
+\* END HELPERS
+
+StateConstraint ==
+    /\ \A s \in safekeepers: 
+        /\ safekeeper_state[s].commit_lsn <= max_commit_lsn
+    /\ \A b \in brokers:
+        /\ \A s \in DOMAIN broker_state[b].sk:
+            /\ broker_state[b].sk[s].commit_lsn <= max_commit_lsn
+
+
+ 
+InitSafekeeper == [prune_lsn |-> 0, commit_lsn |-> 0, rx |-> [b \in brokers |-> NULL] ]
+InitBroker == [sk |-> [s \in safekeepers |-> [prune_lsn |-> 0, commit_lsn |-> 0]]]
+InitPageserver == [last_record_lsn |-> 0, preferred_sk |-> NULL, sk |-> [s \in safekeepers |-> [commit_lsn |-> 0]]]
+InitOnline == safekeepers \cup brokers \cup pageservers
+
+Init ==
+    /\ broker_state = [b \in brokers |-> InitBroker]
+    /\ safekeeper_state = [s \in safekeepers |-> InitSafekeeper]
+    /\ pageserver_state = [p \in pageservers |-> InitPageserver]
+    /\ online = InitOnline
+  
+NodeOnlineOffline ==
+    /\ online' = CHOOSE ss \in SUBSET InitOnline: 
+        /\ Cardinality(ss \cap safekeepers) >= 2
+        /\ Cardinality(ss \cap brokers) >= 2
+        /\ ss \cap pageservers = pageservers \* assume no PS failures for now
+    /\ UNCHANGED <<safekeeper_state,broker_state,pageserver_state>>
+   
+SkCommit(s1, s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ s1 # s2
+    /\ safekeeper_state[s1].commit_lsn = safekeeper_state[s2].commit_lsn
+    /\ LET
+            new_commit_lsn == safekeeper_state[s1].commit_lsn + 1
+       IN
+            safekeeper_state' = [safekeeper_state EXCEPT
+                ![s1].commit_lsn = new_commit_lsn,
+                ![s2].commit_lsn = new_commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPeerRecovery(s1,s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ safekeeper_state[s1].commit_lsn < safekeeper_state[s2].commit_lsn \* s2 has more WAL than s1
+    /\ safekeeper_state[s2].prune_lsn < safekeeper_state[s1].commit_lsn \* s2 has not yet trimmed the WAL the WAL
+    /\ safekeeper_state' = [safekeeper_state EXCEPT![s1].commit_lsn = safekeeper_state[s2].commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPushToBroker(s,b) ==
+    /\ {s, b} \subseteq online 
+    /\ broker_state' = LET
+           broker_side == broker_state[b].sk[s]
+           sk_side == safekeeper_state[s]
+           merged == [broker_side EXCEPT
+                !["commit_lsn"] = Max(broker_side.commit_lsn, sk_side.commit_lsn),
+                !["prune_lsn"] = Max(broker_side.prune_lsn, sk_side.prune_lsn)]
+           IN
+           [broker_state EXCEPT ![b].sk[s] = merged]
+    /\ UNCHANGED <<safekeeper_state, pageserver_state,online>> 
+
+PsRecvBroker(b,p,s) ==
+    /\ {b,p,s} \subseteq online
+    /\ LET
+            bsk == broker_state[b].sk[s]
+            psk == pageserver_state[p].sk[s]
+            updpsk == [psk EXCEPT !["commit_lsn"] = bsk.commit_lsn]
+       IN
+            pageserver_state' = IF bsk.commit_lsn > psk.commit_lsn
+                THEN [pageserver_state EXCEPT ![p].sk[s] = updpsk]
+                ELSE pageserver_state
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>
+    
+    
+SkRecvBroker(b,s) ==
+    /\ {b,s} \subseteq online
+    /\ safekeeper_state' = [safekeeper_state EXCEPT ![s].rx[b] = broker_state[b]] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+
+SkPrune(s) ==
+    /\ {s} \subseteq online
+    /\ LET
+        sk == safekeeper_state[s]
+        skis == {<<b,bi.sk>>: <<b,bi>> \in {<<b,bi>> \in { <<b,sk.rx[b]>>: b \in sk.rx }: bi # NULL} }
+       IN
+        safekeeper_state' = safekeeper_state   
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+    
+
+SksWithNewerWal(p) ==
+    LET
+        ps == pageserver_state[p]
+    IN
+    {s \in DOMAIN ps.sk: ps.sk[s].commit_lsn > ps.last_record_lsn}
+
+PsChooseSk(p) ==
+    /\ {p} \subseteq online
+    /\ SksWithNewerWal(p) # {}
+    /\ pageserver_state' = [pageserver_state EXCEPT![p].preferred_sk = CHOOSE s \in SksWithNewerWal(p): TRUE]
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>    
+                 
+    
+Next ==
+    \/ NodeOnlineOffline
+    \/ \E s1 \in safekeepers: \E s2 \in safekeepers:
+        \/ SkCommit(s1, s2)
+        \/ SkPeerRecovery(s1, s2)
+    \/ \E s \in safekeepers: \E b \in brokers:
+        \/ SkPushToBroker(s, b)
+        \/ SkRecvBroker(b, s)
+    \/ \E s \in safekeepers:
+        \/ SkPrune(s)  
+    \/ \E s \in safekeepers: \E b \in brokers: \E p \in pageservers: PsRecvBroker(b,p,s)
+    \/ \E p \in pageservers: PsChooseSk(p)
+    
+
+Spec == Init /\ [][Next]_<< broker_state, safekeeper_state, pageserver_state,online>>
+
+
+\* invariants
+
+PsLagsSk == \A p \in pageservers: \A s \in DOMAIN pageserver_state[p].sk: 
+        /\ pageserver_state[p].sk[s].commit_lsn <= safekeeper_state[s].commit_lsn
+
+PeerRecoveryIsPossible ==
+    \A laggard \in (safekeepers \cap online): \E donor \in (safekeepers \cap online):
+        /\ (safekeeper_state[laggard].commit_lsn < safekeeper_state[donor].commit_lsn)
+            => safekeeper_state[donor].prune_lsn <= safekeeper_state[laggard].commit_lsn
+        
+
+EventuallyLaggingSkIsNotPreferredSk == <>(
+        LET
+            sks == safekeeper_state
+            lagging_sks == { s \in safekeepers: \A s2 \in safekeepers: sks[s].commit_lsn <= sks[s2].commit_lsn }
+            preferred_sks == {pageserver_state[p].preferred_sk: p \in pageservers}
+        IN
+            preferred_sks \cap lagging_sks = {}
+            
+    )
+
+    
+
+====

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737323805723/MC.cfg

@@ -0,0 +1,54 @@
+\* MV CONSTANT declarations
+CONSTANTS
+b1 = b1
+b2 = b2
+b3 = b3
+\* MV CONSTANT declarations
+CONSTANTS
+s1 = s1
+s2 = s2
+s3 = s3
+\* MV CONSTANT declarations
+CONSTANTS
+p1 = p1
+\* MV CONSTANT declarations
+CONSTANTS
+az1 = az1
+az2 = az2
+az3 = az3
+\* MV CONSTANT definitions
+CONSTANT
+brokers <- const_1737323801657283000
+\* CONSTANT declarations
+CONSTANT NULL = NULL
+\* MV CONSTANT definitions
+CONSTANT
+safekeepers <- const_1737323801657284000
+\* MV CONSTANT definitions
+CONSTANT
+pageservers <- const_1737323801657285000
+\* MV CONSTANT definitions
+CONSTANT
+azs <- const_1737323801657286000
+\* SYMMETRY definition
+SYMMETRY symm_1737323801657287000
+\* CONSTANT definitions
+CONSTANT
+max_commit_lsn <- const_1737323801657288000
+\* CONSTANT definitions
+CONSTANT
+az_mapping <- const_1737323801657289000
+\* CONSTRAINT definition
+CONSTRAINT
+constr_1737323801657290000
+\* SPECIFICATION definition
+SPECIFICATION
+Spec
+\* INVARIANT definition
+INVARIANT
+PsLagsSk
+PeerRecoveryIsPossible
+\* PROPERTY definition
+PROPERTY
+EventuallyLaggingSkIsNotPreferredSk
+\* Generated on Sun Jan 19 22:56:41 CET 2025

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737323805723/MC.out

@@ -0,0 +1,353 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 8 and seed -3590178470709523625 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 22966] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 22:56:42)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 22:56:44.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 2109:1 @!@!@
+Successor state is not completely specified by action SkPrune of the next-state relation. The following variables are not assigned: broker_state, online, pageserver_state.
+
+@!@!@ENDMSG 2109 @!@!@
+@!@!@STARTMSG 2121:1 @!@!@
+The behavior up to this point is:
+@!@!@ENDMSG 2121 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+1: <Initial predicate>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, b3, s1, s2, s3, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+2: <NodeOnlineOffline line 52, col 5 to line 56, col 67 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+3: <SkCommit line 59, col 5 to line 68, col 58 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+4: <SkPrune line 108, col 5 to line 121, col 80 of module replicated>
+/\ pageserver_state = null
+/\ broker_state = null
+/\ online = null
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 1, prune_lsn |-> 1, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2201:0 @!@!@
+The coverage statistics at 2025-01-19 22:56:44
+@!@!@ENDMSG 2201 @!@!@
+@!@!@STARTMSG 2773:0 @!@!@
+<Init line 45, col 1 to line 45, col 4 of module replicated>: 2:2
+@!@!@ENDMSG 2773 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 46, col 5 to line 49, col 26 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<NodeOnlineOffline line 51, col 1 to line 51, col 17 of module replicated>: 1:7
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 52, col 8 to line 55, col 44 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 52, col 18 to line 55, col 44 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 53, col 9 to line 55, col 44 of module replicated: 749
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 53, col 12 to line 53, col 48 of module replicated: 749
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 54, col 12 to line 54, col 44 of module replicated: 301
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 55, col 12 to line 55, col 44 of module replicated: 98
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 52, col 32 to line 52, col 48 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 56, col 8 to line 56, col 67 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkCommit line 58, col 1 to line 58, col 16 of module replicated>: 4:15
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 59, col 8 to line 59, col 32 of module replicated: 62
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 59, col 8 to line 59, col 15 of module replicated: 47:110
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 59, col 27 to line 59, col 32 of module replicated: 47
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 60, col 8 to line 60, col 14 of module replicated: 47
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 8 to line 60, col 9 of module replicated: 32
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 13 to line 60, col 14 of module replicated: 32
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 61, col 8 to line 61, col 72 of module replicated: 34
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 8 to line 61, col 38 of module replicated: 19
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 42 to line 61, col 72 of module replicated: 19
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 65, col 13 to line 67, col 50 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 68, col 8 to line 68, col 58 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPeerRecovery line 70, col 1 to line 70, col 21 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 71, col 8 to line 71, col 32 of module replicated: 46
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 72, col 8 to line 72, col 72 of module replicated: 31
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 73, col 8 to line 73, col 71 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 74, col 8 to line 74, col 102 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 75, col 8 to line 75, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPushToBroker line 77, col 1 to line 77, col 19 of module replicated>: 2:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 78, col 8 to line 78, col 30 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 78, col 8 to line 78, col 13 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 78, col 25 to line 78, col 30 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 79, col 8 to line 86, col 52 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 87, col 8 to line 87, col 62 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsRecvBroker line 89, col 1 to line 89, col 19 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 96, col 13 to line 98, col 37 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 99, col 8 to line 99, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkRecvBroker line 102, col 1 to line 102, col 17 of module replicated>: 5:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 103, col 8 to line 103, col 29 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 103, col 8 to line 103, col 12 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 103, col 24 to line 103, col 29 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 104, col 8 to line 104, col 81 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 105, col 8 to line 105, col 57 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPrune line 107, col 1 to line 107, col 10 of module replicated>: 0:5
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 108, col 8 to line 108, col 27 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 108, col 8 to line 108, col 10 of module replicated: 5:10
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 108, col 22 to line 108, col 27 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 121, col 9 to line 121, col 80 of module replicated: 1
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 121, col 29 to line 121, col 80 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 121, col 30 to line 121, col 45 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 121, col 54 to line 121, col 79 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 121, col 71 to line 121, col 79 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 119, col 22 to line 119, col 42 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 27, col 3 to line 27, col 36 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 119, col 31 to line 119, col 41 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||line 118, col 24 to line 118, col 49 of module replicated: 5:15
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||line 118, col 25 to line 118, col 36 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||line 118, col 45 to line 118, col 48 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||line 111, col 13 to line 117, col 18 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 112, col 17 to line 114, col 84 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||line 112, col 17 to line 112, col 37 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||||||line 114, col 17 to line 114, col 84 of module replicated: 5:10
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 114, col 18 to line 114, col 43 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 114, col 54 to line 114, col 82 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 117, col 13 to line 117, col 18 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsChooseSk line 130, col 1 to line 130, col 13 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 133, col 8 to line 133, col 109 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 134, col 8 to line 134, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PsLagsSk line 156, col 1 to line 156, col 8 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 156, col 13 to line 157, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 156, col 35 to line 157, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 157, col 9 to line 157, col 81 of module replicated: 39
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 156, col 44 to line 156, col 72 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 156, col 22 to line 156, col 32 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PeerRecoveryIsPossible line 159, col 1 to line 159, col 22 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 160, col 5 to line 162, col 88 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 160, col 47 to line 162, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 161, col 9 to line 162, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 161, col 12 to line 162, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 161, col 13 to line 161, col 85 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 162, col 16 to line 162, col 88 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 160, col 61 to line 160, col 83 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 160, col 21 to line 160, col 43 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2202:0 @!@!@
+End of statistics.
+@!@!@ENDMSG 2202 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(4) at 2025-01-19 22:56:44: 73 states generated (1.752 s/min), 13 distinct states found (312 ds/min), 9 states left on queue.
+@!@!@ENDMSG 2200 @!@!@
+@!@!@STARTMSG 2199:0 @!@!@
+73 states generated, 13 distinct states found, 9 states left on queue.
+@!@!@ENDMSG 2199 @!@!@
+@!@!@STARTMSG 2194:0 @!@!@
+The depth of the complete state graph search is 4.
+@!@!@ENDMSG 2194 @!@!@
+@!@!@STARTMSG 2186:0 @!@!@
+Finished in 2516ms at (2025-01-19 22:56:44)
+@!@!@ENDMSG 2186 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737323805723/MC.tla

@@ -0,0 +1,65 @@
+---- MODULE MC ----
+EXTENDS replicated, TLC
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+b1, b2, b3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+s1, s2, s3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+p1
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+az1, az2, az3
+----
+
+\* MV CONSTANT definitions brokers
+const_1737323801657283000 == 
+{b1, b2, b3}
+----
+
+\* MV CONSTANT definitions safekeepers
+const_1737323801657284000 == 
+{s1, s2, s3}
+----
+
+\* MV CONSTANT definitions pageservers
+const_1737323801657285000 == 
+{p1}
+----
+
+\* MV CONSTANT definitions azs
+const_1737323801657286000 == 
+{az1, az2, az3}
+----
+
+\* SYMMETRY definition
+symm_1737323801657287000 == 
+Permutations(const_1737323801657283000) \union Permutations(const_1737323801657284000) \union Permutations(const_1737323801657285000) \union Permutations(const_1737323801657286000)
+----
+
+\* CONSTANT definitions @modelParameterConstants:4max_commit_lsn
+const_1737323801657288000 == 
+2
+----
+
+\* CONSTANT definitions @modelParameterConstants:5az_mapping
+const_1737323801657289000 == 
+[ az1 |-> {b1,s1,p1} , az2 |-> {b2,s2} , az3 |-> {b3,s3}]
+----
+
+\* CONSTRAINT definition @modelParameterContraint:0
+constr_1737323801657290000 ==
+StateConstraint
+----
+=============================================================================
+\* Modification History
+\* Created Sun Jan 19 22:56:41 CET 2025 by cs

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737323805723/MC_TE.out

@@ -0,0 +1,353 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 8 and seed -3590178470709523625 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 22966] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 22:56:42)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 22:56:44.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 2109:1 @!@!@
+Successor state is not completely specified by action SkPrune of the next-state relation. The following variables are not assigned: broker_state, online, pageserver_state.
+
+@!@!@ENDMSG 2109 @!@!@
+@!@!@STARTMSG 2121:1 @!@!@
+The behavior up to this point is:
+@!@!@ENDMSG 2121 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+1: <Initial predicate>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, b3, s1, s2, s3, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+2: <NodeOnlineOffline line 52, col 5 to line 56, col 67 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+3: <SkCommit line 59, col 5 to line 68, col 58 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+4: <SkPrune line 108, col 5 to line 121, col 80 of module replicated>
+/\ pageserver_state = null
+/\ broker_state = null
+/\ online = null
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 1, prune_lsn |-> 1, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2201:0 @!@!@
+The coverage statistics at 2025-01-19 22:56:44
+@!@!@ENDMSG 2201 @!@!@
+@!@!@STARTMSG 2773:0 @!@!@
+<Init line 45, col 1 to line 45, col 4 of module replicated>: 2:2
+@!@!@ENDMSG 2773 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 46, col 5 to line 49, col 26 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<NodeOnlineOffline line 51, col 1 to line 51, col 17 of module replicated>: 1:7
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 52, col 8 to line 55, col 44 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 52, col 18 to line 55, col 44 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 53, col 9 to line 55, col 44 of module replicated: 749
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 53, col 12 to line 53, col 48 of module replicated: 749
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 54, col 12 to line 54, col 44 of module replicated: 301
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 55, col 12 to line 55, col 44 of module replicated: 98
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 52, col 32 to line 52, col 48 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 56, col 8 to line 56, col 67 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkCommit line 58, col 1 to line 58, col 16 of module replicated>: 4:15
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 59, col 8 to line 59, col 32 of module replicated: 62
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 59, col 8 to line 59, col 15 of module replicated: 47:110
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 59, col 27 to line 59, col 32 of module replicated: 47
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 60, col 8 to line 60, col 14 of module replicated: 47
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 8 to line 60, col 9 of module replicated: 32
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 13 to line 60, col 14 of module replicated: 32
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 61, col 8 to line 61, col 72 of module replicated: 34
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 8 to line 61, col 38 of module replicated: 19
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 42 to line 61, col 72 of module replicated: 19
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 65, col 13 to line 67, col 50 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 68, col 8 to line 68, col 58 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPeerRecovery line 70, col 1 to line 70, col 21 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 71, col 8 to line 71, col 32 of module replicated: 46
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 72, col 8 to line 72, col 72 of module replicated: 31
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 73, col 8 to line 73, col 71 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 74, col 8 to line 74, col 102 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 75, col 8 to line 75, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPushToBroker line 77, col 1 to line 77, col 19 of module replicated>: 2:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 78, col 8 to line 78, col 30 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 78, col 8 to line 78, col 13 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 78, col 25 to line 78, col 30 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 79, col 8 to line 86, col 52 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 87, col 8 to line 87, col 62 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsRecvBroker line 89, col 1 to line 89, col 19 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 96, col 13 to line 98, col 37 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 99, col 8 to line 99, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkRecvBroker line 102, col 1 to line 102, col 17 of module replicated>: 5:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 103, col 8 to line 103, col 29 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 103, col 8 to line 103, col 12 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 103, col 24 to line 103, col 29 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 104, col 8 to line 104, col 81 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 105, col 8 to line 105, col 57 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPrune line 107, col 1 to line 107, col 10 of module replicated>: 0:5
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 108, col 8 to line 108, col 27 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 108, col 8 to line 108, col 10 of module replicated: 5:10
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 108, col 22 to line 108, col 27 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 121, col 9 to line 121, col 80 of module replicated: 1
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 121, col 29 to line 121, col 80 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 121, col 30 to line 121, col 45 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 121, col 54 to line 121, col 79 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 121, col 71 to line 121, col 79 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 119, col 22 to line 119, col 42 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 27, col 3 to line 27, col 36 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 119, col 31 to line 119, col 41 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||line 118, col 24 to line 118, col 49 of module replicated: 5:15
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||line 118, col 25 to line 118, col 36 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||line 118, col 45 to line 118, col 48 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||line 111, col 13 to line 117, col 18 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 112, col 17 to line 114, col 84 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||line 112, col 17 to line 112, col 37 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||||||line 114, col 17 to line 114, col 84 of module replicated: 5:10
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 114, col 18 to line 114, col 43 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 114, col 54 to line 114, col 82 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 117, col 13 to line 117, col 18 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsChooseSk line 130, col 1 to line 130, col 13 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 133, col 8 to line 133, col 109 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 134, col 8 to line 134, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PsLagsSk line 156, col 1 to line 156, col 8 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 156, col 13 to line 157, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 156, col 35 to line 157, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 157, col 9 to line 157, col 81 of module replicated: 39
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 156, col 44 to line 156, col 72 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 156, col 22 to line 156, col 32 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PeerRecoveryIsPossible line 159, col 1 to line 159, col 22 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 160, col 5 to line 162, col 88 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 160, col 47 to line 162, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 161, col 9 to line 162, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 161, col 12 to line 162, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 161, col 13 to line 161, col 85 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 162, col 16 to line 162, col 88 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 160, col 61 to line 160, col 83 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 160, col 21 to line 160, col 43 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2202:0 @!@!@
+End of statistics.
+@!@!@ENDMSG 2202 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(4) at 2025-01-19 22:56:44: 73 states generated (1.752 s/min), 13 distinct states found (312 ds/min), 9 states left on queue.
+@!@!@ENDMSG 2200 @!@!@
+@!@!@STARTMSG 2199:0 @!@!@
+73 states generated, 13 distinct states found, 9 states left on queue.
+@!@!@ENDMSG 2199 @!@!@
+@!@!@STARTMSG 2194:0 @!@!@
+The depth of the complete state graph search is 4.
+@!@!@ENDMSG 2194 @!@!@
+@!@!@STARTMSG 2186:0 @!@!@
+Finished in 2516ms at (2025-01-19 22:56:44)
+@!@!@ENDMSG 2186 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737323805723/replicated.tla

@@ -0,0 +1,177 @@
+---- MODULE replicated ----
+
+EXTENDS Integers, FiniteSets
+
+VARIABLES broker_state, safekeeper_state, pageserver_state, online
+
+
+CONSTANT
+    brokers,
+    safekeepers,
+    pageservers,
+    azs,
+    az_mapping
+
+CONSTANT
+    NULL
+    
+CONSTANT
+    max_commit_lsn
+    
+\* HELPERS
+
+Max(a,b) == IF a > b THEN a ELSE b
+
+(* The minimum of a non-empty set of integers *)
+MinOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x <= y
+
+\* END HELPERS
+
+StateConstraint ==
+    /\ \A s \in safekeepers: 
+        /\ safekeeper_state[s].commit_lsn <= max_commit_lsn
+    /\ \A b \in brokers:
+        /\ \A s \in DOMAIN broker_state[b].sk:
+            /\ broker_state[b].sk[s].commit_lsn <= max_commit_lsn
+
+
+ 
+InitSafekeeper == [prune_lsn |-> 0, commit_lsn |-> 0, rx |-> [b \in brokers |-> NULL] ]
+InitBroker == [sk |-> [s \in safekeepers |-> [prune_lsn |-> 0, commit_lsn |-> 0]]]
+InitPageserver == [last_record_lsn |-> 0, preferred_sk |-> NULL, sk |-> [s \in safekeepers |-> [commit_lsn |-> 0]]]
+InitOnline == safekeepers \cup brokers \cup pageservers
+
+Init ==
+    /\ broker_state = [b \in brokers |-> InitBroker]
+    /\ safekeeper_state = [s \in safekeepers |-> InitSafekeeper]
+    /\ pageserver_state = [p \in pageservers |-> InitPageserver]
+    /\ online = InitOnline
+  
+NodeOnlineOffline ==
+    /\ online' = CHOOSE ss \in SUBSET InitOnline: 
+        /\ Cardinality(ss \cap safekeepers) >= 2
+        /\ Cardinality(ss \cap brokers) >= 2
+        /\ ss \cap pageservers = pageservers \* assume no PS failures for now
+    /\ UNCHANGED <<safekeeper_state,broker_state,pageserver_state>>
+   
+SkCommit(s1, s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ s1 # s2
+    /\ safekeeper_state[s1].commit_lsn = safekeeper_state[s2].commit_lsn
+    /\ LET
+            new_commit_lsn == safekeeper_state[s1].commit_lsn + 1
+       IN
+            safekeeper_state' = [safekeeper_state EXCEPT
+                ![s1].commit_lsn = new_commit_lsn,
+                ![s2].commit_lsn = new_commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPeerRecovery(s1,s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ safekeeper_state[s1].commit_lsn < safekeeper_state[s2].commit_lsn \* s2 has more WAL than s1
+    /\ safekeeper_state[s2].prune_lsn < safekeeper_state[s1].commit_lsn \* s2 has not yet trimmed the WAL the WAL
+    /\ safekeeper_state' = [safekeeper_state EXCEPT![s1].commit_lsn = safekeeper_state[s2].commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPushToBroker(s,b) ==
+    /\ {s, b} \subseteq online 
+    /\ broker_state' = LET
+           broker_side == broker_state[b].sk[s]
+           sk_side == safekeeper_state[s]
+           merged == [broker_side EXCEPT
+                !["commit_lsn"] = Max(broker_side.commit_lsn, sk_side.commit_lsn),
+                !["prune_lsn"] = Max(broker_side.prune_lsn, sk_side.prune_lsn)]
+           IN
+           [broker_state EXCEPT ![b].sk[s] = merged]
+    /\ UNCHANGED <<safekeeper_state, pageserver_state,online>> 
+
+PsRecvBroker(b,p,s) ==
+    /\ {b,p,s} \subseteq online
+    /\ LET
+            bsk == broker_state[b].sk[s]
+            psk == pageserver_state[p].sk[s]
+            updpsk == [psk EXCEPT !["commit_lsn"] = bsk.commit_lsn]
+       IN
+            pageserver_state' = IF bsk.commit_lsn > psk.commit_lsn
+                THEN [pageserver_state EXCEPT ![p].sk[s] = updpsk]
+                ELSE pageserver_state
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>
+    
+    
+SkRecvBroker(b,s) ==
+    /\ {b,s} \subseteq online
+    /\ safekeeper_state' = [safekeeper_state EXCEPT ![s].rx[b] = broker_state[b]] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+
+SkPrune(s) ==
+    /\ {s} \subseteq online
+    /\ LET
+        info ==
+            (
+                {safekeeper_state[s]}
+                \cup
+                {safekeeper_state[s].rx[s2]: s2 \in (DOMAIN safekeeper_state[s].rx)}
+            )
+            \
+            {NULL}
+        commit_lsns == {i.commit_lsn: i \in info}
+        prune_lsn == MinOfSet(commit_lsns)
+       IN
+        safekeeper_state' = [safekeeper_state EXCEPT ![s].prune_lsn = prune_lsn] 
+    
+
+SksWithNewerWal(p) ==
+    LET
+        ps == pageserver_state[p]
+    IN
+    {s \in DOMAIN ps.sk: ps.sk[s].commit_lsn > ps.last_record_lsn}
+
+PsChooseSk(p) ==
+    /\ {p} \subseteq online
+    /\ SksWithNewerWal(p) # {}
+    /\ pageserver_state' = [pageserver_state EXCEPT![p].preferred_sk = CHOOSE s \in SksWithNewerWal(p): TRUE]
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>    
+                 
+    
+Next ==
+    \/ NodeOnlineOffline
+    \/ \E s1 \in safekeepers: \E s2 \in safekeepers:
+        \/ SkCommit(s1, s2)
+        \/ SkPeerRecovery(s1, s2)
+    \/ \E s \in safekeepers: \E b \in brokers:
+        \/ SkPushToBroker(s, b)
+        \/ SkRecvBroker(b, s)
+    \/ \E s \in safekeepers:
+        \/ SkPrune(s)  
+    \/ \E s \in safekeepers: \E b \in brokers: \E p \in pageservers: PsRecvBroker(b,p,s)
+    \/ \E p \in pageservers: PsChooseSk(p)
+    
+
+Spec == Init /\ [][Next]_<< broker_state, safekeeper_state, pageserver_state,online>>
+
+
+\* invariants
+
+PsLagsSk == \A p \in pageservers: \A s \in DOMAIN pageserver_state[p].sk: 
+        /\ pageserver_state[p].sk[s].commit_lsn <= safekeeper_state[s].commit_lsn
+
+PeerRecoveryIsPossible ==
+    \A laggard \in (safekeepers \cap online): \E donor \in (safekeepers \cap online):
+        /\ (safekeeper_state[laggard].commit_lsn < safekeeper_state[donor].commit_lsn)
+            => safekeeper_state[donor].prune_lsn <= safekeeper_state[laggard].commit_lsn
+        
+
+EventuallyLaggingSkIsNotPreferredSk == <>(
+        LET
+            sks == safekeeper_state
+            lagging_sks == { s \in safekeepers: \A s2 \in safekeepers: sks[s].commit_lsn <= sks[s2].commit_lsn }
+            preferred_sks == {pageserver_state[p].preferred_sk: p \in pageservers}
+        IN
+            preferred_sks \cap lagging_sks = {}
+            
+    )
+
+    
+
+====

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737323834253/MC.cfg

@@ -0,0 +1,54 @@
+\* MV CONSTANT declarations
+CONSTANTS
+b1 = b1
+b2 = b2
+b3 = b3
+\* MV CONSTANT declarations
+CONSTANTS
+s1 = s1
+s2 = s2
+s3 = s3
+\* MV CONSTANT declarations
+CONSTANTS
+p1 = p1
+\* MV CONSTANT declarations
+CONSTANTS
+az1 = az1
+az2 = az2
+az3 = az3
+\* MV CONSTANT definitions
+CONSTANT
+brokers <- const_1737323830205294000
+\* CONSTANT declarations
+CONSTANT NULL = NULL
+\* MV CONSTANT definitions
+CONSTANT
+safekeepers <- const_1737323830205295000
+\* MV CONSTANT definitions
+CONSTANT
+pageservers <- const_1737323830205296000
+\* MV CONSTANT definitions
+CONSTANT
+azs <- const_1737323830205297000
+\* SYMMETRY definition
+SYMMETRY symm_1737323830205298000
+\* CONSTANT definitions
+CONSTANT
+max_commit_lsn <- const_1737323830205299000
+\* CONSTANT definitions
+CONSTANT
+az_mapping <- const_1737323830205300000
+\* CONSTRAINT definition
+CONSTRAINT
+constr_1737323830205301000
+\* SPECIFICATION definition
+SPECIFICATION
+Spec
+\* INVARIANT definition
+INVARIANT
+PsLagsSk
+PeerRecoveryIsPossible
+\* PROPERTY definition
+PROPERTY
+EventuallyLaggingSkIsNotPreferredSk
+\* Generated on Sun Jan 19 22:57:10 CET 2025

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737323834253/MC.out

@@ -0,0 +1,430 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 103 and seed 5725447841858367243 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 22986] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 22:57:11)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 22:57:12.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 1000:1 @!@!@
+TLC threw an unexpected exception.
+This was probably caused by an error in the spec or model.
+See the User Output or TLC Console for clues to what happened.
+The exception was a java.lang.RuntimeException
+: Attempted to select nonexistent field "commit_lsn" from the record
+[sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])]
+line 118, col 25 to line 118, col 36 of module replicated
+@!@!@ENDMSG 1000 @!@!@
+@!@!@STARTMSG 2121:1 @!@!@
+The behavior up to this point is:
+@!@!@ENDMSG 2121 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+1: <Initial predicate>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, b3, s1, s2, s3, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+2: <NodeOnlineOffline line 52, col 5 to line 56, col 67 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+3: <SkCommit line 59, col 5 to line 68, col 58 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+4: <SkRecvBroker line 103, col 5 to line 105, col 57 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2103:1 @!@!@
+The error occurred when TLC was evaluating the nested
+expressions at the following positions:
+0. Line 108, column 5 to line 122, column 57 in replicated
+1. Line 108, column 8 to line 108, column 27 in replicated
+2. Line 109, column 8 to line 121, column 80 in replicated
+3. Line 121, column 9 to line 121, column 80 in replicated
+4. Line 121, column 29 to line 121, column 80 in replicated
+5. Line 121, column 71 to line 121, column 79 in replicated
+6. Line 119, column 22 to line 119, column 42 in replicated
+7. Line 27, column 3 to line 27, column 36 in replicated
+8. Line 27, column 16 to line 27, column 16 in replicated
+9. Line 119, column 31 to line 119, column 41 in replicated
+10. Line 118, column 24 to line 118, column 49 in replicated
+11. Line 118, column 25 to line 118, column 36 in replicated
+
+
+@!@!@ENDMSG 2103 @!@!@
+@!@!@STARTMSG 2201:0 @!@!@
+The coverage statistics at 2025-01-19 22:57:12
+@!@!@ENDMSG 2201 @!@!@
+@!@!@STARTMSG 2773:0 @!@!@
+<Init line 45, col 1 to line 45, col 4 of module replicated>: 2:2
+@!@!@ENDMSG 2773 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 46, col 5 to line 49, col 26 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<NodeOnlineOffline line 51, col 1 to line 51, col 17 of module replicated>: 1:16
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 52, col 8 to line 55, col 44 of module replicated: 16
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 52, col 18 to line 55, col 44 of module replicated: 16
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 53, col 9 to line 55, col 44 of module replicated: 1712
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 53, col 12 to line 53, col 48 of module replicated: 1712
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 54, col 12 to line 54, col 44 of module replicated: 688
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 55, col 12 to line 55, col 44 of module replicated: 224
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 52, col 32 to line 52, col 48 of module replicated: 16
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 56, col 8 to line 56, col 67 of module replicated: 16
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkCommit line 58, col 1 to line 58, col 16 of module replicated>: 6:37
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 59, col 8 to line 59, col 32 of module replicated: 165
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 59, col 8 to line 59, col 15 of module replicated: 128:301
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 59, col 27 to line 59, col 32 of module replicated: 128
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 60, col 8 to line 60, col 14 of module replicated: 125
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 8 to line 60, col 9 of module replicated: 88
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 13 to line 60, col 14 of module replicated: 88
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 61, col 8 to line 61, col 72 of module replicated: 90
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 8 to line 61, col 38 of module replicated: 53
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 42 to line 61, col 72 of module replicated: 53
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 65, col 13 to line 67, col 50 of module replicated: 37
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 68, col 8 to line 68, col 58 of module replicated: 37
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPeerRecovery line 70, col 1 to line 70, col 21 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 71, col 8 to line 71, col 32 of module replicated: 127
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 72, col 8 to line 72, col 72 of module replicated: 87
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 73, col 8 to line 73, col 71 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 74, col 8 to line 74, col 102 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 75, col 8 to line 75, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPushToBroker line 77, col 1 to line 77, col 19 of module replicated>: 15:83
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 78, col 8 to line 78, col 30 of module replicated: 201
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 78, col 8 to line 78, col 13 of module replicated: 118:298
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 78, col 25 to line 78, col 30 of module replicated: 118
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 79, col 8 to line 86, col 52 of module replicated: 83
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 87, col 8 to line 87, col 62 of module replicated: 83
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsRecvBroker line 89, col 1 to line 89, col 19 of module replicated>: 2:52
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 90, col 8 to line 90, col 31 of module replicated: 124
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 90, col 8 to line 90, col 14 of module replicated: 72:236
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 90, col 26 to line 90, col 31 of module replicated: 72
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 96, col 13 to line 98, col 37 of module replicated: 52
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 96, col 33 to line 98, col 37 of module replicated: 52
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 96, col 36 to line 96, col 66 of module replicated: 52
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 97, col 22 to line 97, col 66 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 98, col 22 to line 98, col 37 of module replicated: 50
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 99, col 8 to line 99, col 58 of module replicated: 52
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkRecvBroker line 102, col 1 to line 102, col 17 of module replicated>: 31:83
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 103, col 8 to line 103, col 29 of module replicated: 201
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 103, col 8 to line 103, col 12 of module replicated: 118:298
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 103, col 24 to line 103, col 29 of module replicated: 118
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 104, col 8 to line 104, col 81 of module replicated: 83
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 105, col 8 to line 105, col 57 of module replicated: 83
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPrune line 107, col 1 to line 107, col 10 of module replicated>: 8:20
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 108, col 8 to line 108, col 27 of module replicated: 49
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 108, col 8 to line 108, col 10 of module replicated: 29:54
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 108, col 22 to line 108, col 27 of module replicated: 29
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 121, col 9 to line 121, col 80 of module replicated: 20
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 121, col 29 to line 121, col 80 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 121, col 30 to line 121, col 45 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 121, col 54 to line 121, col 79 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 121, col 71 to line 121, col 79 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 119, col 22 to line 119, col 42 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 27, col 3 to line 27, col 36 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||line 27, col 19 to line 27, col 36 of module replicated: 20
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||line 27, col 16 to line 27, col 16 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 119, col 31 to line 119, col 41 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||line 118, col 24 to line 118, col 49 of module replicated: 25:60
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||line 118, col 25 to line 118, col 36 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||line 118, col 45 to line 118, col 48 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||line 111, col 13 to line 117, col 18 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 112, col 17 to line 114, col 84 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||line 112, col 17 to line 112, col 37 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||||||line 114, col 17 to line 114, col 84 of module replicated: 25:55
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 114, col 18 to line 114, col 43 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 114, col 54 to line 114, col 82 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 117, col 13 to line 117, col 18 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 122, col 8 to line 122, col 57 of module replicated: 20
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsChooseSk line 131, col 1 to line 131, col 13 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 132, col 8 to line 132, col 27 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 133, col 8 to line 133, col 30 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 133, col 8 to line 133, col 25 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 127, col 15 to line 127, col 33 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 129, col 5 to line 129, col 66 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 129, col 26 to line 129, col 65 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 129, col 12 to line 129, col 23 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 133, col 24 to line 133, col 24 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 133, col 29 to line 133, col 30 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 134, col 8 to line 134, col 109 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 135, col 8 to line 135, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PsLagsSk line 157, col 1 to line 157, col 8 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 157, col 13 to line 158, col 81 of module replicated: 68
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 157, col 35 to line 158, col 81 of module replicated: 68
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 158, col 9 to line 158, col 81 of module replicated: 204
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 157, col 44 to line 157, col 72 of module replicated: 68
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 157, col 22 to line 157, col 32 of module replicated: 68
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PeerRecoveryIsPossible line 160, col 1 to line 160, col 22 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 161, col 5 to line 163, col 88 of module replicated: 68
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 161, col 47 to line 163, col 88 of module replicated: 173
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 162, col 9 to line 163, col 88 of module replicated: 176
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 162, col 12 to line 163, col 88 of module replicated: 176
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 162, col 13 to line 162, col 85 of module replicated: 176
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 163, col 16 to line 163, col 88 of module replicated: 35
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 161, col 61 to line 161, col 83 of module replicated: 173
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 161, col 21 to line 161, col 43 of module replicated: 68
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2202:0 @!@!@
+End of statistics.
+@!@!@ENDMSG 2202 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(5) at 2025-01-19 22:57:12: 273 states generated (6.319 s/min), 64 distinct states found (1.481 ds/min), 52 states left on queue.
+@!@!@ENDMSG 2200 @!@!@
+@!@!@STARTMSG 2199:0 @!@!@
+273 states generated, 64 distinct states found, 52 states left on queue.
+@!@!@ENDMSG 2199 @!@!@
+@!@!@STARTMSG 2186:0 @!@!@
+Finished in 2610ms at (2025-01-19 22:57:12)
+@!@!@ENDMSG 2186 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737323834253/MC.tla

@@ -0,0 +1,65 @@
+---- MODULE MC ----
+EXTENDS replicated, TLC
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+b1, b2, b3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+s1, s2, s3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+p1
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+az1, az2, az3
+----
+
+\* MV CONSTANT definitions brokers
+const_1737323830205294000 == 
+{b1, b2, b3}
+----
+
+\* MV CONSTANT definitions safekeepers
+const_1737323830205295000 == 
+{s1, s2, s3}
+----
+
+\* MV CONSTANT definitions pageservers
+const_1737323830205296000 == 
+{p1}
+----
+
+\* MV CONSTANT definitions azs
+const_1737323830205297000 == 
+{az1, az2, az3}
+----
+
+\* SYMMETRY definition
+symm_1737323830205298000 == 
+Permutations(const_1737323830205294000) \union Permutations(const_1737323830205295000) \union Permutations(const_1737323830205296000) \union Permutations(const_1737323830205297000)
+----
+
+\* CONSTANT definitions @modelParameterConstants:4max_commit_lsn
+const_1737323830205299000 == 
+2
+----
+
+\* CONSTANT definitions @modelParameterConstants:5az_mapping
+const_1737323830205300000 == 
+[ az1 |-> {b1,s1,p1} , az2 |-> {b2,s2} , az3 |-> {b3,s3}]
+----
+
+\* CONSTRAINT definition @modelParameterContraint:0
+constr_1737323830205301000 ==
+StateConstraint
+----
+=============================================================================
+\* Modification History
+\* Created Sun Jan 19 22:57:10 CET 2025 by cs

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737323834253/MC_TE.out

@@ -0,0 +1,430 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 103 and seed 5725447841858367243 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 22986] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 22:57:11)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 22:57:12.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 1000:1 @!@!@
+TLC threw an unexpected exception.
+This was probably caused by an error in the spec or model.
+See the User Output or TLC Console for clues to what happened.
+The exception was a java.lang.RuntimeException
+: Attempted to select nonexistent field "commit_lsn" from the record
+[sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])]
+line 118, col 25 to line 118, col 36 of module replicated
+@!@!@ENDMSG 1000 @!@!@
+@!@!@STARTMSG 2121:1 @!@!@
+The behavior up to this point is:
+@!@!@ENDMSG 2121 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+1: <Initial predicate>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, b3, s1, s2, s3, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+2: <NodeOnlineOffline line 52, col 5 to line 56, col 67 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+3: <SkCommit line 59, col 5 to line 68, col 58 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+4: <SkRecvBroker line 103, col 5 to line 105, col 57 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2103:1 @!@!@
+The error occurred when TLC was evaluating the nested
+expressions at the following positions:
+0. Line 108, column 5 to line 122, column 57 in replicated
+1. Line 108, column 8 to line 108, column 27 in replicated
+2. Line 109, column 8 to line 121, column 80 in replicated
+3. Line 121, column 9 to line 121, column 80 in replicated
+4. Line 121, column 29 to line 121, column 80 in replicated
+5. Line 121, column 71 to line 121, column 79 in replicated
+6. Line 119, column 22 to line 119, column 42 in replicated
+7. Line 27, column 3 to line 27, column 36 in replicated
+8. Line 27, column 16 to line 27, column 16 in replicated
+9. Line 119, column 31 to line 119, column 41 in replicated
+10. Line 118, column 24 to line 118, column 49 in replicated
+11. Line 118, column 25 to line 118, column 36 in replicated
+
+
+@!@!@ENDMSG 2103 @!@!@
+@!@!@STARTMSG 2201:0 @!@!@
+The coverage statistics at 2025-01-19 22:57:12
+@!@!@ENDMSG 2201 @!@!@
+@!@!@STARTMSG 2773:0 @!@!@
+<Init line 45, col 1 to line 45, col 4 of module replicated>: 2:2
+@!@!@ENDMSG 2773 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 46, col 5 to line 49, col 26 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<NodeOnlineOffline line 51, col 1 to line 51, col 17 of module replicated>: 1:16
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 52, col 8 to line 55, col 44 of module replicated: 16
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 52, col 18 to line 55, col 44 of module replicated: 16
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 53, col 9 to line 55, col 44 of module replicated: 1712
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 53, col 12 to line 53, col 48 of module replicated: 1712
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 54, col 12 to line 54, col 44 of module replicated: 688
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 55, col 12 to line 55, col 44 of module replicated: 224
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 52, col 32 to line 52, col 48 of module replicated: 16
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 56, col 8 to line 56, col 67 of module replicated: 16
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkCommit line 58, col 1 to line 58, col 16 of module replicated>: 6:37
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 59, col 8 to line 59, col 32 of module replicated: 165
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 59, col 8 to line 59, col 15 of module replicated: 128:301
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 59, col 27 to line 59, col 32 of module replicated: 128
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 60, col 8 to line 60, col 14 of module replicated: 125
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 8 to line 60, col 9 of module replicated: 88
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 13 to line 60, col 14 of module replicated: 88
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 61, col 8 to line 61, col 72 of module replicated: 90
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 8 to line 61, col 38 of module replicated: 53
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 42 to line 61, col 72 of module replicated: 53
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 65, col 13 to line 67, col 50 of module replicated: 37
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 68, col 8 to line 68, col 58 of module replicated: 37
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPeerRecovery line 70, col 1 to line 70, col 21 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 71, col 8 to line 71, col 32 of module replicated: 127
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 72, col 8 to line 72, col 72 of module replicated: 87
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 73, col 8 to line 73, col 71 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 74, col 8 to line 74, col 102 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 75, col 8 to line 75, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPushToBroker line 77, col 1 to line 77, col 19 of module replicated>: 15:83
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 78, col 8 to line 78, col 30 of module replicated: 201
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 78, col 8 to line 78, col 13 of module replicated: 118:298
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 78, col 25 to line 78, col 30 of module replicated: 118
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 79, col 8 to line 86, col 52 of module replicated: 83
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 87, col 8 to line 87, col 62 of module replicated: 83
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsRecvBroker line 89, col 1 to line 89, col 19 of module replicated>: 2:52
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 90, col 8 to line 90, col 31 of module replicated: 124
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 90, col 8 to line 90, col 14 of module replicated: 72:236
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 90, col 26 to line 90, col 31 of module replicated: 72
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 96, col 13 to line 98, col 37 of module replicated: 52
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 96, col 33 to line 98, col 37 of module replicated: 52
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 96, col 36 to line 96, col 66 of module replicated: 52
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 97, col 22 to line 97, col 66 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 98, col 22 to line 98, col 37 of module replicated: 50
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 99, col 8 to line 99, col 58 of module replicated: 52
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkRecvBroker line 102, col 1 to line 102, col 17 of module replicated>: 31:83
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 103, col 8 to line 103, col 29 of module replicated: 201
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 103, col 8 to line 103, col 12 of module replicated: 118:298
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 103, col 24 to line 103, col 29 of module replicated: 118
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 104, col 8 to line 104, col 81 of module replicated: 83
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 105, col 8 to line 105, col 57 of module replicated: 83
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPrune line 107, col 1 to line 107, col 10 of module replicated>: 8:20
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 108, col 8 to line 108, col 27 of module replicated: 49
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 108, col 8 to line 108, col 10 of module replicated: 29:54
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 108, col 22 to line 108, col 27 of module replicated: 29
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 121, col 9 to line 121, col 80 of module replicated: 20
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 121, col 29 to line 121, col 80 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 121, col 30 to line 121, col 45 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 121, col 54 to line 121, col 79 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 121, col 71 to line 121, col 79 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 119, col 22 to line 119, col 42 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 27, col 3 to line 27, col 36 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||line 27, col 19 to line 27, col 36 of module replicated: 20
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||line 27, col 16 to line 27, col 16 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 119, col 31 to line 119, col 41 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||line 118, col 24 to line 118, col 49 of module replicated: 25:60
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||line 118, col 25 to line 118, col 36 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||line 118, col 45 to line 118, col 48 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||line 111, col 13 to line 117, col 18 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 112, col 17 to line 114, col 84 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||line 112, col 17 to line 112, col 37 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||||||line 114, col 17 to line 114, col 84 of module replicated: 25:55
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 114, col 18 to line 114, col 43 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 114, col 54 to line 114, col 82 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 117, col 13 to line 117, col 18 of module replicated: 25
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 122, col 8 to line 122, col 57 of module replicated: 20
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsChooseSk line 131, col 1 to line 131, col 13 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 132, col 8 to line 132, col 27 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 133, col 8 to line 133, col 30 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 133, col 8 to line 133, col 25 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 127, col 15 to line 127, col 33 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 129, col 5 to line 129, col 66 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 129, col 26 to line 129, col 65 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 129, col 12 to line 129, col 23 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 133, col 24 to line 133, col 24 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 133, col 29 to line 133, col 30 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 134, col 8 to line 134, col 109 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 135, col 8 to line 135, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PsLagsSk line 157, col 1 to line 157, col 8 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 157, col 13 to line 158, col 81 of module replicated: 68
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 157, col 35 to line 158, col 81 of module replicated: 68
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 158, col 9 to line 158, col 81 of module replicated: 204
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 157, col 44 to line 157, col 72 of module replicated: 68
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 157, col 22 to line 157, col 32 of module replicated: 68
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PeerRecoveryIsPossible line 160, col 1 to line 160, col 22 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 161, col 5 to line 163, col 88 of module replicated: 68
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 161, col 47 to line 163, col 88 of module replicated: 173
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 162, col 9 to line 163, col 88 of module replicated: 176
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 162, col 12 to line 163, col 88 of module replicated: 176
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 162, col 13 to line 162, col 85 of module replicated: 176
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 163, col 16 to line 163, col 88 of module replicated: 35
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 161, col 61 to line 161, col 83 of module replicated: 173
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 161, col 21 to line 161, col 43 of module replicated: 68
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2202:0 @!@!@
+End of statistics.
+@!@!@ENDMSG 2202 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(5) at 2025-01-19 22:57:12: 273 states generated (6.319 s/min), 64 distinct states found (1.481 ds/min), 52 states left on queue.
+@!@!@ENDMSG 2200 @!@!@
+@!@!@STARTMSG 2199:0 @!@!@
+273 states generated, 64 distinct states found, 52 states left on queue.
+@!@!@ENDMSG 2199 @!@!@
+@!@!@STARTMSG 2186:0 @!@!@
+Finished in 2610ms at (2025-01-19 22:57:12)
+@!@!@ENDMSG 2186 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737323834253/replicated.tla

@@ -0,0 +1,178 @@
+---- MODULE replicated ----
+
+EXTENDS Integers, FiniteSets
+
+VARIABLES broker_state, safekeeper_state, pageserver_state, online
+
+
+CONSTANT
+    brokers,
+    safekeepers,
+    pageservers,
+    azs,
+    az_mapping
+
+CONSTANT
+    NULL
+    
+CONSTANT
+    max_commit_lsn
+    
+\* HELPERS
+
+Max(a,b) == IF a > b THEN a ELSE b
+
+(* The minimum of a non-empty set of integers *)
+MinOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x <= y
+
+\* END HELPERS
+
+StateConstraint ==
+    /\ \A s \in safekeepers: 
+        /\ safekeeper_state[s].commit_lsn <= max_commit_lsn
+    /\ \A b \in brokers:
+        /\ \A s \in DOMAIN broker_state[b].sk:
+            /\ broker_state[b].sk[s].commit_lsn <= max_commit_lsn
+
+
+ 
+InitSafekeeper == [prune_lsn |-> 0, commit_lsn |-> 0, rx |-> [b \in brokers |-> NULL] ]
+InitBroker == [sk |-> [s \in safekeepers |-> [prune_lsn |-> 0, commit_lsn |-> 0]]]
+InitPageserver == [last_record_lsn |-> 0, preferred_sk |-> NULL, sk |-> [s \in safekeepers |-> [commit_lsn |-> 0]]]
+InitOnline == safekeepers \cup brokers \cup pageservers
+
+Init ==
+    /\ broker_state = [b \in brokers |-> InitBroker]
+    /\ safekeeper_state = [s \in safekeepers |-> InitSafekeeper]
+    /\ pageserver_state = [p \in pageservers |-> InitPageserver]
+    /\ online = InitOnline
+  
+NodeOnlineOffline ==
+    /\ online' = CHOOSE ss \in SUBSET InitOnline: 
+        /\ Cardinality(ss \cap safekeepers) >= 2
+        /\ Cardinality(ss \cap brokers) >= 2
+        /\ ss \cap pageservers = pageservers \* assume no PS failures for now
+    /\ UNCHANGED <<safekeeper_state,broker_state,pageserver_state>>
+   
+SkCommit(s1, s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ s1 # s2
+    /\ safekeeper_state[s1].commit_lsn = safekeeper_state[s2].commit_lsn
+    /\ LET
+            new_commit_lsn == safekeeper_state[s1].commit_lsn + 1
+       IN
+            safekeeper_state' = [safekeeper_state EXCEPT
+                ![s1].commit_lsn = new_commit_lsn,
+                ![s2].commit_lsn = new_commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPeerRecovery(s1,s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ safekeeper_state[s1].commit_lsn < safekeeper_state[s2].commit_lsn \* s2 has more WAL than s1
+    /\ safekeeper_state[s2].prune_lsn < safekeeper_state[s1].commit_lsn \* s2 has not yet trimmed the WAL the WAL
+    /\ safekeeper_state' = [safekeeper_state EXCEPT![s1].commit_lsn = safekeeper_state[s2].commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPushToBroker(s,b) ==
+    /\ {s, b} \subseteq online 
+    /\ broker_state' = LET
+           broker_side == broker_state[b].sk[s]
+           sk_side == safekeeper_state[s]
+           merged == [broker_side EXCEPT
+                !["commit_lsn"] = Max(broker_side.commit_lsn, sk_side.commit_lsn),
+                !["prune_lsn"] = Max(broker_side.prune_lsn, sk_side.prune_lsn)]
+           IN
+           [broker_state EXCEPT ![b].sk[s] = merged]
+    /\ UNCHANGED <<safekeeper_state, pageserver_state,online>> 
+
+PsRecvBroker(b,p,s) ==
+    /\ {b,p,s} \subseteq online
+    /\ LET
+            bsk == broker_state[b].sk[s]
+            psk == pageserver_state[p].sk[s]
+            updpsk == [psk EXCEPT !["commit_lsn"] = bsk.commit_lsn]
+       IN
+            pageserver_state' = IF bsk.commit_lsn > psk.commit_lsn
+                THEN [pageserver_state EXCEPT ![p].sk[s] = updpsk]
+                ELSE pageserver_state
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>
+    
+    
+SkRecvBroker(b,s) ==
+    /\ {b,s} \subseteq online
+    /\ safekeeper_state' = [safekeeper_state EXCEPT ![s].rx[b] = broker_state[b]] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+
+SkPrune(s) ==
+    /\ {s} \subseteq online
+    /\ LET
+        info ==
+            (
+                {safekeeper_state[s]}
+                \cup
+                {safekeeper_state[s].rx[s2]: s2 \in (DOMAIN safekeeper_state[s].rx)}
+            )
+            \
+            {NULL}
+        commit_lsns == {i.commit_lsn: i \in info}
+        prune_lsn == MinOfSet(commit_lsns)
+       IN
+        safekeeper_state' = [safekeeper_state EXCEPT ![s].prune_lsn = prune_lsn] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+    
+
+SksWithNewerWal(p) ==
+    LET
+        ps == pageserver_state[p]
+    IN
+    {s \in DOMAIN ps.sk: ps.sk[s].commit_lsn > ps.last_record_lsn}
+
+PsChooseSk(p) ==
+    /\ {p} \subseteq online
+    /\ SksWithNewerWal(p) # {}
+    /\ pageserver_state' = [pageserver_state EXCEPT![p].preferred_sk = CHOOSE s \in SksWithNewerWal(p): TRUE]
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>    
+                 
+    
+Next ==
+    \/ NodeOnlineOffline
+    \/ \E s1 \in safekeepers: \E s2 \in safekeepers:
+        \/ SkCommit(s1, s2)
+        \/ SkPeerRecovery(s1, s2)
+    \/ \E s \in safekeepers: \E b \in brokers:
+        \/ SkPushToBroker(s, b)
+        \/ SkRecvBroker(b, s)
+    \/ \E s \in safekeepers:
+        \/ SkPrune(s)  
+    \/ \E s \in safekeepers: \E b \in brokers: \E p \in pageservers: PsRecvBroker(b,p,s)
+    \/ \E p \in pageservers: PsChooseSk(p)
+    
+
+Spec == Init /\ [][Next]_<< broker_state, safekeeper_state, pageserver_state,online>>
+
+
+\* invariants
+
+PsLagsSk == \A p \in pageservers: \A s \in DOMAIN pageserver_state[p].sk: 
+        /\ pageserver_state[p].sk[s].commit_lsn <= safekeeper_state[s].commit_lsn
+
+PeerRecoveryIsPossible ==
+    \A laggard \in (safekeepers \cap online): \E donor \in (safekeepers \cap online):
+        /\ (safekeeper_state[laggard].commit_lsn < safekeeper_state[donor].commit_lsn)
+            => safekeeper_state[donor].prune_lsn <= safekeeper_state[laggard].commit_lsn
+        
+
+EventuallyLaggingSkIsNotPreferredSk == <>(
+        LET
+            sks == safekeeper_state
+            lagging_sks == { s \in safekeepers: \A s2 \in safekeepers: sks[s].commit_lsn <= sks[s2].commit_lsn }
+            preferred_sks == {pageserver_state[p].preferred_sk: p \in pageservers}
+        IN
+            preferred_sks \cap lagging_sks = {}
+            
+    )
+
+    
+
+====

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737324141888/MC.cfg

@@ -0,0 +1,54 @@
+\* MV CONSTANT declarations
+CONSTANTS
+b1 = b1
+b2 = b2
+b3 = b3
+\* MV CONSTANT declarations
+CONSTANTS
+s1 = s1
+s2 = s2
+s3 = s3
+\* MV CONSTANT declarations
+CONSTANTS
+p1 = p1
+\* MV CONSTANT declarations
+CONSTANTS
+az1 = az1
+az2 = az2
+az3 = az3
+\* MV CONSTANT definitions
+CONSTANT
+brokers <- const_1737324137833305000
+\* CONSTANT declarations
+CONSTANT NULL = NULL
+\* MV CONSTANT definitions
+CONSTANT
+safekeepers <- const_1737324137833306000
+\* MV CONSTANT definitions
+CONSTANT
+pageservers <- const_1737324137833307000
+\* MV CONSTANT definitions
+CONSTANT
+azs <- const_1737324137833308000
+\* SYMMETRY definition
+SYMMETRY symm_1737324137833309000
+\* CONSTANT definitions
+CONSTANT
+max_commit_lsn <- const_1737324137833310000
+\* CONSTANT definitions
+CONSTANT
+az_mapping <- const_1737324137833311000
+\* CONSTRAINT definition
+CONSTRAINT
+constr_1737324137833312000
+\* SPECIFICATION definition
+SPECIFICATION
+Spec
+\* INVARIANT definition
+INVARIANT
+PsLagsSk
+PeerRecoveryIsPossible
+\* PROPERTY definition
+PROPERTY
+EventuallyLaggingSkIsNotPreferredSk
+\* Generated on Sun Jan 19 23:02:17 CET 2025

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737324141888/MC.out

@@ -0,0 +1,368 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 60 and seed 1436414801931403111 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 23214] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 23:02:19)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 23:02:20.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 1000:1 @!@!@
+TLC threw an unexpected exception.
+This was probably caused by an error in the spec or model.
+See the User Output or TLC Console for clues to what happened.
+The exception was a java.lang.RuntimeException
+: Attempted to select field "sk" from a non-record value NULL
+line 111, col 36 to line 111, col 40 of module replicated
+@!@!@ENDMSG 1000 @!@!@
+@!@!@STARTMSG 2121:1 @!@!@
+The behavior up to this point is:
+@!@!@ENDMSG 2121 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+1: <Initial predicate>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, b3, s1, s2, s3, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+2: <NodeOnlineOffline line 52, col 5 to line 56, col 67 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2103:1 @!@!@
+The error occurred when TLC was evaluating the nested
+expressions at the following positions:
+0. Line 108, column 5 to line 124, column 57 in replicated
+1. Line 108, column 8 to line 108, column 27 in replicated
+2. Line 109, column 8 to line 123, column 80 in replicated
+3. Line 123, column 9 to line 123, column 80 in replicated
+4. Line 123, column 29 to line 123, column 80 in replicated
+5. Line 123, column 71 to line 123, column 79 in replicated
+6. Line 121, column 22 to line 121, column 42 in replicated
+7. Line 27, column 3 to line 27, column 36 in replicated
+8. Line 27, column 16 to line 27, column 16 in replicated
+9. Line 121, column 31 to line 121, column 41 in replicated
+10. Line 120, column 24 to line 120, column 49 in replicated
+11. Line 120, column 45 to line 120, column 48 in replicated
+12. Line 113, column 13 to line 119, column 18 in replicated
+13. Line 114, column 17 to line 116, column 32 in replicated
+14. Line 116, column 17 to line 116, column 32 in replicated
+15. Line 111, column 29 to line 111, column 62 in replicated
+16. Line 111, column 35 to line 111, column 62 in replicated
+17. Line 111, column 36 to line 111, column 40 in replicated
+
+
+@!@!@ENDMSG 2103 @!@!@
+@!@!@STARTMSG 2201:0 @!@!@
+The coverage statistics at 2025-01-19 23:02:20
+@!@!@ENDMSG 2201 @!@!@
+@!@!@STARTMSG 2773:0 @!@!@
+<Init line 45, col 1 to line 45, col 4 of module replicated>: 2:2
+@!@!@ENDMSG 2773 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 46, col 5 to line 49, col 26 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<NodeOnlineOffline line 51, col 1 to line 51, col 17 of module replicated>: 1:6
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 52, col 8 to line 55, col 44 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 52, col 18 to line 55, col 44 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 53, col 9 to line 55, col 44 of module replicated: 642
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 53, col 12 to line 53, col 48 of module replicated: 642
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 54, col 12 to line 54, col 44 of module replicated: 258
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 55, col 12 to line 55, col 44 of module replicated: 84
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 52, col 32 to line 52, col 48 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 56, col 8 to line 56, col 67 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkCommit line 58, col 1 to line 58, col 16 of module replicated>: 4:14
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 59, col 8 to line 59, col 32 of module replicated: 59
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 59, col 8 to line 59, col 15 of module replicated: 45:105
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 59, col 27 to line 59, col 32 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 60, col 8 to line 60, col 14 of module replicated: 44
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 8 to line 60, col 9 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 13 to line 60, col 14 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 61, col 8 to line 61, col 72 of module replicated: 32
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 8 to line 61, col 38 of module replicated: 18
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 42 to line 61, col 72 of module replicated: 18
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 65, col 13 to line 67, col 50 of module replicated: 14
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 68, col 8 to line 68, col 58 of module replicated: 14
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPeerRecovery line 70, col 1 to line 70, col 21 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 71, col 8 to line 71, col 32 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 72, col 8 to line 72, col 72 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 73, col 8 to line 73, col 71 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 74, col 8 to line 74, col 102 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 75, col 8 to line 75, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPushToBroker line 77, col 1 to line 77, col 19 of module replicated>: 2:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 78, col 8 to line 78, col 30 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 78, col 8 to line 78, col 13 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 78, col 25 to line 78, col 30 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 79, col 8 to line 86, col 52 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 87, col 8 to line 87, col 62 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsRecvBroker line 89, col 1 to line 89, col 19 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 96, col 13 to line 98, col 37 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 99, col 8 to line 99, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkRecvBroker line 102, col 1 to line 102, col 17 of module replicated>: 5:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 103, col 8 to line 103, col 29 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 103, col 8 to line 103, col 12 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 103, col 24 to line 103, col 29 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 104, col 8 to line 104, col 81 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 105, col 8 to line 105, col 57 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPrune line 107, col 1 to line 107, col 10 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 108, col 8 to line 108, col 27 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 123, col 9 to line 123, col 80 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 123, col 29 to line 123, col 80 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 123, col 30 to line 123, col 45 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 123, col 54 to line 123, col 79 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 123, col 71 to line 123, col 79 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 121, col 22 to line 121, col 42 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 27, col 3 to line 27, col 36 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 121, col 31 to line 121, col 41 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||line 120, col 24 to line 120, col 49 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||line 120, col 45 to line 120, col 48 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||line 113, col 13 to line 119, col 18 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 114, col 17 to line 116, col 32 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||line 114, col 17 to line 114, col 37 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||line 116, col 17 to line 116, col 32 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 111, col 29 to line 111, col 62 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||line 111, col 35 to line 111, col 62 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||||line 111, col 36 to line 111, col 40 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||||line 111, col 50 to line 111, col 61 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||||||||||line 110, col 25 to line 110, col 90 of module replicated: 5:10
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||||||line 110, col 26 to line 110, col 50 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||||||line 110, col 60 to line 110, col 88 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 124, col 8 to line 124, col 57 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsChooseSk line 133, col 1 to line 133, col 13 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 136, col 8 to line 136, col 109 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 137, col 8 to line 137, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PsLagsSk line 159, col 1 to line 159, col 8 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 159, col 13 to line 160, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 159, col 35 to line 160, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 160, col 9 to line 160, col 81 of module replicated: 39
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 159, col 44 to line 159, col 72 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 159, col 22 to line 159, col 32 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PeerRecoveryIsPossible line 162, col 1 to line 162, col 22 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 163, col 5 to line 165, col 88 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 47 to line 165, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 164, col 9 to line 165, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 164, col 12 to line 165, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 164, col 13 to line 164, col 85 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 165, col 16 to line 165, col 88 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 163, col 61 to line 163, col 83 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 21 to line 163, col 43 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2202:0 @!@!@
+End of statistics.
+@!@!@ENDMSG 2202 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(4) at 2025-01-19 23:02:20: 69 states generated (1.718 s/min), 13 distinct states found (323 ds/min), 9 states left on queue.
+@!@!@ENDMSG 2200 @!@!@
+@!@!@STARTMSG 2199:0 @!@!@
+69 states generated, 13 distinct states found, 9 states left on queue.
+@!@!@ENDMSG 2199 @!@!@
+@!@!@STARTMSG 2186:0 @!@!@
+Finished in 2426ms at (2025-01-19 23:02:20)
+@!@!@ENDMSG 2186 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737324141888/MC.tla

@@ -0,0 +1,65 @@
+---- MODULE MC ----
+EXTENDS replicated, TLC
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+b1, b2, b3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+s1, s2, s3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+p1
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+az1, az2, az3
+----
+
+\* MV CONSTANT definitions brokers
+const_1737324137833305000 == 
+{b1, b2, b3}
+----
+
+\* MV CONSTANT definitions safekeepers
+const_1737324137833306000 == 
+{s1, s2, s3}
+----
+
+\* MV CONSTANT definitions pageservers
+const_1737324137833307000 == 
+{p1}
+----
+
+\* MV CONSTANT definitions azs
+const_1737324137833308000 == 
+{az1, az2, az3}
+----
+
+\* SYMMETRY definition
+symm_1737324137833309000 == 
+Permutations(const_1737324137833305000) \union Permutations(const_1737324137833306000) \union Permutations(const_1737324137833307000) \union Permutations(const_1737324137833308000)
+----
+
+\* CONSTANT definitions @modelParameterConstants:4max_commit_lsn
+const_1737324137833310000 == 
+2
+----
+
+\* CONSTANT definitions @modelParameterConstants:5az_mapping
+const_1737324137833311000 == 
+[ az1 |-> {b1,s1,p1} , az2 |-> {b2,s2} , az3 |-> {b3,s3}]
+----
+
+\* CONSTRAINT definition @modelParameterContraint:0
+constr_1737324137833312000 ==
+StateConstraint
+----
+=============================================================================
+\* Modification History
+\* Created Sun Jan 19 23:02:17 CET 2025 by cs

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737324141888/MC_TE.out

@@ -0,0 +1,368 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 60 and seed 1436414801931403111 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 23214] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 23:02:19)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 23:02:20.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 1000:1 @!@!@
+TLC threw an unexpected exception.
+This was probably caused by an error in the spec or model.
+See the User Output or TLC Console for clues to what happened.
+The exception was a java.lang.RuntimeException
+: Attempted to select field "sk" from a non-record value NULL
+line 111, col 36 to line 111, col 40 of module replicated
+@!@!@ENDMSG 1000 @!@!@
+@!@!@STARTMSG 2121:1 @!@!@
+The behavior up to this point is:
+@!@!@ENDMSG 2121 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+1: <Initial predicate>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, b3, s1, s2, s3, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+2: <NodeOnlineOffline line 52, col 5 to line 56, col 67 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2103:1 @!@!@
+The error occurred when TLC was evaluating the nested
+expressions at the following positions:
+0. Line 108, column 5 to line 124, column 57 in replicated
+1. Line 108, column 8 to line 108, column 27 in replicated
+2. Line 109, column 8 to line 123, column 80 in replicated
+3. Line 123, column 9 to line 123, column 80 in replicated
+4. Line 123, column 29 to line 123, column 80 in replicated
+5. Line 123, column 71 to line 123, column 79 in replicated
+6. Line 121, column 22 to line 121, column 42 in replicated
+7. Line 27, column 3 to line 27, column 36 in replicated
+8. Line 27, column 16 to line 27, column 16 in replicated
+9. Line 121, column 31 to line 121, column 41 in replicated
+10. Line 120, column 24 to line 120, column 49 in replicated
+11. Line 120, column 45 to line 120, column 48 in replicated
+12. Line 113, column 13 to line 119, column 18 in replicated
+13. Line 114, column 17 to line 116, column 32 in replicated
+14. Line 116, column 17 to line 116, column 32 in replicated
+15. Line 111, column 29 to line 111, column 62 in replicated
+16. Line 111, column 35 to line 111, column 62 in replicated
+17. Line 111, column 36 to line 111, column 40 in replicated
+
+
+@!@!@ENDMSG 2103 @!@!@
+@!@!@STARTMSG 2201:0 @!@!@
+The coverage statistics at 2025-01-19 23:02:20
+@!@!@ENDMSG 2201 @!@!@
+@!@!@STARTMSG 2773:0 @!@!@
+<Init line 45, col 1 to line 45, col 4 of module replicated>: 2:2
+@!@!@ENDMSG 2773 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 46, col 5 to line 49, col 26 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<NodeOnlineOffline line 51, col 1 to line 51, col 17 of module replicated>: 1:6
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 52, col 8 to line 55, col 44 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 52, col 18 to line 55, col 44 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 53, col 9 to line 55, col 44 of module replicated: 642
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 53, col 12 to line 53, col 48 of module replicated: 642
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 54, col 12 to line 54, col 44 of module replicated: 258
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 55, col 12 to line 55, col 44 of module replicated: 84
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 52, col 32 to line 52, col 48 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 56, col 8 to line 56, col 67 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkCommit line 58, col 1 to line 58, col 16 of module replicated>: 4:14
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 59, col 8 to line 59, col 32 of module replicated: 59
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 59, col 8 to line 59, col 15 of module replicated: 45:105
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 59, col 27 to line 59, col 32 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 60, col 8 to line 60, col 14 of module replicated: 44
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 8 to line 60, col 9 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 13 to line 60, col 14 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 61, col 8 to line 61, col 72 of module replicated: 32
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 8 to line 61, col 38 of module replicated: 18
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 42 to line 61, col 72 of module replicated: 18
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 65, col 13 to line 67, col 50 of module replicated: 14
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 68, col 8 to line 68, col 58 of module replicated: 14
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPeerRecovery line 70, col 1 to line 70, col 21 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 71, col 8 to line 71, col 32 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 72, col 8 to line 72, col 72 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 73, col 8 to line 73, col 71 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 74, col 8 to line 74, col 102 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 75, col 8 to line 75, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPushToBroker line 77, col 1 to line 77, col 19 of module replicated>: 2:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 78, col 8 to line 78, col 30 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 78, col 8 to line 78, col 13 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 78, col 25 to line 78, col 30 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 79, col 8 to line 86, col 52 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 87, col 8 to line 87, col 62 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsRecvBroker line 89, col 1 to line 89, col 19 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 96, col 13 to line 98, col 37 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 99, col 8 to line 99, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkRecvBroker line 102, col 1 to line 102, col 17 of module replicated>: 5:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 103, col 8 to line 103, col 29 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 103, col 8 to line 103, col 12 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 103, col 24 to line 103, col 29 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 104, col 8 to line 104, col 81 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 105, col 8 to line 105, col 57 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPrune line 107, col 1 to line 107, col 10 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 108, col 8 to line 108, col 27 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 123, col 9 to line 123, col 80 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 123, col 29 to line 123, col 80 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 123, col 30 to line 123, col 45 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 123, col 54 to line 123, col 79 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 123, col 71 to line 123, col 79 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 121, col 22 to line 121, col 42 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 27, col 3 to line 27, col 36 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 121, col 31 to line 121, col 41 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||line 120, col 24 to line 120, col 49 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||line 120, col 45 to line 120, col 48 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||line 113, col 13 to line 119, col 18 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 114, col 17 to line 116, col 32 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||line 114, col 17 to line 114, col 37 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||line 116, col 17 to line 116, col 32 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 111, col 29 to line 111, col 62 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||line 111, col 35 to line 111, col 62 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||||line 111, col 36 to line 111, col 40 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||||line 111, col 50 to line 111, col 61 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||||||||||line 110, col 25 to line 110, col 90 of module replicated: 5:10
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||||||line 110, col 26 to line 110, col 50 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||||||line 110, col 60 to line 110, col 88 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 124, col 8 to line 124, col 57 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsChooseSk line 133, col 1 to line 133, col 13 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 136, col 8 to line 136, col 109 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 137, col 8 to line 137, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PsLagsSk line 159, col 1 to line 159, col 8 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 159, col 13 to line 160, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 159, col 35 to line 160, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 160, col 9 to line 160, col 81 of module replicated: 39
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 159, col 44 to line 159, col 72 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 159, col 22 to line 159, col 32 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PeerRecoveryIsPossible line 162, col 1 to line 162, col 22 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 163, col 5 to line 165, col 88 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 47 to line 165, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 164, col 9 to line 165, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 164, col 12 to line 165, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 164, col 13 to line 164, col 85 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 165, col 16 to line 165, col 88 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 163, col 61 to line 163, col 83 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 21 to line 163, col 43 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2202:0 @!@!@
+End of statistics.
+@!@!@ENDMSG 2202 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(4) at 2025-01-19 23:02:20: 69 states generated (1.718 s/min), 13 distinct states found (323 ds/min), 9 states left on queue.
+@!@!@ENDMSG 2200 @!@!@
+@!@!@STARTMSG 2199:0 @!@!@
+69 states generated, 13 distinct states found, 9 states left on queue.
+@!@!@ENDMSG 2199 @!@!@
+@!@!@STARTMSG 2186:0 @!@!@
+Finished in 2426ms at (2025-01-19 23:02:20)
+@!@!@ENDMSG 2186 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737324141888/replicated.tla

@@ -0,0 +1,180 @@
+---- MODULE replicated ----
+
+EXTENDS Integers, FiniteSets
+
+VARIABLES broker_state, safekeeper_state, pageserver_state, online
+
+
+CONSTANT
+    brokers,
+    safekeepers,
+    pageservers,
+    azs,
+    az_mapping
+
+CONSTANT
+    NULL
+    
+CONSTANT
+    max_commit_lsn
+    
+\* HELPERS
+
+Max(a,b) == IF a > b THEN a ELSE b
+
+(* The minimum of a non-empty set of integers *)
+MinOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x <= y
+
+\* END HELPERS
+
+StateConstraint ==
+    /\ \A s \in safekeepers: 
+        /\ safekeeper_state[s].commit_lsn <= max_commit_lsn
+    /\ \A b \in brokers:
+        /\ \A s \in DOMAIN broker_state[b].sk:
+            /\ broker_state[b].sk[s].commit_lsn <= max_commit_lsn
+
+
+ 
+InitSafekeeper == [prune_lsn |-> 0, commit_lsn |-> 0, rx |-> [b \in brokers |-> NULL] ]
+InitBroker == [sk |-> [s \in safekeepers |-> [prune_lsn |-> 0, commit_lsn |-> 0]]]
+InitPageserver == [last_record_lsn |-> 0, preferred_sk |-> NULL, sk |-> [s \in safekeepers |-> [commit_lsn |-> 0]]]
+InitOnline == safekeepers \cup brokers \cup pageservers
+
+Init ==
+    /\ broker_state = [b \in brokers |-> InitBroker]
+    /\ safekeeper_state = [s \in safekeepers |-> InitSafekeeper]
+    /\ pageserver_state = [p \in pageservers |-> InitPageserver]
+    /\ online = InitOnline
+  
+NodeOnlineOffline ==
+    /\ online' = CHOOSE ss \in SUBSET InitOnline: 
+        /\ Cardinality(ss \cap safekeepers) >= 2
+        /\ Cardinality(ss \cap brokers) >= 2
+        /\ ss \cap pageservers = pageservers \* assume no PS failures for now
+    /\ UNCHANGED <<safekeeper_state,broker_state,pageserver_state>>
+   
+SkCommit(s1, s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ s1 # s2
+    /\ safekeeper_state[s1].commit_lsn = safekeeper_state[s2].commit_lsn
+    /\ LET
+            new_commit_lsn == safekeeper_state[s1].commit_lsn + 1
+       IN
+            safekeeper_state' = [safekeeper_state EXCEPT
+                ![s1].commit_lsn = new_commit_lsn,
+                ![s2].commit_lsn = new_commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPeerRecovery(s1,s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ safekeeper_state[s1].commit_lsn < safekeeper_state[s2].commit_lsn \* s2 has more WAL than s1
+    /\ safekeeper_state[s2].prune_lsn < safekeeper_state[s1].commit_lsn \* s2 has not yet trimmed the WAL the WAL
+    /\ safekeeper_state' = [safekeeper_state EXCEPT![s1].commit_lsn = safekeeper_state[s2].commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPushToBroker(s,b) ==
+    /\ {s, b} \subseteq online 
+    /\ broker_state' = LET
+           broker_side == broker_state[b].sk[s]
+           sk_side == safekeeper_state[s]
+           merged == [broker_side EXCEPT
+                !["commit_lsn"] = Max(broker_side.commit_lsn, sk_side.commit_lsn),
+                !["prune_lsn"] = Max(broker_side.prune_lsn, sk_side.prune_lsn)]
+           IN
+           [broker_state EXCEPT ![b].sk[s] = merged]
+    /\ UNCHANGED <<safekeeper_state, pageserver_state,online>> 
+
+PsRecvBroker(b,p,s) ==
+    /\ {b,p,s} \subseteq online
+    /\ LET
+            bsk == broker_state[b].sk[s]
+            psk == pageserver_state[p].sk[s]
+            updpsk == [psk EXCEPT !["commit_lsn"] = bsk.commit_lsn]
+       IN
+            pageserver_state' = IF bsk.commit_lsn > psk.commit_lsn
+                THEN [pageserver_state EXCEPT ![p].sk[s] = updpsk]
+                ELSE pageserver_state
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>
+    
+    
+SkRecvBroker(b,s) ==
+    /\ {b,s} \subseteq online
+    /\ safekeeper_state' = [safekeeper_state EXCEPT ![s].rx[b] = broker_state[b]] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+
+SkPrune(s) ==
+    /\ {s} \subseteq online
+    /\ LET
+        broker_infos == {safekeeper_state[s].rx[b]: b \in (DOMAIN safekeeper_state[s].rx)}
+        broker_infos_sks == UNION {bi.sk: bi \in broker_infos}
+        info ==
+            (
+                {safekeeper_state[s]}
+                \cup
+                broker_infos_sks
+            )
+            \
+            {NULL}
+        commit_lsns == {i.commit_lsn: i \in info}
+        prune_lsn == MinOfSet(commit_lsns)
+       IN
+        safekeeper_state' = [safekeeper_state EXCEPT ![s].prune_lsn = prune_lsn] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+    
+
+SksWithNewerWal(p) ==
+    LET
+        ps == pageserver_state[p]
+    IN
+    {s \in DOMAIN ps.sk: ps.sk[s].commit_lsn > ps.last_record_lsn}
+
+PsChooseSk(p) ==
+    /\ {p} \subseteq online
+    /\ SksWithNewerWal(p) # {}
+    /\ pageserver_state' = [pageserver_state EXCEPT![p].preferred_sk = CHOOSE s \in SksWithNewerWal(p): TRUE]
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>    
+                 
+    
+Next ==
+    \/ NodeOnlineOffline
+    \/ \E s1 \in safekeepers: \E s2 \in safekeepers:
+        \/ SkCommit(s1, s2)
+        \/ SkPeerRecovery(s1, s2)
+    \/ \E s \in safekeepers: \E b \in brokers:
+        \/ SkPushToBroker(s, b)
+        \/ SkRecvBroker(b, s)
+    \/ \E s \in safekeepers:
+        \/ SkPrune(s)  
+    \/ \E s \in safekeepers: \E b \in brokers: \E p \in pageservers: PsRecvBroker(b,p,s)
+    \/ \E p \in pageservers: PsChooseSk(p)
+    
+
+Spec == Init /\ [][Next]_<< broker_state, safekeeper_state, pageserver_state,online>>
+
+
+\* invariants
+
+PsLagsSk == \A p \in pageservers: \A s \in DOMAIN pageserver_state[p].sk: 
+        /\ pageserver_state[p].sk[s].commit_lsn <= safekeeper_state[s].commit_lsn
+
+PeerRecoveryIsPossible ==
+    \A laggard \in (safekeepers \cap online): \E donor \in (safekeepers \cap online):
+        /\ (safekeeper_state[laggard].commit_lsn < safekeeper_state[donor].commit_lsn)
+            => safekeeper_state[donor].prune_lsn <= safekeeper_state[laggard].commit_lsn
+        
+
+EventuallyLaggingSkIsNotPreferredSk == <>(
+        LET
+            sks == safekeeper_state
+            lagging_sks == { s \in safekeepers: \A s2 \in safekeepers: sks[s].commit_lsn <= sks[s2].commit_lsn }
+            preferred_sks == {pageserver_state[p].preferred_sk: p \in pageservers}
+        IN
+            preferred_sks \cap lagging_sks = {}
+            
+    )
+
+    
+
+====

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737324186169/MC.cfg

@@ -0,0 +1,54 @@
+\* MV CONSTANT declarations
+CONSTANTS
+b1 = b1
+b2 = b2
+b3 = b3
+\* MV CONSTANT declarations
+CONSTANTS
+s1 = s1
+s2 = s2
+s3 = s3
+\* MV CONSTANT declarations
+CONSTANTS
+p1 = p1
+\* MV CONSTANT declarations
+CONSTANTS
+az1 = az1
+az2 = az2
+az3 = az3
+\* MV CONSTANT definitions
+CONSTANT
+brokers <- const_1737324182115316000
+\* CONSTANT declarations
+CONSTANT NULL = NULL
+\* MV CONSTANT definitions
+CONSTANT
+safekeepers <- const_1737324182115317000
+\* MV CONSTANT definitions
+CONSTANT
+pageservers <- const_1737324182115318000
+\* MV CONSTANT definitions
+CONSTANT
+azs <- const_1737324182115319000
+\* SYMMETRY definition
+SYMMETRY symm_1737324182115320000
+\* CONSTANT definitions
+CONSTANT
+max_commit_lsn <- const_1737324182115321000
+\* CONSTANT definitions
+CONSTANT
+az_mapping <- const_1737324182115322000
+\* CONSTRAINT definition
+CONSTRAINT
+constr_1737324182115323000
+\* SPECIFICATION definition
+SPECIFICATION
+Spec
+\* INVARIANT definition
+INVARIANT
+PsLagsSk
+PeerRecoveryIsPossible
+\* PROPERTY definition
+PROPERTY
+EventuallyLaggingSkIsNotPreferredSk
+\* Generated on Sun Jan 19 23:03:02 CET 2025

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737324186169/MC.out

@@ -0,0 +1,442 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 58 and seed 2509155719358048991 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 23239] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 23:03:03)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 23:03:04.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 1000:1 @!@!@
+TLC threw an unexpected exception.
+This was probably caused by an error in the spec or model.
+See the User Output or TLC Console for clues to what happened.
+The exception was a java.lang.RuntimeException
+: Attempted to enumerate UNION(s), but some element of s is nonenumerable.
+@!@!@ENDMSG 1000 @!@!@
+@!@!@STARTMSG 2121:1 @!@!@
+The behavior up to this point is:
+@!@!@ENDMSG 2121 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+1: <Initial predicate>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, b3, s1, s2, s3, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+2: <NodeOnlineOffline line 52, col 5 to line 56, col 67 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+3: <SkCommit line 59, col 5 to line 68, col 58 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+4: <SkRecvBroker line 103, col 5 to line 105, col 57 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2147:1 @!@!@
+TLC was unable to fingerprint.
+
+Fingerprint Stack Trace:
+2) line 111, col 29 to line 111, col 74 of module replicated
+1) line 114, col 17 to line 116, col 32 of module replicated
+0) line 113, col 13 to line 119, col 18 of module replicated
+
+Reason:
+Attempted to enumerate UNION(s), but some element of s is nonenumerable.
+@!@!@ENDMSG 2147 @!@!@
+@!@!@STARTMSG 2201:0 @!@!@
+The coverage statistics at 2025-01-19 23:03:04
+@!@!@ENDMSG 2201 @!@!@
+@!@!@STARTMSG 2773:0 @!@!@
+<Init line 45, col 1 to line 45, col 4 of module replicated>: 2:2
+@!@!@ENDMSG 2773 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 46, col 5 to line 49, col 26 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<NodeOnlineOffline line 51, col 1 to line 51, col 17 of module replicated>: 1:15
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 52, col 8 to line 55, col 44 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 52, col 18 to line 55, col 44 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 53, col 9 to line 55, col 44 of module replicated: 1605
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 53, col 12 to line 53, col 48 of module replicated: 1605
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 54, col 12 to line 54, col 44 of module replicated: 645
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 55, col 12 to line 55, col 44 of module replicated: 210
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 52, col 32 to line 52, col 48 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 56, col 8 to line 56, col 67 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkCommit line 58, col 1 to line 58, col 16 of module replicated>: 6:31
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 59, col 8 to line 59, col 32 of module replicated: 150
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 59, col 8 to line 59, col 15 of module replicated: 119:277
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 59, col 27 to line 59, col 32 of module replicated: 119
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 60, col 8 to line 60, col 14 of module replicated: 110
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 8 to line 60, col 9 of module replicated: 79
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 13 to line 60, col 14 of module replicated: 79
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 61, col 8 to line 61, col 72 of module replicated: 78
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 8 to line 61, col 38 of module replicated: 47
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 42 to line 61, col 72 of module replicated: 47
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 65, col 13 to line 67, col 50 of module replicated: 31
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 68, col 8 to line 68, col 58 of module replicated: 31
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPeerRecovery line 70, col 1 to line 70, col 21 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 71, col 8 to line 71, col 32 of module replicated: 118
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 72, col 8 to line 72, col 72 of module replicated: 78
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 73, col 8 to line 73, col 71 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 74, col 8 to line 74, col 102 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 75, col 8 to line 75, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPushToBroker line 77, col 1 to line 77, col 19 of module replicated>: 16:74
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 78, col 8 to line 78, col 30 of module replicated: 183
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 78, col 8 to line 78, col 13 of module replicated: 109:271
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 78, col 25 to line 78, col 30 of module replicated: 109
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 79, col 8 to line 86, col 52 of module replicated: 74
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 87, col 8 to line 87, col 62 of module replicated: 74
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsRecvBroker line 89, col 1 to line 89, col 19 of module replicated>: 2:52
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 90, col 8 to line 90, col 31 of module replicated: 124
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 90, col 8 to line 90, col 14 of module replicated: 72:236
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 90, col 26 to line 90, col 31 of module replicated: 72
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 96, col 13 to line 98, col 37 of module replicated: 52
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 96, col 33 to line 98, col 37 of module replicated: 52
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 96, col 36 to line 96, col 66 of module replicated: 52
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 97, col 22 to line 97, col 66 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 98, col 22 to line 98, col 37 of module replicated: 50
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 99, col 8 to line 99, col 58 of module replicated: 52
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkRecvBroker line 102, col 1 to line 102, col 17 of module replicated>: 27:74
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 103, col 8 to line 103, col 29 of module replicated: 183
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 103, col 8 to line 103, col 12 of module replicated: 109:271
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 103, col 24 to line 103, col 29 of module replicated: 109
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 104, col 8 to line 104, col 81 of module replicated: 74
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 105, col 8 to line 105, col 57 of module replicated: 74
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPrune line 107, col 1 to line 107, col 10 of module replicated>: 8:20
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 108, col 8 to line 108, col 27 of module replicated: 48
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 108, col 8 to line 108, col 10 of module replicated: 28:52
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 108, col 22 to line 108, col 27 of module replicated: 28
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 123, col 9 to line 123, col 80 of module replicated: 20
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 123, col 29 to line 123, col 80 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 123, col 30 to line 123, col 45 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 123, col 54 to line 123, col 79 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 123, col 71 to line 123, col 79 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 121, col 22 to line 121, col 42 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 27, col 3 to line 27, col 36 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||line 27, col 19 to line 27, col 36 of module replicated: 20
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||line 27, col 16 to line 27, col 16 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 121, col 31 to line 121, col 41 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||line 120, col 24 to line 120, col 49 of module replicated: 24:60
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||line 120, col 25 to line 120, col 36 of module replicated: 20
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||line 120, col 45 to line 120, col 48 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||line 113, col 13 to line 119, col 18 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 114, col 17 to line 116, col 32 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||line 114, col 17 to line 114, col 37 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||line 116, col 17 to line 116, col 32 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 111, col 29 to line 111, col 74 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||||||||line 111, col 35 to line 111, col 74 of module replicated: 24:24
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||||line 111, col 36 to line 111, col 40 of module replicated: 4
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||||line 111, col 51 to line 111, col 71 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||||line 111, col 51 to line 111, col 62 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  ||||||||||||||line 110, col 25 to line 110, col 90 of module replicated: 24:28
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||||||line 110, col 26 to line 110, col 50 of module replicated: 72
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||||||line 110, col 60 to line 110, col 88 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||||line 111, col 66 to line 111, col 71 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 119, col 13 to line 119, col 18 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 124, col 8 to line 124, col 57 of module replicated: 20
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsChooseSk line 133, col 1 to line 133, col 13 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 134, col 8 to line 134, col 27 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 135, col 8 to line 135, col 30 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 135, col 8 to line 135, col 25 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 129, col 15 to line 129, col 33 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 131, col 5 to line 131, col 66 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 131, col 26 to line 131, col 65 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 131, col 12 to line 131, col 23 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 135, col 24 to line 135, col 24 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 135, col 29 to line 135, col 30 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 136, col 8 to line 136, col 109 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 137, col 8 to line 137, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PsLagsSk line 159, col 1 to line 159, col 8 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 159, col 13 to line 160, col 81 of module replicated: 65
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 159, col 35 to line 160, col 81 of module replicated: 65
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 160, col 9 to line 160, col 81 of module replicated: 195
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 159, col 44 to line 159, col 72 of module replicated: 65
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 159, col 22 to line 159, col 32 of module replicated: 65
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PeerRecoveryIsPossible line 162, col 1 to line 162, col 22 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 163, col 5 to line 165, col 88 of module replicated: 65
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 47 to line 165, col 88 of module replicated: 164
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 164, col 9 to line 165, col 88 of module replicated: 167
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 164, col 12 to line 165, col 88 of module replicated: 167
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 164, col 13 to line 164, col 85 of module replicated: 167
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 165, col 16 to line 165, col 88 of module replicated: 35
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 163, col 61 to line 163, col 83 of module replicated: 164
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 21 to line 163, col 43 of module replicated: 65
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2202:0 @!@!@
+End of statistics.
+@!@!@ENDMSG 2202 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(5) at 2025-01-19 23:03:04: 248 states generated (5.740 s/min), 61 distinct states found (1.412 ds/min), 50 states left on queue.
+@!@!@ENDMSG 2200 @!@!@
+@!@!@STARTMSG 2199:0 @!@!@
+248 states generated, 61 distinct states found, 50 states left on queue.
+@!@!@ENDMSG 2199 @!@!@
+@!@!@STARTMSG 2186:0 @!@!@
+Finished in 2611ms at (2025-01-19 23:03:04)
+@!@!@ENDMSG 2186 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737324186169/MC.tla

@@ -0,0 +1,65 @@
+---- MODULE MC ----
+EXTENDS replicated, TLC
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+b1, b2, b3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+s1, s2, s3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+p1
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+az1, az2, az3
+----
+
+\* MV CONSTANT definitions brokers
+const_1737324182115316000 == 
+{b1, b2, b3}
+----
+
+\* MV CONSTANT definitions safekeepers
+const_1737324182115317000 == 
+{s1, s2, s3}
+----
+
+\* MV CONSTANT definitions pageservers
+const_1737324182115318000 == 
+{p1}
+----
+
+\* MV CONSTANT definitions azs
+const_1737324182115319000 == 
+{az1, az2, az3}
+----
+
+\* SYMMETRY definition
+symm_1737324182115320000 == 
+Permutations(const_1737324182115316000) \union Permutations(const_1737324182115317000) \union Permutations(const_1737324182115318000) \union Permutations(const_1737324182115319000)
+----
+
+\* CONSTANT definitions @modelParameterConstants:4max_commit_lsn
+const_1737324182115321000 == 
+2
+----
+
+\* CONSTANT definitions @modelParameterConstants:5az_mapping
+const_1737324182115322000 == 
+[ az1 |-> {b1,s1,p1} , az2 |-> {b2,s2} , az3 |-> {b3,s3}]
+----
+
+\* CONSTRAINT definition @modelParameterContraint:0
+constr_1737324182115323000 ==
+StateConstraint
+----
+=============================================================================
+\* Modification History
+\* Created Sun Jan 19 23:03:02 CET 2025 by cs

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737324186169/MC_TE.out

@@ -0,0 +1,442 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 58 and seed 2509155719358048991 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 23239] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 23:03:03)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 23:03:04.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 1000:1 @!@!@
+TLC threw an unexpected exception.
+This was probably caused by an error in the spec or model.
+See the User Output or TLC Console for clues to what happened.
+The exception was a java.lang.RuntimeException
+: Attempted to enumerate UNION(s), but some element of s is nonenumerable.
+@!@!@ENDMSG 1000 @!@!@
+@!@!@STARTMSG 2121:1 @!@!@
+The behavior up to this point is:
+@!@!@ENDMSG 2121 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+1: <Initial predicate>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, b3, s1, s2, s3, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+2: <NodeOnlineOffline line 52, col 5 to line 56, col 67 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+3: <SkCommit line 59, col 5 to line 68, col 58 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+4: <SkRecvBroker line 103, col 5 to line 105, col 57 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2147:1 @!@!@
+TLC was unable to fingerprint.
+
+Fingerprint Stack Trace:
+2) line 111, col 29 to line 111, col 74 of module replicated
+1) line 114, col 17 to line 116, col 32 of module replicated
+0) line 113, col 13 to line 119, col 18 of module replicated
+
+Reason:
+Attempted to enumerate UNION(s), but some element of s is nonenumerable.
+@!@!@ENDMSG 2147 @!@!@
+@!@!@STARTMSG 2201:0 @!@!@
+The coverage statistics at 2025-01-19 23:03:04
+@!@!@ENDMSG 2201 @!@!@
+@!@!@STARTMSG 2773:0 @!@!@
+<Init line 45, col 1 to line 45, col 4 of module replicated>: 2:2
+@!@!@ENDMSG 2773 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 46, col 5 to line 49, col 26 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<NodeOnlineOffline line 51, col 1 to line 51, col 17 of module replicated>: 1:15
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 52, col 8 to line 55, col 44 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 52, col 18 to line 55, col 44 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 53, col 9 to line 55, col 44 of module replicated: 1605
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 53, col 12 to line 53, col 48 of module replicated: 1605
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 54, col 12 to line 54, col 44 of module replicated: 645
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 55, col 12 to line 55, col 44 of module replicated: 210
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 52, col 32 to line 52, col 48 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 56, col 8 to line 56, col 67 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkCommit line 58, col 1 to line 58, col 16 of module replicated>: 6:31
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 59, col 8 to line 59, col 32 of module replicated: 150
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 59, col 8 to line 59, col 15 of module replicated: 119:277
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 59, col 27 to line 59, col 32 of module replicated: 119
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 60, col 8 to line 60, col 14 of module replicated: 110
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 8 to line 60, col 9 of module replicated: 79
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 13 to line 60, col 14 of module replicated: 79
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 61, col 8 to line 61, col 72 of module replicated: 78
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 8 to line 61, col 38 of module replicated: 47
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 42 to line 61, col 72 of module replicated: 47
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 65, col 13 to line 67, col 50 of module replicated: 31
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 68, col 8 to line 68, col 58 of module replicated: 31
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPeerRecovery line 70, col 1 to line 70, col 21 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 71, col 8 to line 71, col 32 of module replicated: 118
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 72, col 8 to line 72, col 72 of module replicated: 78
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 73, col 8 to line 73, col 71 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 74, col 8 to line 74, col 102 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 75, col 8 to line 75, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPushToBroker line 77, col 1 to line 77, col 19 of module replicated>: 16:74
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 78, col 8 to line 78, col 30 of module replicated: 183
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 78, col 8 to line 78, col 13 of module replicated: 109:271
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 78, col 25 to line 78, col 30 of module replicated: 109
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 79, col 8 to line 86, col 52 of module replicated: 74
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 87, col 8 to line 87, col 62 of module replicated: 74
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsRecvBroker line 89, col 1 to line 89, col 19 of module replicated>: 2:52
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 90, col 8 to line 90, col 31 of module replicated: 124
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 90, col 8 to line 90, col 14 of module replicated: 72:236
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 90, col 26 to line 90, col 31 of module replicated: 72
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 96, col 13 to line 98, col 37 of module replicated: 52
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 96, col 33 to line 98, col 37 of module replicated: 52
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 96, col 36 to line 96, col 66 of module replicated: 52
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 97, col 22 to line 97, col 66 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 98, col 22 to line 98, col 37 of module replicated: 50
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 99, col 8 to line 99, col 58 of module replicated: 52
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkRecvBroker line 102, col 1 to line 102, col 17 of module replicated>: 27:74
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 103, col 8 to line 103, col 29 of module replicated: 183
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 103, col 8 to line 103, col 12 of module replicated: 109:271
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 103, col 24 to line 103, col 29 of module replicated: 109
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 104, col 8 to line 104, col 81 of module replicated: 74
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 105, col 8 to line 105, col 57 of module replicated: 74
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPrune line 107, col 1 to line 107, col 10 of module replicated>: 8:20
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 108, col 8 to line 108, col 27 of module replicated: 48
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 108, col 8 to line 108, col 10 of module replicated: 28:52
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 108, col 22 to line 108, col 27 of module replicated: 28
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 123, col 9 to line 123, col 80 of module replicated: 20
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 123, col 29 to line 123, col 80 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 123, col 30 to line 123, col 45 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 123, col 54 to line 123, col 79 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 123, col 71 to line 123, col 79 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 121, col 22 to line 121, col 42 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 27, col 3 to line 27, col 36 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||line 27, col 19 to line 27, col 36 of module replicated: 20
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||line 27, col 16 to line 27, col 16 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 121, col 31 to line 121, col 41 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||line 120, col 24 to line 120, col 49 of module replicated: 24:60
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||line 120, col 25 to line 120, col 36 of module replicated: 20
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||line 120, col 45 to line 120, col 48 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||line 113, col 13 to line 119, col 18 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 114, col 17 to line 116, col 32 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||line 114, col 17 to line 114, col 37 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||line 116, col 17 to line 116, col 32 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 111, col 29 to line 111, col 74 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||||||||line 111, col 35 to line 111, col 74 of module replicated: 24:24
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||||line 111, col 36 to line 111, col 40 of module replicated: 4
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||||line 111, col 51 to line 111, col 71 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||||line 111, col 51 to line 111, col 62 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  ||||||||||||||line 110, col 25 to line 110, col 90 of module replicated: 24:28
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||||||line 110, col 26 to line 110, col 50 of module replicated: 72
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||||||line 110, col 60 to line 110, col 88 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||||line 111, col 66 to line 111, col 71 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 119, col 13 to line 119, col 18 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 124, col 8 to line 124, col 57 of module replicated: 20
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsChooseSk line 133, col 1 to line 133, col 13 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 134, col 8 to line 134, col 27 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 135, col 8 to line 135, col 30 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 135, col 8 to line 135, col 25 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 129, col 15 to line 129, col 33 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 131, col 5 to line 131, col 66 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 131, col 26 to line 131, col 65 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 131, col 12 to line 131, col 23 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 135, col 24 to line 135, col 24 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 135, col 29 to line 135, col 30 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 136, col 8 to line 136, col 109 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 137, col 8 to line 137, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PsLagsSk line 159, col 1 to line 159, col 8 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 159, col 13 to line 160, col 81 of module replicated: 65
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 159, col 35 to line 160, col 81 of module replicated: 65
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 160, col 9 to line 160, col 81 of module replicated: 195
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 159, col 44 to line 159, col 72 of module replicated: 65
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 159, col 22 to line 159, col 32 of module replicated: 65
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PeerRecoveryIsPossible line 162, col 1 to line 162, col 22 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 163, col 5 to line 165, col 88 of module replicated: 65
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 47 to line 165, col 88 of module replicated: 164
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 164, col 9 to line 165, col 88 of module replicated: 167
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 164, col 12 to line 165, col 88 of module replicated: 167
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 164, col 13 to line 164, col 85 of module replicated: 167
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 165, col 16 to line 165, col 88 of module replicated: 35
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 163, col 61 to line 163, col 83 of module replicated: 164
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 21 to line 163, col 43 of module replicated: 65
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2202:0 @!@!@
+End of statistics.
+@!@!@ENDMSG 2202 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(5) at 2025-01-19 23:03:04: 248 states generated (5.740 s/min), 61 distinct states found (1.412 ds/min), 50 states left on queue.
+@!@!@ENDMSG 2200 @!@!@
+@!@!@STARTMSG 2199:0 @!@!@
+248 states generated, 61 distinct states found, 50 states left on queue.
+@!@!@ENDMSG 2199 @!@!@
+@!@!@STARTMSG 2186:0 @!@!@
+Finished in 2611ms at (2025-01-19 23:03:04)
+@!@!@ENDMSG 2186 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737324186169/replicated.tla

@@ -0,0 +1,180 @@
+---- MODULE replicated ----
+
+EXTENDS Integers, FiniteSets
+
+VARIABLES broker_state, safekeeper_state, pageserver_state, online
+
+
+CONSTANT
+    brokers,
+    safekeepers,
+    pageservers,
+    azs,
+    az_mapping
+
+CONSTANT
+    NULL
+    
+CONSTANT
+    max_commit_lsn
+    
+\* HELPERS
+
+Max(a,b) == IF a > b THEN a ELSE b
+
+(* The minimum of a non-empty set of integers *)
+MinOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x <= y
+
+\* END HELPERS
+
+StateConstraint ==
+    /\ \A s \in safekeepers: 
+        /\ safekeeper_state[s].commit_lsn <= max_commit_lsn
+    /\ \A b \in brokers:
+        /\ \A s \in DOMAIN broker_state[b].sk:
+            /\ broker_state[b].sk[s].commit_lsn <= max_commit_lsn
+
+
+ 
+InitSafekeeper == [prune_lsn |-> 0, commit_lsn |-> 0, rx |-> [b \in brokers |-> NULL] ]
+InitBroker == [sk |-> [s \in safekeepers |-> [prune_lsn |-> 0, commit_lsn |-> 0]]]
+InitPageserver == [last_record_lsn |-> 0, preferred_sk |-> NULL, sk |-> [s \in safekeepers |-> [commit_lsn |-> 0]]]
+InitOnline == safekeepers \cup brokers \cup pageservers
+
+Init ==
+    /\ broker_state = [b \in brokers |-> InitBroker]
+    /\ safekeeper_state = [s \in safekeepers |-> InitSafekeeper]
+    /\ pageserver_state = [p \in pageservers |-> InitPageserver]
+    /\ online = InitOnline
+  
+NodeOnlineOffline ==
+    /\ online' = CHOOSE ss \in SUBSET InitOnline: 
+        /\ Cardinality(ss \cap safekeepers) >= 2
+        /\ Cardinality(ss \cap brokers) >= 2
+        /\ ss \cap pageservers = pageservers \* assume no PS failures for now
+    /\ UNCHANGED <<safekeeper_state,broker_state,pageserver_state>>
+   
+SkCommit(s1, s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ s1 # s2
+    /\ safekeeper_state[s1].commit_lsn = safekeeper_state[s2].commit_lsn
+    /\ LET
+            new_commit_lsn == safekeeper_state[s1].commit_lsn + 1
+       IN
+            safekeeper_state' = [safekeeper_state EXCEPT
+                ![s1].commit_lsn = new_commit_lsn,
+                ![s2].commit_lsn = new_commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPeerRecovery(s1,s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ safekeeper_state[s1].commit_lsn < safekeeper_state[s2].commit_lsn \* s2 has more WAL than s1
+    /\ safekeeper_state[s2].prune_lsn < safekeeper_state[s1].commit_lsn \* s2 has not yet trimmed the WAL the WAL
+    /\ safekeeper_state' = [safekeeper_state EXCEPT![s1].commit_lsn = safekeeper_state[s2].commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPushToBroker(s,b) ==
+    /\ {s, b} \subseteq online 
+    /\ broker_state' = LET
+           broker_side == broker_state[b].sk[s]
+           sk_side == safekeeper_state[s]
+           merged == [broker_side EXCEPT
+                !["commit_lsn"] = Max(broker_side.commit_lsn, sk_side.commit_lsn),
+                !["prune_lsn"] = Max(broker_side.prune_lsn, sk_side.prune_lsn)]
+           IN
+           [broker_state EXCEPT ![b].sk[s] = merged]
+    /\ UNCHANGED <<safekeeper_state, pageserver_state,online>> 
+
+PsRecvBroker(b,p,s) ==
+    /\ {b,p,s} \subseteq online
+    /\ LET
+            bsk == broker_state[b].sk[s]
+            psk == pageserver_state[p].sk[s]
+            updpsk == [psk EXCEPT !["commit_lsn"] = bsk.commit_lsn]
+       IN
+            pageserver_state' = IF bsk.commit_lsn > psk.commit_lsn
+                THEN [pageserver_state EXCEPT ![p].sk[s] = updpsk]
+                ELSE pageserver_state
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>
+    
+    
+SkRecvBroker(b,s) ==
+    /\ {b,s} \subseteq online
+    /\ safekeeper_state' = [safekeeper_state EXCEPT ![s].rx[b] = broker_state[b]] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+
+SkPrune(s) ==
+    /\ {s} \subseteq online
+    /\ LET
+        broker_infos == {safekeeper_state[s].rx[b]: b \in (DOMAIN safekeeper_state[s].rx)}
+        broker_infos_sks == UNION {bi.sk: bi \in (broker_infos \ {NULL}) }
+        info ==
+            (
+                {safekeeper_state[s]}
+                \cup
+                broker_infos_sks
+            )
+            \
+            {NULL}
+        commit_lsns == {i.commit_lsn: i \in info}
+        prune_lsn == MinOfSet(commit_lsns)
+       IN
+        safekeeper_state' = [safekeeper_state EXCEPT ![s].prune_lsn = prune_lsn] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+    
+
+SksWithNewerWal(p) ==
+    LET
+        ps == pageserver_state[p]
+    IN
+    {s \in DOMAIN ps.sk: ps.sk[s].commit_lsn > ps.last_record_lsn}
+
+PsChooseSk(p) ==
+    /\ {p} \subseteq online
+    /\ SksWithNewerWal(p) # {}
+    /\ pageserver_state' = [pageserver_state EXCEPT![p].preferred_sk = CHOOSE s \in SksWithNewerWal(p): TRUE]
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>    
+                 
+    
+Next ==
+    \/ NodeOnlineOffline
+    \/ \E s1 \in safekeepers: \E s2 \in safekeepers:
+        \/ SkCommit(s1, s2)
+        \/ SkPeerRecovery(s1, s2)
+    \/ \E s \in safekeepers: \E b \in brokers:
+        \/ SkPushToBroker(s, b)
+        \/ SkRecvBroker(b, s)
+    \/ \E s \in safekeepers:
+        \/ SkPrune(s)  
+    \/ \E s \in safekeepers: \E b \in brokers: \E p \in pageservers: PsRecvBroker(b,p,s)
+    \/ \E p \in pageservers: PsChooseSk(p)
+    
+
+Spec == Init /\ [][Next]_<< broker_state, safekeeper_state, pageserver_state,online>>
+
+
+\* invariants
+
+PsLagsSk == \A p \in pageservers: \A s \in DOMAIN pageserver_state[p].sk: 
+        /\ pageserver_state[p].sk[s].commit_lsn <= safekeeper_state[s].commit_lsn
+
+PeerRecoveryIsPossible ==
+    \A laggard \in (safekeepers \cap online): \E donor \in (safekeepers \cap online):
+        /\ (safekeeper_state[laggard].commit_lsn < safekeeper_state[donor].commit_lsn)
+            => safekeeper_state[donor].prune_lsn <= safekeeper_state[laggard].commit_lsn
+        
+
+EventuallyLaggingSkIsNotPreferredSk == <>(
+        LET
+            sks == safekeeper_state
+            lagging_sks == { s \in safekeepers: \A s2 \in safekeepers: sks[s].commit_lsn <= sks[s2].commit_lsn }
+            preferred_sks == {pageserver_state[p].preferred_sk: p \in pageservers}
+        IN
+            preferred_sks \cap lagging_sks = {}
+            
+    )
+
+    
+
+====

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737324315582/MC.cfg

@@ -0,0 +1,54 @@
+\* MV CONSTANT declarations
+CONSTANTS
+b1 = b1
+b2 = b2
+b3 = b3
+\* MV CONSTANT declarations
+CONSTANTS
+s1 = s1
+s2 = s2
+s3 = s3
+\* MV CONSTANT declarations
+CONSTANTS
+p1 = p1
+\* MV CONSTANT declarations
+CONSTANTS
+az1 = az1
+az2 = az2
+az3 = az3
+\* MV CONSTANT definitions
+CONSTANT
+brokers <- const_1737324311514327000
+\* CONSTANT declarations
+CONSTANT NULL = NULL
+\* MV CONSTANT definitions
+CONSTANT
+safekeepers <- const_1737324311514328000
+\* MV CONSTANT definitions
+CONSTANT
+pageservers <- const_1737324311514329000
+\* MV CONSTANT definitions
+CONSTANT
+azs <- const_1737324311514330000
+\* SYMMETRY definition
+SYMMETRY symm_1737324311514331000
+\* CONSTANT definitions
+CONSTANT
+max_commit_lsn <- const_1737324311514332000
+\* CONSTANT definitions
+CONSTANT
+az_mapping <- const_1737324311514333000
+\* CONSTRAINT definition
+CONSTRAINT
+constr_1737324311514334000
+\* SPECIFICATION definition
+SPECIFICATION
+Spec
+\* INVARIANT definition
+INVARIANT
+PsLagsSk
+PeerRecoveryIsPossible
+\* PROPERTY definition
+PROPERTY
+EventuallyLaggingSkIsNotPreferredSk
+\* Generated on Sun Jan 19 23:05:11 CET 2025

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737324315582/MC.out

@@ -0,0 +1,324 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 120 and seed -4351583933909868251 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 23327] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 23:05:12)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 23:05:13.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 1000:1 @!@!@
+TLC threw an unexpected exception.
+This was probably caused by an error in the spec or model.
+See the User Output or TLC Console for clues to what happened.
+The exception was a java.lang.RuntimeException
+: Attempted to select field "rx" from a non-record value s1
+line 111, col 50 to line 111, col 53 of module replicated
+@!@!@ENDMSG 1000 @!@!@
+@!@!@STARTMSG 2121:1 @!@!@
+The behavior up to this point is:
+@!@!@ENDMSG 2121 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+1: <Initial predicate>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, b3, s1, s2, s3, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+2: <NodeOnlineOffline line 52, col 5 to line 56, col 67 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+3: <SkCommit line 59, col 5 to line 68, col 58 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2103:1 @!@!@
+The error occurred when TLC was evaluating the nested
+expressions at the following positions:
+0. Line 108, column 5 to line 125, column 57 in replicated
+1. Line 108, column 8 to line 108, column 27 in replicated
+2. Line 109, column 8 to line 124, column 80 in replicated
+3. Line 124, column 9 to line 124, column 80 in replicated
+4. Line 124, column 29 to line 124, column 80 in replicated
+5. Line 124, column 71 to line 124, column 79 in replicated
+6. Line 122, column 22 to line 122, column 42 in replicated
+7. Line 27, column 3 to line 27, column 36 in replicated
+8. Line 27, column 16 to line 27, column 16 in replicated
+9. Line 122, column 31 to line 122, column 41 in replicated
+10. Line 121, column 24 to line 121, column 49 in replicated
+11. Line 121, column 45 to line 121, column 48 in replicated
+12. Line 114, column 13 to line 120, column 18 in replicated
+13. Line 115, column 17 to line 117, column 32 in replicated
+14. Line 117, column 17 to line 117, column 32 in replicated
+15. Line 112, column 29 to line 112, column 74 in replicated
+16. Line 112, column 35 to line 112, column 74 in replicated
+17. Line 112, column 51 to line 112, column 71 in replicated
+18. Line 112, column 51 to line 112, column 62 in replicated
+19. Line 111, column 25 to line 111, column 64 in replicated
+20. Line 111, column 25 to line 111, column 55 in replicated
+21. Line 111, column 43 to line 111, column 53 in replicated
+22. Line 111, column 50 to line 111, column 53 in replicated
+
+
+@!@!@ENDMSG 2103 @!@!@
+@!@!@STARTMSG 2201:0 @!@!@
+The coverage statistics at 2025-01-19 23:05:14
+@!@!@ENDMSG 2201 @!@!@
+@!@!@STARTMSG 2773:0 @!@!@
+<Init line 45, col 1 to line 45, col 4 of module replicated>: 2:2
+@!@!@ENDMSG 2773 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 46, col 5 to line 49, col 26 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<NodeOnlineOffline line 51, col 1 to line 51, col 17 of module replicated>: 1:7
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 52, col 8 to line 55, col 44 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 52, col 18 to line 55, col 44 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 53, col 9 to line 55, col 44 of module replicated: 749
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 53, col 12 to line 53, col 48 of module replicated: 749
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 54, col 12 to line 54, col 44 of module replicated: 301
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 55, col 12 to line 55, col 44 of module replicated: 98
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 52, col 32 to line 52, col 48 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 56, col 8 to line 56, col 67 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkCommit line 58, col 1 to line 58, col 16 of module replicated>: 4:15
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 59, col 8 to line 59, col 32 of module replicated: 62
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 59, col 8 to line 59, col 15 of module replicated: 47:110
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 59, col 27 to line 59, col 32 of module replicated: 47
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 60, col 8 to line 60, col 14 of module replicated: 47
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 8 to line 60, col 9 of module replicated: 32
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 13 to line 60, col 14 of module replicated: 32
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 61, col 8 to line 61, col 72 of module replicated: 34
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 8 to line 61, col 38 of module replicated: 19
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 42 to line 61, col 72 of module replicated: 19
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 65, col 13 to line 67, col 50 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 68, col 8 to line 68, col 58 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPeerRecovery line 70, col 1 to line 70, col 21 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 71, col 8 to line 71, col 32 of module replicated: 46
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 72, col 8 to line 72, col 72 of module replicated: 31
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 73, col 8 to line 73, col 71 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 74, col 8 to line 74, col 102 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 75, col 8 to line 75, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPushToBroker line 77, col 1 to line 77, col 19 of module replicated>: 2:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 78, col 8 to line 78, col 30 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 78, col 8 to line 78, col 13 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 78, col 25 to line 78, col 30 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 79, col 8 to line 86, col 52 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 87, col 8 to line 87, col 62 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsRecvBroker line 89, col 1 to line 89, col 19 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 96, col 13 to line 98, col 37 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 99, col 8 to line 99, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkRecvBroker line 102, col 1 to line 102, col 17 of module replicated>: 5:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 103, col 8 to line 103, col 29 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 103, col 8 to line 103, col 12 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 103, col 24 to line 103, col 29 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 104, col 8 to line 104, col 81 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 105, col 8 to line 105, col 57 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPrune line 107, col 1 to line 107, col 10 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 108, col 8 to line 108, col 27 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 124, col 9 to line 124, col 80 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 124, col 29 to line 124, col 80 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 125, col 8 to line 125, col 57 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsChooseSk line 134, col 1 to line 134, col 13 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 137, col 8 to line 137, col 109 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 138, col 8 to line 138, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PsLagsSk line 160, col 1 to line 160, col 8 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 160, col 13 to line 161, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 160, col 35 to line 161, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 161, col 9 to line 161, col 81 of module replicated: 39
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 160, col 44 to line 160, col 72 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 160, col 22 to line 160, col 32 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PeerRecoveryIsPossible line 163, col 1 to line 163, col 22 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 164, col 5 to line 166, col 88 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 164, col 47 to line 166, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 165, col 9 to line 166, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 165, col 12 to line 166, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 165, col 13 to line 165, col 85 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 166, col 16 to line 166, col 88 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 164, col 61 to line 164, col 83 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 164, col 21 to line 164, col 43 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2202:0 @!@!@
+End of statistics.
+@!@!@ENDMSG 2202 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(4) at 2025-01-19 23:05:14: 69 states generated (1.732 s/min), 13 distinct states found (326 ds/min), 9 states left on queue.
+@!@!@ENDMSG 2200 @!@!@
+@!@!@STARTMSG 2199:0 @!@!@
+69 states generated, 13 distinct states found, 9 states left on queue.
+@!@!@ENDMSG 2199 @!@!@
+@!@!@STARTMSG 2186:0 @!@!@
+Finished in 2406ms at (2025-01-19 23:05:14)
+@!@!@ENDMSG 2186 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737324315582/MC.tla

@@ -0,0 +1,65 @@
+---- MODULE MC ----
+EXTENDS replicated, TLC
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+b1, b2, b3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+s1, s2, s3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+p1
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+az1, az2, az3
+----
+
+\* MV CONSTANT definitions brokers
+const_1737324311514327000 == 
+{b1, b2, b3}
+----
+
+\* MV CONSTANT definitions safekeepers
+const_1737324311514328000 == 
+{s1, s2, s3}
+----
+
+\* MV CONSTANT definitions pageservers
+const_1737324311514329000 == 
+{p1}
+----
+
+\* MV CONSTANT definitions azs
+const_1737324311514330000 == 
+{az1, az2, az3}
+----
+
+\* SYMMETRY definition
+symm_1737324311514331000 == 
+Permutations(const_1737324311514327000) \union Permutations(const_1737324311514328000) \union Permutations(const_1737324311514329000) \union Permutations(const_1737324311514330000)
+----
+
+\* CONSTANT definitions @modelParameterConstants:4max_commit_lsn
+const_1737324311514332000 == 
+2
+----
+
+\* CONSTANT definitions @modelParameterConstants:5az_mapping
+const_1737324311514333000 == 
+[ az1 |-> {b1,s1,p1} , az2 |-> {b2,s2} , az3 |-> {b3,s3}]
+----
+
+\* CONSTRAINT definition @modelParameterContraint:0
+constr_1737324311514334000 ==
+StateConstraint
+----
+=============================================================================
+\* Modification History
+\* Created Sun Jan 19 23:05:11 CET 2025 by cs

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737324315582/MC_TE.out

@@ -0,0 +1,324 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 120 and seed -4351583933909868251 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 23327] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 23:05:12)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 23:05:13.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 1000:1 @!@!@
+TLC threw an unexpected exception.
+This was probably caused by an error in the spec or model.
+See the User Output or TLC Console for clues to what happened.
+The exception was a java.lang.RuntimeException
+: Attempted to select field "rx" from a non-record value s1
+line 111, col 50 to line 111, col 53 of module replicated
+@!@!@ENDMSG 1000 @!@!@
+@!@!@STARTMSG 2121:1 @!@!@
+The behavior up to this point is:
+@!@!@ENDMSG 2121 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+1: <Initial predicate>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, b3, s1, s2, s3, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+2: <NodeOnlineOffline line 52, col 5 to line 56, col 67 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+3: <SkCommit line 59, col 5 to line 68, col 58 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2103:1 @!@!@
+The error occurred when TLC was evaluating the nested
+expressions at the following positions:
+0. Line 108, column 5 to line 125, column 57 in replicated
+1. Line 108, column 8 to line 108, column 27 in replicated
+2. Line 109, column 8 to line 124, column 80 in replicated
+3. Line 124, column 9 to line 124, column 80 in replicated
+4. Line 124, column 29 to line 124, column 80 in replicated
+5. Line 124, column 71 to line 124, column 79 in replicated
+6. Line 122, column 22 to line 122, column 42 in replicated
+7. Line 27, column 3 to line 27, column 36 in replicated
+8. Line 27, column 16 to line 27, column 16 in replicated
+9. Line 122, column 31 to line 122, column 41 in replicated
+10. Line 121, column 24 to line 121, column 49 in replicated
+11. Line 121, column 45 to line 121, column 48 in replicated
+12. Line 114, column 13 to line 120, column 18 in replicated
+13. Line 115, column 17 to line 117, column 32 in replicated
+14. Line 117, column 17 to line 117, column 32 in replicated
+15. Line 112, column 29 to line 112, column 74 in replicated
+16. Line 112, column 35 to line 112, column 74 in replicated
+17. Line 112, column 51 to line 112, column 71 in replicated
+18. Line 112, column 51 to line 112, column 62 in replicated
+19. Line 111, column 25 to line 111, column 64 in replicated
+20. Line 111, column 25 to line 111, column 55 in replicated
+21. Line 111, column 43 to line 111, column 53 in replicated
+22. Line 111, column 50 to line 111, column 53 in replicated
+
+
+@!@!@ENDMSG 2103 @!@!@
+@!@!@STARTMSG 2201:0 @!@!@
+The coverage statistics at 2025-01-19 23:05:14
+@!@!@ENDMSG 2201 @!@!@
+@!@!@STARTMSG 2773:0 @!@!@
+<Init line 45, col 1 to line 45, col 4 of module replicated>: 2:2
+@!@!@ENDMSG 2773 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 46, col 5 to line 49, col 26 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<NodeOnlineOffline line 51, col 1 to line 51, col 17 of module replicated>: 1:7
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 52, col 8 to line 55, col 44 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 52, col 18 to line 55, col 44 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 53, col 9 to line 55, col 44 of module replicated: 749
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 53, col 12 to line 53, col 48 of module replicated: 749
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 54, col 12 to line 54, col 44 of module replicated: 301
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 55, col 12 to line 55, col 44 of module replicated: 98
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 52, col 32 to line 52, col 48 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 56, col 8 to line 56, col 67 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkCommit line 58, col 1 to line 58, col 16 of module replicated>: 4:15
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 59, col 8 to line 59, col 32 of module replicated: 62
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 59, col 8 to line 59, col 15 of module replicated: 47:110
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 59, col 27 to line 59, col 32 of module replicated: 47
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 60, col 8 to line 60, col 14 of module replicated: 47
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 8 to line 60, col 9 of module replicated: 32
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 60, col 13 to line 60, col 14 of module replicated: 32
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 61, col 8 to line 61, col 72 of module replicated: 34
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 8 to line 61, col 38 of module replicated: 19
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 61, col 42 to line 61, col 72 of module replicated: 19
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 65, col 13 to line 67, col 50 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 68, col 8 to line 68, col 58 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPeerRecovery line 70, col 1 to line 70, col 21 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 71, col 8 to line 71, col 32 of module replicated: 46
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 72, col 8 to line 72, col 72 of module replicated: 31
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 73, col 8 to line 73, col 71 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 74, col 8 to line 74, col 102 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 75, col 8 to line 75, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPushToBroker line 77, col 1 to line 77, col 19 of module replicated>: 2:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 78, col 8 to line 78, col 30 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 78, col 8 to line 78, col 13 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 78, col 25 to line 78, col 30 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 79, col 8 to line 86, col 52 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 87, col 8 to line 87, col 62 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsRecvBroker line 89, col 1 to line 89, col 19 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 96, col 13 to line 98, col 37 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 99, col 8 to line 99, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkRecvBroker line 102, col 1 to line 102, col 17 of module replicated>: 5:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 103, col 8 to line 103, col 29 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 103, col 8 to line 103, col 12 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 103, col 24 to line 103, col 29 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 104, col 8 to line 104, col 81 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 105, col 8 to line 105, col 57 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPrune line 107, col 1 to line 107, col 10 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 108, col 8 to line 108, col 27 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 124, col 9 to line 124, col 80 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 124, col 29 to line 124, col 80 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 125, col 8 to line 125, col 57 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsChooseSk line 134, col 1 to line 134, col 13 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 137, col 8 to line 137, col 109 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 138, col 8 to line 138, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PsLagsSk line 160, col 1 to line 160, col 8 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 160, col 13 to line 161, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 160, col 35 to line 161, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 161, col 9 to line 161, col 81 of module replicated: 39
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 160, col 44 to line 160, col 72 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 160, col 22 to line 160, col 32 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PeerRecoveryIsPossible line 163, col 1 to line 163, col 22 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 164, col 5 to line 166, col 88 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 164, col 47 to line 166, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 165, col 9 to line 166, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 165, col 12 to line 166, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 165, col 13 to line 165, col 85 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 166, col 16 to line 166, col 88 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 164, col 61 to line 164, col 83 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 164, col 21 to line 164, col 43 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2202:0 @!@!@
+End of statistics.
+@!@!@ENDMSG 2202 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(4) at 2025-01-19 23:05:14: 69 states generated (1.732 s/min), 13 distinct states found (326 ds/min), 9 states left on queue.
+@!@!@ENDMSG 2200 @!@!@
+@!@!@STARTMSG 2199:0 @!@!@
+69 states generated, 13 distinct states found, 9 states left on queue.
+@!@!@ENDMSG 2199 @!@!@
+@!@!@STARTMSG 2186:0 @!@!@
+Finished in 2406ms at (2025-01-19 23:05:14)
+@!@!@ENDMSG 2186 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737324315582/replicated.tla

@@ -0,0 +1,181 @@
+---- MODULE replicated ----
+
+EXTENDS Integers, FiniteSets
+
+VARIABLES broker_state, safekeeper_state, pageserver_state, online
+
+
+CONSTANT
+    brokers,
+    safekeepers,
+    pageservers,
+    azs,
+    az_mapping
+
+CONSTANT
+    NULL
+    
+CONSTANT
+    max_commit_lsn
+    
+\* HELPERS
+
+Max(a,b) == IF a > b THEN a ELSE b
+
+(* The minimum of a non-empty set of integers *)
+MinOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x <= y
+
+\* END HELPERS
+
+StateConstraint ==
+    /\ \A s \in safekeepers: 
+        /\ safekeeper_state[s].commit_lsn <= max_commit_lsn
+    /\ \A b \in brokers:
+        /\ \A s \in DOMAIN broker_state[b].sk:
+            /\ broker_state[b].sk[s].commit_lsn <= max_commit_lsn
+
+
+ 
+InitSafekeeper == [prune_lsn |-> 0, commit_lsn |-> 0, rx |-> [b \in brokers |-> NULL] ]
+InitBroker == [sk |-> [s \in safekeepers |-> [prune_lsn |-> 0, commit_lsn |-> 0]]]
+InitPageserver == [last_record_lsn |-> 0, preferred_sk |-> NULL, sk |-> [s \in safekeepers |-> [commit_lsn |-> 0]]]
+InitOnline == safekeepers \cup brokers \cup pageservers
+
+Init ==
+    /\ broker_state = [b \in brokers |-> InitBroker]
+    /\ safekeeper_state = [s \in safekeepers |-> InitSafekeeper]
+    /\ pageserver_state = [p \in pageservers |-> InitPageserver]
+    /\ online = InitOnline
+  
+NodeOnlineOffline ==
+    /\ online' = CHOOSE ss \in SUBSET InitOnline: 
+        /\ Cardinality(ss \cap safekeepers) >= 2
+        /\ Cardinality(ss \cap brokers) >= 2
+        /\ ss \cap pageservers = pageservers \* assume no PS failures for now
+    /\ UNCHANGED <<safekeeper_state,broker_state,pageserver_state>>
+   
+SkCommit(s1, s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ s1 # s2
+    /\ safekeeper_state[s1].commit_lsn = safekeeper_state[s2].commit_lsn
+    /\ LET
+            new_commit_lsn == safekeeper_state[s1].commit_lsn + 1
+       IN
+            safekeeper_state' = [safekeeper_state EXCEPT
+                ![s1].commit_lsn = new_commit_lsn,
+                ![s2].commit_lsn = new_commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPeerRecovery(s1,s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ safekeeper_state[s1].commit_lsn < safekeeper_state[s2].commit_lsn \* s2 has more WAL than s1
+    /\ safekeeper_state[s2].prune_lsn < safekeeper_state[s1].commit_lsn \* s2 has not yet trimmed the WAL the WAL
+    /\ safekeeper_state' = [safekeeper_state EXCEPT![s1].commit_lsn = safekeeper_state[s2].commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPushToBroker(s,b) ==
+    /\ {s, b} \subseteq online 
+    /\ broker_state' = LET
+           broker_side == broker_state[b].sk[s]
+           sk_side == safekeeper_state[s]
+           merged == [broker_side EXCEPT
+                !["commit_lsn"] = Max(broker_side.commit_lsn, sk_side.commit_lsn),
+                !["prune_lsn"] = Max(broker_side.prune_lsn, sk_side.prune_lsn)]
+           IN
+           [broker_state EXCEPT ![b].sk[s] = merged]
+    /\ UNCHANGED <<safekeeper_state, pageserver_state,online>> 
+
+PsRecvBroker(b,p,s) ==
+    /\ {b,p,s} \subseteq online
+    /\ LET
+            bsk == broker_state[b].sk[s]
+            psk == pageserver_state[p].sk[s]
+            updpsk == [psk EXCEPT !["commit_lsn"] = bsk.commit_lsn]
+       IN
+            pageserver_state' = IF bsk.commit_lsn > psk.commit_lsn
+                THEN [pageserver_state EXCEPT ![p].sk[s] = updpsk]
+                ELSE pageserver_state
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>
+    
+    
+SkRecvBroker(b,s) ==
+    /\ {b,s} \subseteq online
+    /\ safekeeper_state' = [safekeeper_state EXCEPT ![s].rx[b] = broker_state[b]] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+
+SkPrune(s) ==
+    /\ {s} \subseteq online
+    /\ LET
+        sk == safekeeper_state[s]
+        broker_infos == {sk.rx[b]: b \in (DOMAIN s.rx)} \ {NULL}
+        broker_infos_sks == UNION {bi.sk: bi \in (broker_infos \ {NULL}) }
+        info ==
+            (
+                {safekeeper_state[s]}
+                \cup
+                broker_infos_sks
+            )
+            \
+            {NULL}
+        commit_lsns == {i.commit_lsn: i \in info}
+        prune_lsn == MinOfSet(commit_lsns)
+       IN
+        safekeeper_state' = [safekeeper_state EXCEPT ![s].prune_lsn = prune_lsn] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+    
+
+SksWithNewerWal(p) ==
+    LET
+        ps == pageserver_state[p]
+    IN
+    {s \in DOMAIN ps.sk: ps.sk[s].commit_lsn > ps.last_record_lsn}
+
+PsChooseSk(p) ==
+    /\ {p} \subseteq online
+    /\ SksWithNewerWal(p) # {}
+    /\ pageserver_state' = [pageserver_state EXCEPT![p].preferred_sk = CHOOSE s \in SksWithNewerWal(p): TRUE]
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>    
+                 
+    
+Next ==
+    \/ NodeOnlineOffline
+    \/ \E s1 \in safekeepers: \E s2 \in safekeepers:
+        \/ SkCommit(s1, s2)
+        \/ SkPeerRecovery(s1, s2)
+    \/ \E s \in safekeepers: \E b \in brokers:
+        \/ SkPushToBroker(s, b)
+        \/ SkRecvBroker(b, s)
+    \/ \E s \in safekeepers:
+        \/ SkPrune(s)  
+    \/ \E s \in safekeepers: \E b \in brokers: \E p \in pageservers: PsRecvBroker(b,p,s)
+    \/ \E p \in pageservers: PsChooseSk(p)
+    
+
+Spec == Init /\ [][Next]_<< broker_state, safekeeper_state, pageserver_state,online>>
+
+
+\* invariants
+
+PsLagsSk == \A p \in pageservers: \A s \in DOMAIN pageserver_state[p].sk: 
+        /\ pageserver_state[p].sk[s].commit_lsn <= safekeeper_state[s].commit_lsn
+
+PeerRecoveryIsPossible ==
+    \A laggard \in (safekeepers \cap online): \E donor \in (safekeepers \cap online):
+        /\ (safekeeper_state[laggard].commit_lsn < safekeeper_state[donor].commit_lsn)
+            => safekeeper_state[donor].prune_lsn <= safekeeper_state[laggard].commit_lsn
+        
+
+EventuallyLaggingSkIsNotPreferredSk == <>(
+        LET
+            sks == safekeeper_state
+            lagging_sks == { s \in safekeepers: \A s2 \in safekeepers: sks[s].commit_lsn <= sks[s2].commit_lsn }
+            preferred_sks == {pageserver_state[p].preferred_sk: p \in pageservers}
+        IN
+            preferred_sks \cap lagging_sks = {}
+            
+    )
+
+    
+
+====

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737326115252/MC.cfg

@@ -0,0 +1,54 @@
+\* MV CONSTANT declarations
+CONSTANTS
+b1 = b1
+b2 = b2
+b3 = b3
+\* MV CONSTANT declarations
+CONSTANTS
+s1 = s1
+s2 = s2
+s3 = s3
+\* MV CONSTANT declarations
+CONSTANTS
+p1 = p1
+\* MV CONSTANT declarations
+CONSTANTS
+az1 = az1
+az2 = az2
+az3 = az3
+\* MV CONSTANT definitions
+CONSTANT
+brokers <- const_1737326111193338000
+\* CONSTANT declarations
+CONSTANT NULL = NULL
+\* MV CONSTANT definitions
+CONSTANT
+safekeepers <- const_1737326111193339000
+\* MV CONSTANT definitions
+CONSTANT
+pageservers <- const_1737326111193340000
+\* MV CONSTANT definitions
+CONSTANT
+azs <- const_1737326111193341000
+\* SYMMETRY definition
+SYMMETRY symm_1737326111193342000
+\* CONSTANT definitions
+CONSTANT
+max_commit_lsn <- const_1737326111193343000
+\* CONSTANT definitions
+CONSTANT
+az_mapping <- const_1737326111193344000
+\* CONSTRAINT definition
+CONSTRAINT
+constr_1737326111193345000
+\* SPECIFICATION definition
+SPECIFICATION
+Spec
+\* INVARIANT definition
+INVARIANT
+PsLagsSk
+PeerRecoveryIsPossible
+\* PROPERTY definition
+PROPERTY
+EventuallyLaggingSkIsNotPreferredSk
+\* Generated on Sun Jan 19 23:35:11 CET 2025

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737326115252/MC.out

@@ -0,0 +1,312 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 75 and seed 8173044349253312442 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 24585] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 23:35:12)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 23:35:13.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 1000:1 @!@!@
+TLC threw an unexpected exception.
+This was probably caused by an error in the spec or model.
+See the User Output or TLC Console for clues to what happened.
+The exception was a java.lang.RuntimeException
+: Attempted to select field "rx" from a non-record value s1
+line 116, col 50 to line 116, col 53 of module replicated
+@!@!@ENDMSG 1000 @!@!@
+@!@!@STARTMSG 2121:1 @!@!@
+The behavior up to this point is:
+@!@!@ENDMSG 2121 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+1: <Initial predicate>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, b3, s1, s2, s3, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+2: <NodeOnlineOffline line 56, col 5 to line 60, col 67 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2103:1 @!@!@
+The error occurred when TLC was evaluating the nested
+expressions at the following positions:
+0. Line 112, column 5 to line 128, column 57 in replicated
+1. Line 112, column 8 to line 112, column 27 in replicated
+2. Line 113, column 8 to line 127, column 80 in replicated
+3. Line 127, column 9 to line 127, column 80 in replicated
+4. Line 127, column 29 to line 127, column 80 in replicated
+5. Line 127, column 71 to line 127, column 79 in replicated
+6. Line 125, column 22 to line 125, column 46 in replicated
+7. Line 27, column 3 to line 27, column 36 in replicated
+8. Line 27, column 16 to line 27, column 16 in replicated
+9. Line 125, column 31 to line 125, column 45 in replicated
+10. Line 118, column 28 to line 124, column 13 in replicated
+11. Line 123, column 27 to line 123, column 35 in replicated
+12. Line 117, column 22 to line 117, column 64 in replicated
+13. Line 117, column 29 to line 117, column 63 in replicated
+14. Line 117, column 51 to line 117, column 62 in replicated
+15. Line 116, column 25 to line 116, column 64 in replicated
+16. Line 116, column 25 to line 116, column 55 in replicated
+17. Line 116, column 43 to line 116, column 53 in replicated
+18. Line 116, column 50 to line 116, column 53 in replicated
+
+
+@!@!@ENDMSG 2103 @!@!@
+@!@!@STARTMSG 2201:0 @!@!@
+The coverage statistics at 2025-01-19 23:35:14
+@!@!@ENDMSG 2201 @!@!@
+@!@!@STARTMSG 2773:0 @!@!@
+<Init line 49, col 1 to line 49, col 4 of module replicated>: 2:2
+@!@!@ENDMSG 2773 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 50, col 5 to line 53, col 26 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<NodeOnlineOffline line 55, col 1 to line 55, col 17 of module replicated>: 1:6
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 56, col 8 to line 59, col 44 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 56, col 18 to line 59, col 44 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 57, col 9 to line 59, col 44 of module replicated: 642
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 57, col 12 to line 57, col 48 of module replicated: 642
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 58, col 12 to line 58, col 44 of module replicated: 258
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 59, col 12 to line 59, col 44 of module replicated: 84
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 56, col 32 to line 56, col 48 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 60, col 8 to line 60, col 67 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkCommit line 62, col 1 to line 62, col 16 of module replicated>: 4:14
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 63, col 8 to line 63, col 32 of module replicated: 59
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 63, col 8 to line 63, col 15 of module replicated: 45:105
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 63, col 27 to line 63, col 32 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 64, col 8 to line 64, col 14 of module replicated: 44
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 64, col 8 to line 64, col 9 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 64, col 13 to line 64, col 14 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 65, col 8 to line 65, col 72 of module replicated: 32
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 65, col 8 to line 65, col 38 of module replicated: 18
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 65, col 42 to line 65, col 72 of module replicated: 18
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 69, col 13 to line 71, col 50 of module replicated: 14
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 72, col 8 to line 72, col 58 of module replicated: 14
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPeerRecovery line 74, col 1 to line 74, col 21 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 75, col 8 to line 75, col 32 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 76, col 8 to line 76, col 72 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 77, col 8 to line 77, col 71 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 78, col 8 to line 78, col 102 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 79, col 8 to line 79, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPushToBroker line 81, col 1 to line 81, col 19 of module replicated>: 2:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 82, col 8 to line 82, col 30 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 82, col 8 to line 82, col 13 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 82, col 25 to line 82, col 30 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 83, col 8 to line 90, col 52 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 91, col 8 to line 91, col 62 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsRecvBroker line 93, col 1 to line 93, col 19 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 100, col 13 to line 102, col 37 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 103, col 8 to line 103, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkRecvBroker line 106, col 1 to line 106, col 17 of module replicated>: 5:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 107, col 8 to line 107, col 29 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 107, col 8 to line 107, col 12 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 107, col 24 to line 107, col 29 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 108, col 8 to line 108, col 81 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 109, col 8 to line 109, col 57 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPrune line 111, col 1 to line 111, col 10 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 112, col 8 to line 112, col 27 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 127, col 9 to line 127, col 80 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 127, col 29 to line 127, col 80 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 128, col 8 to line 128, col 57 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsChooseSk line 137, col 1 to line 137, col 13 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 140, col 8 to line 140, col 109 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 141, col 8 to line 141, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PsLagsSk line 163, col 1 to line 163, col 8 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 163, col 13 to line 164, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 35 to line 164, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 164, col 9 to line 164, col 81 of module replicated: 39
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 163, col 44 to line 163, col 72 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 22 to line 163, col 32 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PeerRecoveryIsPossible line 166, col 1 to line 166, col 22 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 167, col 5 to line 169, col 88 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 167, col 47 to line 169, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 168, col 9 to line 169, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 168, col 12 to line 169, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 168, col 13 to line 168, col 85 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 169, col 16 to line 169, col 88 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 167, col 61 to line 167, col 83 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 167, col 21 to line 167, col 43 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2202:0 @!@!@
+End of statistics.
+@!@!@ENDMSG 2202 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(4) at 2025-01-19 23:35:14: 69 states generated (1.538 s/min), 13 distinct states found (289 ds/min), 9 states left on queue.
+@!@!@ENDMSG 2200 @!@!@
+@!@!@STARTMSG 2199:0 @!@!@
+69 states generated, 13 distinct states found, 9 states left on queue.
+@!@!@ENDMSG 2199 @!@!@
+@!@!@STARTMSG 2186:0 @!@!@
+Finished in 2708ms at (2025-01-19 23:35:14)
+@!@!@ENDMSG 2186 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737326115252/MC.tla

@@ -0,0 +1,65 @@
+---- MODULE MC ----
+EXTENDS replicated, TLC
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+b1, b2, b3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+s1, s2, s3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+p1
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+az1, az2, az3
+----
+
+\* MV CONSTANT definitions brokers
+const_1737326111193338000 == 
+{b1, b2, b3}
+----
+
+\* MV CONSTANT definitions safekeepers
+const_1737326111193339000 == 
+{s1, s2, s3}
+----
+
+\* MV CONSTANT definitions pageservers
+const_1737326111193340000 == 
+{p1}
+----
+
+\* MV CONSTANT definitions azs
+const_1737326111193341000 == 
+{az1, az2, az3}
+----
+
+\* SYMMETRY definition
+symm_1737326111193342000 == 
+Permutations(const_1737326111193338000) \union Permutations(const_1737326111193339000) \union Permutations(const_1737326111193340000) \union Permutations(const_1737326111193341000)
+----
+
+\* CONSTANT definitions @modelParameterConstants:4max_commit_lsn
+const_1737326111193343000 == 
+2
+----
+
+\* CONSTANT definitions @modelParameterConstants:5az_mapping
+const_1737326111193344000 == 
+[ az1 |-> {b1,s1,p1} , az2 |-> {b2,s2} , az3 |-> {b3,s3}]
+----
+
+\* CONSTRAINT definition @modelParameterContraint:0
+constr_1737326111193345000 ==
+StateConstraint
+----
+=============================================================================
+\* Modification History
+\* Created Sun Jan 19 23:35:11 CET 2025 by cs

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737326115252/MC_TE.out

@@ -0,0 +1,312 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 75 and seed 8173044349253312442 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 24585] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 23:35:12)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 23:35:13.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 1000:1 @!@!@
+TLC threw an unexpected exception.
+This was probably caused by an error in the spec or model.
+See the User Output or TLC Console for clues to what happened.
+The exception was a java.lang.RuntimeException
+: Attempted to select field "rx" from a non-record value s1
+line 116, col 50 to line 116, col 53 of module replicated
+@!@!@ENDMSG 1000 @!@!@
+@!@!@STARTMSG 2121:1 @!@!@
+The behavior up to this point is:
+@!@!@ENDMSG 2121 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+1: <Initial predicate>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, b3, s1, s2, s3, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+2: <NodeOnlineOffline line 56, col 5 to line 60, col 67 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2103:1 @!@!@
+The error occurred when TLC was evaluating the nested
+expressions at the following positions:
+0. Line 112, column 5 to line 128, column 57 in replicated
+1. Line 112, column 8 to line 112, column 27 in replicated
+2. Line 113, column 8 to line 127, column 80 in replicated
+3. Line 127, column 9 to line 127, column 80 in replicated
+4. Line 127, column 29 to line 127, column 80 in replicated
+5. Line 127, column 71 to line 127, column 79 in replicated
+6. Line 125, column 22 to line 125, column 46 in replicated
+7. Line 27, column 3 to line 27, column 36 in replicated
+8. Line 27, column 16 to line 27, column 16 in replicated
+9. Line 125, column 31 to line 125, column 45 in replicated
+10. Line 118, column 28 to line 124, column 13 in replicated
+11. Line 123, column 27 to line 123, column 35 in replicated
+12. Line 117, column 22 to line 117, column 64 in replicated
+13. Line 117, column 29 to line 117, column 63 in replicated
+14. Line 117, column 51 to line 117, column 62 in replicated
+15. Line 116, column 25 to line 116, column 64 in replicated
+16. Line 116, column 25 to line 116, column 55 in replicated
+17. Line 116, column 43 to line 116, column 53 in replicated
+18. Line 116, column 50 to line 116, column 53 in replicated
+
+
+@!@!@ENDMSG 2103 @!@!@
+@!@!@STARTMSG 2201:0 @!@!@
+The coverage statistics at 2025-01-19 23:35:14
+@!@!@ENDMSG 2201 @!@!@
+@!@!@STARTMSG 2773:0 @!@!@
+<Init line 49, col 1 to line 49, col 4 of module replicated>: 2:2
+@!@!@ENDMSG 2773 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 50, col 5 to line 53, col 26 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<NodeOnlineOffline line 55, col 1 to line 55, col 17 of module replicated>: 1:6
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 56, col 8 to line 59, col 44 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 56, col 18 to line 59, col 44 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 57, col 9 to line 59, col 44 of module replicated: 642
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 57, col 12 to line 57, col 48 of module replicated: 642
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 58, col 12 to line 58, col 44 of module replicated: 258
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 59, col 12 to line 59, col 44 of module replicated: 84
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 56, col 32 to line 56, col 48 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 60, col 8 to line 60, col 67 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkCommit line 62, col 1 to line 62, col 16 of module replicated>: 4:14
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 63, col 8 to line 63, col 32 of module replicated: 59
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 63, col 8 to line 63, col 15 of module replicated: 45:105
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 63, col 27 to line 63, col 32 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 64, col 8 to line 64, col 14 of module replicated: 44
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 64, col 8 to line 64, col 9 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 64, col 13 to line 64, col 14 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 65, col 8 to line 65, col 72 of module replicated: 32
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 65, col 8 to line 65, col 38 of module replicated: 18
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 65, col 42 to line 65, col 72 of module replicated: 18
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 69, col 13 to line 71, col 50 of module replicated: 14
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 72, col 8 to line 72, col 58 of module replicated: 14
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPeerRecovery line 74, col 1 to line 74, col 21 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 75, col 8 to line 75, col 32 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 76, col 8 to line 76, col 72 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 77, col 8 to line 77, col 71 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 78, col 8 to line 78, col 102 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 79, col 8 to line 79, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPushToBroker line 81, col 1 to line 81, col 19 of module replicated>: 2:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 82, col 8 to line 82, col 30 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 82, col 8 to line 82, col 13 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 82, col 25 to line 82, col 30 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 83, col 8 to line 90, col 52 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 91, col 8 to line 91, col 62 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsRecvBroker line 93, col 1 to line 93, col 19 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 100, col 13 to line 102, col 37 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 103, col 8 to line 103, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkRecvBroker line 106, col 1 to line 106, col 17 of module replicated>: 5:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 107, col 8 to line 107, col 29 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 107, col 8 to line 107, col 12 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 107, col 24 to line 107, col 29 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 108, col 8 to line 108, col 81 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 109, col 8 to line 109, col 57 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPrune line 111, col 1 to line 111, col 10 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 112, col 8 to line 112, col 27 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 127, col 9 to line 127, col 80 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 127, col 29 to line 127, col 80 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 128, col 8 to line 128, col 57 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsChooseSk line 137, col 1 to line 137, col 13 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 140, col 8 to line 140, col 109 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 141, col 8 to line 141, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PsLagsSk line 163, col 1 to line 163, col 8 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 163, col 13 to line 164, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 35 to line 164, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 164, col 9 to line 164, col 81 of module replicated: 39
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 163, col 44 to line 163, col 72 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 22 to line 163, col 32 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PeerRecoveryIsPossible line 166, col 1 to line 166, col 22 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 167, col 5 to line 169, col 88 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 167, col 47 to line 169, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 168, col 9 to line 169, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 168, col 12 to line 169, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 168, col 13 to line 168, col 85 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 169, col 16 to line 169, col 88 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 167, col 61 to line 167, col 83 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 167, col 21 to line 167, col 43 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2202:0 @!@!@
+End of statistics.
+@!@!@ENDMSG 2202 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(4) at 2025-01-19 23:35:14: 69 states generated (1.538 s/min), 13 distinct states found (289 ds/min), 9 states left on queue.
+@!@!@ENDMSG 2200 @!@!@
+@!@!@STARTMSG 2199:0 @!@!@
+69 states generated, 13 distinct states found, 9 states left on queue.
+@!@!@ENDMSG 2199 @!@!@
+@!@!@STARTMSG 2186:0 @!@!@
+Finished in 2708ms at (2025-01-19 23:35:14)
+@!@!@ENDMSG 2186 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737326115252/replicated.tla

@@ -0,0 +1,184 @@
+---- MODULE replicated ----
+
+EXTENDS Integers, FiniteSets
+
+VARIABLES broker_state, safekeeper_state, pageserver_state, online
+
+
+CONSTANT
+    brokers,
+    safekeepers,
+    pageservers,
+    azs,
+    az_mapping
+
+CONSTANT
+    NULL
+    
+CONSTANT
+    max_commit_lsn
+    
+\* HELPERS
+
+Max(a,b) == IF a > b THEN a ELSE b
+
+(* The minimum of a non-empty set of integers *)
+MinOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x <= y
+  
+(* The minimum of a non-empty set of integers *)
+MaxOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x >= y
+
+\* END HELPERS
+
+StateConstraint ==
+    /\ \A s \in safekeepers: 
+        /\ safekeeper_state[s].commit_lsn <= max_commit_lsn
+    /\ \A b \in brokers:
+        /\ \A s \in DOMAIN broker_state[b].sk:
+            /\ broker_state[b].sk[s].commit_lsn <= max_commit_lsn
+
+
+ 
+InitSafekeeper == [prune_lsn |-> 0, commit_lsn |-> 0, rx |-> [b \in brokers |-> NULL] ]
+InitBroker == [sk |-> [s \in safekeepers |-> [prune_lsn |-> 0, commit_lsn |-> 0]]]
+InitPageserver == [last_record_lsn |-> 0, preferred_sk |-> NULL, sk |-> [s \in safekeepers |-> [commit_lsn |-> 0]]]
+InitOnline == safekeepers \cup brokers \cup pageservers
+
+Init ==
+    /\ broker_state = [b \in brokers |-> InitBroker]
+    /\ safekeeper_state = [s \in safekeepers |-> InitSafekeeper]
+    /\ pageserver_state = [p \in pageservers |-> InitPageserver]
+    /\ online = InitOnline
+  
+NodeOnlineOffline ==
+    /\ online' = CHOOSE ss \in SUBSET InitOnline: 
+        /\ Cardinality(ss \cap safekeepers) >= 2
+        /\ Cardinality(ss \cap brokers) >= 2
+        /\ ss \cap pageservers = pageservers \* assume no PS failures for now
+    /\ UNCHANGED <<safekeeper_state,broker_state,pageserver_state>>
+   
+SkCommit(s1, s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ s1 # s2
+    /\ safekeeper_state[s1].commit_lsn = safekeeper_state[s2].commit_lsn
+    /\ LET
+            new_commit_lsn == safekeeper_state[s1].commit_lsn + 1
+       IN
+            safekeeper_state' = [safekeeper_state EXCEPT
+                ![s1].commit_lsn = new_commit_lsn,
+                ![s2].commit_lsn = new_commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPeerRecovery(s1,s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ safekeeper_state[s1].commit_lsn < safekeeper_state[s2].commit_lsn \* s2 has more WAL than s1
+    /\ safekeeper_state[s2].prune_lsn < safekeeper_state[s1].commit_lsn \* s2 has not yet trimmed the WAL the WAL
+    /\ safekeeper_state' = [safekeeper_state EXCEPT![s1].commit_lsn = safekeeper_state[s2].commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPushToBroker(s,b) ==
+    /\ {s, b} \subseteq online 
+    /\ broker_state' = LET
+           broker_side == broker_state[b].sk[s]
+           sk_side == safekeeper_state[s]
+           merged == [broker_side EXCEPT
+                !["commit_lsn"] = Max(broker_side.commit_lsn, sk_side.commit_lsn),
+                !["prune_lsn"] = Max(broker_side.prune_lsn, sk_side.prune_lsn)]
+           IN
+           [broker_state EXCEPT ![b].sk[s] = merged]
+    /\ UNCHANGED <<safekeeper_state, pageserver_state,online>> 
+
+PsRecvBroker(b,p,s) ==
+    /\ {b,p,s} \subseteq online
+    /\ LET
+            bsk == broker_state[b].sk[s]
+            psk == pageserver_state[p].sk[s]
+            updpsk == [psk EXCEPT !["commit_lsn"] = bsk.commit_lsn]
+       IN
+            pageserver_state' = IF bsk.commit_lsn > psk.commit_lsn
+                THEN [pageserver_state EXCEPT ![p].sk[s] = updpsk]
+                ELSE pageserver_state
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>
+    
+    
+SkRecvBroker(b,s) ==
+    /\ {b,s} \subseteq online
+    /\ safekeeper_state' = [safekeeper_state EXCEPT ![s].rx[b] = broker_state[b]] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+
+SkPrune(s) ==
+    /\ {s} \subseteq online
+    /\ LET
+        sk == safekeeper_state[s]
+        
+        broker_infos == {sk.rx[b]: b \in (DOMAIN s.rx)} \ {NULL}
+        known_sks == UNION ({DOMAIN bi.sk: bi \in broker_infos})
+        max_commit_lsns == {
+                MaxOfSet(
+                    {
+                        sk.rx[b].sk[s2].commit_lsn: b \in DOMAIN sk.rx
+                    }
+                ): s2 \in known_sks
+            }
+        prune_lsn == MinOfSet(max_commit_lsns)
+       IN
+        safekeeper_state' = [safekeeper_state EXCEPT ![s].prune_lsn = prune_lsn] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+    
+
+SksWithNewerWal(p) ==
+    LET
+        ps == pageserver_state[p]
+    IN
+    {s \in DOMAIN ps.sk: ps.sk[s].commit_lsn > ps.last_record_lsn}
+
+PsChooseSk(p) ==
+    /\ {p} \subseteq online
+    /\ SksWithNewerWal(p) # {}
+    /\ pageserver_state' = [pageserver_state EXCEPT![p].preferred_sk = CHOOSE s \in SksWithNewerWal(p): TRUE]
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>    
+                 
+    
+Next ==
+    \/ NodeOnlineOffline
+    \/ \E s1 \in safekeepers: \E s2 \in safekeepers:
+        \/ SkCommit(s1, s2)
+        \/ SkPeerRecovery(s1, s2)
+    \/ \E s \in safekeepers: \E b \in brokers:
+        \/ SkPushToBroker(s, b)
+        \/ SkRecvBroker(b, s)
+    \/ \E s \in safekeepers:
+        \/ SkPrune(s)  
+    \/ \E s \in safekeepers: \E b \in brokers: \E p \in pageservers: PsRecvBroker(b,p,s)
+    \/ \E p \in pageservers: PsChooseSk(p)
+    
+
+Spec == Init /\ [][Next]_<< broker_state, safekeeper_state, pageserver_state,online>>
+
+
+\* invariants
+
+PsLagsSk == \A p \in pageservers: \A s \in DOMAIN pageserver_state[p].sk: 
+        /\ pageserver_state[p].sk[s].commit_lsn <= safekeeper_state[s].commit_lsn
+
+PeerRecoveryIsPossible ==
+    \A laggard \in (safekeepers \cap online): \E donor \in (safekeepers \cap online):
+        /\ (safekeeper_state[laggard].commit_lsn < safekeeper_state[donor].commit_lsn)
+            => safekeeper_state[donor].prune_lsn <= safekeeper_state[laggard].commit_lsn
+        
+
+EventuallyLaggingSkIsNotPreferredSk == <>(
+        LET
+            sks == safekeeper_state
+            lagging_sks == { s \in safekeepers: \A s2 \in safekeepers: sks[s].commit_lsn <= sks[s2].commit_lsn }
+            preferred_sks == {pageserver_state[p].preferred_sk: p \in pageservers}
+        IN
+            preferred_sks \cap lagging_sks = {}
+            
+    )
+
+    
+
+====

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737326135419/MC.cfg

@@ -0,0 +1,54 @@
+\* MV CONSTANT declarations
+CONSTANTS
+b1 = b1
+b2 = b2
+b3 = b3
+\* MV CONSTANT declarations
+CONSTANTS
+s1 = s1
+s2 = s2
+s3 = s3
+\* MV CONSTANT declarations
+CONSTANTS
+p1 = p1
+\* MV CONSTANT declarations
+CONSTANTS
+az1 = az1
+az2 = az2
+az3 = az3
+\* MV CONSTANT definitions
+CONSTANT
+brokers <- const_1737326131354349000
+\* CONSTANT declarations
+CONSTANT NULL = NULL
+\* MV CONSTANT definitions
+CONSTANT
+safekeepers <- const_1737326131354350000
+\* MV CONSTANT definitions
+CONSTANT
+pageservers <- const_1737326131354351000
+\* MV CONSTANT definitions
+CONSTANT
+azs <- const_1737326131354352000
+\* SYMMETRY definition
+SYMMETRY symm_1737326131354353000
+\* CONSTANT definitions
+CONSTANT
+max_commit_lsn <- const_1737326131354354000
+\* CONSTANT definitions
+CONSTANT
+az_mapping <- const_1737326131354355000
+\* CONSTRAINT definition
+CONSTRAINT
+constr_1737326131354356000
+\* SPECIFICATION definition
+SPECIFICATION
+Spec
+\* INVARIANT definition
+INVARIANT
+PsLagsSk
+PeerRecoveryIsPossible
+\* PROPERTY definition
+PROPERTY
+EventuallyLaggingSkIsNotPreferredSk
+\* Generated on Sun Jan 19 23:35:31 CET 2025

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737326135419/MC.out

@@ -0,0 +1,358 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 0 and seed 849358144369345749 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 24597] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 23:35:32)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 23:35:33.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 1000:1 @!@!@
+TLC threw an unexpected exception.
+This was probably caused by an error in the spec or model.
+See the User Output or TLC Console for clues to what happened.
+The exception was a java.lang.RuntimeException
+: Attempted to compute the value of an expression of form
+CHOOSE x \in S: P, but no element of S satisfied P.
+line 27, col 3 to line 27, col 36 of module replicated
+@!@!@ENDMSG 1000 @!@!@
+@!@!@STARTMSG 2121:1 @!@!@
+The behavior up to this point is:
+@!@!@ENDMSG 2121 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+1: <Initial predicate>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, b3, s1, s2, s3, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+2: <NodeOnlineOffline line 56, col 5 to line 60, col 67 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+3: <SkCommit line 63, col 5 to line 72, col 58 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2103:1 @!@!@
+The error occurred when TLC was evaluating the nested
+expressions at the following positions:
+0. Line 112, column 5 to line 128, column 57 in replicated
+1. Line 112, column 8 to line 112, column 27 in replicated
+2. Line 113, column 8 to line 127, column 80 in replicated
+3. Line 127, column 9 to line 127, column 80 in replicated
+4. Line 127, column 29 to line 127, column 80 in replicated
+5. Line 127, column 71 to line 127, column 79 in replicated
+6. Line 125, column 22 to line 125, column 46 in replicated
+7. Line 27, column 3 to line 27, column 36 in replicated
+
+
+@!@!@ENDMSG 2103 @!@!@
+@!@!@STARTMSG 2201:0 @!@!@
+The coverage statistics at 2025-01-19 23:35:33
+@!@!@ENDMSG 2201 @!@!@
+@!@!@STARTMSG 2773:0 @!@!@
+<Init line 49, col 1 to line 49, col 4 of module replicated>: 2:2
+@!@!@ENDMSG 2773 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 50, col 5 to line 53, col 26 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<NodeOnlineOffline line 55, col 1 to line 55, col 17 of module replicated>: 1:7
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 56, col 8 to line 59, col 44 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 56, col 18 to line 59, col 44 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 57, col 9 to line 59, col 44 of module replicated: 749
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 57, col 12 to line 57, col 48 of module replicated: 749
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 58, col 12 to line 58, col 44 of module replicated: 301
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 59, col 12 to line 59, col 44 of module replicated: 98
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 56, col 32 to line 56, col 48 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 60, col 8 to line 60, col 67 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkCommit line 62, col 1 to line 62, col 16 of module replicated>: 4:15
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 63, col 8 to line 63, col 32 of module replicated: 62
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 63, col 8 to line 63, col 15 of module replicated: 47:110
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 63, col 27 to line 63, col 32 of module replicated: 47
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 64, col 8 to line 64, col 14 of module replicated: 47
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 64, col 8 to line 64, col 9 of module replicated: 32
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 64, col 13 to line 64, col 14 of module replicated: 32
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 65, col 8 to line 65, col 72 of module replicated: 34
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 65, col 8 to line 65, col 38 of module replicated: 19
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 65, col 42 to line 65, col 72 of module replicated: 19
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 69, col 13 to line 71, col 50 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 72, col 8 to line 72, col 58 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPeerRecovery line 74, col 1 to line 74, col 21 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 75, col 8 to line 75, col 32 of module replicated: 46
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 76, col 8 to line 76, col 72 of module replicated: 31
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 77, col 8 to line 77, col 71 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 78, col 8 to line 78, col 102 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 79, col 8 to line 79, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPushToBroker line 81, col 1 to line 81, col 19 of module replicated>: 2:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 82, col 8 to line 82, col 30 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 82, col 8 to line 82, col 13 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 82, col 25 to line 82, col 30 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 83, col 8 to line 90, col 52 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 91, col 8 to line 91, col 62 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsRecvBroker line 93, col 1 to line 93, col 19 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 100, col 13 to line 102, col 37 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 103, col 8 to line 103, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkRecvBroker line 106, col 1 to line 106, col 17 of module replicated>: 5:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 107, col 8 to line 107, col 29 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 107, col 8 to line 107, col 12 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 107, col 24 to line 107, col 29 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 108, col 8 to line 108, col 81 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 109, col 8 to line 109, col 57 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPrune line 111, col 1 to line 111, col 10 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 112, col 8 to line 112, col 27 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 127, col 9 to line 127, col 80 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 127, col 29 to line 127, col 80 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 127, col 30 to line 127, col 45 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 127, col 54 to line 127, col 79 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 127, col 71 to line 127, col 79 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 125, col 22 to line 125, col 46 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 27, col 3 to line 27, col 36 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 125, col 31 to line 125, col 45 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||line 118, col 28 to line 124, col 13 of module replicated: 5:5
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||line 123, col 27 to line 123, col 35 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||line 117, col 22 to line 117, col 64 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  ||||||||line 117, col 29 to line 117, col 63 of module replicated: 5:5
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||line 117, col 51 to line 117, col 62 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 116, col 25 to line 116, col 65 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||||||||line 116, col 25 to line 116, col 56 of module replicated: 5:5
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||||line 116, col 26 to line 116, col 33 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||||line 116, col 43 to line 116, col 54 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||line 116, col 60 to line 116, col 65 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 128, col 8 to line 128, col 57 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsChooseSk line 137, col 1 to line 137, col 13 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 140, col 8 to line 140, col 109 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 141, col 8 to line 141, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PsLagsSk line 163, col 1 to line 163, col 8 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 163, col 13 to line 164, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 35 to line 164, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 164, col 9 to line 164, col 81 of module replicated: 39
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 163, col 44 to line 163, col 72 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 22 to line 163, col 32 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PeerRecoveryIsPossible line 166, col 1 to line 166, col 22 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 167, col 5 to line 169, col 88 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 167, col 47 to line 169, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 168, col 9 to line 169, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 168, col 12 to line 169, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 168, col 13 to line 168, col 85 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 169, col 16 to line 169, col 88 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 167, col 61 to line 167, col 83 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 167, col 21 to line 167, col 43 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2202:0 @!@!@
+End of statistics.
+@!@!@ENDMSG 2202 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(4) at 2025-01-19 23:35:33: 69 states generated (1.712 s/min), 13 distinct states found (322 ds/min), 9 states left on queue.
+@!@!@ENDMSG 2200 @!@!@
+@!@!@STARTMSG 2199:0 @!@!@
+69 states generated, 13 distinct states found, 9 states left on queue.
+@!@!@ENDMSG 2199 @!@!@
+@!@!@STARTMSG 2186:0 @!@!@
+Finished in 2433ms at (2025-01-19 23:35:33)
+@!@!@ENDMSG 2186 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737326135419/MC.tla

@@ -0,0 +1,65 @@
+---- MODULE MC ----
+EXTENDS replicated, TLC
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+b1, b2, b3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+s1, s2, s3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+p1
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+az1, az2, az3
+----
+
+\* MV CONSTANT definitions brokers
+const_1737326131354349000 == 
+{b1, b2, b3}
+----
+
+\* MV CONSTANT definitions safekeepers
+const_1737326131354350000 == 
+{s1, s2, s3}
+----
+
+\* MV CONSTANT definitions pageservers
+const_1737326131354351000 == 
+{p1}
+----
+
+\* MV CONSTANT definitions azs
+const_1737326131354352000 == 
+{az1, az2, az3}
+----
+
+\* SYMMETRY definition
+symm_1737326131354353000 == 
+Permutations(const_1737326131354349000) \union Permutations(const_1737326131354350000) \union Permutations(const_1737326131354351000) \union Permutations(const_1737326131354352000)
+----
+
+\* CONSTANT definitions @modelParameterConstants:4max_commit_lsn
+const_1737326131354354000 == 
+2
+----
+
+\* CONSTANT definitions @modelParameterConstants:5az_mapping
+const_1737326131354355000 == 
+[ az1 |-> {b1,s1,p1} , az2 |-> {b2,s2} , az3 |-> {b3,s3}]
+----
+
+\* CONSTRAINT definition @modelParameterContraint:0
+constr_1737326131354356000 ==
+StateConstraint
+----
+=============================================================================
+\* Modification History
+\* Created Sun Jan 19 23:35:31 CET 2025 by cs

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737326135419/MC_TE.out

@@ -0,0 +1,358 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 0 and seed 849358144369345749 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 24597] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 23:35:32)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 23:35:33.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 1000:1 @!@!@
+TLC threw an unexpected exception.
+This was probably caused by an error in the spec or model.
+See the User Output or TLC Console for clues to what happened.
+The exception was a java.lang.RuntimeException
+: Attempted to compute the value of an expression of form
+CHOOSE x \in S: P, but no element of S satisfied P.
+line 27, col 3 to line 27, col 36 of module replicated
+@!@!@ENDMSG 1000 @!@!@
+@!@!@STARTMSG 2121:1 @!@!@
+The behavior up to this point is:
+@!@!@ENDMSG 2121 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+1: <Initial predicate>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, b3, s1, s2, s3, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+2: <NodeOnlineOffline line 56, col 5 to line 60, col 67 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+3: <SkCommit line 63, col 5 to line 72, col 58 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 1, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2103:1 @!@!@
+The error occurred when TLC was evaluating the nested
+expressions at the following positions:
+0. Line 112, column 5 to line 128, column 57 in replicated
+1. Line 112, column 8 to line 112, column 27 in replicated
+2. Line 113, column 8 to line 127, column 80 in replicated
+3. Line 127, column 9 to line 127, column 80 in replicated
+4. Line 127, column 29 to line 127, column 80 in replicated
+5. Line 127, column 71 to line 127, column 79 in replicated
+6. Line 125, column 22 to line 125, column 46 in replicated
+7. Line 27, column 3 to line 27, column 36 in replicated
+
+
+@!@!@ENDMSG 2103 @!@!@
+@!@!@STARTMSG 2201:0 @!@!@
+The coverage statistics at 2025-01-19 23:35:33
+@!@!@ENDMSG 2201 @!@!@
+@!@!@STARTMSG 2773:0 @!@!@
+<Init line 49, col 1 to line 49, col 4 of module replicated>: 2:2
+@!@!@ENDMSG 2773 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 50, col 5 to line 53, col 26 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<NodeOnlineOffline line 55, col 1 to line 55, col 17 of module replicated>: 1:7
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 56, col 8 to line 59, col 44 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 56, col 18 to line 59, col 44 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 57, col 9 to line 59, col 44 of module replicated: 749
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 57, col 12 to line 57, col 48 of module replicated: 749
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 58, col 12 to line 58, col 44 of module replicated: 301
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 59, col 12 to line 59, col 44 of module replicated: 98
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 56, col 32 to line 56, col 48 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 60, col 8 to line 60, col 67 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkCommit line 62, col 1 to line 62, col 16 of module replicated>: 4:15
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 63, col 8 to line 63, col 32 of module replicated: 62
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 63, col 8 to line 63, col 15 of module replicated: 47:110
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 63, col 27 to line 63, col 32 of module replicated: 47
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 64, col 8 to line 64, col 14 of module replicated: 47
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 64, col 8 to line 64, col 9 of module replicated: 32
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 64, col 13 to line 64, col 14 of module replicated: 32
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 65, col 8 to line 65, col 72 of module replicated: 34
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 65, col 8 to line 65, col 38 of module replicated: 19
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 65, col 42 to line 65, col 72 of module replicated: 19
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 69, col 13 to line 71, col 50 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 72, col 8 to line 72, col 58 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPeerRecovery line 74, col 1 to line 74, col 21 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 75, col 8 to line 75, col 32 of module replicated: 46
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 76, col 8 to line 76, col 72 of module replicated: 31
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 77, col 8 to line 77, col 71 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 78, col 8 to line 78, col 102 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 79, col 8 to line 79, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPushToBroker line 81, col 1 to line 81, col 19 of module replicated>: 2:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 82, col 8 to line 82, col 30 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 82, col 8 to line 82, col 13 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 82, col 25 to line 82, col 30 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 83, col 8 to line 90, col 52 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 91, col 8 to line 91, col 62 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsRecvBroker line 93, col 1 to line 93, col 19 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 100, col 13 to line 102, col 37 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 103, col 8 to line 103, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkRecvBroker line 106, col 1 to line 106, col 17 of module replicated>: 5:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 107, col 8 to line 107, col 29 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 107, col 8 to line 107, col 12 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 107, col 24 to line 107, col 29 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 108, col 8 to line 108, col 81 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 109, col 8 to line 109, col 57 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPrune line 111, col 1 to line 111, col 10 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 112, col 8 to line 112, col 27 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 127, col 9 to line 127, col 80 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 127, col 29 to line 127, col 80 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 127, col 30 to line 127, col 45 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 127, col 54 to line 127, col 79 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 127, col 71 to line 127, col 79 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 125, col 22 to line 125, col 46 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 27, col 3 to line 27, col 36 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 125, col 31 to line 125, col 45 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||line 118, col 28 to line 124, col 13 of module replicated: 5:5
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||line 123, col 27 to line 123, col 35 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||line 117, col 22 to line 117, col 64 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  ||||||||line 117, col 29 to line 117, col 63 of module replicated: 5:5
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||line 117, col 51 to line 117, col 62 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 116, col 25 to line 116, col 65 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||||||||line 116, col 25 to line 116, col 56 of module replicated: 5:5
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||||line 116, col 26 to line 116, col 33 of module replicated: 15
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||||line 116, col 43 to line 116, col 54 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||line 116, col 60 to line 116, col 65 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 128, col 8 to line 128, col 57 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsChooseSk line 137, col 1 to line 137, col 13 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 140, col 8 to line 140, col 109 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 141, col 8 to line 141, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PsLagsSk line 163, col 1 to line 163, col 8 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 163, col 13 to line 164, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 35 to line 164, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 164, col 9 to line 164, col 81 of module replicated: 39
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 163, col 44 to line 163, col 72 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 22 to line 163, col 32 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PeerRecoveryIsPossible line 166, col 1 to line 166, col 22 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 167, col 5 to line 169, col 88 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 167, col 47 to line 169, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 168, col 9 to line 169, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 168, col 12 to line 169, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 168, col 13 to line 168, col 85 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 169, col 16 to line 169, col 88 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 167, col 61 to line 167, col 83 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 167, col 21 to line 167, col 43 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2202:0 @!@!@
+End of statistics.
+@!@!@ENDMSG 2202 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(4) at 2025-01-19 23:35:33: 69 states generated (1.712 s/min), 13 distinct states found (322 ds/min), 9 states left on queue.
+@!@!@ENDMSG 2200 @!@!@
+@!@!@STARTMSG 2199:0 @!@!@
+69 states generated, 13 distinct states found, 9 states left on queue.
+@!@!@ENDMSG 2199 @!@!@
+@!@!@STARTMSG 2186:0 @!@!@
+Finished in 2433ms at (2025-01-19 23:35:33)
+@!@!@ENDMSG 2186 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737326135419/replicated.tla

@@ -0,0 +1,184 @@
+---- MODULE replicated ----
+
+EXTENDS Integers, FiniteSets
+
+VARIABLES broker_state, safekeeper_state, pageserver_state, online
+
+
+CONSTANT
+    brokers,
+    safekeepers,
+    pageservers,
+    azs,
+    az_mapping
+
+CONSTANT
+    NULL
+    
+CONSTANT
+    max_commit_lsn
+    
+\* HELPERS
+
+Max(a,b) == IF a > b THEN a ELSE b
+
+(* The minimum of a non-empty set of integers *)
+MinOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x <= y
+  
+(* The minimum of a non-empty set of integers *)
+MaxOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x >= y
+
+\* END HELPERS
+
+StateConstraint ==
+    /\ \A s \in safekeepers: 
+        /\ safekeeper_state[s].commit_lsn <= max_commit_lsn
+    /\ \A b \in brokers:
+        /\ \A s \in DOMAIN broker_state[b].sk:
+            /\ broker_state[b].sk[s].commit_lsn <= max_commit_lsn
+
+
+ 
+InitSafekeeper == [prune_lsn |-> 0, commit_lsn |-> 0, rx |-> [b \in brokers |-> NULL] ]
+InitBroker == [sk |-> [s \in safekeepers |-> [prune_lsn |-> 0, commit_lsn |-> 0]]]
+InitPageserver == [last_record_lsn |-> 0, preferred_sk |-> NULL, sk |-> [s \in safekeepers |-> [commit_lsn |-> 0]]]
+InitOnline == safekeepers \cup brokers \cup pageservers
+
+Init ==
+    /\ broker_state = [b \in brokers |-> InitBroker]
+    /\ safekeeper_state = [s \in safekeepers |-> InitSafekeeper]
+    /\ pageserver_state = [p \in pageservers |-> InitPageserver]
+    /\ online = InitOnline
+  
+NodeOnlineOffline ==
+    /\ online' = CHOOSE ss \in SUBSET InitOnline: 
+        /\ Cardinality(ss \cap safekeepers) >= 2
+        /\ Cardinality(ss \cap brokers) >= 2
+        /\ ss \cap pageservers = pageservers \* assume no PS failures for now
+    /\ UNCHANGED <<safekeeper_state,broker_state,pageserver_state>>
+   
+SkCommit(s1, s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ s1 # s2
+    /\ safekeeper_state[s1].commit_lsn = safekeeper_state[s2].commit_lsn
+    /\ LET
+            new_commit_lsn == safekeeper_state[s1].commit_lsn + 1
+       IN
+            safekeeper_state' = [safekeeper_state EXCEPT
+                ![s1].commit_lsn = new_commit_lsn,
+                ![s2].commit_lsn = new_commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPeerRecovery(s1,s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ safekeeper_state[s1].commit_lsn < safekeeper_state[s2].commit_lsn \* s2 has more WAL than s1
+    /\ safekeeper_state[s2].prune_lsn < safekeeper_state[s1].commit_lsn \* s2 has not yet trimmed the WAL the WAL
+    /\ safekeeper_state' = [safekeeper_state EXCEPT![s1].commit_lsn = safekeeper_state[s2].commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPushToBroker(s,b) ==
+    /\ {s, b} \subseteq online 
+    /\ broker_state' = LET
+           broker_side == broker_state[b].sk[s]
+           sk_side == safekeeper_state[s]
+           merged == [broker_side EXCEPT
+                !["commit_lsn"] = Max(broker_side.commit_lsn, sk_side.commit_lsn),
+                !["prune_lsn"] = Max(broker_side.prune_lsn, sk_side.prune_lsn)]
+           IN
+           [broker_state EXCEPT ![b].sk[s] = merged]
+    /\ UNCHANGED <<safekeeper_state, pageserver_state,online>> 
+
+PsRecvBroker(b,p,s) ==
+    /\ {b,p,s} \subseteq online
+    /\ LET
+            bsk == broker_state[b].sk[s]
+            psk == pageserver_state[p].sk[s]
+            updpsk == [psk EXCEPT !["commit_lsn"] = bsk.commit_lsn]
+       IN
+            pageserver_state' = IF bsk.commit_lsn > psk.commit_lsn
+                THEN [pageserver_state EXCEPT ![p].sk[s] = updpsk]
+                ELSE pageserver_state
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>
+    
+    
+SkRecvBroker(b,s) ==
+    /\ {b,s} \subseteq online
+    /\ safekeeper_state' = [safekeeper_state EXCEPT ![s].rx[b] = broker_state[b]] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+
+SkPrune(s) ==
+    /\ {s} \subseteq online
+    /\ LET
+        sk == safekeeper_state[s]
+        
+        broker_infos == {sk.rx[b]: b \in (DOMAIN sk.rx)} \ {NULL}
+        known_sks == UNION ({DOMAIN bi.sk: bi \in broker_infos})
+        max_commit_lsns == {
+                MaxOfSet(
+                    {
+                        sk.rx[b].sk[s2].commit_lsn: b \in DOMAIN sk.rx
+                    }
+                ): s2 \in known_sks
+            }
+        prune_lsn == MinOfSet(max_commit_lsns)
+       IN
+        safekeeper_state' = [safekeeper_state EXCEPT ![s].prune_lsn = prune_lsn] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+    
+
+SksWithNewerWal(p) ==
+    LET
+        ps == pageserver_state[p]
+    IN
+    {s \in DOMAIN ps.sk: ps.sk[s].commit_lsn > ps.last_record_lsn}
+
+PsChooseSk(p) ==
+    /\ {p} \subseteq online
+    /\ SksWithNewerWal(p) # {}
+    /\ pageserver_state' = [pageserver_state EXCEPT![p].preferred_sk = CHOOSE s \in SksWithNewerWal(p): TRUE]
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>    
+                 
+    
+Next ==
+    \/ NodeOnlineOffline
+    \/ \E s1 \in safekeepers: \E s2 \in safekeepers:
+        \/ SkCommit(s1, s2)
+        \/ SkPeerRecovery(s1, s2)
+    \/ \E s \in safekeepers: \E b \in brokers:
+        \/ SkPushToBroker(s, b)
+        \/ SkRecvBroker(b, s)
+    \/ \E s \in safekeepers:
+        \/ SkPrune(s)  
+    \/ \E s \in safekeepers: \E b \in brokers: \E p \in pageservers: PsRecvBroker(b,p,s)
+    \/ \E p \in pageservers: PsChooseSk(p)
+    
+
+Spec == Init /\ [][Next]_<< broker_state, safekeeper_state, pageserver_state,online>>
+
+
+\* invariants
+
+PsLagsSk == \A p \in pageservers: \A s \in DOMAIN pageserver_state[p].sk: 
+        /\ pageserver_state[p].sk[s].commit_lsn <= safekeeper_state[s].commit_lsn
+
+PeerRecoveryIsPossible ==
+    \A laggard \in (safekeepers \cap online): \E donor \in (safekeepers \cap online):
+        /\ (safekeeper_state[laggard].commit_lsn < safekeeper_state[donor].commit_lsn)
+            => safekeeper_state[donor].prune_lsn <= safekeeper_state[laggard].commit_lsn
+        
+
+EventuallyLaggingSkIsNotPreferredSk == <>(
+        LET
+            sks == safekeeper_state
+            lagging_sks == { s \in safekeepers: \A s2 \in safekeepers: sks[s].commit_lsn <= sks[s2].commit_lsn }
+            preferred_sks == {pageserver_state[p].preferred_sk: p \in pageservers}
+        IN
+            preferred_sks \cap lagging_sks = {}
+            
+    )
+
+    
+
+====

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737326274268/MC.cfg

@@ -0,0 +1,54 @@
+\* MV CONSTANT declarations
+CONSTANTS
+b1 = b1
+b2 = b2
+b3 = b3
+\* MV CONSTANT declarations
+CONSTANTS
+s1 = s1
+s2 = s2
+s3 = s3
+\* MV CONSTANT declarations
+CONSTANTS
+p1 = p1
+\* MV CONSTANT declarations
+CONSTANTS
+az1 = az1
+az2 = az2
+az3 = az3
+\* MV CONSTANT definitions
+CONSTANT
+brokers <- const_1737326270219360000
+\* CONSTANT declarations
+CONSTANT NULL = NULL
+\* MV CONSTANT definitions
+CONSTANT
+safekeepers <- const_1737326270219361000
+\* MV CONSTANT definitions
+CONSTANT
+pageservers <- const_1737326270219362000
+\* MV CONSTANT definitions
+CONSTANT
+azs <- const_1737326270219363000
+\* SYMMETRY definition
+SYMMETRY symm_1737326270219364000
+\* CONSTANT definitions
+CONSTANT
+max_commit_lsn <- const_1737326270219365000
+\* CONSTANT definitions
+CONSTANT
+az_mapping <- const_1737326270219366000
+\* CONSTRAINT definition
+CONSTRAINT
+constr_1737326270219367000
+\* SPECIFICATION definition
+SPECIFICATION
+Spec
+\* INVARIANT definition
+INVARIANT
+PsLagsSk
+PeerRecoveryIsPossible
+\* PROPERTY definition
+PROPERTY
+EventuallyLaggingSkIsNotPreferredSk
+\* Generated on Sun Jan 19 23:37:50 CET 2025

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737326274268/MC.out

@@ -0,0 +1,450 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 125 and seed -2645522394156634911 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 24715] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 23:37:51)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 23:37:52.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 1000:1 @!@!@
+TLC threw an unexpected exception.
+This was probably caused by an error in the spec or model.
+See the User Output or TLC Console for clues to what happened.
+The exception was a java.lang.RuntimeException
+: Attempted to select field "sk" from a non-record value NULL
+line 121, col 25 to line 121, col 35 of module replicated
+@!@!@ENDMSG 1000 @!@!@
+@!@!@STARTMSG 2121:1 @!@!@
+The behavior up to this point is:
+@!@!@ENDMSG 2121 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+1: <Initial predicate>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, b3, s1, s2, s3, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+2: <NodeOnlineOffline line 56, col 5 to line 60, col 67 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+3: <SkRecvBroker line 107, col 5 to line 109, col 57 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2103:1 @!@!@
+The error occurred when TLC was evaluating the nested
+expressions at the following positions:
+0. Line 112, column 5 to line 128, column 57 in replicated
+1. Line 112, column 8 to line 112, column 27 in replicated
+2. Line 113, column 8 to line 127, column 80 in replicated
+3. Line 127, column 9 to line 127, column 80 in replicated
+4. Line 127, column 29 to line 127, column 80 in replicated
+5. Line 127, column 71 to line 127, column 79 in replicated
+6. Line 125, column 22 to line 125, column 82 in replicated
+7. Line 125, column 25 to line 125, column 44 in replicated
+8. Line 125, column 25 to line 125, column 39 in replicated
+9. Line 118, column 28 to line 124, column 13 in replicated
+10. Line 119, column 17 to line 123, column 17 in replicated
+11. Line 31, column 3 to line 31, column 36 in replicated
+12. Line 31, column 16 to line 31, column 16 in replicated
+13. Line 120, column 21 to line 122, column 21 in replicated
+14. Line 121, column 25 to line 121, column 50 in replicated
+15. Line 121, column 25 to line 121, column 39 in replicated
+16. Line 121, column 25 to line 121, column 35 in replicated
+
+
+@!@!@ENDMSG 2103 @!@!@
+@!@!@STARTMSG 2201:0 @!@!@
+The coverage statistics at 2025-01-19 23:37:53
+@!@!@ENDMSG 2201 @!@!@
+@!@!@STARTMSG 2773:0 @!@!@
+<Init line 49, col 1 to line 49, col 4 of module replicated>: 2:2
+@!@!@ENDMSG 2773 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 50, col 5 to line 53, col 26 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<NodeOnlineOffline line 55, col 1 to line 55, col 17 of module replicated>: 1:12
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 56, col 8 to line 59, col 44 of module replicated: 12
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 56, col 18 to line 59, col 44 of module replicated: 12
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 57, col 9 to line 59, col 44 of module replicated: 1284
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 57, col 12 to line 57, col 48 of module replicated: 1284
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 58, col 12 to line 58, col 44 of module replicated: 516
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 59, col 12 to line 59, col 44 of module replicated: 168
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 56, col 32 to line 56, col 48 of module replicated: 12
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 60, col 8 to line 60, col 67 of module replicated: 12
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkCommit line 62, col 1 to line 62, col 16 of module replicated>: 5:26
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 63, col 8 to line 63, col 32 of module replicated: 125
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 63, col 8 to line 63, col 15 of module replicated: 99:234
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 63, col 27 to line 63, col 32 of module replicated: 99
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 64, col 8 to line 64, col 14 of module replicated: 95
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 64, col 8 to line 64, col 9 of module replicated: 69
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 64, col 13 to line 64, col 14 of module replicated: 69
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 65, col 8 to line 65, col 72 of module replicated: 68
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 65, col 8 to line 65, col 38 of module replicated: 42
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 65, col 42 to line 65, col 72 of module replicated: 42
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 69, col 13 to line 71, col 50 of module replicated: 26
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 72, col 8 to line 72, col 58 of module replicated: 26
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPeerRecovery line 74, col 1 to line 74, col 21 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 75, col 8 to line 75, col 32 of module replicated: 99
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 76, col 8 to line 76, col 72 of module replicated: 69
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 77, col 8 to line 77, col 71 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 78, col 8 to line 78, col 102 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 79, col 8 to line 79, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPushToBroker line 81, col 1 to line 81, col 19 of module replicated>: 9:66
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 82, col 8 to line 82, col 30 of module replicated: 157
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 82, col 8 to line 82, col 13 of module replicated: 91:233
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 82, col 25 to line 82, col 30 of module replicated: 91
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 83, col 8 to line 90, col 52 of module replicated: 66
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 91, col 8 to line 91, col 62 of module replicated: 66
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsRecvBroker line 93, col 1 to line 93, col 19 of module replicated>: 1:48
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 94, col 8 to line 94, col 31 of module replicated: 111
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 94, col 8 to line 94, col 14 of module replicated: 63:213
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 94, col 26 to line 94, col 31 of module replicated: 63
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 100, col 13 to line 102, col 37 of module replicated: 48
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 100, col 33 to line 102, col 37 of module replicated: 48
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 100, col 36 to line 100, col 66 of module replicated: 48
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 101, col 22 to line 101, col 66 of module replicated: 1
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 102, col 22 to line 102, col 37 of module replicated: 47
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 103, col 8 to line 103, col 58 of module replicated: 48
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkRecvBroker line 106, col 1 to line 106, col 17 of module replicated>: 22:66
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 107, col 8 to line 107, col 29 of module replicated: 157
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 107, col 8 to line 107, col 12 of module replicated: 91:233
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 107, col 24 to line 107, col 29 of module replicated: 91
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 108, col 8 to line 108, col 81 of module replicated: 66
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 109, col 8 to line 109, col 57 of module replicated: 66
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPrune line 111, col 1 to line 111, col 10 of module replicated>: 0:18
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 112, col 8 to line 112, col 27 of module replicated: 42
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 112, col 8 to line 112, col 10 of module replicated: 24:45
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 112, col 22 to line 112, col 27 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 127, col 9 to line 127, col 80 of module replicated: 18
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 127, col 29 to line 127, col 80 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 127, col 30 to line 127, col 45 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 127, col 54 to line 127, col 79 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 127, col 71 to line 127, col 79 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 125, col 22 to line 125, col 82 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 125, col 25 to line 125, col 44 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||line 125, col 25 to line 125, col 39 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||line 118, col 28 to line 124, col 13 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||line 119, col 17 to line 123, col 17 of module replicated: 3
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 31, col 3 to line 31, col 36 of module replicated: 3
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 120, col 21 to line 122, col 21 of module replicated: 3
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||line 121, col 25 to line 121, col 50 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 121, col 25 to line 121, col 39 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||line 121, col 25 to line 121, col 35 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||line 121, col 37 to line 121, col 38 of module replicated: 3
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||line 121, col 59 to line 121, col 70 of module replicated: 3
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||line 123, col 27 to line 123, col 35 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 117, col 22 to line 117, col 64 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||||||line 117, col 29 to line 117, col 63 of module replicated: 21:24
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 117, col 30 to line 117, col 41 of module replicated: 3
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 117, col 51 to line 117, col 62 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||line 116, col 25 to line 116, col 65 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  ||||||||||||line 116, col 25 to line 116, col 56 of module replicated: 21:24
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||||line 116, col 26 to line 116, col 33 of module replicated: 63
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||||line 116, col 43 to line 116, col 54 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||||line 116, col 60 to line 116, col 65 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||line 125, col 43 to line 125, col 44 of module replicated: 18
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 128, col 8 to line 128, col 57 of module replicated: 18
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsChooseSk line 137, col 1 to line 137, col 13 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 138, col 8 to line 138, col 27 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 139, col 8 to line 139, col 30 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 139, col 8 to line 139, col 25 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 133, col 15 to line 133, col 33 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 135, col 5 to line 135, col 66 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 135, col 26 to line 135, col 65 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 135, col 12 to line 135, col 23 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 139, col 24 to line 139, col 24 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 139, col 29 to line 139, col 30 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 140, col 8 to line 140, col 109 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 141, col 8 to line 141, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PsLagsSk line 163, col 1 to line 163, col 8 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 163, col 13 to line 164, col 81 of module replicated: 43
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 35 to line 164, col 81 of module replicated: 43
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 164, col 9 to line 164, col 81 of module replicated: 129
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 163, col 44 to line 163, col 72 of module replicated: 43
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 22 to line 163, col 32 of module replicated: 43
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PeerRecoveryIsPossible line 166, col 1 to line 166, col 22 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 167, col 5 to line 169, col 88 of module replicated: 43
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 167, col 47 to line 169, col 88 of module replicated: 116
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 168, col 9 to line 169, col 88 of module replicated: 116
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 168, col 12 to line 169, col 88 of module replicated: 116
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 168, col 13 to line 168, col 85 of module replicated: 116
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 169, col 16 to line 169, col 88 of module replicated: 28
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 167, col 61 to line 167, col 83 of module replicated: 116
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 167, col 21 to line 167, col 43 of module replicated: 43
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2202:0 @!@!@
+End of statistics.
+@!@!@ENDMSG 2202 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(5) at 2025-01-19 23:37:53: 220 states generated (4.929 s/min), 39 distinct states found (873 ds/min), 30 states left on queue.
+@!@!@ENDMSG 2200 @!@!@
+@!@!@STARTMSG 2199:0 @!@!@
+220 states generated, 39 distinct states found, 30 states left on queue.
+@!@!@ENDMSG 2199 @!@!@
+@!@!@STARTMSG 2186:0 @!@!@
+Finished in 2693ms at (2025-01-19 23:37:53)
+@!@!@ENDMSG 2186 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737326274268/MC.tla

@@ -0,0 +1,65 @@
+---- MODULE MC ----
+EXTENDS replicated, TLC
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+b1, b2, b3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+s1, s2, s3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+p1
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+az1, az2, az3
+----
+
+\* MV CONSTANT definitions brokers
+const_1737326270219360000 == 
+{b1, b2, b3}
+----
+
+\* MV CONSTANT definitions safekeepers
+const_1737326270219361000 == 
+{s1, s2, s3}
+----
+
+\* MV CONSTANT definitions pageservers
+const_1737326270219362000 == 
+{p1}
+----
+
+\* MV CONSTANT definitions azs
+const_1737326270219363000 == 
+{az1, az2, az3}
+----
+
+\* SYMMETRY definition
+symm_1737326270219364000 == 
+Permutations(const_1737326270219360000) \union Permutations(const_1737326270219361000) \union Permutations(const_1737326270219362000) \union Permutations(const_1737326270219363000)
+----
+
+\* CONSTANT definitions @modelParameterConstants:4max_commit_lsn
+const_1737326270219365000 == 
+2
+----
+
+\* CONSTANT definitions @modelParameterConstants:5az_mapping
+const_1737326270219366000 == 
+[ az1 |-> {b1,s1,p1} , az2 |-> {b2,s2} , az3 |-> {b3,s3}]
+----
+
+\* CONSTRAINT definition @modelParameterContraint:0
+constr_1737326270219367000 ==
+StateConstraint
+----
+=============================================================================
+\* Modification History
+\* Created Sun Jan 19 23:37:50 CET 2025 by cs

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737326274268/MC_TE.out

@@ -0,0 +1,450 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 125 and seed -2645522394156634911 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 24715] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 23:37:51)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 23:37:52.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 1000:1 @!@!@
+TLC threw an unexpected exception.
+This was probably caused by an error in the spec or model.
+See the User Output or TLC Console for clues to what happened.
+The exception was a java.lang.RuntimeException
+: Attempted to select field "sk" from a non-record value NULL
+line 121, col 25 to line 121, col 35 of module replicated
+@!@!@ENDMSG 1000 @!@!@
+@!@!@STARTMSG 2121:1 @!@!@
+The behavior up to this point is:
+@!@!@ENDMSG 2121 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+1: <Initial predicate>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, b3, s1, s2, s3, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+2: <NodeOnlineOffline line 56, col 5 to line 60, col 67 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+3: <SkRecvBroker line 107, col 5 to line 109, col 57 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2103:1 @!@!@
+The error occurred when TLC was evaluating the nested
+expressions at the following positions:
+0. Line 112, column 5 to line 128, column 57 in replicated
+1. Line 112, column 8 to line 112, column 27 in replicated
+2. Line 113, column 8 to line 127, column 80 in replicated
+3. Line 127, column 9 to line 127, column 80 in replicated
+4. Line 127, column 29 to line 127, column 80 in replicated
+5. Line 127, column 71 to line 127, column 79 in replicated
+6. Line 125, column 22 to line 125, column 82 in replicated
+7. Line 125, column 25 to line 125, column 44 in replicated
+8. Line 125, column 25 to line 125, column 39 in replicated
+9. Line 118, column 28 to line 124, column 13 in replicated
+10. Line 119, column 17 to line 123, column 17 in replicated
+11. Line 31, column 3 to line 31, column 36 in replicated
+12. Line 31, column 16 to line 31, column 16 in replicated
+13. Line 120, column 21 to line 122, column 21 in replicated
+14. Line 121, column 25 to line 121, column 50 in replicated
+15. Line 121, column 25 to line 121, column 39 in replicated
+16. Line 121, column 25 to line 121, column 35 in replicated
+
+
+@!@!@ENDMSG 2103 @!@!@
+@!@!@STARTMSG 2201:0 @!@!@
+The coverage statistics at 2025-01-19 23:37:53
+@!@!@ENDMSG 2201 @!@!@
+@!@!@STARTMSG 2773:0 @!@!@
+<Init line 49, col 1 to line 49, col 4 of module replicated>: 2:2
+@!@!@ENDMSG 2773 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 50, col 5 to line 53, col 26 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<NodeOnlineOffline line 55, col 1 to line 55, col 17 of module replicated>: 1:12
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 56, col 8 to line 59, col 44 of module replicated: 12
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 56, col 18 to line 59, col 44 of module replicated: 12
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 57, col 9 to line 59, col 44 of module replicated: 1284
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 57, col 12 to line 57, col 48 of module replicated: 1284
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 58, col 12 to line 58, col 44 of module replicated: 516
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 59, col 12 to line 59, col 44 of module replicated: 168
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 56, col 32 to line 56, col 48 of module replicated: 12
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 60, col 8 to line 60, col 67 of module replicated: 12
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkCommit line 62, col 1 to line 62, col 16 of module replicated>: 5:26
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 63, col 8 to line 63, col 32 of module replicated: 125
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 63, col 8 to line 63, col 15 of module replicated: 99:234
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 63, col 27 to line 63, col 32 of module replicated: 99
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 64, col 8 to line 64, col 14 of module replicated: 95
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 64, col 8 to line 64, col 9 of module replicated: 69
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 64, col 13 to line 64, col 14 of module replicated: 69
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 65, col 8 to line 65, col 72 of module replicated: 68
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 65, col 8 to line 65, col 38 of module replicated: 42
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 65, col 42 to line 65, col 72 of module replicated: 42
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 69, col 13 to line 71, col 50 of module replicated: 26
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 72, col 8 to line 72, col 58 of module replicated: 26
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPeerRecovery line 74, col 1 to line 74, col 21 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 75, col 8 to line 75, col 32 of module replicated: 99
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 76, col 8 to line 76, col 72 of module replicated: 69
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 77, col 8 to line 77, col 71 of module replicated: 8
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 78, col 8 to line 78, col 102 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 79, col 8 to line 79, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPushToBroker line 81, col 1 to line 81, col 19 of module replicated>: 9:66
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 82, col 8 to line 82, col 30 of module replicated: 157
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 82, col 8 to line 82, col 13 of module replicated: 91:233
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 82, col 25 to line 82, col 30 of module replicated: 91
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 83, col 8 to line 90, col 52 of module replicated: 66
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 91, col 8 to line 91, col 62 of module replicated: 66
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsRecvBroker line 93, col 1 to line 93, col 19 of module replicated>: 1:48
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 94, col 8 to line 94, col 31 of module replicated: 111
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 94, col 8 to line 94, col 14 of module replicated: 63:213
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 94, col 26 to line 94, col 31 of module replicated: 63
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 100, col 13 to line 102, col 37 of module replicated: 48
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 100, col 33 to line 102, col 37 of module replicated: 48
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 100, col 36 to line 100, col 66 of module replicated: 48
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 101, col 22 to line 101, col 66 of module replicated: 1
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 102, col 22 to line 102, col 37 of module replicated: 47
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 103, col 8 to line 103, col 58 of module replicated: 48
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkRecvBroker line 106, col 1 to line 106, col 17 of module replicated>: 22:66
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 107, col 8 to line 107, col 29 of module replicated: 157
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 107, col 8 to line 107, col 12 of module replicated: 91:233
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 107, col 24 to line 107, col 29 of module replicated: 91
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 108, col 8 to line 108, col 81 of module replicated: 66
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 109, col 8 to line 109, col 57 of module replicated: 66
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPrune line 111, col 1 to line 111, col 10 of module replicated>: 0:18
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 112, col 8 to line 112, col 27 of module replicated: 42
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 112, col 8 to line 112, col 10 of module replicated: 24:45
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 112, col 22 to line 112, col 27 of module replicated: 24
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 127, col 9 to line 127, col 80 of module replicated: 18
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 127, col 29 to line 127, col 80 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 127, col 30 to line 127, col 45 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 127, col 54 to line 127, col 79 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 127, col 71 to line 127, col 79 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 125, col 22 to line 125, col 82 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 125, col 25 to line 125, col 44 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||line 125, col 25 to line 125, col 39 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||line 118, col 28 to line 124, col 13 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||line 119, col 17 to line 123, col 17 of module replicated: 3
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 31, col 3 to line 31, col 36 of module replicated: 3
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 120, col 21 to line 122, col 21 of module replicated: 3
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||line 121, col 25 to line 121, col 50 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 121, col 25 to line 121, col 39 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||line 121, col 25 to line 121, col 35 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||line 121, col 37 to line 121, col 38 of module replicated: 3
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||line 121, col 59 to line 121, col 70 of module replicated: 3
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||line 123, col 27 to line 123, col 35 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||line 117, col 22 to line 117, col 64 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |||||||||line 117, col 29 to line 117, col 63 of module replicated: 21:24
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 117, col 30 to line 117, col 41 of module replicated: 3
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||line 117, col 51 to line 117, col 62 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||line 116, col 25 to line 116, col 65 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  ||||||||||||line 116, col 25 to line 116, col 56 of module replicated: 21:24
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||||line 116, col 26 to line 116, col 33 of module replicated: 63
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||||||||||line 116, col 43 to line 116, col 54 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||||||||||line 116, col 60 to line 116, col 65 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||||line 125, col 43 to line 125, col 44 of module replicated: 18
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 128, col 8 to line 128, col 57 of module replicated: 18
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsChooseSk line 137, col 1 to line 137, col 13 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 138, col 8 to line 138, col 27 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 139, col 8 to line 139, col 30 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 139, col 8 to line 139, col 25 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 133, col 15 to line 133, col 33 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 135, col 5 to line 135, col 66 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 135, col 26 to line 135, col 65 of module replicated: 21
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 135, col 12 to line 135, col 23 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 139, col 24 to line 139, col 24 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 139, col 29 to line 139, col 30 of module replicated: 7
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 140, col 8 to line 140, col 109 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 141, col 8 to line 141, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PsLagsSk line 163, col 1 to line 163, col 8 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 163, col 13 to line 164, col 81 of module replicated: 43
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 35 to line 164, col 81 of module replicated: 43
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 164, col 9 to line 164, col 81 of module replicated: 129
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 163, col 44 to line 163, col 72 of module replicated: 43
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 163, col 22 to line 163, col 32 of module replicated: 43
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PeerRecoveryIsPossible line 166, col 1 to line 166, col 22 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 167, col 5 to line 169, col 88 of module replicated: 43
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 167, col 47 to line 169, col 88 of module replicated: 116
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 168, col 9 to line 169, col 88 of module replicated: 116
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 168, col 12 to line 169, col 88 of module replicated: 116
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 168, col 13 to line 168, col 85 of module replicated: 116
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 169, col 16 to line 169, col 88 of module replicated: 28
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 167, col 61 to line 167, col 83 of module replicated: 116
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 167, col 21 to line 167, col 43 of module replicated: 43
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2202:0 @!@!@
+End of statistics.
+@!@!@ENDMSG 2202 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(5) at 2025-01-19 23:37:53: 220 states generated (4.929 s/min), 39 distinct states found (873 ds/min), 30 states left on queue.
+@!@!@ENDMSG 2200 @!@!@
+@!@!@STARTMSG 2199:0 @!@!@
+220 states generated, 39 distinct states found, 30 states left on queue.
+@!@!@ENDMSG 2199 @!@!@
+@!@!@STARTMSG 2186:0 @!@!@
+Finished in 2693ms at (2025-01-19 23:37:53)
+@!@!@ENDMSG 2186 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737326274268/replicated.tla

@@ -0,0 +1,184 @@
+---- MODULE replicated ----
+
+EXTENDS Integers, FiniteSets
+
+VARIABLES broker_state, safekeeper_state, pageserver_state, online
+
+
+CONSTANT
+    brokers,
+    safekeepers,
+    pageservers,
+    azs,
+    az_mapping
+
+CONSTANT
+    NULL
+    
+CONSTANT
+    max_commit_lsn
+    
+\* HELPERS
+
+Max(a,b) == IF a > b THEN a ELSE b
+
+(* The minimum of a non-empty set of integers *)
+MinOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x <= y
+  
+(* The minimum of a non-empty set of integers *)
+MaxOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x >= y
+
+\* END HELPERS
+
+StateConstraint ==
+    /\ \A s \in safekeepers: 
+        /\ safekeeper_state[s].commit_lsn <= max_commit_lsn
+    /\ \A b \in brokers:
+        /\ \A s \in DOMAIN broker_state[b].sk:
+            /\ broker_state[b].sk[s].commit_lsn <= max_commit_lsn
+
+
+ 
+InitSafekeeper == [prune_lsn |-> 0, commit_lsn |-> 0, rx |-> [b \in brokers |-> NULL] ]
+InitBroker == [sk |-> [s \in safekeepers |-> [prune_lsn |-> 0, commit_lsn |-> 0]]]
+InitPageserver == [last_record_lsn |-> 0, preferred_sk |-> NULL, sk |-> [s \in safekeepers |-> [commit_lsn |-> 0]]]
+InitOnline == safekeepers \cup brokers \cup pageservers
+
+Init ==
+    /\ broker_state = [b \in brokers |-> InitBroker]
+    /\ safekeeper_state = [s \in safekeepers |-> InitSafekeeper]
+    /\ pageserver_state = [p \in pageservers |-> InitPageserver]
+    /\ online = InitOnline
+  
+NodeOnlineOffline ==
+    /\ online' = CHOOSE ss \in SUBSET InitOnline: 
+        /\ Cardinality(ss \cap safekeepers) >= 2
+        /\ Cardinality(ss \cap brokers) >= 2
+        /\ ss \cap pageservers = pageservers \* assume no PS failures for now
+    /\ UNCHANGED <<safekeeper_state,broker_state,pageserver_state>>
+   
+SkCommit(s1, s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ s1 # s2
+    /\ safekeeper_state[s1].commit_lsn = safekeeper_state[s2].commit_lsn
+    /\ LET
+            new_commit_lsn == safekeeper_state[s1].commit_lsn + 1
+       IN
+            safekeeper_state' = [safekeeper_state EXCEPT
+                ![s1].commit_lsn = new_commit_lsn,
+                ![s2].commit_lsn = new_commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPeerRecovery(s1,s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ safekeeper_state[s1].commit_lsn < safekeeper_state[s2].commit_lsn \* s2 has more WAL than s1
+    /\ safekeeper_state[s2].prune_lsn < safekeeper_state[s1].commit_lsn \* s2 has not yet trimmed the WAL the WAL
+    /\ safekeeper_state' = [safekeeper_state EXCEPT![s1].commit_lsn = safekeeper_state[s2].commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPushToBroker(s,b) ==
+    /\ {s, b} \subseteq online 
+    /\ broker_state' = LET
+           broker_side == broker_state[b].sk[s]
+           sk_side == safekeeper_state[s]
+           merged == [broker_side EXCEPT
+                !["commit_lsn"] = Max(broker_side.commit_lsn, sk_side.commit_lsn),
+                !["prune_lsn"] = Max(broker_side.prune_lsn, sk_side.prune_lsn)]
+           IN
+           [broker_state EXCEPT ![b].sk[s] = merged]
+    /\ UNCHANGED <<safekeeper_state, pageserver_state,online>> 
+
+PsRecvBroker(b,p,s) ==
+    /\ {b,p,s} \subseteq online
+    /\ LET
+            bsk == broker_state[b].sk[s]
+            psk == pageserver_state[p].sk[s]
+            updpsk == [psk EXCEPT !["commit_lsn"] = bsk.commit_lsn]
+       IN
+            pageserver_state' = IF bsk.commit_lsn > psk.commit_lsn
+                THEN [pageserver_state EXCEPT ![p].sk[s] = updpsk]
+                ELSE pageserver_state
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>
+    
+    
+SkRecvBroker(b,s) ==
+    /\ {b,s} \subseteq online
+    /\ safekeeper_state' = [safekeeper_state EXCEPT ![s].rx[b] = broker_state[b]] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+
+SkPrune(s) ==
+    /\ {s} \subseteq online
+    /\ LET
+        sk == safekeeper_state[s]
+        
+        broker_infos == {sk.rx[b]: b \in (DOMAIN sk.rx)} \ {NULL}
+        known_sks == UNION ({DOMAIN bi.sk: bi \in broker_infos})
+        max_commit_lsns == {
+                MaxOfSet(
+                    {
+                        sk.rx[b].sk[s2].commit_lsn: b \in DOMAIN sk.rx
+                    }
+                ): s2 \in known_sks
+            }
+        prune_lsn == IF max_commit_lsns # {} THEN MinOfSet(max_commit_lsns) ELSE 0
+       IN
+        safekeeper_state' = [safekeeper_state EXCEPT ![s].prune_lsn = prune_lsn] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+    
+
+SksWithNewerWal(p) ==
+    LET
+        ps == pageserver_state[p]
+    IN
+    {s \in DOMAIN ps.sk: ps.sk[s].commit_lsn > ps.last_record_lsn}
+
+PsChooseSk(p) ==
+    /\ {p} \subseteq online
+    /\ SksWithNewerWal(p) # {}
+    /\ pageserver_state' = [pageserver_state EXCEPT![p].preferred_sk = CHOOSE s \in SksWithNewerWal(p): TRUE]
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>    
+                 
+    
+Next ==
+    \/ NodeOnlineOffline
+    \/ \E s1 \in safekeepers: \E s2 \in safekeepers:
+        \/ SkCommit(s1, s2)
+        \/ SkPeerRecovery(s1, s2)
+    \/ \E s \in safekeepers: \E b \in brokers:
+        \/ SkPushToBroker(s, b)
+        \/ SkRecvBroker(b, s)
+    \/ \E s \in safekeepers:
+        \/ SkPrune(s)  
+    \/ \E s \in safekeepers: \E b \in brokers: \E p \in pageservers: PsRecvBroker(b,p,s)
+    \/ \E p \in pageservers: PsChooseSk(p)
+    
+
+Spec == Init /\ [][Next]_<< broker_state, safekeeper_state, pageserver_state,online>>
+
+
+\* invariants
+
+PsLagsSk == \A p \in pageservers: \A s \in DOMAIN pageserver_state[p].sk: 
+        /\ pageserver_state[p].sk[s].commit_lsn <= safekeeper_state[s].commit_lsn
+
+PeerRecoveryIsPossible ==
+    \A laggard \in (safekeepers \cap online): \E donor \in (safekeepers \cap online):
+        /\ (safekeeper_state[laggard].commit_lsn < safekeeper_state[donor].commit_lsn)
+            => safekeeper_state[donor].prune_lsn <= safekeeper_state[laggard].commit_lsn
+        
+
+EventuallyLaggingSkIsNotPreferredSk == <>(
+        LET
+            sks == safekeeper_state
+            lagging_sks == { s \in safekeepers: \A s2 \in safekeepers: sks[s].commit_lsn <= sks[s2].commit_lsn }
+            preferred_sks == {pageserver_state[p].preferred_sk: p \in pageservers}
+        IN
+            preferred_sks \cap lagging_sks = {}
+            
+    )
+
+    
+
+====

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737327144010/MC.cfg

@@ -0,0 +1,54 @@
+\* MV CONSTANT declarations
+CONSTANTS
+b1 = b1
+b2 = b2
+b3 = b3
+\* MV CONSTANT declarations
+CONSTANTS
+s1 = s1
+s2 = s2
+s3 = s3
+\* MV CONSTANT declarations
+CONSTANTS
+p1 = p1
+\* MV CONSTANT declarations
+CONSTANTS
+az1 = az1
+az2 = az2
+az3 = az3
+\* MV CONSTANT definitions
+CONSTANT
+brokers <- const_1737327139928371000
+\* CONSTANT declarations
+CONSTANT NULL = NULL
+\* MV CONSTANT definitions
+CONSTANT
+safekeepers <- const_1737327139928372000
+\* MV CONSTANT definitions
+CONSTANT
+pageservers <- const_1737327139928373000
+\* MV CONSTANT definitions
+CONSTANT
+azs <- const_1737327139928374000
+\* SYMMETRY definition
+SYMMETRY symm_1737327139928375000
+\* CONSTANT definitions
+CONSTANT
+max_commit_lsn <- const_1737327139928376000
+\* CONSTANT definitions
+CONSTANT
+az_mapping <- const_1737327139928377000
+\* CONSTRAINT definition
+CONSTRAINT
+constr_1737327139928378000
+\* SPECIFICATION definition
+SPECIFICATION
+Spec
+\* INVARIANT definition
+INVARIANT
+PsLagsSk
+PeerRecoveryIsPossible
+\* PROPERTY definition
+PROPERTY
+EventuallyLaggingSkIsNotPreferredSk
+\* Generated on Sun Jan 19 23:52:19 CET 2025

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737327144010/MC.out

@@ -0,0 +1,297 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 19 and seed -4531850586577240157 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 25317] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 23:52:21)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 23:52:22.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 2109:1 @!@!@
+Successor state is not completely specified by action SkPrune of the next-state relation. The following variable is not assigned: safekeeper_state.
+
+@!@!@ENDMSG 2109 @!@!@
+@!@!@STARTMSG 2121:1 @!@!@
+The behavior up to this point is:
+@!@!@ENDMSG 2121 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+1: <Initial predicate>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, b3, s1, s2, s3, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+2: <NodeOnlineOffline line 56, col 5 to line 60, col 67 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+3: <SkPrune line 112, col 5 to line 118, col 57 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = null
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2201:0 @!@!@
+The coverage statistics at 2025-01-19 23:52:22
+@!@!@ENDMSG 2201 @!@!@
+@!@!@STARTMSG 2773:0 @!@!@
+<Init line 49, col 1 to line 49, col 4 of module replicated>: 2:2
+@!@!@ENDMSG 2773 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 50, col 5 to line 53, col 26 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<NodeOnlineOffline line 55, col 1 to line 55, col 17 of module replicated>: 1:6
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 56, col 8 to line 59, col 44 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 56, col 18 to line 59, col 44 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 57, col 9 to line 59, col 44 of module replicated: 642
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 57, col 12 to line 57, col 48 of module replicated: 642
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 58, col 12 to line 58, col 44 of module replicated: 258
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 59, col 12 to line 59, col 44 of module replicated: 84
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 56, col 32 to line 56, col 48 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 60, col 8 to line 60, col 67 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkCommit line 62, col 1 to line 62, col 16 of module replicated>: 4:14
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 63, col 8 to line 63, col 32 of module replicated: 59
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 63, col 8 to line 63, col 15 of module replicated: 45:105
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 63, col 27 to line 63, col 32 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 64, col 8 to line 64, col 14 of module replicated: 44
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 64, col 8 to line 64, col 9 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 64, col 13 to line 64, col 14 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 65, col 8 to line 65, col 72 of module replicated: 32
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 65, col 8 to line 65, col 38 of module replicated: 18
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 65, col 42 to line 65, col 72 of module replicated: 18
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 69, col 13 to line 71, col 50 of module replicated: 14
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 72, col 8 to line 72, col 58 of module replicated: 14
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPeerRecovery line 74, col 1 to line 74, col 21 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 75, col 8 to line 75, col 32 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 76, col 8 to line 76, col 72 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 77, col 8 to line 77, col 71 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 78, col 8 to line 78, col 102 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 79, col 8 to line 79, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPushToBroker line 81, col 1 to line 81, col 19 of module replicated>: 2:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 82, col 8 to line 82, col 30 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 82, col 8 to line 82, col 13 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 82, col 25 to line 82, col 30 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 83, col 8 to line 90, col 52 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 91, col 8 to line 91, col 62 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsRecvBroker line 93, col 1 to line 93, col 19 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 100, col 13 to line 102, col 37 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 103, col 8 to line 103, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkRecvBroker line 106, col 1 to line 106, col 17 of module replicated>: 5:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 107, col 8 to line 107, col 29 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 107, col 8 to line 107, col 12 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 107, col 24 to line 107, col 29 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 108, col 8 to line 108, col 81 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 109, col 8 to line 109, col 57 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPrune line 111, col 1 to line 111, col 10 of module replicated>: 0:5
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 112, col 8 to line 112, col 27 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 112, col 8 to line 112, col 10 of module replicated: 5:10
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 112, col 22 to line 112, col 27 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 117, col 9 to line 117, col 12 of module replicated: 1
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 118, col 8 to line 118, col 57 of module replicated: 1
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsChooseSk line 127, col 1 to line 127, col 13 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 130, col 8 to line 130, col 109 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 131, col 8 to line 131, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PsLagsSk line 153, col 1 to line 153, col 8 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 153, col 13 to line 154, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 153, col 35 to line 154, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 154, col 9 to line 154, col 81 of module replicated: 39
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 153, col 44 to line 153, col 72 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 153, col 22 to line 153, col 32 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PeerRecoveryIsPossible line 156, col 1 to line 156, col 22 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 157, col 5 to line 159, col 88 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 157, col 47 to line 159, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 158, col 9 to line 159, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 158, col 12 to line 159, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 158, col 13 to line 158, col 85 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 159, col 16 to line 159, col 88 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 157, col 61 to line 157, col 83 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 157, col 21 to line 157, col 43 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2202:0 @!@!@
+End of statistics.
+@!@!@ENDMSG 2202 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(4) at 2025-01-19 23:52:22: 73 states generated (1.649 s/min), 13 distinct states found (293 ds/min), 9 states left on queue.
+@!@!@ENDMSG 2200 @!@!@
+@!@!@STARTMSG 2199:0 @!@!@
+73 states generated, 13 distinct states found, 9 states left on queue.
+@!@!@ENDMSG 2199 @!@!@
+@!@!@STARTMSG 2194:0 @!@!@
+The depth of the complete state graph search is 4.
+@!@!@ENDMSG 2194 @!@!@
+@!@!@STARTMSG 2186:0 @!@!@
+Finished in 2670ms at (2025-01-19 23:52:22)
+@!@!@ENDMSG 2186 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737327144010/MC.tla

@@ -0,0 +1,65 @@
+---- MODULE MC ----
+EXTENDS replicated, TLC
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+b1, b2, b3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+s1, s2, s3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+p1
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+az1, az2, az3
+----
+
+\* MV CONSTANT definitions brokers
+const_1737327139928371000 == 
+{b1, b2, b3}
+----
+
+\* MV CONSTANT definitions safekeepers
+const_1737327139928372000 == 
+{s1, s2, s3}
+----
+
+\* MV CONSTANT definitions pageservers
+const_1737327139928373000 == 
+{p1}
+----
+
+\* MV CONSTANT definitions azs
+const_1737327139928374000 == 
+{az1, az2, az3}
+----
+
+\* SYMMETRY definition
+symm_1737327139928375000 == 
+Permutations(const_1737327139928371000) \union Permutations(const_1737327139928372000) \union Permutations(const_1737327139928373000) \union Permutations(const_1737327139928374000)
+----
+
+\* CONSTANT definitions @modelParameterConstants:4max_commit_lsn
+const_1737327139928376000 == 
+2
+----
+
+\* CONSTANT definitions @modelParameterConstants:5az_mapping
+const_1737327139928377000 == 
+[ az1 |-> {b1,s1,p1} , az2 |-> {b2,s2} , az3 |-> {b3,s3}]
+----
+
+\* CONSTRAINT definition @modelParameterContraint:0
+constr_1737327139928378000 ==
+StateConstraint
+----
+=============================================================================
+\* Modification History
+\* Created Sun Jan 19 23:52:19 CET 2025 by cs

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737327144010/MC_TE.out

@@ -0,0 +1,297 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 19 and seed -4531850586577240157 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 25317] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 23:52:21)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 23:52:22.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 2109:1 @!@!@
+Successor state is not completely specified by action SkPrune of the next-state relation. The following variable is not assigned: safekeeper_state.
+
+@!@!@ENDMSG 2109 @!@!@
+@!@!@STARTMSG 2121:1 @!@!@
+The behavior up to this point is:
+@!@!@ENDMSG 2121 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+1: <Initial predicate>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, b3, s1, s2, s3, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+2: <NodeOnlineOffline line 56, col 5 to line 60, col 67 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0, rx |-> (b1 :> NULL @@ b2 :> NULL @@ b3 :> NULL)])
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2217:4 @!@!@
+3: <SkPrune line 112, col 5 to line 118, col 57 of module replicated>
+/\ pageserver_state = (p1 :> [sk |-> (s1 :> [commit_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0]), last_record_lsn |-> 0, preferred_sk |-> NULL])
+/\ broker_state = (b1 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b2 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])] @@ b3 :> [sk |-> (s1 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s2 :> [commit_lsn |-> 0, prune_lsn |-> 0] @@ s3 :> [commit_lsn |-> 0, prune_lsn |-> 0])])
+/\ online = {b1, b2, s1, s2, p1}
+/\ safekeeper_state = null
+
+@!@!@ENDMSG 2217 @!@!@
+@!@!@STARTMSG 2201:0 @!@!@
+The coverage statistics at 2025-01-19 23:52:22
+@!@!@ENDMSG 2201 @!@!@
+@!@!@STARTMSG 2773:0 @!@!@
+<Init line 49, col 1 to line 49, col 4 of module replicated>: 2:2
+@!@!@ENDMSG 2773 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 50, col 5 to line 53, col 26 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<NodeOnlineOffline line 55, col 1 to line 55, col 17 of module replicated>: 1:6
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 56, col 8 to line 59, col 44 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 56, col 18 to line 59, col 44 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 57, col 9 to line 59, col 44 of module replicated: 642
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 57, col 12 to line 57, col 48 of module replicated: 642
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 58, col 12 to line 58, col 44 of module replicated: 258
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 59, col 12 to line 59, col 44 of module replicated: 84
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 56, col 32 to line 56, col 48 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 60, col 8 to line 60, col 67 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkCommit line 62, col 1 to line 62, col 16 of module replicated>: 4:14
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 63, col 8 to line 63, col 32 of module replicated: 59
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 63, col 8 to line 63, col 15 of module replicated: 45:105
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 63, col 27 to line 63, col 32 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 64, col 8 to line 64, col 14 of module replicated: 44
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 64, col 8 to line 64, col 9 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 64, col 13 to line 64, col 14 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 65, col 8 to line 65, col 72 of module replicated: 32
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 65, col 8 to line 65, col 38 of module replicated: 18
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 65, col 42 to line 65, col 72 of module replicated: 18
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 69, col 13 to line 71, col 50 of module replicated: 14
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 72, col 8 to line 72, col 58 of module replicated: 14
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPeerRecovery line 74, col 1 to line 74, col 21 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 75, col 8 to line 75, col 32 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 76, col 8 to line 76, col 72 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 77, col 8 to line 77, col 71 of module replicated: 2
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 78, col 8 to line 78, col 102 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 79, col 8 to line 79, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPushToBroker line 81, col 1 to line 81, col 19 of module replicated>: 2:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 82, col 8 to line 82, col 30 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 82, col 8 to line 82, col 13 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 82, col 25 to line 82, col 30 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 83, col 8 to line 90, col 52 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 91, col 8 to line 91, col 62 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsRecvBroker line 93, col 1 to line 93, col 19 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 100, col 13 to line 102, col 37 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 103, col 8 to line 103, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkRecvBroker line 106, col 1 to line 106, col 17 of module replicated>: 5:30
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 107, col 8 to line 107, col 29 of module replicated: 75
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 107, col 8 to line 107, col 12 of module replicated: 45:111
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 107, col 24 to line 107, col 29 of module replicated: 45
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 108, col 8 to line 108, col 81 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 109, col 8 to line 109, col 57 of module replicated: 30
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<SkPrune line 111, col 1 to line 111, col 10 of module replicated>: 0:5
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 112, col 8 to line 112, col 27 of module replicated: 6
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2775:0 @!@!@
+  |line 112, col 8 to line 112, col 10 of module replicated: 5:10
+@!@!@ENDMSG 2775 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 112, col 22 to line 112, col 27 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 117, col 9 to line 117, col 12 of module replicated: 1
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 118, col 8 to line 118, col 57 of module replicated: 1
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2772:0 @!@!@
+<PsChooseSk line 127, col 1 to line 127, col 13 of module replicated>: 0:0
+@!@!@ENDMSG 2772 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 130, col 8 to line 130, col 109 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 131, col 8 to line 131, col 58 of module replicated: 0
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PsLagsSk line 153, col 1 to line 153, col 8 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 153, col 13 to line 154, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 153, col 35 to line 154, col 81 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 154, col 9 to line 154, col 81 of module replicated: 39
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 153, col 44 to line 153, col 72 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 153, col 22 to line 153, col 32 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2774:0 @!@!@
+<PeerRecoveryIsPossible line 156, col 1 to line 156, col 22 of module replicated>
+@!@!@ENDMSG 2774 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  line 157, col 5 to line 159, col 88 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 157, col 47 to line 159, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 158, col 9 to line 159, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |||line 158, col 12 to line 159, col 88 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 158, col 13 to line 158, col 85 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||||line 159, col 16 to line 159, col 88 of module replicated: 5
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  ||line 157, col 61 to line 157, col 83 of module replicated: 33
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2221:0 @!@!@
+  |line 157, col 21 to line 157, col 43 of module replicated: 13
+@!@!@ENDMSG 2221 @!@!@
+@!@!@STARTMSG 2202:0 @!@!@
+End of statistics.
+@!@!@ENDMSG 2202 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(4) at 2025-01-19 23:52:22: 73 states generated (1.649 s/min), 13 distinct states found (293 ds/min), 9 states left on queue.
+@!@!@ENDMSG 2200 @!@!@
+@!@!@STARTMSG 2199:0 @!@!@
+73 states generated, 13 distinct states found, 9 states left on queue.
+@!@!@ENDMSG 2199 @!@!@
+@!@!@STARTMSG 2194:0 @!@!@
+The depth of the complete state graph search is 4.
+@!@!@ENDMSG 2194 @!@!@
+@!@!@STARTMSG 2186:0 @!@!@
+Finished in 2670ms at (2025-01-19 23:52:22)
+@!@!@ENDMSG 2186 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737327144010/replicated.tla

@@ -0,0 +1,174 @@
+---- MODULE replicated ----
+
+EXTENDS Integers, FiniteSets
+
+VARIABLES broker_state, safekeeper_state, pageserver_state, online
+
+
+CONSTANT
+    brokers,
+    safekeepers,
+    pageservers,
+    azs,
+    az_mapping
+
+CONSTANT
+    NULL
+    
+CONSTANT
+    max_commit_lsn
+    
+\* HELPERS
+
+Max(a,b) == IF a > b THEN a ELSE b
+
+(* The minimum of a non-empty set of integers *)
+MinOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x <= y
+  
+(* The minimum of a non-empty set of integers *)
+MaxOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x >= y
+
+\* END HELPERS
+
+StateConstraint ==
+    /\ \A s \in safekeepers: 
+        /\ safekeeper_state[s].commit_lsn <= max_commit_lsn
+    /\ \A b \in brokers:
+        /\ \A s \in DOMAIN broker_state[b].sk:
+            /\ broker_state[b].sk[s].commit_lsn <= max_commit_lsn
+
+
+ 
+InitSafekeeper == [prune_lsn |-> 0, commit_lsn |-> 0, rx |-> [b \in brokers |-> NULL] ]
+InitBroker == [sk |-> [s \in safekeepers |-> [prune_lsn |-> 0, commit_lsn |-> 0]]]
+InitPageserver == [last_record_lsn |-> 0, preferred_sk |-> NULL, sk |-> [s \in safekeepers |-> [commit_lsn |-> 0]]]
+InitOnline == safekeepers \cup brokers \cup pageservers
+
+Init ==
+    /\ broker_state = [b \in brokers |-> InitBroker]
+    /\ safekeeper_state = [s \in safekeepers |-> InitSafekeeper]
+    /\ pageserver_state = [p \in pageservers |-> InitPageserver]
+    /\ online = InitOnline
+  
+NodeOnlineOffline ==
+    /\ online' = CHOOSE ss \in SUBSET InitOnline: 
+        /\ Cardinality(ss \cap safekeepers) >= 2
+        /\ Cardinality(ss \cap brokers) >= 2
+        /\ ss \cap pageservers = pageservers \* assume no PS failures for now
+    /\ UNCHANGED <<safekeeper_state,broker_state,pageserver_state>>
+   
+SkCommit(s1, s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ s1 # s2
+    /\ safekeeper_state[s1].commit_lsn = safekeeper_state[s2].commit_lsn
+    /\ LET
+            new_commit_lsn == safekeeper_state[s1].commit_lsn + 1
+       IN
+            safekeeper_state' = [safekeeper_state EXCEPT
+                ![s1].commit_lsn = new_commit_lsn,
+                ![s2].commit_lsn = new_commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPeerRecovery(s1,s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ safekeeper_state[s1].commit_lsn < safekeeper_state[s2].commit_lsn \* s2 has more WAL than s1
+    /\ safekeeper_state[s2].prune_lsn < safekeeper_state[s1].commit_lsn \* s2 has not yet trimmed the WAL the WAL
+    /\ safekeeper_state' = [safekeeper_state EXCEPT![s1].commit_lsn = safekeeper_state[s2].commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPushToBroker(s,b) ==
+    /\ {s, b} \subseteq online 
+    /\ broker_state' = LET
+           broker_side == broker_state[b].sk[s]
+           sk_side == safekeeper_state[s]
+           merged == [broker_side EXCEPT
+                !["commit_lsn"] = Max(broker_side.commit_lsn, sk_side.commit_lsn),
+                !["prune_lsn"] = Max(broker_side.prune_lsn, sk_side.prune_lsn)]
+           IN
+           [broker_state EXCEPT ![b].sk[s] = merged]
+    /\ UNCHANGED <<safekeeper_state, pageserver_state,online>> 
+
+PsRecvBroker(b,p,s) ==
+    /\ {b,p,s} \subseteq online
+    /\ LET
+            bsk == broker_state[b].sk[s]
+            psk == pageserver_state[p].sk[s]
+            updpsk == [psk EXCEPT !["commit_lsn"] = bsk.commit_lsn]
+       IN
+            pageserver_state' = IF bsk.commit_lsn > psk.commit_lsn
+                THEN [pageserver_state EXCEPT ![p].sk[s] = updpsk]
+                ELSE pageserver_state
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>
+    
+    
+SkRecvBroker(b,s) ==
+    /\ {b,s} \subseteq online
+    /\ safekeeper_state' = [safekeeper_state EXCEPT ![s].rx[b] = broker_state[b]] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+
+SkPrune(s) ==
+    /\ {s} \subseteq online
+    /\ LET
+        sk == safekeeper_state[s]
+        skis == {<<b,bi.sk>>: <<b,bi>> \in {<<b,bi>> \in { <<b,sk.rx[b]>>: b \in sk.rx }: bi # NULL} }
+       IN
+        TRUE   
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+    
+
+SksWithNewerWal(p) ==
+    LET
+        ps == pageserver_state[p]
+    IN
+    {s \in DOMAIN ps.sk: ps.sk[s].commit_lsn > ps.last_record_lsn}
+
+PsChooseSk(p) ==
+    /\ {p} \subseteq online
+    /\ SksWithNewerWal(p) # {}
+    /\ pageserver_state' = [pageserver_state EXCEPT![p].preferred_sk = CHOOSE s \in SksWithNewerWal(p): TRUE]
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>    
+                 
+    
+Next ==
+    \/ NodeOnlineOffline
+    \/ \E s1 \in safekeepers: \E s2 \in safekeepers:
+        \/ SkCommit(s1, s2)
+        \/ SkPeerRecovery(s1, s2)
+    \/ \E s \in safekeepers: \E b \in brokers:
+        \/ SkPushToBroker(s, b)
+        \/ SkRecvBroker(b, s)
+    \/ \E s \in safekeepers:
+        \/ SkPrune(s)  
+    \/ \E s \in safekeepers: \E b \in brokers: \E p \in pageservers: PsRecvBroker(b,p,s)
+    \/ \E p \in pageservers: PsChooseSk(p)
+    
+
+Spec == Init /\ [][Next]_<< broker_state, safekeeper_state, pageserver_state,online>>
+
+
+\* invariants
+
+PsLagsSk == \A p \in pageservers: \A s \in DOMAIN pageserver_state[p].sk: 
+        /\ pageserver_state[p].sk[s].commit_lsn <= safekeeper_state[s].commit_lsn
+
+PeerRecoveryIsPossible ==
+    \A laggard \in (safekeepers \cap online): \E donor \in (safekeepers \cap online):
+        /\ (safekeeper_state[laggard].commit_lsn < safekeeper_state[donor].commit_lsn)
+            => safekeeper_state[donor].prune_lsn <= safekeeper_state[laggard].commit_lsn
+        
+
+EventuallyLaggingSkIsNotPreferredSk == <>(
+        LET
+            sks == safekeeper_state
+            lagging_sks == { s \in safekeepers: \A s2 \in safekeepers: sks[s].commit_lsn <= sks[s2].commit_lsn }
+            preferred_sks == {pageserver_state[p].preferred_sk: p \in pageservers}
+        IN
+            preferred_sks \cap lagging_sks = {}
+            
+    )
+
+    
+
+====

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737327172537/MC.cfg

@@ -0,0 +1,54 @@
+\* MV CONSTANT declarations
+CONSTANTS
+b1 = b1
+b2 = b2
+b3 = b3
+\* MV CONSTANT declarations
+CONSTANTS
+s1 = s1
+s2 = s2
+s3 = s3
+\* MV CONSTANT declarations
+CONSTANTS
+p1 = p1
+\* MV CONSTANT declarations
+CONSTANTS
+az1 = az1
+az2 = az2
+az3 = az3
+\* MV CONSTANT definitions
+CONSTANT
+brokers <- const_1737327160682382000
+\* CONSTANT declarations
+CONSTANT NULL = NULL
+\* MV CONSTANT definitions
+CONSTANT
+safekeepers <- const_1737327160682383000
+\* MV CONSTANT definitions
+CONSTANT
+pageservers <- const_1737327160682384000
+\* MV CONSTANT definitions
+CONSTANT
+azs <- const_1737327160682385000
+\* SYMMETRY definition
+SYMMETRY symm_1737327160682386000
+\* CONSTANT definitions
+CONSTANT
+max_commit_lsn <- const_1737327160682387000
+\* CONSTANT definitions
+CONSTANT
+az_mapping <- const_1737327160682388000
+\* CONSTRAINT definition
+CONSTRAINT
+constr_1737327160682389000
+\* SPECIFICATION definition
+SPECIFICATION
+Spec
+\* INVARIANT definition
+INVARIANT
+PsLagsSk
+PeerRecoveryIsPossible
+\* PROPERTY definition
+PROPERTY
+EventuallyLaggingSkIsNotPreferredSk
+\* Generated on Sun Jan 19 23:52:40 CET 2025

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737327172537/MC.out

@@ -0,0 +1,53 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 121 and seed -1130553646976177579 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 25330] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 23:52:41)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 23:52:43.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 2192:0 @!@!@
+Checking temporal properties for the current state space with 1627 total distinct states at (2025-01-19 23:52:46)
+@!@!@ENDMSG 2192 @!@!@
+@!@!@STARTMSG 2267:0 @!@!@
+Finished checking temporal properties in 00s at 2025-01-19 23:52:46
+@!@!@ENDMSG 2267 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(9) at 2025-01-19 23:52:46: 45.678 states generated (45.678 s/min), 7.117 distinct states found (7.117 ds/min), 5.486 states left on queue.
+@!@!@ENDMSG 2200 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737327172537/MC.tla

@@ -0,0 +1,65 @@
+---- MODULE MC ----
+EXTENDS replicated, TLC
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+b1, b2, b3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+s1, s2, s3
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+p1
+----
+
+\* MV CONSTANT declarations@modelParameterConstants
+CONSTANTS
+az1, az2, az3
+----
+
+\* MV CONSTANT definitions brokers
+const_1737327160682382000 == 
+{b1, b2, b3}
+----
+
+\* MV CONSTANT definitions safekeepers
+const_1737327160682383000 == 
+{s1, s2, s3}
+----
+
+\* MV CONSTANT definitions pageservers
+const_1737327160682384000 == 
+{p1}
+----
+
+\* MV CONSTANT definitions azs
+const_1737327160682385000 == 
+{az1, az2, az3}
+----
+
+\* SYMMETRY definition
+symm_1737327160682386000 == 
+Permutations(const_1737327160682382000) \union Permutations(const_1737327160682383000) \union Permutations(const_1737327160682384000) \union Permutations(const_1737327160682385000)
+----
+
+\* CONSTANT definitions @modelParameterConstants:4max_commit_lsn
+const_1737327160682387000 == 
+2
+----
+
+\* CONSTANT definitions @modelParameterConstants:5az_mapping
+const_1737327160682388000 == 
+[ az1 |-> {b1,s1,p1} , az2 |-> {b2,s2} , az3 |-> {b3,s3}]
+----
+
+\* CONSTRAINT definition @modelParameterContraint:0
+constr_1737327160682389000 ==
+StateConstraint
+----
+=============================================================================
+\* Modification History
+\* Created Sun Jan 19 23:52:40 CET 2025 by cs

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737327172537/MC_TE.out

@@ -0,0 +1,53 @@
+@!@!@STARTMSG 2262:0 @!@!@
+TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)
+@!@!@ENDMSG 2262 @!@!@
+@!@!@STARTMSG 2187:0 @!@!@
+Running breadth-first search Model-Checking with fp 121 and seed -1130553646976177579 with 4 workers on 8 cores with 2428MB heap and 5460MB offheap memory [pid: 25330] (Mac OS X 10.16 x86_64, AdoptOpenJDK 14.0.1 x86_64, OffHeapDiskFPSet, DiskStateQueue).
+@!@!@ENDMSG 2187 @!@!@
+@!@!@STARTMSG 2220:0 @!@!@
+Starting SANY...
+@!@!@ENDMSG 2220 @!@!@
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/MC.tla
+Parsing file /Users/cs/src/neon/storage_broker/spec/replicated/replicated.toolbox/Model_1/replicated.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/TLC.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Integers.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/FiniteSets.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Naturals.tla
+Parsing file /Applications/TLA+ Toolbox.app/Contents/Eclipse/plugins/org.lamport.tlatools_1.0.0.202408081326/tla2sany/StandardModules/Sequences.tla
+Semantic processing of module Naturals
+Semantic processing of module Integers
+Semantic processing of module Sequences
+Semantic processing of module FiniteSets
+Semantic processing of module replicated
+Semantic processing of module TLC
+Semantic processing of module MC
+@!@!@STARTMSG 2219:0 @!@!@
+SANY finished.
+@!@!@ENDMSG 2219 @!@!@
+@!@!@STARTMSG 2185:0 @!@!@
+Starting... (2025-01-19 23:52:41)
+@!@!@ENDMSG 2185 @!@!@
+@!@!@STARTMSG 2279:3 @!@!@
+Declaring symmetry during liveness checking is dangerous. It might cause TLC to miss violations of the stated liveness properties. Please check liveness without symmetry defined.
+@!@!@ENDMSG 2279 @!@!@
+@!@!@STARTMSG 2284:3 @!@!@
+Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details.
+@!@!@ENDMSG 2284 @!@!@
+@!@!@STARTMSG 2212:0 @!@!@
+Implied-temporal checking--satisfiability problem has 1 branches.
+@!@!@ENDMSG 2212 @!@!@
+@!@!@STARTMSG 2189:0 @!@!@
+Computing initial states...
+@!@!@ENDMSG 2189 @!@!@
+@!@!@STARTMSG 2190:0 @!@!@
+Finished computing initial states: 1 distinct state generated at 2025-01-19 23:52:43.
+@!@!@ENDMSG 2190 @!@!@
+@!@!@STARTMSG 2192:0 @!@!@
+Checking temporal properties for the current state space with 1627 total distinct states at (2025-01-19 23:52:46)
+@!@!@ENDMSG 2192 @!@!@
+@!@!@STARTMSG 2267:0 @!@!@
+Finished checking temporal properties in 00s at 2025-01-19 23:52:46
+@!@!@ENDMSG 2267 @!@!@
+@!@!@STARTMSG 2200:0 @!@!@
+Progress(9) at 2025-01-19 23:52:46: 45.678 states generated (45.678 s/min), 7.117 distinct states found (7.117 ds/min), 5.486 states left on queue.
+@!@!@ENDMSG 2200 @!@!@

storage_broker/spec/replicated/replicated.toolbox/Model_1_SnapShot_1737327172537/replicated.tla

@@ -0,0 +1,174 @@
+---- MODULE replicated ----
+
+EXTENDS Integers, FiniteSets
+
+VARIABLES broker_state, safekeeper_state, pageserver_state, online
+
+
+CONSTANT
+    brokers,
+    safekeepers,
+    pageservers,
+    azs,
+    az_mapping
+
+CONSTANT
+    NULL
+    
+CONSTANT
+    max_commit_lsn
+    
+\* HELPERS
+
+Max(a,b) == IF a > b THEN a ELSE b
+
+(* The minimum of a non-empty set of integers *)
+MinOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x <= y
+  
+(* The minimum of a non-empty set of integers *)
+MaxOfSet(S) == 
+  CHOOSE x \in S: \A y \in S: x >= y
+
+\* END HELPERS
+
+StateConstraint ==
+    /\ \A s \in safekeepers: 
+        /\ safekeeper_state[s].commit_lsn <= max_commit_lsn
+    /\ \A b \in brokers:
+        /\ \A s \in DOMAIN broker_state[b].sk:
+            /\ broker_state[b].sk[s].commit_lsn <= max_commit_lsn
+
+
+ 
+InitSafekeeper == [prune_lsn |-> 0, commit_lsn |-> 0, rx |-> [b \in brokers |-> NULL] ]
+InitBroker == [sk |-> [s \in safekeepers |-> [prune_lsn |-> 0, commit_lsn |-> 0]]]
+InitPageserver == [last_record_lsn |-> 0, preferred_sk |-> NULL, sk |-> [s \in safekeepers |-> [commit_lsn |-> 0]]]
+InitOnline == safekeepers \cup brokers \cup pageservers
+
+Init ==
+    /\ broker_state = [b \in brokers |-> InitBroker]
+    /\ safekeeper_state = [s \in safekeepers |-> InitSafekeeper]
+    /\ pageserver_state = [p \in pageservers |-> InitPageserver]
+    /\ online = InitOnline
+  
+NodeOnlineOffline ==
+    /\ online' = CHOOSE ss \in SUBSET InitOnline: 
+        /\ Cardinality(ss \cap safekeepers) >= 2
+        /\ Cardinality(ss \cap brokers) >= 2
+        /\ ss \cap pageservers = pageservers \* assume no PS failures for now
+    /\ UNCHANGED <<safekeeper_state,broker_state,pageserver_state>>
+   
+SkCommit(s1, s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ s1 # s2
+    /\ safekeeper_state[s1].commit_lsn = safekeeper_state[s2].commit_lsn
+    /\ LET
+            new_commit_lsn == safekeeper_state[s1].commit_lsn + 1
+       IN
+            safekeeper_state' = [safekeeper_state EXCEPT
+                ![s1].commit_lsn = new_commit_lsn,
+                ![s2].commit_lsn = new_commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPeerRecovery(s1,s2) ==
+    /\ {s1, s2} \subseteq online 
+    /\ safekeeper_state[s1].commit_lsn < safekeeper_state[s2].commit_lsn \* s2 has more WAL than s1
+    /\ safekeeper_state[s2].prune_lsn < safekeeper_state[s1].commit_lsn \* s2 has not yet trimmed the WAL the WAL
+    /\ safekeeper_state' = [safekeeper_state EXCEPT![s1].commit_lsn = safekeeper_state[s2].commit_lsn]
+    /\ UNCHANGED <<broker_state, pageserver_state,online>>
+
+SkPushToBroker(s,b) ==
+    /\ {s, b} \subseteq online 
+    /\ broker_state' = LET
+           broker_side == broker_state[b].sk[s]
+           sk_side == safekeeper_state[s]
+           merged == [broker_side EXCEPT
+                !["commit_lsn"] = Max(broker_side.commit_lsn, sk_side.commit_lsn),
+                !["prune_lsn"] = Max(broker_side.prune_lsn, sk_side.prune_lsn)]
+           IN
+           [broker_state EXCEPT ![b].sk[s] = merged]
+    /\ UNCHANGED <<safekeeper_state, pageserver_state,online>> 
+
+PsRecvBroker(b,p,s) ==
+    /\ {b,p,s} \subseteq online
+    /\ LET
+            bsk == broker_state[b].sk[s]
+            psk == pageserver_state[p].sk[s]
+            updpsk == [psk EXCEPT !["commit_lsn"] = bsk.commit_lsn]
+       IN
+            pageserver_state' = IF bsk.commit_lsn > psk.commit_lsn
+                THEN [pageserver_state EXCEPT ![p].sk[s] = updpsk]
+                ELSE pageserver_state
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>
+    
+    
+SkRecvBroker(b,s) ==
+    /\ {b,s} \subseteq online
+    /\ safekeeper_state' = [safekeeper_state EXCEPT ![s].rx[b] = broker_state[b]] 
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+
+SkPrune(s) ==
+    /\ {s} \subseteq online
+    /\ LET
+        sk == safekeeper_state[s]
+        skis == {<<b,bi.sk>>: <<b,bi>> \in {<<b,bi>> \in { <<b,sk.rx[b]>>: b \in sk.rx }: bi # NULL} }
+       IN
+        safekeeper_state' = safekeeper_state   
+    /\ UNCHANGED <<pageserver_state,broker_state,online>>
+    
+
+SksWithNewerWal(p) ==
+    LET
+        ps == pageserver_state[p]
+    IN
+    {s \in DOMAIN ps.sk: ps.sk[s].commit_lsn > ps.last_record_lsn}
+
+PsChooseSk(p) ==
+    /\ {p} \subseteq online
+    /\ SksWithNewerWal(p) # {}
+    /\ pageserver_state' = [pageserver_state EXCEPT![p].preferred_sk = CHOOSE s \in SksWithNewerWal(p): TRUE]
+    /\ UNCHANGED <<safekeeper_state, broker_state,online>>    
+                 
+    
+Next ==
+    \/ NodeOnlineOffline
+    \/ \E s1 \in safekeepers: \E s2 \in safekeepers:
+        \/ SkCommit(s1, s2)
+        \/ SkPeerRecovery(s1, s2)
+    \/ \E s \in safekeepers: \E b \in brokers:
+        \/ SkPushToBroker(s, b)
+        \/ SkRecvBroker(b, s)
+    \/ \E s \in safekeepers:
+        \/ SkPrune(s)  
+    \/ \E s \in safekeepers: \E b \in brokers: \E p \in pageservers: PsRecvBroker(b,p,s)
+    \/ \E p \in pageservers: PsChooseSk(p)
+    
+
+Spec == Init /\ [][Next]_<< broker_state, safekeeper_state, pageserver_state,online>>
+
+
+\* invariants
+
+PsLagsSk == \A p \in pageservers: \A s \in DOMAIN pageserver_state[p].sk: 
+        /\ pageserver_state[p].sk[s].commit_lsn <= safekeeper_state[s].commit_lsn
+
+PeerRecoveryIsPossible ==
+    \A laggard \in (safekeepers \cap online): \E donor \in (safekeepers \cap online):
+        /\ (safekeeper_state[laggard].commit_lsn < safekeeper_state[donor].commit_lsn)
+            => safekeeper_state[donor].prune_lsn <= safekeeper_state[laggard].commit_lsn
+        
+
+EventuallyLaggingSkIsNotPreferredSk == <>(
+        LET
+            sks == safekeeper_state
+            lagging_sks == { s \in safekeepers: \A s2 \in safekeepers: sks[s].commit_lsn <= sks[s2].commit_lsn }
+            preferred_sks == {pageserver_state[p].preferred_sk: p \in pageservers}
+        IN
+            preferred_sks \cap lagging_sks = {}
+            
+    )
+
+    
+
+====

storage_broker/spec/replicated/replicated.toolbox/replicated___Model_1.launch

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.lamport.tla.toolbox.tool.tlc.modelCheck">
+    <stringAttribute key="TLCCmdLineParameters" value=""/>
+    <intAttribute key="collectCoverage" value="1"/>
+    <stringAttribute key="configurationName" value="Model_1"/>
+    <booleanAttribute key="deferLiveness" value="false"/>
+    <intAttribute key="dfidDepth" value="100"/>
+    <booleanAttribute key="dfidMode" value="false"/>
+    <intAttribute key="distributedFPSetCount" value="0"/>
+    <stringAttribute key="distributedNetworkInterface" value="100.65.224.26"/>
+    <intAttribute key="distributedNodesCount" value="1"/>
+    <stringAttribute key="distributedTLC" value="off"/>
+    <stringAttribute key="distributedTLCVMArgs" value=""/>
+    <intAttribute key="fpBits" value="1"/>
+    <intAttribute key="fpIndex" value="96"/>
+    <booleanAttribute key="fpIndexRandom" value="true"/>
+    <intAttribute key="maxHeapSize" value="25"/>
+    <intAttribute key="maxSetSize" value="1000000"/>
+    <booleanAttribute key="mcMode" value="true"/>
+    <stringAttribute key="modelBehaviorInit" value=""/>
+    <stringAttribute key="modelBehaviorNext" value=""/>
+    <stringAttribute key="modelBehaviorSpec" value="Spec"/>
+    <intAttribute key="modelBehaviorSpecType" value="1"/>
+    <stringAttribute key="modelBehaviorVars" value="pageserver_state, broker_state, online, safekeeper_state"/>
+    <stringAttribute key="modelComments" value=""/>
+    <booleanAttribute key="modelCorrectnessCheckDeadlock" value="true"/>
+    <listAttribute key="modelCorrectnessInvariants">
+        <listEntry value="1PsLagsSk"/>
+        <listEntry value="1PeerRecoveryIsPossible"/>
+    </listAttribute>
+    <listAttribute key="modelCorrectnessProperties">
+        <listEntry value="1EventuallyLaggingSkIsNotPreferredSk"/>
+    </listAttribute>
+    <intAttribute key="modelEditorOpenTabs" value="14"/>
+    <stringAttribute key="modelExpressionEval" value=""/>
+    <stringAttribute key="modelParameterActionConstraint" value=""/>
+    <listAttribute key="modelParameterConstants">
+        <listEntry value="brokers;;{b1, b2, b3};1;1"/>
+        <listEntry value="NULL;;NULL;1;0"/>
+        <listEntry value="safekeepers;;{s1, s2, s3};1;1"/>
+        <listEntry value="pageservers;;{p1};1;1"/>
+        <listEntry value="max_commit_lsn;;2;0;0"/>
+        <listEntry value="az_mapping;;[ az1 |-&gt; {b1,s1,p1} , az2 |-&gt; {b2,s2} , az3 |-&gt; {b3,s3}];0;0"/>
+        <listEntry value="azs;;{az1, az2, az3};1;1"/>
+    </listAttribute>
+    <stringAttribute key="modelParameterContraint" value="StateConstraint"/>
+    <listAttribute key="modelParameterDefinitions"/>
+    <stringAttribute key="modelParameterModelValues" value="{}"/>
+    <stringAttribute key="modelParameterNewDefinitions" value=""/>
+    <intAttribute key="modelVersion" value="20191005"/>
+    <intAttribute key="numberOfWorkers" value="4"/>
+    <booleanAttribute key="recover" value="false"/>
+    <stringAttribute key="result.mail.address" value=""/>
+    <intAttribute key="simuAril" value="-1"/>
+    <intAttribute key="simuDepth" value="100"/>
+    <intAttribute key="simuSeed" value="-1"/>
+    <stringAttribute key="specName" value="replicated"/>
+    <stringAttribute key="tlcResourcesProfile" value="local custom"/>
+    <stringAttribute key="view" value=""/>
+    <booleanAttribute key="visualizeStateGraph" value="false"/>
+</launchConfiguration>

storage_broker/spec/replicated/replicated.toolbox/replicated___Model_1_SnapShot_1737323805723.launch

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.lamport.tla.toolbox.tool.tlc.modelCheck">
+    <stringAttribute key="TLCCmdLineParameters" value=""/>
+    <intAttribute key="collectCoverage" value="1"/>
+    <stringAttribute key="configurationName" value="Model_1_SnapShot_1737323805723"/>
+    <booleanAttribute key="deferLiveness" value="false"/>
+    <intAttribute key="dfidDepth" value="100"/>
+    <booleanAttribute key="dfidMode" value="false"/>
+    <intAttribute key="distributedFPSetCount" value="0"/>
+    <stringAttribute key="distributedNetworkInterface" value="100.65.224.26"/>
+    <intAttribute key="distributedNodesCount" value="1"/>
+    <stringAttribute key="distributedTLC" value="off"/>
+    <stringAttribute key="distributedTLCVMArgs" value=""/>
+    <intAttribute key="fpBits" value="1"/>
+    <intAttribute key="fpIndex" value="96"/>
+    <booleanAttribute key="fpIndexRandom" value="true"/>
+    <intAttribute key="maxHeapSize" value="25"/>
+    <intAttribute key="maxSetSize" value="1000000"/>
+    <booleanAttribute key="mcMode" value="true"/>
+    <stringAttribute key="modelBehaviorInit" value=""/>
+    <stringAttribute key="modelBehaviorNext" value=""/>
+    <stringAttribute key="modelBehaviorSpec" value="Spec"/>
+    <intAttribute key="modelBehaviorSpecType" value="1"/>
+    <stringAttribute key="modelBehaviorVars" value="pageserver_state, broker_state, online, safekeeper_state"/>
+    <stringAttribute key="modelComments" value=""/>
+    <booleanAttribute key="modelCorrectnessCheckDeadlock" value="true"/>
+    <listAttribute key="modelCorrectnessInvariants">
+        <listEntry value="1PsLagsSk"/>
+        <listEntry value="1PeerRecoveryIsPossible"/>
+    </listAttribute>
+    <listAttribute key="modelCorrectnessProperties">
+        <listEntry value="1EventuallyLaggingSkIsNotPreferredSk"/>
+    </listAttribute>
+    <intAttribute key="modelEditorOpenTabs" value="14"/>
+    <stringAttribute key="modelExpressionEval" value=""/>
+    <stringAttribute key="modelParameterActionConstraint" value=""/>
+    <listAttribute key="modelParameterConstants">
+        <listEntry value="brokers;;{b1, b2, b3};1;1"/>
+        <listEntry value="NULL;;NULL;1;0"/>
+        <listEntry value="safekeepers;;{s1, s2, s3};1;1"/>
+        <listEntry value="pageservers;;{p1};1;1"/>
+        <listEntry value="max_commit_lsn;;2;0;0"/>
+        <listEntry value="az_mapping;;[ az1 |-&gt; {b1,s1,p1} , az2 |-&gt; {b2,s2} , az3 |-&gt; {b3,s3}];0;0"/>
+        <listEntry value="azs;;{az1, az2, az3};1;1"/>
+    </listAttribute>
+    <stringAttribute key="modelParameterContraint" value="StateConstraint"/>
+    <listAttribute key="modelParameterDefinitions"/>
+    <stringAttribute key="modelParameterModelValues" value="{}"/>
+    <stringAttribute key="modelParameterNewDefinitions" value=""/>
+    <intAttribute key="modelVersion" value="20191005"/>
+    <intAttribute key="numberOfWorkers" value="4"/>
+    <booleanAttribute key="recover" value="false"/>
+    <stringAttribute key="result.mail.address" value=""/>
+    <intAttribute key="simuAril" value="-1"/>
+    <intAttribute key="simuDepth" value="100"/>
+    <intAttribute key="simuSeed" value="-1"/>
+    <stringAttribute key="specName" value="replicated"/>
+    <stringAttribute key="tlcResourcesProfile" value="local custom"/>
+    <stringAttribute key="view" value=""/>
+    <booleanAttribute key="visualizeStateGraph" value="false"/>
+</launchConfiguration>

storage_broker/spec/replicated/replicated.toolbox/replicated___Model_1_SnapShot_1737323834253.launch

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.lamport.tla.toolbox.tool.tlc.modelCheck">
+    <stringAttribute key="TLCCmdLineParameters" value=""/>
+    <intAttribute key="collectCoverage" value="1"/>
+    <stringAttribute key="configurationName" value="Model_1_SnapShot_1737323834253"/>
+    <booleanAttribute key="deferLiveness" value="false"/>
+    <intAttribute key="dfidDepth" value="100"/>
+    <booleanAttribute key="dfidMode" value="false"/>
+    <intAttribute key="distributedFPSetCount" value="0"/>
+    <stringAttribute key="distributedNetworkInterface" value="100.65.224.26"/>
+    <intAttribute key="distributedNodesCount" value="1"/>
+    <stringAttribute key="distributedTLC" value="off"/>
+    <stringAttribute key="distributedTLCVMArgs" value=""/>
+    <intAttribute key="fpBits" value="1"/>
+    <intAttribute key="fpIndex" value="96"/>
+    <booleanAttribute key="fpIndexRandom" value="true"/>
+    <intAttribute key="maxHeapSize" value="25"/>
+    <intAttribute key="maxSetSize" value="1000000"/>
+    <booleanAttribute key="mcMode" value="true"/>
+    <stringAttribute key="modelBehaviorInit" value=""/>
+    <stringAttribute key="modelBehaviorNext" value=""/>
+    <stringAttribute key="modelBehaviorSpec" value="Spec"/>
+    <intAttribute key="modelBehaviorSpecType" value="1"/>
+    <stringAttribute key="modelBehaviorVars" value="pageserver_state, broker_state, online, safekeeper_state"/>
+    <stringAttribute key="modelComments" value=""/>
+    <booleanAttribute key="modelCorrectnessCheckDeadlock" value="true"/>
+    <listAttribute key="modelCorrectnessInvariants">
+        <listEntry value="1PsLagsSk"/>
+        <listEntry value="1PeerRecoveryIsPossible"/>
+    </listAttribute>
+    <listAttribute key="modelCorrectnessProperties">
+        <listEntry value="1EventuallyLaggingSkIsNotPreferredSk"/>
+    </listAttribute>
+    <intAttribute key="modelEditorOpenTabs" value="14"/>
+    <stringAttribute key="modelExpressionEval" value=""/>
+    <stringAttribute key="modelParameterActionConstraint" value=""/>
+    <listAttribute key="modelParameterConstants">
+        <listEntry value="brokers;;{b1, b2, b3};1;1"/>
+        <listEntry value="NULL;;NULL;1;0"/>
+        <listEntry value="safekeepers;;{s1, s2, s3};1;1"/>
+        <listEntry value="pageservers;;{p1};1;1"/>
+        <listEntry value="max_commit_lsn;;2;0;0"/>
+        <listEntry value="az_mapping;;[ az1 |-&gt; {b1,s1,p1} , az2 |-&gt; {b2,s2} , az3 |-&gt; {b3,s3}];0;0"/>
+        <listEntry value="azs;;{az1, az2, az3};1;1"/>
+    </listAttribute>
+    <stringAttribute key="modelParameterContraint" value="StateConstraint"/>
+    <listAttribute key="modelParameterDefinitions"/>
+    <stringAttribute key="modelParameterModelValues" value="{}"/>
+    <stringAttribute key="modelParameterNewDefinitions" value=""/>
+    <intAttribute key="modelVersion" value="20191005"/>
+    <intAttribute key="numberOfWorkers" value="4"/>
+    <booleanAttribute key="recover" value="false"/>
+    <stringAttribute key="result.mail.address" value=""/>
+    <intAttribute key="simuAril" value="-1"/>
+    <intAttribute key="simuDepth" value="100"/>
+    <intAttribute key="simuSeed" value="-1"/>
+    <stringAttribute key="specName" value="replicated"/>
+    <stringAttribute key="tlcResourcesProfile" value="local custom"/>
+    <stringAttribute key="view" value=""/>
+    <booleanAttribute key="visualizeStateGraph" value="false"/>
+</launchConfiguration>

storage_broker/spec/replicated/replicated.toolbox/replicated___Model_1_SnapShot_1737324141888.launch

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.lamport.tla.toolbox.tool.tlc.modelCheck">
+    <stringAttribute key="TLCCmdLineParameters" value=""/>
+    <intAttribute key="collectCoverage" value="1"/>
+    <stringAttribute key="configurationName" value="Model_1_SnapShot_1737324141888"/>
+    <booleanAttribute key="deferLiveness" value="false"/>
+    <intAttribute key="dfidDepth" value="100"/>
+    <booleanAttribute key="dfidMode" value="false"/>
+    <intAttribute key="distributedFPSetCount" value="0"/>
+    <stringAttribute key="distributedNetworkInterface" value="100.65.224.26"/>
+    <intAttribute key="distributedNodesCount" value="1"/>
+    <stringAttribute key="distributedTLC" value="off"/>
+    <stringAttribute key="distributedTLCVMArgs" value=""/>
+    <intAttribute key="fpBits" value="1"/>
+    <intAttribute key="fpIndex" value="96"/>
+    <booleanAttribute key="fpIndexRandom" value="true"/>
+    <intAttribute key="maxHeapSize" value="25"/>
+    <intAttribute key="maxSetSize" value="1000000"/>
+    <booleanAttribute key="mcMode" value="true"/>
+    <stringAttribute key="modelBehaviorInit" value=""/>
+    <stringAttribute key="modelBehaviorNext" value=""/>
+    <stringAttribute key="modelBehaviorSpec" value="Spec"/>
+    <intAttribute key="modelBehaviorSpecType" value="1"/>
+    <stringAttribute key="modelBehaviorVars" value="pageserver_state, broker_state, online, safekeeper_state"/>
+    <stringAttribute key="modelComments" value=""/>
+    <booleanAttribute key="modelCorrectnessCheckDeadlock" value="true"/>
+    <listAttribute key="modelCorrectnessInvariants">
+        <listEntry value="1PsLagsSk"/>
+        <listEntry value="1PeerRecoveryIsPossible"/>
+    </listAttribute>
+    <listAttribute key="modelCorrectnessProperties">
+        <listEntry value="1EventuallyLaggingSkIsNotPreferredSk"/>
+    </listAttribute>
+    <intAttribute key="modelEditorOpenTabs" value="14"/>
+    <stringAttribute key="modelExpressionEval" value=""/>
+    <stringAttribute key="modelParameterActionConstraint" value=""/>
+    <listAttribute key="modelParameterConstants">
+        <listEntry value="brokers;;{b1, b2, b3};1;1"/>
+        <listEntry value="NULL;;NULL;1;0"/>
+        <listEntry value="safekeepers;;{s1, s2, s3};1;1"/>
+        <listEntry value="pageservers;;{p1};1;1"/>
+        <listEntry value="max_commit_lsn;;2;0;0"/>
+        <listEntry value="az_mapping;;[ az1 |-&gt; {b1,s1,p1} , az2 |-&gt; {b2,s2} , az3 |-&gt; {b3,s3}];0;0"/>
+        <listEntry value="azs;;{az1, az2, az3};1;1"/>
+    </listAttribute>
+    <stringAttribute key="modelParameterContraint" value="StateConstraint"/>
+    <listAttribute key="modelParameterDefinitions"/>
+    <stringAttribute key="modelParameterModelValues" value="{}"/>
+    <stringAttribute key="modelParameterNewDefinitions" value=""/>
+    <intAttribute key="modelVersion" value="20191005"/>
+    <intAttribute key="numberOfWorkers" value="4"/>
+    <booleanAttribute key="recover" value="false"/>
+    <stringAttribute key="result.mail.address" value=""/>
+    <intAttribute key="simuAril" value="-1"/>
+    <intAttribute key="simuDepth" value="100"/>
+    <intAttribute key="simuSeed" value="-1"/>
+    <stringAttribute key="specName" value="replicated"/>
+    <stringAttribute key="tlcResourcesProfile" value="local custom"/>
+    <stringAttribute key="view" value=""/>
+    <booleanAttribute key="visualizeStateGraph" value="false"/>
+</launchConfiguration>

storage_broker/spec/replicated/replicated.toolbox/replicated___Model_1_SnapShot_1737324186169.launch

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.lamport.tla.toolbox.tool.tlc.modelCheck">
+    <stringAttribute key="TLCCmdLineParameters" value=""/>
+    <intAttribute key="collectCoverage" value="1"/>
+    <stringAttribute key="configurationName" value="Model_1_SnapShot_1737324186169"/>
+    <booleanAttribute key="deferLiveness" value="false"/>
+    <intAttribute key="dfidDepth" value="100"/>
+    <booleanAttribute key="dfidMode" value="false"/>
+    <intAttribute key="distributedFPSetCount" value="0"/>
+    <stringAttribute key="distributedNetworkInterface" value="100.65.224.26"/>
+    <intAttribute key="distributedNodesCount" value="1"/>
+    <stringAttribute key="distributedTLC" value="off"/>
+    <stringAttribute key="distributedTLCVMArgs" value=""/>
+    <intAttribute key="fpBits" value="1"/>
+    <intAttribute key="fpIndex" value="96"/>
+    <booleanAttribute key="fpIndexRandom" value="true"/>
+    <intAttribute key="maxHeapSize" value="25"/>
+    <intAttribute key="maxSetSize" value="1000000"/>
+    <booleanAttribute key="mcMode" value="true"/>
+    <stringAttribute key="modelBehaviorInit" value=""/>
+    <stringAttribute key="modelBehaviorNext" value=""/>
+    <stringAttribute key="modelBehaviorSpec" value="Spec"/>
+    <intAttribute key="modelBehaviorSpecType" value="1"/>
+    <stringAttribute key="modelBehaviorVars" value="pageserver_state, broker_state, online, safekeeper_state"/>
+    <stringAttribute key="modelComments" value=""/>
+    <booleanAttribute key="modelCorrectnessCheckDeadlock" value="true"/>
+    <listAttribute key="modelCorrectnessInvariants">
+        <listEntry value="1PsLagsSk"/>
+        <listEntry value="1PeerRecoveryIsPossible"/>
+    </listAttribute>
+    <listAttribute key="modelCorrectnessProperties">
+        <listEntry value="1EventuallyLaggingSkIsNotPreferredSk"/>
+    </listAttribute>
+    <intAttribute key="modelEditorOpenTabs" value="14"/>
+    <stringAttribute key="modelExpressionEval" value=""/>
+    <stringAttribute key="modelParameterActionConstraint" value=""/>
+    <listAttribute key="modelParameterConstants">
+        <listEntry value="brokers;;{b1, b2, b3};1;1"/>
+        <listEntry value="NULL;;NULL;1;0"/>
+        <listEntry value="safekeepers;;{s1, s2, s3};1;1"/>
+        <listEntry value="pageservers;;{p1};1;1"/>
+        <listEntry value="max_commit_lsn;;2;0;0"/>
+        <listEntry value="az_mapping;;[ az1 |-&gt; {b1,s1,p1} , az2 |-&gt; {b2,s2} , az3 |-&gt; {b3,s3}];0;0"/>
+        <listEntry value="azs;;{az1, az2, az3};1;1"/>
+    </listAttribute>
+    <stringAttribute key="modelParameterContraint" value="StateConstraint"/>
+    <listAttribute key="modelParameterDefinitions"/>
+    <stringAttribute key="modelParameterModelValues" value="{}"/>
+    <stringAttribute key="modelParameterNewDefinitions" value=""/>
+    <intAttribute key="modelVersion" value="20191005"/>
+    <intAttribute key="numberOfWorkers" value="4"/>
+    <booleanAttribute key="recover" value="false"/>
+    <stringAttribute key="result.mail.address" value=""/>
+    <intAttribute key="simuAril" value="-1"/>
+    <intAttribute key="simuDepth" value="100"/>
+    <intAttribute key="simuSeed" value="-1"/>
+    <stringAttribute key="specName" value="replicated"/>
+    <stringAttribute key="tlcResourcesProfile" value="local custom"/>
+    <stringAttribute key="view" value=""/>
+    <booleanAttribute key="visualizeStateGraph" value="false"/>
+</launchConfiguration>

storage_broker/spec/replicated/replicated.toolbox/replicated___Model_1_SnapShot_1737324315582.launch

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.lamport.tla.toolbox.tool.tlc.modelCheck">
+    <stringAttribute key="TLCCmdLineParameters" value=""/>
+    <intAttribute key="collectCoverage" value="1"/>
+    <stringAttribute key="configurationName" value="Model_1_SnapShot_1737324315582"/>
+    <booleanAttribute key="deferLiveness" value="false"/>
+    <intAttribute key="dfidDepth" value="100"/>
+    <booleanAttribute key="dfidMode" value="false"/>
+    <intAttribute key="distributedFPSetCount" value="0"/>
+    <stringAttribute key="distributedNetworkInterface" value="100.65.224.26"/>
+    <intAttribute key="distributedNodesCount" value="1"/>
+    <stringAttribute key="distributedTLC" value="off"/>
+    <stringAttribute key="distributedTLCVMArgs" value=""/>
+    <intAttribute key="fpBits" value="1"/>
+    <intAttribute key="fpIndex" value="96"/>
+    <booleanAttribute key="fpIndexRandom" value="true"/>
+    <intAttribute key="maxHeapSize" value="25"/>
+    <intAttribute key="maxSetSize" value="1000000"/>
+    <booleanAttribute key="mcMode" value="true"/>
+    <stringAttribute key="modelBehaviorInit" value=""/>
+    <stringAttribute key="modelBehaviorNext" value=""/>
+    <stringAttribute key="modelBehaviorSpec" value="Spec"/>
+    <intAttribute key="modelBehaviorSpecType" value="1"/>
+    <stringAttribute key="modelBehaviorVars" value="pageserver_state, broker_state, online, safekeeper_state"/>
+    <stringAttribute key="modelComments" value=""/>
+    <booleanAttribute key="modelCorrectnessCheckDeadlock" value="true"/>
+    <listAttribute key="modelCorrectnessInvariants">
+        <listEntry value="1PsLagsSk"/>
+        <listEntry value="1PeerRecoveryIsPossible"/>
+    </listAttribute>
+    <listAttribute key="modelCorrectnessProperties">
+        <listEntry value="1EventuallyLaggingSkIsNotPreferredSk"/>
+    </listAttribute>
+    <intAttribute key="modelEditorOpenTabs" value="14"/>
+    <stringAttribute key="modelExpressionEval" value=""/>
+    <stringAttribute key="modelParameterActionConstraint" value=""/>
+    <listAttribute key="modelParameterConstants">
+        <listEntry value="brokers;;{b1, b2, b3};1;1"/>
+        <listEntry value="NULL;;NULL;1;0"/>
+        <listEntry value="safekeepers;;{s1, s2, s3};1;1"/>
+        <listEntry value="pageservers;;{p1};1;1"/>
+        <listEntry value="max_commit_lsn;;2;0;0"/>
+        <listEntry value="az_mapping;;[ az1 |-&gt; {b1,s1,p1} , az2 |-&gt; {b2,s2} , az3 |-&gt; {b3,s3}];0;0"/>
+        <listEntry value="azs;;{az1, az2, az3};1;1"/>
+    </listAttribute>
+    <stringAttribute key="modelParameterContraint" value="StateConstraint"/>
+    <listAttribute key="modelParameterDefinitions"/>
+    <stringAttribute key="modelParameterModelValues" value="{}"/>
+    <stringAttribute key="modelParameterNewDefinitions" value=""/>
+    <intAttribute key="modelVersion" value="20191005"/>
+    <intAttribute key="numberOfWorkers" value="4"/>
+    <booleanAttribute key="recover" value="false"/>
+    <stringAttribute key="result.mail.address" value=""/>
+    <intAttribute key="simuAril" value="-1"/>
+    <intAttribute key="simuDepth" value="100"/>
+    <intAttribute key="simuSeed" value="-1"/>
+    <stringAttribute key="specName" value="replicated"/>
+    <stringAttribute key="tlcResourcesProfile" value="local custom"/>
+    <stringAttribute key="view" value=""/>
+    <booleanAttribute key="visualizeStateGraph" value="false"/>
+</launchConfiguration>

storage_broker/spec/replicated/replicated.toolbox/replicated___Model_1_SnapShot_1737326115252.launch

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.lamport.tla.toolbox.tool.tlc.modelCheck">
+    <stringAttribute key="TLCCmdLineParameters" value=""/>
+    <intAttribute key="collectCoverage" value="1"/>
+    <stringAttribute key="configurationName" value="Model_1_SnapShot_1737326115252"/>
+    <booleanAttribute key="deferLiveness" value="false"/>
+    <intAttribute key="dfidDepth" value="100"/>
+    <booleanAttribute key="dfidMode" value="false"/>
+    <intAttribute key="distributedFPSetCount" value="0"/>
+    <stringAttribute key="distributedNetworkInterface" value="100.65.224.26"/>
+    <intAttribute key="distributedNodesCount" value="1"/>
+    <stringAttribute key="distributedTLC" value="off"/>
+    <stringAttribute key="distributedTLCVMArgs" value=""/>
+    <intAttribute key="fpBits" value="1"/>
+    <intAttribute key="fpIndex" value="96"/>
+    <booleanAttribute key="fpIndexRandom" value="true"/>
+    <intAttribute key="maxHeapSize" value="25"/>
+    <intAttribute key="maxSetSize" value="1000000"/>
+    <booleanAttribute key="mcMode" value="true"/>
+    <stringAttribute key="modelBehaviorInit" value=""/>
+    <stringAttribute key="modelBehaviorNext" value=""/>
+    <stringAttribute key="modelBehaviorSpec" value="Spec"/>
+    <intAttribute key="modelBehaviorSpecType" value="1"/>
+    <stringAttribute key="modelBehaviorVars" value="pageserver_state, broker_state, online, safekeeper_state"/>
+    <stringAttribute key="modelComments" value=""/>
+    <booleanAttribute key="modelCorrectnessCheckDeadlock" value="true"/>
+    <listAttribute key="modelCorrectnessInvariants">
+        <listEntry value="1PsLagsSk"/>
+        <listEntry value="1PeerRecoveryIsPossible"/>
+    </listAttribute>
+    <listAttribute key="modelCorrectnessProperties">
+        <listEntry value="1EventuallyLaggingSkIsNotPreferredSk"/>
+    </listAttribute>
+    <intAttribute key="modelEditorOpenTabs" value="14"/>
+    <stringAttribute key="modelExpressionEval" value=""/>
+    <stringAttribute key="modelParameterActionConstraint" value=""/>
+    <listAttribute key="modelParameterConstants">
+        <listEntry value="brokers;;{b1, b2, b3};1;1"/>
+        <listEntry value="NULL;;NULL;1;0"/>
+        <listEntry value="safekeepers;;{s1, s2, s3};1;1"/>
+        <listEntry value="pageservers;;{p1};1;1"/>
+        <listEntry value="max_commit_lsn;;2;0;0"/>
+        <listEntry value="az_mapping;;[ az1 |-&gt; {b1,s1,p1} , az2 |-&gt; {b2,s2} , az3 |-&gt; {b3,s3}];0;0"/>
+        <listEntry value="azs;;{az1, az2, az3};1;1"/>
+    </listAttribute>
+    <stringAttribute key="modelParameterContraint" value="StateConstraint"/>
+    <listAttribute key="modelParameterDefinitions"/>
+    <stringAttribute key="modelParameterModelValues" value="{}"/>
+    <stringAttribute key="modelParameterNewDefinitions" value=""/>
+    <intAttribute key="modelVersion" value="20191005"/>
+    <intAttribute key="numberOfWorkers" value="4"/>
+    <booleanAttribute key="recover" value="false"/>
+    <stringAttribute key="result.mail.address" value=""/>
+    <intAttribute key="simuAril" value="-1"/>
+    <intAttribute key="simuDepth" value="100"/>
+    <intAttribute key="simuSeed" value="-1"/>
+    <stringAttribute key="specName" value="replicated"/>
+    <stringAttribute key="tlcResourcesProfile" value="local custom"/>
+    <stringAttribute key="view" value=""/>
+    <booleanAttribute key="visualizeStateGraph" value="false"/>
+</launchConfiguration>

storage_broker/spec/replicated/replicated.toolbox/replicated___Model_1_SnapShot_1737326135419.launch

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.lamport.tla.toolbox.tool.tlc.modelCheck">
+    <stringAttribute key="TLCCmdLineParameters" value=""/>
+    <intAttribute key="collectCoverage" value="1"/>
+    <stringAttribute key="configurationName" value="Model_1_SnapShot_1737326135419"/>
+    <booleanAttribute key="deferLiveness" value="false"/>
+    <intAttribute key="dfidDepth" value="100"/>
+    <booleanAttribute key="dfidMode" value="false"/>
+    <intAttribute key="distributedFPSetCount" value="0"/>
+    <stringAttribute key="distributedNetworkInterface" value="100.65.224.26"/>
+    <intAttribute key="distributedNodesCount" value="1"/>
+    <stringAttribute key="distributedTLC" value="off"/>
+    <stringAttribute key="distributedTLCVMArgs" value=""/>
+    <intAttribute key="fpBits" value="1"/>
+    <intAttribute key="fpIndex" value="96"/>
+    <booleanAttribute key="fpIndexRandom" value="true"/>
+    <intAttribute key="maxHeapSize" value="25"/>
+    <intAttribute key="maxSetSize" value="1000000"/>
+    <booleanAttribute key="mcMode" value="true"/>
+    <stringAttribute key="modelBehaviorInit" value=""/>
+    <stringAttribute key="modelBehaviorNext" value=""/>
+    <stringAttribute key="modelBehaviorSpec" value="Spec"/>
+    <intAttribute key="modelBehaviorSpecType" value="1"/>
+    <stringAttribute key="modelBehaviorVars" value="pageserver_state, broker_state, online, safekeeper_state"/>
+    <stringAttribute key="modelComments" value=""/>
+    <booleanAttribute key="modelCorrectnessCheckDeadlock" value="true"/>
+    <listAttribute key="modelCorrectnessInvariants">
+        <listEntry value="1PsLagsSk"/>
+        <listEntry value="1PeerRecoveryIsPossible"/>
+    </listAttribute>
+    <listAttribute key="modelCorrectnessProperties">
+        <listEntry value="1EventuallyLaggingSkIsNotPreferredSk"/>
+    </listAttribute>
+    <intAttribute key="modelEditorOpenTabs" value="14"/>
+    <stringAttribute key="modelExpressionEval" value=""/>
+    <stringAttribute key="modelParameterActionConstraint" value=""/>
+    <listAttribute key="modelParameterConstants">
+        <listEntry value="brokers;;{b1, b2, b3};1;1"/>
+        <listEntry value="NULL;;NULL;1;0"/>
+        <listEntry value="safekeepers;;{s1, s2, s3};1;1"/>
+        <listEntry value="pageservers;;{p1};1;1"/>
+        <listEntry value="max_commit_lsn;;2;0;0"/>
+        <listEntry value="az_mapping;;[ az1 |-&gt; {b1,s1,p1} , az2 |-&gt; {b2,s2} , az3 |-&gt; {b3,s3}];0;0"/>
+        <listEntry value="azs;;{az1, az2, az3};1;1"/>
+    </listAttribute>
+    <stringAttribute key="modelParameterContraint" value="StateConstraint"/>
+    <listAttribute key="modelParameterDefinitions"/>
+    <stringAttribute key="modelParameterModelValues" value="{}"/>
+    <stringAttribute key="modelParameterNewDefinitions" value=""/>
+    <intAttribute key="modelVersion" value="20191005"/>
+    <intAttribute key="numberOfWorkers" value="4"/>
+    <booleanAttribute key="recover" value="false"/>
+    <stringAttribute key="result.mail.address" value=""/>
+    <intAttribute key="simuAril" value="-1"/>
+    <intAttribute key="simuDepth" value="100"/>
+    <intAttribute key="simuSeed" value="-1"/>
+    <stringAttribute key="specName" value="replicated"/>
+    <stringAttribute key="tlcResourcesProfile" value="local custom"/>
+    <stringAttribute key="view" value=""/>
+    <booleanAttribute key="visualizeStateGraph" value="false"/>
+</launchConfiguration>

storage_broker/spec/replicated/replicated.toolbox/replicated___Model_1_SnapShot_1737326274268.launch

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.lamport.tla.toolbox.tool.tlc.modelCheck">
+    <stringAttribute key="TLCCmdLineParameters" value=""/>
+    <intAttribute key="collectCoverage" value="1"/>
+    <stringAttribute key="configurationName" value="Model_1_SnapShot_1737326274268"/>
+    <booleanAttribute key="deferLiveness" value="false"/>
+    <intAttribute key="dfidDepth" value="100"/>
+    <booleanAttribute key="dfidMode" value="false"/>
+    <intAttribute key="distributedFPSetCount" value="0"/>
+    <stringAttribute key="distributedNetworkInterface" value="100.65.224.26"/>
+    <intAttribute key="distributedNodesCount" value="1"/>
+    <stringAttribute key="distributedTLC" value="off"/>
+    <stringAttribute key="distributedTLCVMArgs" value=""/>
+    <intAttribute key="fpBits" value="1"/>
+    <intAttribute key="fpIndex" value="96"/>
+    <booleanAttribute key="fpIndexRandom" value="true"/>
+    <intAttribute key="maxHeapSize" value="25"/>
+    <intAttribute key="maxSetSize" value="1000000"/>
+    <booleanAttribute key="mcMode" value="true"/>
+    <stringAttribute key="modelBehaviorInit" value=""/>
+    <stringAttribute key="modelBehaviorNext" value=""/>
+    <stringAttribute key="modelBehaviorSpec" value="Spec"/>
+    <intAttribute key="modelBehaviorSpecType" value="1"/>
+    <stringAttribute key="modelBehaviorVars" value="pageserver_state, broker_state, online, safekeeper_state"/>
+    <stringAttribute key="modelComments" value=""/>
+    <booleanAttribute key="modelCorrectnessCheckDeadlock" value="true"/>
+    <listAttribute key="modelCorrectnessInvariants">
+        <listEntry value="1PsLagsSk"/>
+        <listEntry value="1PeerRecoveryIsPossible"/>
+    </listAttribute>
+    <listAttribute key="modelCorrectnessProperties">
+        <listEntry value="1EventuallyLaggingSkIsNotPreferredSk"/>
+    </listAttribute>
+    <intAttribute key="modelEditorOpenTabs" value="14"/>
+    <stringAttribute key="modelExpressionEval" value=""/>
+    <stringAttribute key="modelParameterActionConstraint" value=""/>
+    <listAttribute key="modelParameterConstants">
+        <listEntry value="brokers;;{b1, b2, b3};1;1"/>
+        <listEntry value="NULL;;NULL;1;0"/>
+        <listEntry value="safekeepers;;{s1, s2, s3};1;1"/>
+        <listEntry value="pageservers;;{p1};1;1"/>
+        <listEntry value="max_commit_lsn;;2;0;0"/>
+        <listEntry value="az_mapping;;[ az1 |-&gt; {b1,s1,p1} , az2 |-&gt; {b2,s2} , az3 |-&gt; {b3,s3}];0;0"/>
+        <listEntry value="azs;;{az1, az2, az3};1;1"/>
+    </listAttribute>
+    <stringAttribute key="modelParameterContraint" value="StateConstraint"/>
+    <listAttribute key="modelParameterDefinitions"/>
+    <stringAttribute key="modelParameterModelValues" value="{}"/>
+    <stringAttribute key="modelParameterNewDefinitions" value=""/>
+    <intAttribute key="modelVersion" value="20191005"/>
+    <intAttribute key="numberOfWorkers" value="4"/>
+    <booleanAttribute key="recover" value="false"/>
+    <stringAttribute key="result.mail.address" value=""/>
+    <intAttribute key="simuAril" value="-1"/>
+    <intAttribute key="simuDepth" value="100"/>
+    <intAttribute key="simuSeed" value="-1"/>
+    <stringAttribute key="specName" value="replicated"/>
+    <stringAttribute key="tlcResourcesProfile" value="local custom"/>
+    <stringAttribute key="view" value=""/>
+    <booleanAttribute key="visualizeStateGraph" value="false"/>
+</launchConfiguration>

storage_broker/spec/replicated/replicated.toolbox/replicated___Model_1_SnapShot_1737327144010.launch

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.lamport.tla.toolbox.tool.tlc.modelCheck">
+    <stringAttribute key="TLCCmdLineParameters" value=""/>
+    <intAttribute key="collectCoverage" value="1"/>
+    <stringAttribute key="configurationName" value="Model_1_SnapShot_1737327144010"/>
+    <booleanAttribute key="deferLiveness" value="false"/>
+    <intAttribute key="dfidDepth" value="100"/>
+    <booleanAttribute key="dfidMode" value="false"/>
+    <intAttribute key="distributedFPSetCount" value="0"/>
+    <stringAttribute key="distributedNetworkInterface" value="100.65.224.26"/>
+    <intAttribute key="distributedNodesCount" value="1"/>
+    <stringAttribute key="distributedTLC" value="off"/>
+    <stringAttribute key="distributedTLCVMArgs" value=""/>
+    <intAttribute key="fpBits" value="1"/>
+    <intAttribute key="fpIndex" value="96"/>
+    <booleanAttribute key="fpIndexRandom" value="true"/>
+    <intAttribute key="maxHeapSize" value="25"/>
+    <intAttribute key="maxSetSize" value="1000000"/>
+    <booleanAttribute key="mcMode" value="true"/>
+    <stringAttribute key="modelBehaviorInit" value=""/>
+    <stringAttribute key="modelBehaviorNext" value=""/>
+    <stringAttribute key="modelBehaviorSpec" value="Spec"/>
+    <intAttribute key="modelBehaviorSpecType" value="1"/>
+    <stringAttribute key="modelBehaviorVars" value="pageserver_state, broker_state, online, safekeeper_state"/>
+    <stringAttribute key="modelComments" value=""/>
+    <booleanAttribute key="modelCorrectnessCheckDeadlock" value="true"/>
+    <listAttribute key="modelCorrectnessInvariants">
+        <listEntry value="1PsLagsSk"/>
+        <listEntry value="1PeerRecoveryIsPossible"/>
+    </listAttribute>
+    <listAttribute key="modelCorrectnessProperties">
+        <listEntry value="1EventuallyLaggingSkIsNotPreferredSk"/>
+    </listAttribute>
+    <intAttribute key="modelEditorOpenTabs" value="14"/>
+    <stringAttribute key="modelExpressionEval" value=""/>
+    <stringAttribute key="modelParameterActionConstraint" value=""/>
+    <listAttribute key="modelParameterConstants">
+        <listEntry value="brokers;;{b1, b2, b3};1;1"/>
+        <listEntry value="NULL;;NULL;1;0"/>
+        <listEntry value="safekeepers;;{s1, s2, s3};1;1"/>
+        <listEntry value="pageservers;;{p1};1;1"/>
+        <listEntry value="max_commit_lsn;;2;0;0"/>
+        <listEntry value="az_mapping;;[ az1 |-&gt; {b1,s1,p1} , az2 |-&gt; {b2,s2} , az3 |-&gt; {b3,s3}];0;0"/>
+        <listEntry value="azs;;{az1, az2, az3};1;1"/>
+    </listAttribute>
+    <stringAttribute key="modelParameterContraint" value="StateConstraint"/>
+    <listAttribute key="modelParameterDefinitions"/>
+    <stringAttribute key="modelParameterModelValues" value="{}"/>
+    <stringAttribute key="modelParameterNewDefinitions" value=""/>
+    <intAttribute key="modelVersion" value="20191005"/>
+    <intAttribute key="numberOfWorkers" value="4"/>
+    <booleanAttribute key="recover" value="false"/>
+    <stringAttribute key="result.mail.address" value=""/>
+    <intAttribute key="simuAril" value="-1"/>
+    <intAttribute key="simuDepth" value="100"/>
+    <intAttribute key="simuSeed" value="-1"/>
+    <stringAttribute key="specName" value="replicated"/>
+    <stringAttribute key="tlcResourcesProfile" value="local custom"/>
+    <stringAttribute key="view" value=""/>
+    <booleanAttribute key="visualizeStateGraph" value="false"/>
+</launchConfiguration>

storage_broker/spec/replicated/replicated.toolbox/replicated___Model_1_SnapShot_1737327172537.launch

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.lamport.tla.toolbox.tool.tlc.modelCheck">
+    <stringAttribute key="TLCCmdLineParameters" value=""/>
+    <intAttribute key="collectCoverage" value="1"/>
+    <stringAttribute key="configurationName" value="Model_1_SnapShot_1737327172537"/>
+    <booleanAttribute key="deferLiveness" value="false"/>
+    <intAttribute key="dfidDepth" value="100"/>
+    <booleanAttribute key="dfidMode" value="false"/>
+    <intAttribute key="distributedFPSetCount" value="0"/>
+    <stringAttribute key="distributedNetworkInterface" value="100.65.224.26"/>
+    <intAttribute key="distributedNodesCount" value="1"/>
+    <stringAttribute key="distributedTLC" value="off"/>
+    <stringAttribute key="distributedTLCVMArgs" value=""/>
+    <intAttribute key="fpBits" value="1"/>
+    <intAttribute key="fpIndex" value="96"/>
+    <booleanAttribute key="fpIndexRandom" value="true"/>
+    <intAttribute key="maxHeapSize" value="25"/>
+    <intAttribute key="maxSetSize" value="1000000"/>
+    <booleanAttribute key="mcMode" value="true"/>
+    <stringAttribute key="modelBehaviorInit" value=""/>
+    <stringAttribute key="modelBehaviorNext" value=""/>
+    <stringAttribute key="modelBehaviorSpec" value="Spec"/>
+    <intAttribute key="modelBehaviorSpecType" value="1"/>
+    <stringAttribute key="modelBehaviorVars" value="pageserver_state, broker_state, online, safekeeper_state"/>
+    <stringAttribute key="modelComments" value=""/>
+    <booleanAttribute key="modelCorrectnessCheckDeadlock" value="true"/>
+    <listAttribute key="modelCorrectnessInvariants">
+        <listEntry value="1PsLagsSk"/>
+        <listEntry value="1PeerRecoveryIsPossible"/>
+    </listAttribute>
+    <listAttribute key="modelCorrectnessProperties">
+        <listEntry value="1EventuallyLaggingSkIsNotPreferredSk"/>
+    </listAttribute>
+    <intAttribute key="modelEditorOpenTabs" value="14"/>
+    <stringAttribute key="modelExpressionEval" value=""/>
+    <stringAttribute key="modelParameterActionConstraint" value=""/>
+    <listAttribute key="modelParameterConstants">
+        <listEntry value="brokers;;{b1, b2, b3};1;1"/>
+        <listEntry value="NULL;;NULL;1;0"/>
+        <listEntry value="safekeepers;;{s1, s2, s3};1;1"/>
+        <listEntry value="pageservers;;{p1};1;1"/>
+        <listEntry value="max_commit_lsn;;2;0;0"/>
+        <listEntry value="az_mapping;;[ az1 |-&gt; {b1,s1,p1} , az2 |-&gt; {b2,s2} , az3 |-&gt; {b3,s3}];0;0"/>
+        <listEntry value="azs;;{az1, az2, az3};1;1"/>
+    </listAttribute>
+    <stringAttribute key="modelParameterContraint" value="StateConstraint"/>
+    <listAttribute key="modelParameterDefinitions"/>
+    <stringAttribute key="modelParameterModelValues" value="{}"/>
+    <stringAttribute key="modelParameterNewDefinitions" value=""/>
+    <intAttribute key="modelVersion" value="20191005"/>
+    <intAttribute key="numberOfWorkers" value="4"/>
+    <booleanAttribute key="recover" value="false"/>
+    <stringAttribute key="result.mail.address" value=""/>
+    <intAttribute key="simuAril" value="-1"/>
+    <intAttribute key="simuDepth" value="100"/>
+    <intAttribute key="simuSeed" value="-1"/>
+    <stringAttribute key="specName" value="replicated"/>
+    <stringAttribute key="tlcResourcesProfile" value="local custom"/>
+    <stringAttribute key="view" value=""/>
+    <booleanAttribute key="visualizeStateGraph" value="false"/>
+</launchConfiguration>