@@ -67,7 +67,6 @@ def sync(_chain, peer):
6767 bft_node .bft_peer_sync_loop (chain , engine , interval = 3 , stop_event = stop )
6868
6969 self .assertIn ("unavailable" , "\\ n" .join (captured .output ))
70-
7170 self .assertEqual (contacted , ["unavailable" , "healthy" ])
7271 self .assertEqual (engine .snapshots , [[{"height" : 0 }]])
7372 self .assertEqual (engine .retries , 1 )
@@ -111,6 +110,61 @@ def retry_pending_finalization(self):
111110 self .assertIn ("BFT chain catch-up failed" , log_output )
112111 self .assertIn ("BFT pending finalization retry failed" , log_output )
113112
113+ def test_validator_set_recovery_is_retried_after_transient_failure (self ):
114+ chain = _Chain ()
115+ chain .validators = [{"public_key" : "validator-a" , "voting_power" : 1 }]
116+ chain .validators_hash = "new-validator-set"
117+ chain .validator_activation_height = 2
118+ stop = _StopAfterCycles (2 )
119+
120+ class RecoveringEngine (_Engine ):
121+ def __init__ (self ):
122+ super ().__init__ ()
123+ self .validator_set_hash = "old-validator-set"
124+ self .validator_activation_height = 1
125+ self .pending_validator_recovery = True
126+ self .activations = []
127+
128+ def activate_validator_set (
129+ self , validators , signer , activation_height , chain_height
130+ ):
131+ self .activations .append (
132+ (validators , signer , activation_height , chain_height )
133+ )
134+ self .validator_set_hash = chain .validators_hash
135+ self .validator_activation_height = activation_height
136+ self .pending_validator_recovery = False
137+
138+ engine = RecoveringEngine ()
139+ recovery_calls = []
140+
141+ def recover ():
142+ recovery_calls .append (True )
143+ if len (recovery_calls ) == 1 :
144+ raise OSError ("temporary validator-state recovery failure" )
145+ return {
146+ "validators" : chain .validators ,
147+ "signer" : None ,
148+ "activation_height" : chain .validator_activation_height ,
149+ "chain_height" : len (chain .blocks ),
150+ }
151+
152+ with patch .object (bft_node , "synchronize_peer" , return_value = True ):
153+ with self .assertLogs (bft_node .LOGGER , level = "ERROR" ) as captured :
154+ bft_node .bft_peer_sync_loop (
155+ chain , engine , interval = 3 , stop_event = stop ,
156+ on_validator_set = recover ,
157+ )
158+
159+ self .assertEqual (len (recovery_calls ), 2 )
160+ self .assertEqual (len (engine .activations ), 1 )
161+ self .assertEqual (engine .activations [0 ][2 :], (2 , 1 ))
162+ self .assertEqual (engine .snapshots , [[{"height" : 0 }]])
163+ self .assertIn (
164+ "BFT chain catch-up failed; will retry next cycle" ,
165+ "\\ n" .join (captured .output ),
166+ )
167+
114168
115169if __name__ == "__main__" :
116170 unittest .main ()
0 commit comments