Skip to content

Commit afbbcae

Browse files
authored
Test validator-set recovery after transient failure (#60)
1 parent a2cd805 commit afbbcae

1 file changed

Lines changed: 55 additions & 1 deletion

File tree

‎tests/test_bft_peer_sync.py‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@ def sync(_chain, peer):
6767
bft_node.bft_peer_sync_loop(chain, engine, interval=3, stop_event=stop)
6868

6969
self.assertIn("unavailable", "\\n".join(captured.output))
70-
7170
self.assertEqual(contacted, ["unavailable", "healthy"])
7271
self.assertEqual(engine.snapshots, [[{"height": 0}]])
7372
self.assertEqual(engine.retries, 1)
@@ -111,6 +110,61 @@ def retry_pending_finalization(self):
111110
self.assertIn("BFT chain catch-up failed", log_output)
112111
self.assertIn("BFT pending finalization retry failed", log_output)
113112

113+
def test_validator_set_recovery_is_retried_after_transient_failure(self):
114+
chain = _Chain()
115+
chain.validators = [{"public_key": "validator-a", "voting_power": 1}]
116+
chain.validators_hash = "new-validator-set"
117+
chain.validator_activation_height = 2
118+
stop = _StopAfterCycles(2)
119+
120+
class RecoveringEngine(_Engine):
121+
def __init__(self):
122+
super().__init__()
123+
self.validator_set_hash = "old-validator-set"
124+
self.validator_activation_height = 1
125+
self.pending_validator_recovery = True
126+
self.activations = []
127+
128+
def activate_validator_set(
129+
self, validators, signer, activation_height, chain_height
130+
):
131+
self.activations.append(
132+
(validators, signer, activation_height, chain_height)
133+
)
134+
self.validator_set_hash = chain.validators_hash
135+
self.validator_activation_height = activation_height
136+
self.pending_validator_recovery = False
137+
138+
engine = RecoveringEngine()
139+
recovery_calls = []
140+
141+
def recover():
142+
recovery_calls.append(True)
143+
if len(recovery_calls) == 1:
144+
raise OSError("temporary validator-state recovery failure")
145+
return {
146+
"validators": chain.validators,
147+
"signer": None,
148+
"activation_height": chain.validator_activation_height,
149+
"chain_height": len(chain.blocks),
150+
}
151+
152+
with patch.object(bft_node, "synchronize_peer", return_value=True):
153+
with self.assertLogs(bft_node.LOGGER, level="ERROR") as captured:
154+
bft_node.bft_peer_sync_loop(
155+
chain, engine, interval=3, stop_event=stop,
156+
on_validator_set=recover,
157+
)
158+
159+
self.assertEqual(len(recovery_calls), 2)
160+
self.assertEqual(len(engine.activations), 1)
161+
self.assertEqual(engine.activations[0][2:], (2, 1))
162+
self.assertEqual(engine.snapshots, [[{"height": 0}]])
163+
self.assertIn(
164+
"BFT chain catch-up failed; will retry next cycle",
165+
"\\n".join(captured.output),
166+
)
167+
114168

115169
if __name__ == "__main__":
116170
unittest.main()

0 commit comments

Comments
 (0)