[backup] Bound restore dispatch and adapt oversized transactions

fdbclient/ClientKnobs.cpp

@@ -303,6 +303,7 @@ void ClientKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	init( BACKUP_LOGFILE_BLOCK_SIZE,        1024 * 1024);
 	init( BACKUP_DISPATCH_ADDTASK_SIZE,             50 );
 	init( RESTORE_DISPATCH_ADDTASK_SIZE,           150 );
+	init( RESTORE_DISPATCH_TX_SIZE,      4 * 1024 * 1024 ); if( randomize && buggify() ) RESTORE_DISPATCH_TX_SIZE = 1 * 1024;
 	init( RESTORE_DISPATCH_BATCH_SIZE,           30000 ); if( randomize && buggify() ) RESTORE_DISPATCH_BATCH_SIZE = 20;
 	init (RESTORE_PARTITIONED_BATCH_VERSION_SIZE, 10000000); // each step restores 10s worth of data
 	init( RESTORE_WRITE_TX_SIZE,            256 * 1024 );

fdbclient/include/fdbclient/Knobs.h

@@ -235,6 +235,7 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ClientKnobs : public KnobsImpl<ClientKno
 	int BACKUP_DISPATCH_ADDTASK_SIZE;
 	bool BACKUP_ALLOW_DRYRUN;
 	int RESTORE_DISPATCH_ADDTASK_SIZE;
+	int RESTORE_DISPATCH_TX_SIZE; // Target bytes for partitioned restore range-task dispatch.
 	int RESTORE_DISPATCH_BATCH_SIZE;
 	int RESTORE_WRITE_TX_SIZE;
 	int RESTORE_PARTITIONED_BATCH_VERSION_SIZE;

fdbserver/workloads/RestoreWriterRecovery.cpp

@@ -0,0 +1,56 @@
+/*
+ * RestoreWriterRecovery.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/tester/workloads.h"
+
+namespace fileBackup {
+Future<Void> testRestoreWriterRecovery(Database cx);
+Future<Void> testRestoreDispatchRecovery(Database cx);
+} // namespace fileBackup
+
+struct RestoreWriterRecoveryWorkload : TestWorkload {
+	static constexpr auto NAME = "RestoreWriterRecovery";
+	explicit RestoreWriterRecoveryWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
+
+	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.insert("all"); }
+	Future<Void> setup(Database const& cx) override { return Void(); }
+	Future<Void> start(Database const& cx) override {
+		return clientId == 0 ? fileBackup::testRestoreWriterRecovery(cx) : Void();
+	}
+	Future<bool> check(Database const& cx) override { return true; }
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+};
+
+WorkloadFactory<RestoreWriterRecoveryWorkload> RestoreWriterRecoveryWorkloadFactory;
+
+struct RestoreDispatchRecoveryWorkload : TestWorkload {
+	static constexpr auto NAME = "RestoreDispatchRecovery";
+	explicit RestoreDispatchRecoveryWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
+
+	void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.insert("all"); }
+	Future<Void> setup(Database const& cx) override { return Void(); }
+	Future<Void> start(Database const& cx) override {
+		return clientId == 0 ? fileBackup::testRestoreDispatchRecovery(cx) : Void();
+	}
+	Future<bool> check(Database const& cx) override { return true; }
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+};
+
+WorkloadFactory<RestoreDispatchRecoveryWorkload> RestoreDispatchRecoveryWorkloadFactory;

tests/CMakeLists.txt

@@ -257,6 +257,8 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES fast/ReadHotDetectionCorrectness.toml IGNORE) # TODO re-enable once read hot detection is enabled.
   add_fdb_test(TEST_FILES fast/ReportConflictingKeys.toml)
   add_fdb_test(TEST_FILES fast/RESTUnit.toml IGNORE)
+  add_fdb_test(TEST_FILES fast/RestoreDispatchRecovery.toml)
+  add_fdb_test(TEST_FILES fast/RestoreWriterRecovery.toml)
   add_fdb_test(TEST_FILES fast/SelectorCorrectness.toml)
   add_fdb_test(TEST_FILES fast/ShardedRocksNondeterministicTest.toml)
   add_fdb_test(TEST_FILES fast/StalePeerTest.toml)

tests/fast/RestoreDispatchRecovery.toml

@@ -0,0 +1,15 @@
+testClass = "Backup"
+buggify = false
+
+[[knobs]]
+# Let the injected age failures, rather than the byte target, bound dispatch.
+restore_dispatch_tx_size = 4194304
+
+[configuration]
+tenantModes = ['disabled']
+
+[[test]]
+testTitle = 'RestoreDispatchRecovery'
+
+    [[test.workload]]
+    testName = 'RestoreDispatchRecovery'

tests/fast/RestoreWriterRecovery.toml

@@ -0,0 +1,16 @@
+testClass = "Backup"
+
+[[knobs]]
+# The test caps writer transactions at 4 KiB, below their initial 16 KiB budget.
+restore_write_tx_size = 16384
+backup_logfile_block_size = 65536
+mutation_block_size = 10000
+
+[configuration]
+tenantModes = ['disabled']
+
+[[test]]
+testTitle = 'RestoreWriterRecovery'
+
+    [[test.workload]]
+    testName = 'RestoreWriterRecovery'

tests/slow/BackupCorrectnessPartitioned.toml

@@ -1,5 +1,12 @@
 testClass = "Backup"
 
+# Exercise file/block continuations and transaction-byte limits.
+[[knobs]]
+backup_rangefile_block_size = 4096
+restore_dispatch_addtask_size = 2
+restore_dispatch_tx_size = 4096
+restore_write_tx_size = 4096
+
 [configuration]
 
 [[test]]
@@ -18,5 +25,3 @@ simBackupAgents = 'BackupToFile'
     backupAfter = 10.0
     restoreAfter = 60.0
     backupRangesCount = -1
-
-