backport: v0.26 bitcoin#27549, bitcoin#27425, bitcoin#23813, bitcoin#27892

src/Makefile.test_util.include

@@ -43,6 +43,7 @@ libtest_util_a_SOURCES = \
   test/util/masternode.cpp \
   test/util/mining.cpp \
   test/util/net.cpp \
+  test/util/random.cpp \
   test/util/script.cpp \
   test/util/setup_common.cpp \
   test/util/str.cpp \

src/rpc/blockchain.cpp

@@ -488,7 +488,8 @@ static RPCHelpMan getblockfrompeer()
         "We must have the header for this block, e.g. using submitheader.\n"
         "Subsequent calls for the same block may cause the response from the previous peer to be ignored.\n"
         "Peers generally ignore requests for a stale block that they never fully verified, or one that is more than a month old.\n"
-        "When a peer does not respond with a block, we will disconnect.\n\n"
+        "When a peer does not respond with a block, we will disconnect.\n"
+        "Note: The block could be re-pruned as soon as it is received.\n\n"
         "Returns an empty JSON object if the request was successfully scheduled.",
         {
             {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The block hash to try to fetch"},

src/test/denialofservice_tests.cpp

@@ -13,6 +13,7 @@
 #include <script/signingprovider.h>
 #include <script/standard.h>
 #include <test/util/net.h>
+#include <test/util/random.h>
 #include <test/util/setup_common.h>
 #include <timedata.h>
 #include <util/string.h>

src/test/fuzz/addrman.cpp

@@ -280,13 +280,21 @@ FUZZ_TARGET(addrman, .init = initialize_addrman)
             });
     }
     const AddrMan& const_addr_man{addr_man};
+    std::optional<Network> network;
+    if (fuzzed_data_provider.ConsumeBool()) {
+        network = fuzzed_data_provider.PickValueInArray(ALL_NETWORKS);
+    }
     (void)const_addr_man.GetAddr(
-        /* max_addresses */ fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096),
-        /* max_pct */ fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096),
-        /* network */ std::nullopt,
-        /* filtered =*/ fuzzed_data_provider.ConsumeBool());
-    (void)const_addr_man.Select(fuzzed_data_provider.ConsumeBool());
-    (void)const_addr_man.Size();
+        /*max_addresses=*/fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096),
+        /*max_pct=*/fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096),
+        network,
+        /*filtered=*/fuzzed_data_provider.ConsumeBool());
+    (void)const_addr_man.Select(fuzzed_data_provider.ConsumeBool(), network);
+    std::optional<bool> in_new;
+    if (fuzzed_data_provider.ConsumeBool()) {
+        in_new = fuzzed_data_provider.ConsumeBool();
+    }
+    (void)const_addr_man.Size(network, in_new);
     CDataStream data_stream(SER_NETWORK, PROTOCOL_VERSION);
     data_stream << const_addr_man;
 }

src/test/miniscript_tests.cpp

@@ -5,6 +5,7 @@
 
 #include <string>
 
+#include <test/util/random.h>
 #include <test/util/setup_common.h>
 #include <boost/test/unit_test.hpp>
 

src/test/net_peer_connection_tests.cpp

@@ -14,6 +14,7 @@
 #include <random.h>
 #include <test/util/logging.h>
 #include <test/util/net.h>
+#include <test/util/random.h>
 #include <test/util/setup_common.h>
 #include <tinyformat.h>
 #include <version.h>

src/test/random_tests.cpp

@@ -4,6 +4,7 @@
 
 #include <random.h>
 
+#include <test/util/random.h>
 #include <test/util/setup_common.h>
 #include <util/time.h>
 

src/test/util/random.cpp

@@ -0,0 +1,33 @@
+// Copyright (c) 2023 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <test/util/random.h>
+
+#include <logging.h>
+#include <random.h>
+#include <uint256.h>
+
+#include <cstdlib>
+#include <string>
+
+FastRandomContext g_insecure_rand_ctx;
+
+/** Return the unsigned from the environment var if available, otherwise 0 */
+static uint256 GetUintFromEnv(const std::string& env_name)
+{
+    const char* num = std::getenv(env_name.c_str());
+    if (!num) return {};
+    return uint256S(num);
+}
+
+void Seed(FastRandomContext& ctx)
+{
+    // Should be enough to get the seed once for the process
+    static uint256 seed{};
+    static const std::string RANDOM_CTX_SEED{"RANDOM_CTX_SEED"};
+    if (seed.IsNull()) seed = GetUintFromEnv(RANDOM_CTX_SEED);
+    if (seed.IsNull()) seed = GetRandHash();
+    LogPrintf("%s: Setting random seed for current tests to %s=%s\n", __func__, RANDOM_CTX_SEED, seed.GetHex());
+    ctx = FastRandomContext(seed);
+}

src/test/util/random.h

@@ -7,11 +7,41 @@
 
 #include <consensus/amount.h>
 #include <random.h>
-#include <test/util/setup_common.h>
 #include <uint256.h>
 
 #include <cstdint>
 
+/**
+ * This global and the helpers that use it are not thread-safe.
+ *
+ * If thread-safety is needed, the global could be made thread_local (given
+ * that thread_local is supported on all architectures we support) or a
+ * per-thread instance could be used in the multi-threaded test.
+ */
+extern FastRandomContext g_insecure_rand_ctx;
+
+/**
+ * Flag to make GetRand in random.h return the same number
+ */
+extern bool g_mock_deterministic_tests;
+
+enum class SeedRand {
+    ZEROS, //!< Seed with a compile time constant of zeros
+    SEED,  //!< Call the Seed() helper
+};
+
+/** Seed the given random ctx or use the seed passed in via an environment var */
+void Seed(FastRandomContext& ctx);
+
+static inline void SeedInsecureRand(SeedRand seed = SeedRand::SEED)
+{
+    if (seed == SeedRand::ZEROS) {
+        g_insecure_rand_ctx = FastRandomContext(/*fDeterministic=*/true);
+    } else {
+        Seed(g_insecure_rand_ctx);
+    }
+}
+
 static inline uint32_t InsecureRand32()
 {
     return g_insecure_rand_ctx.rand32();

src/test/util/setup_common.cpp

@@ -40,6 +40,7 @@
 #include <policy/fees_args.h>
 #include <policy/settings.h>
 #include <pow.h>
+#include <random.h>
 #include <rpc/blockchain.h>
 #include <rpc/register.h>
 #include <rpc/server.h>
@@ -49,6 +50,7 @@
 #include <streams.h>
 #include <test/util/index.h>
 #include <test/util/net.h>
+#include <test/util/random.h>
 #include <test/util/txmempool.h>
 #include <timedata.h>
 #include <txdb.h>
@@ -105,29 +107,9 @@ using node::VerifyLoadedChainstate;
 
 const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr;
 
-FastRandomContext g_insecure_rand_ctx;
 /** Random context to get unique temp data dirs. Separate from g_insecure_rand_ctx, which can be seeded from a const env var */
 static FastRandomContext g_insecure_rand_ctx_temp_path;
 
-/** Return the unsigned from the environment var if available, otherwise 0 */
-static uint256 GetUintFromEnv(const std::string& env_name)
-{
-    const char* num = std::getenv(env_name.c_str());
-    if (!num) return {};
-    return uint256S(num);
-}
-
-void Seed(FastRandomContext& ctx)
-{
-    // Should be enough to get the seed once for the process
-    static uint256 seed{};
-    static const std::string RANDOM_CTX_SEED{"RANDOM_CTX_SEED"};
-    if (seed.IsNull()) seed = GetUintFromEnv(RANDOM_CTX_SEED);
-    if (seed.IsNull()) seed = GetRandHash();
-    LogPrintf("%s: Setting random seed for current tests to %s=%s\n", __func__, RANDOM_CTX_SEED, seed.GetHex());
-    ctx = FastRandomContext(seed);
-}
-
 std::ostream& operator<<(std::ostream& os, const uint256& num)
 {
     os << num.ToString();

src/test/util/setup_common.h

@@ -13,7 +13,6 @@
 #include <node/context.h> // IWYU pragma: export
 #include <primitives/transaction.h>
 #include <pubkey.h>
-#include <random.h>
 #include <util/check.h>
 #include <util/fs.h>
 #include <util/string.h>
@@ -32,6 +31,7 @@ struct Params;
 };
 
 class Chainstate;
+class FastRandomContext;
 
 /** This is connected to the logger. Can be used to redirect logs to any other log */
 extern const std::function<void(const std::string&)> G_TEST_LOG_FUN;
@@ -48,36 +48,6 @@ std::ostream& operator<<(typename std::enable_if<std::is_enum<T>::value, std::os
 }
 } // namespace std
 
-/**
- * This global and the helpers that use it are not thread-safe.
- *
- * If thread-safety is needed, a per-thread instance could be
- * used in the multi-threaded test.
- */
-extern FastRandomContext g_insecure_rand_ctx;
-
-/**
- * Flag to make GetRand in random.h return the same number
- */
-extern bool g_mock_deterministic_tests;
-
-enum class SeedRand {
-    ZEROS, //!< Seed with a compile time constant of zeros
-    SEED,  //!< Call the Seed() helper
-};
-
-/** Seed the given random ctx or use the seed passed in via an environment var */
-void Seed(FastRandomContext& ctx);
-
-static inline void SeedInsecureRand(SeedRand seed = SeedRand::SEED)
-{
-    if (seed == SeedRand::ZEROS) {
-        g_insecure_rand_ctx = FastRandomContext(/*fDeterministic=*/true);
-    } else {
-        Seed(g_insecure_rand_ctx);
-    }
-}
-
 static constexpr CAmount CENT{1000000};
 
 /** Initialize components during initialization (NodeContext-friendly aliases) */

src/util/translation.h

@@ -49,22 +49,18 @@ inline bilingual_str Untranslated(std::string original) { return {original, orig
 
 // Provide an overload of tinyformat::format which can take bilingual_str arguments.
 namespace tinyformat {
-inline std::string TranslateArg(const bilingual_str& arg, bool translated)
-{
-    return translated ? arg.translated : arg.original;
-}
-
-template <typename T>
-inline T const& TranslateArg(const T& arg, bool translated)
-{
-    return arg;
-}
-
 template <typename... Args>
 bilingual_str format(const bilingual_str& fmt, const Args&... args)
 {
-    return bilingual_str{tfm::format(fmt.original, TranslateArg(args, false)...),
-                         tfm::format(fmt.translated, TranslateArg(args, true)...)};
+    const auto translate_arg{[](const auto& arg, bool translated) -> const auto& {
+        if constexpr (std::is_same_v<decltype(arg), const bilingual_str&>) {
+            return translated ? arg.translated : arg.original;
+        } else {
+            return arg;
+        }
+    }};
+    return bilingual_str{tfm::format(fmt.original, translate_arg(args, false)...),
+                         tfm::format(fmt.translated, translate_arg(args, true)...)};
 }
 } // namespace tinyformat
 

src/wallet/test/platformkeys_tests.cpp

@@ -10,6 +10,7 @@
 #include <script/descriptor.h>
 #include <script/signingprovider.h>
 #include <script/standard.h>
+#include <test/util/random.h>
 #include <test/util/setup_common.h>
 #include <util/strencodings.h>
 #include <wallet/bip39.h>

src/wallet/test/wallet_tests.cpp

@@ -26,6 +26,7 @@
 #include <rpc/server.h>
 #include <test/util/logging.h>
 #include <test/util/masternode.h>
+#include <test/util/random.h>
 #include <test/util/setup_common.h>
 #include <txmempool.h>
 #include <util/translation.h>

test/functional/rpc_getblockfrompeer.py

@@ -17,20 +17,26 @@
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
     assert_equal,
+    assert_greater_than,
     assert_raises_rpc_error,
 )
 
 
 class GetBlockFromPeerTest(BitcoinTestFramework):
     def set_test_params(self):
-        self.num_nodes = 2
+        self.num_nodes = 3
+        self.extra_args = [
+            [],
+            [],
+            ["-fastprune", "-prune=1"]
+        ]
 
     def setup_network(self):
         self.setup_nodes()
 
-    def check_for_block(self, hash):
+    def check_for_block(self, node, hash):
         try:
-            self.nodes[0].getblock(hash)
+            self.nodes[node].getblock(hash)
             return True
         except JSONRPCException:
             return False
@@ -47,7 +53,7 @@ def run_test(self):
 
         self.log.info("Connect nodes to sync headers")
         self.connect_nodes(0, 1)
-        self.sync_blocks()
+        self.sync_blocks(self.nodes[0:2])
 
         self.log.info("Node 0 should only have the header for node 1's block 3")
         x = next(filter(lambda x: x['hash'] == short_tip, self.nodes[0].getchaintips()))
@@ -73,7 +79,7 @@ def run_test(self):
 
         self.log.info("Successful fetch")
         result = self.nodes[0].getblockfrompeer(short_tip, peer_0_peer_1_id)
-        self.wait_until(lambda: self.check_for_block(short_tip), timeout=1)
+        self.wait_until(lambda: self.check_for_block(node=0, hash=short_tip), timeout=1)
         assert_equal(result, {})
 
         self.log.info("Don't fetch blocks we already have")
@@ -104,6 +110,49 @@ def run_test(self):
         assert_raises_rpc_error(-1, error_msg, self.nodes[1].getblockfrompeer, blockhash, node1_interface_id)
         self.stop_node(1, expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
 
+        self.log.info("Connect pruned node")
+        # We need to generate more blocks to be able to prune
+        self.connect_nodes(0, 2)
+        pruned_node = self.nodes[2]
+        # Dash blocks are smaller than Bitcoin's, so many more of them fit into a
+        # (-fastprune) block file. More blocks are needed before the first block
+        # file wraps and can be pruned, and the resulting prune heights are not the
+        # same as upstream, hence they are not compared against fixed values here.
+        self.generate(self.nodes[0], 900, sync_fun=self.no_op)
+        self.sync_blocks([self.nodes[0], pruned_node])
+        pruneheight = pruned_node.pruneblockchain(800)
+        assert_greater_than(pruneheight, 2)
+        # Ensure the block is actually pruned
+        pruned_block = self.nodes[0].getblockhash(2)
+        assert_raises_rpc_error(-1, "Block not available (pruned data)", pruned_node.getblock, pruned_block)
+
+        self.log.info("Fetch pruned block")
+        peers = pruned_node.getpeerinfo()
+        assert_equal(len(peers), 1)
+        pruned_node_peer_0_id = peers[0]["id"]
+        result = pruned_node.getblockfrompeer(pruned_block, pruned_node_peer_0_id)
+        self.wait_until(lambda: self.check_for_block(node=2, hash=pruned_block), timeout=1)
+        assert_equal(result, {})
+
+        self.log.info("Fetched block persists after next pruning event")
+        tip_at_fetch = pruned_node.getblockcount()
+        self.generate(self.nodes[0], 250, sync_fun=self.no_op)
+        self.sync_blocks([self.nodes[0], pruned_node])
+        # Prune below the height the tip had when the block was fetched: the block
+        # file the re-fetched block was written to still holds recent blocks and is
+        # therefore kept.
+        prev_pruneheight = pruneheight
+        pruneheight = pruned_node.pruneblockchain(tip_at_fetch - 55)
+        assert_greater_than(pruneheight, prev_pruneheight)
+        assert_equal(pruned_node.getblock(pruned_block)["hash"], pruned_block)
+
+        self.log.info("Fetched block can be pruned again when prune height exceeds the height of the tip at the time when the block was fetched")
+        self.generate(self.nodes[0], 1200, sync_fun=self.no_op)
+        self.sync_blocks([self.nodes[0], pruned_node])
+        assert_greater_than(pruned_node.pruneblockchain(tip_at_fetch + 1000), tip_at_fetch)
+        assert_raises_rpc_error(-1, "Block not available (pruned data)", pruned_node.getblock, pruned_block)
+        self.stop_node(2, expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
+
 
 if __name__ == '__main__':
     GetBlockFromPeerTest().main()