diff --git a/src/.clang-tidy b/src/.clang-tidy index 8f1b842bb0ce..e22cf9378e95 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -5,6 +5,7 @@ bugprone-use-after-move, misc-unused-using-decls, modernize-use-default-member-init, modernize-use-emplace, +modernize-use-noexcept, modernize-use-nullptr, performance-*, -performance-avoid-endl, diff --git a/src/Makefile.am b/src/Makefile.am index 079979e2aec5..20df7196d91a 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1043,6 +1043,7 @@ libbitcoin_util_a_SOURCES = \ random.cpp \ randomenv.cpp \ stacktraces.cpp \ + streams.cpp \ support/cleanse.cpp \ support/lockedpool.cpp \ sync.cpp \ @@ -1373,6 +1374,7 @@ libdashkernel_la_SOURCES = \ script/standard.cpp \ shutdown.cpp \ spork.cpp \ + streams.cpp \ support/cleanse.cpp \ support/lockedpool.cpp \ sync.cpp \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index b97660a74532..5dae5a3d7cc1 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -60,7 +60,8 @@ bench_bench_dash_SOURCES = \ bench/strencodings.cpp \ bench/string_cast.cpp \ bench/util_time.cpp \ - bench/verify_script.cpp + bench/verify_script.cpp \ + bench/xor.cpp nodist_bench_bench_dash_SOURCES = $(GENERATED_BENCH_FILES) diff --git a/src/bench/bench.h b/src/bench/bench.h index 78196134e781..6065ddf3fc91 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -14,7 +14,7 @@ #include #include -#include +#include // IWYU pragma: export /* * Usage: diff --git a/src/bench/xor.cpp b/src/bench/xor.cpp new file mode 100644 index 000000000000..edda74214a29 --- /dev/null +++ b/src/bench/xor.cpp @@ -0,0 +1,24 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit/. + +#include + +#include +#include + +#include +#include + +static void Xor(benchmark::Bench& bench) +{ + FastRandomContext frc{/*fDeterministic=*/true}; + auto data{frc.randbytes(1024)}; + auto key{frc.randbytes(31)}; + + bench.batch(data.size()).unit("byte").run([&] { + util::Xor(data, key); + }); +} + +BENCHMARK(Xor, benchmark::PriorityLevel::HIGH); diff --git a/src/hash.h b/src/hash.h index 8fa9319732f9..3a79b7c0f879 100644 --- a/src/hash.h +++ b/src/hash.h @@ -161,7 +161,6 @@ class CHashWriter : public HashWriter template CHashWriter& operator<<(const T& obj) { - // Serialize to this stream ::Serialize(*this, obj); return (*this); } @@ -196,7 +195,6 @@ class CHashVerifier : public CHashWriter template CHashVerifier& operator>>(T&& obj) { - // Unserialize from this stream ::Unserialize(*this, obj); return (*this); } diff --git a/src/streams.cpp b/src/streams.cpp new file mode 100644 index 000000000000..6921dad67733 --- /dev/null +++ b/src/streams.cpp @@ -0,0 +1,66 @@ +// Copyright (c) 2009-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit/. + +#include +#include + +#include + +std::size_t AutoFile::detail_fread(Span dst) +{ + if (!m_file) throw std::ios_base::failure("AutoFile::read: file handle is nullptr"); + if (m_xor.empty()) { + return std::fread(dst.data(), 1, dst.size(), m_file); + } else { + const auto init_pos{std::ftell(m_file)}; + if (init_pos < 0) throw std::ios_base::failure("AutoFile::read: ftell failed"); + std::size_t ret{std::fread(dst.data(), 1, dst.size(), m_file)}; + util::Xor(dst.subspan(0, ret), m_xor, init_pos); + return ret; + } +} + +void AutoFile::read(Span dst) +{ + if (detail_fread(dst) != dst.size()) { + throw std::ios_base::failure(feof() ? "AutoFile::read: end of file" : "AutoFile::read: fread failed"); + } +} + +void AutoFile::ignore(size_t nSize) +{ + if (!m_file) throw std::ios_base::failure("AutoFile::ignore: file handle is nullptr"); + unsigned char data[4096]; + while (nSize > 0) { + size_t nNow = std::min(nSize, sizeof(data)); + if (std::fread(data, 1, nNow, m_file) != nNow) { + throw std::ios_base::failure(feof() ? "AutoFile::ignore: end of file" : "AutoFile::ignore: fread failed"); + } + nSize -= nNow; + } +} + +void AutoFile::write(Span src) +{ + if (!m_file) throw std::ios_base::failure("AutoFile::write: file handle is nullptr"); + if (m_xor.empty()) { + if (std::fwrite(src.data(), 1, src.size(), m_file) != src.size()) { + throw std::ios_base::failure("AutoFile::write: write failed"); + } + } else { + auto current_pos{std::ftell(m_file)}; + if (current_pos < 0) throw std::ios_base::failure("AutoFile::write: ftell failed"); + std::array buf; + while (src.size() > 0) { + auto buf_now{Span{buf}.first(std::min(src.size(), buf.size()))}; + std::copy(src.begin(), src.begin() + buf_now.size(), buf_now.begin()); + util::Xor(buf_now, m_xor, current_pos); + if (std::fwrite(buf_now.data(), 1, buf_now.size(), m_file) != buf_now.size()) { + throw std::ios_base::failure{"XorFile::write: failed"}; + } + src = src.subspan(buf_now.size()); + current_pos += buf_now.size(); + } + } +} diff --git a/src/streams.h b/src/streams.h index 368ce70eebac..aa0dd9b85adf 100644 --- a/src/streams.h +++ b/src/streams.h @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -23,6 +24,27 @@ #include #include +namespace util { +inline void Xor(Span write, Span key, size_t key_offset = 0) +{ + if (key.size() == 0) { + return; + } + key_offset %= key.size(); + + for (size_t i = 0, j = key_offset; i != write.size(); i++) { + write[i] ^= key[j++]; + + // This potentially acts on very many bytes of data, so it's + // important that we calculate `j`, i.e. the `key` index in this + // way instead of doing a %, which would effectively be a division + // for each byte Xor'd -- much slower than need be. + if (j == key.size()) + j = 0; + } +} +} // namespace util + template class OverrideStream { @@ -37,7 +59,6 @@ class OverrideStream template OverrideStream& operator<<(const T& obj) { - // Serialize to this stream ::Serialize(*this, obj); return (*this); } @@ -45,7 +66,6 @@ class OverrideStream template OverrideStream& operator>>(T&& obj) { - // Unserialize from this stream ::Unserialize(*this, obj); return (*this); } @@ -111,7 +131,6 @@ class CVectorWriter template CVectorWriter& operator<<(const T& obj) { - // Serialize to this stream ::Serialize(*this, obj); return (*this); } @@ -156,7 +175,6 @@ class SpanReader template SpanReader& operator>>(T&& obj) { - // Unserialize from this stream ::Unserialize(*this, obj); return (*this); } @@ -309,7 +327,6 @@ class DataStream template DataStream& operator<<(const T& obj) { - // Serialize to this stream ::Serialize(*this, obj); return (*this); } @@ -317,7 +334,6 @@ class DataStream template DataStream& operator>>(T&& obj) { - // Unserialize from this stream ::Unserialize(*this, obj); return (*this); } @@ -329,20 +345,7 @@ class DataStream */ void Xor(const std::vector& key) { - if (key.size() == 0) { - return; - } - - for (size_type i = 0, j = 0; i != size(); i++) { - vch[i] ^= std::byte{key[j++]}; - - // This potentially acts on very many bytes of data, so it's - // important that we calculate `j`, i.e. the `key` index in this - // way instead of doing a %, which would effectively be a division - // for each byte Xor'd -- much slower than need be. - if (j == key.size()) - j = 0; - } + util::Xor(MakeWritableByteSpan(*this), MakeByteSpan(key)); } }; @@ -484,7 +487,6 @@ class BitStreamWriter } }; - /** Non-refcounted RAII wrapper for FILE* * * Will automatically close the file when it goes out of scope if not null. @@ -494,81 +496,60 @@ class BitStreamWriter class AutoFile { protected: - FILE* file; + std::FILE* m_file; + const std::vector m_xor; public: - explicit AutoFile(FILE* filenew) : file{filenew} {} + explicit AutoFile(std::FILE* file, std::vector data_xor={}) : m_file{file}, m_xor{std::move(data_xor)} {} - ~AutoFile() - { - fclose(); - } + ~AutoFile() { fclose(); } // Disallow copies AutoFile(const AutoFile&) = delete; AutoFile& operator=(const AutoFile&) = delete; + bool feof() const { return std::feof(m_file); } + int fclose() { - int retval{0}; - if (file) { - retval = ::fclose(file); - file = nullptr; - } - return retval; + if (auto rel{release()}) return std::fclose(rel); + return 0; } /** Get wrapped FILE* with transfer of ownership. * @note This will invalidate the AutoFile object, and makes it the responsibility of the caller * of this function to clean up the returned FILE*. */ - FILE* release() { FILE* ret = file; file = nullptr; return ret; } + std::FILE* release() + { + std::FILE* ret{m_file}; + m_file = nullptr; + return ret; + } /** Get wrapped FILE* without transfer of ownership. * @note Ownership of the FILE* will remain with this class. Use this only if the scope of the * AutoFile outlives use of the passed pointer. */ - FILE* Get() const { return file; } + std::FILE* Get() const { return m_file; } /** Return true if the wrapped FILE* is nullptr, false otherwise. */ - bool IsNull() const { return (file == nullptr); } + bool IsNull() const { return m_file == nullptr; } + + /** Implementation detail, only used internally. */ + std::size_t detail_fread(Span dst); // // Stream subset // - void read(Span dst) - { - if (!file) throw std::ios_base::failure("AutoFile::read: file handle is nullptr"); - if (fread(dst.data(), 1, dst.size(), file) != dst.size()) { - throw std::ios_base::failure(feof(file) ? "AutoFile::read: end of file" : "AutoFile::read: fread failed"); - } - } - - void ignore(size_t nSize) - { - if (!file) throw std::ios_base::failure("AutoFile::ignore: file handle is nullptr"); - unsigned char data[4096]; - while (nSize > 0) { - size_t nNow = std::min(nSize, sizeof(data)); - if (fread(data, 1, nNow, file) != nNow) - throw std::ios_base::failure(feof(file) ? "AutoFile::ignore: end of file" : "AutoFile::read: fread failed"); - nSize -= nNow; - } - } - - void write(Span src) - { - if (!file) throw std::ios_base::failure("AutoFile::write: file handle is nullptr"); - if (fwrite(src.data(), 1, src.size(), file) != src.size()) { - throw std::ios_base::failure("AutoFile::write: write failed"); - } - } + void read(Span dst); + void ignore(size_t nSize); + void write(Span src); template AutoFile& operator<<(const T& obj) { - if (!file) throw std::ios_base::failure("AutoFile::operator<<: file handle is nullptr"); ::Serialize(*this, obj); return *this; } @@ -576,7 +557,6 @@ class AutoFile template AutoFile& operator>>(T&& obj) { - if (!file) throw std::ios_base::failure("AutoFile::operator>>: file handle is nullptr"); ::Unserialize(*this, obj); return *this; } @@ -589,16 +569,13 @@ class CAutoFile : public AutoFile const int nVersion; public: - CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) : AutoFile{filenew}, nType(nTypeIn), nVersion(nVersionIn) {} + explicit CAutoFile(std::FILE* file, int type, int version, std::vector data_xor = {}) : AutoFile{file, std::move(data_xor)}, nType{type}, nVersion{version} {} int GetType() const { return nType; } int GetVersion() const { return nVersion; } template CAutoFile& operator<<(const T& obj) { - // Serialize to this stream - if (!file) - throw std::ios_base::failure("CAutoFile::operator<<: file handle is nullptr"); ::Serialize(*this, obj); return (*this); } @@ -606,9 +583,6 @@ class CAutoFile : public AutoFile template CAutoFile& operator>>(T&& obj) { - // Unserialize from this stream - if (!file) - throw std::ios_base::failure("CAutoFile::operator>>: file handle is nullptr"); ::Unserialize(*this, obj); return (*this); } @@ -757,7 +731,6 @@ class CBufferedFile template CBufferedFile& operator>>(T&& obj) { - // Unserialize from this stream ::Unserialize(*this, obj); return (*this); } diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 36892b069886..80d6fb68a094 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include @@ -13,6 +14,55 @@ using namespace std::string_literals; BOOST_FIXTURE_TEST_SUITE(streams_tests, BasicTestingSetup) +BOOST_AUTO_TEST_CASE(xor_file) +{ + fs::path xor_path{m_args.GetDataDirBase() / "test_xor.bin"}; + auto raw_file{[&](const auto& mode) { return fsbridge::fopen(xor_path, mode); }}; + const std::vector test1{1, 2, 3}; + const std::vector test2{4, 5}; + const std::vector xor_pat{std::byte{0xff}, std::byte{0x00}}; + { + // Check errors for missing file + AutoFile xor_file{raw_file("rb"), xor_pat}; + BOOST_CHECK_EXCEPTION(xor_file << std::byte{}, std::ios_base::failure, HasReason{"AutoFile::write: file handle is nullpt"}); + BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: file handle is nullpt"}); + BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: file handle is nullpt"}); + } + { + AutoFile xor_file{raw_file("wbx"), xor_pat}; + xor_file << test1 << test2; + } + { + // Read raw from disk + AutoFile non_xor_file{raw_file("rb")}; + std::vector raw(7); + non_xor_file >> Span{raw}; + BOOST_CHECK_EQUAL(HexStr(raw), "fc01fd03fd04fa"); + // Check that no padding exists + BOOST_CHECK_EXCEPTION(non_xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"}); + } + { + AutoFile xor_file{raw_file("rb"), xor_pat}; + std::vector read1, read2; + xor_file >> read1 >> read2; + BOOST_CHECK_EQUAL(HexStr(read1), HexStr(test1)); + BOOST_CHECK_EQUAL(HexStr(read2), HexStr(test2)); + // Check that eof was reached + BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"}); + } + { + AutoFile xor_file{raw_file("rb"), xor_pat}; + std::vector read2; + // Check that ignore works + xor_file.ignore(4); + xor_file >> read2; + BOOST_CHECK_EQUAL(HexStr(read2), HexStr(test2)); + // Check that ignore and read fail now + BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"}); + BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"}); + } +} + BOOST_AUTO_TEST_CASE(streams_vector_writer) { unsigned char a(1); diff --git a/src/validation.cpp b/src/validation.cpp index c58b1338f48a..d9e8fd69c5e4 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -6139,7 +6139,7 @@ static void FlushSnapshotToDisk(CCoinsViewCache& coins_cache, bool snapshot_load struct StopHashingException : public std::exception { - const char* what() const throw() override + const char* what() const noexcept override { return "ComputeUTXOStats interrupted by shutdown."; } diff --git a/test/functional/wallet_descriptor.py b/test/functional/wallet_descriptor.py index 081ad6276c50..0bdecfafe9e6 100755 --- a/test/functional/wallet_descriptor.py +++ b/test/functional/wallet_descriptor.py @@ -199,9 +199,11 @@ def run_test(self): self.nodes[0].createwallet(wallet_name="crashme", descriptors=True) self.nodes[0].unloadwallet("crashme") wallet_db = os.path.join(self.nodes[0].datadir, self.chain, "wallets", "crashme", self.wallet_data_filename) - with sqlite3.connect(wallet_db) as conn: + conn = sqlite3.connect(wallet_db) + with conn: # add "cscript" entry: key type is uint160 (20 bytes), value type is CScript (zero-length here) conn.execute('INSERT INTO main VALUES(?, ?)', (b'\x07cscript' + b'\x00'*20, b'\x00')) + conn.close() assert_raises_rpc_error(-4, "Unexpected legacy entry in descriptor wallet found.", self.nodes[0].loadwallet, "crashme") diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index 889f8d3b60ae..b5d187076518 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -104,9 +104,13 @@ def wait_p2p(): # Set correct m_best_block_time, which is used in ResubmitWalletTransactions node.syncwithvalidationinterfacequeue() - # Evict these txs from the mempool evict_time = block_time + 60 * 60 * DEFAULT_MEMPOOL_EXPIRY_HOURS + 5 - node.setmocktime(evict_time) + # Flush out currently scheduled resubmit attempt now so that there can't be one right between eviction and check. + with node.assert_debug_log(['resubmit 2 unconfirmed transactions']): + node.setmocktime(evict_time) + node.mockscheduler(60) + + # Evict these txs from the mempool indep_send = node.send(outputs=[{node.getnewaddress(): 1}], options={"inputs": [indep_utxo]}) node.getmempoolentry(indep_send["txid"]) assert_raises_rpc_error(-5, "Transaction not in mempool", node.getmempoolentry, txid)