diff --git a/src/index/base.cpp b/src/index/base.cpp index 57fb9e42f362..33f87c3b0304 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -23,6 +23,7 @@ using node::PruneLockInfo; using node::ReadBlockFromDisk; using node::fPruneMode; +using node::g_indexes_ready_to_sync; constexpr uint8_t DB_BEST_BLOCK{'B'}; @@ -82,16 +83,21 @@ bool BaseIndex::Init() if (locator.IsNull()) { SetBestBlockIndex(nullptr); } else { - SetBestBlockIndex(m_chainstate->FindForkInGlobalIndex(locator)); + // Setting the best block to the locator's top block. If it is not part of the + // best chain, we will rewind to the fork point during index sync + const CBlockIndex* locator_index{m_chainstate->m_blockman.LookupBlockIndex(locator.vHave.at(0))}; + if (!locator_index) { + return InitError(strprintf(Untranslated("%s: best block of the index not found. Please rebuild the index."), GetName())); + } + SetBestBlockIndex(locator_index); } - // Note: this will latch to true immediately if the user starts up with an empty - // datadir and an index enabled. If this is the case, indexation will happen solely - // via `BlockConnected` signals until, possibly, the next restart. - m_synced = m_best_block_index.load() == active_chain.Tip(); - if (!m_synced) { + // Skip pruning check if indexes are not ready to sync (because reindex-chainstate has wiped the chain). + const CBlockIndex* start_block = m_best_block_index.load(); + bool synced = start_block == active_chain.Tip(); + if (!synced && g_indexes_ready_to_sync) { bool prune_violation = false; - if (!m_best_block_index) { + if (!start_block) { // index is not built yet // make sure we have all block data back to the genesis prune_violation = m_chainstate->m_blockman.GetFirstStoredBlock(*active_chain.Tip()) != active_chain.Genesis(); @@ -99,7 +105,7 @@ bool BaseIndex::Init() // in case the index has a best block set and is not fully synced // check if we have the required blocks to continue building the index else { - const CBlockIndex* block_to_test = m_best_block_index.load(); + const CBlockIndex* block_to_test = start_block; if (!active_chain.Contains(block_to_test)) { // if the bestblock is not part of the mainchain, find the fork // and make sure we have all data down to the fork @@ -123,6 +129,16 @@ bool BaseIndex::Init() return InitError(strprintf(Untranslated("%s best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)"), GetName())); } } + + // Child init + if (!CustomInit(start_block ? std::make_optional(interfaces::BlockKey{start_block->GetBlockHash(), start_block->nHeight}) : std::nullopt)) { + return false; + } + + // Note: this will latch to true immediately if the user starts up with an empty + // datadir and an index enabled. If this is the case, indexation will happen solely + // via `BlockConnected` signals until, possibly, the next restart. + m_synced = synced; return true; } @@ -144,6 +160,12 @@ static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev, CChain& void BaseIndex::ThreadSync() { + // Wait for a possible reindex-chainstate to finish until continuing + // with the index sync + while (!g_indexes_ready_to_sync) { + if (!m_interrupt.sleep_for(std::chrono::milliseconds(500))) return; + } + const CBlockIndex* pindex = m_best_block_index.load(); if (!m_synced) { auto& consensus_params = Params().GetConsensus(); @@ -408,11 +430,6 @@ bool BaseIndex::Start() RegisterValidationInterface(this); if (!Init()) return false; - const CBlockIndex* index = m_best_block_index.load(); - if (!CustomInit(index ? std::make_optional(interfaces::BlockKey{index->GetBlockHash(), index->nHeight}) : std::nullopt)) { - return false; - } - m_thread_sync = std::thread(&util::TraceThread, GetName(), [this] { ThreadSync(); }); return true; } diff --git a/src/init.cpp b/src/init.cpp index b30195204c95..92d6e4b2713d 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -174,6 +174,7 @@ using node::ThreadImport; using node::VerifyLoadedChainstate; using node::fPruneMode; using node::fReindex; +using node::g_indexes_ready_to_sync; using node::nPruneTarget; #ifdef ENABLE_WALLET using wallet::DEFAULT_DISABLE_WALLET; @@ -635,7 +636,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-addressindex", strprintf("Maintain a full address index, used to query for the balance, txids and unspent outputs for addresses (default: %u)", DEFAULT_ADDRESSINDEX), ArgsManager::ALLOW_ANY, OptionsCategory::INDEXING); argsman.AddArg("-reindex", "Rebuild chain state and block index from the blk*.dat files on disk. This will also rebuild active optional indexes.", ArgsManager::ALLOW_ANY, OptionsCategory::INDEXING); - argsman.AddArg("-reindex-chainstate", "Rebuild chain state from the currently indexed blocks. When in pruning mode or if blocks on disk might be corrupted, use full -reindex instead. Deactivate all optional indexes before running this.", ArgsManager::ALLOW_ANY, OptionsCategory::INDEXING); + argsman.AddArg("-reindex-chainstate", "Rebuild chain state from the currently indexed blocks. When in pruning mode or if blocks on disk might be corrupted, use full -reindex instead.", ArgsManager::ALLOW_ANY, OptionsCategory::INDEXING); argsman.AddArg("-spentindex", strprintf("Maintain a full spent index, used to query the spending txid and input index for an outpoint (default: %u)", DEFAULT_SPENTINDEX), ArgsManager::ALLOW_ANY, OptionsCategory::INDEXING); argsman.AddArg("-timestampindex", strprintf("Maintain a timestamp index for block hashes, used to query blocks hashes by a range of timestamps (default: %u)", DEFAULT_TIMESTAMPINDEX), ArgsManager::ALLOW_ANY, OptionsCategory::INDEXING); argsman.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), ArgsManager::ALLOW_ANY, OptionsCategory::INDEXING); @@ -1359,19 +1360,6 @@ bool AppInitParameterInteraction(const ArgsManager& args) if (args.GetBoolArg("-peerbloomfilters", DEFAULT_PEERBLOOMFILTERS)) nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM); - if (args.GetBoolArg("-reindex-chainstate", false)) { - // indexes that must be deactivated to prevent index corruption, see #24630 - if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) { - return InitError(_("-reindex-chainstate option is not compatible with -coinstatsindex. Please temporarily disable coinstatsindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.")); - } - if (g_enabled_filter_types.count(BlockFilterType::BASIC_FILTER)) { - return InitError(_("-reindex-chainstate option is not compatible with -blockfilterindex. Please temporarily disable blockfilterindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.")); - } - if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - return InitError(_("-reindex-chainstate option is not compatible with -txindex. Please temporarily disable txindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.")); - } - } - try { const bool fQuorumVvecRequestsEnabled{llmq::GetEnabledQuorumVvecSyncEntries(args).size() > 0}; if (!args.GetBoolArg("-llmq-data-recovery", llmq::DEFAULT_ENABLE_QUORUM_DATA_RECOVERY) && fQuorumVvecRequestsEnabled) { @@ -2225,6 +2213,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) node.peerman->AddExtraHandler(std::make_unique(node.peerman.get(), *node.govman, *node.mn_sync, *node.connman, *node.netfulfilledman)); // ********************************************************* Step 8: start indexers + + // If reindex-chainstate was specified, delay syncing indexes until ThreadImport has reindexed the chain + if (!fReindexChainState) g_indexes_ready_to_sync = true; if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { g_txindex = std::make_unique(interfaces::MakeChain(node), cache_sizes.tx_index, false, fReindex); if (!g_txindex->Start()) { diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index f04e2e6e7892..625e77f84758 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -26,6 +26,7 @@ namespace node { std::atomic_bool fImporting(false); std::atomic_bool fReindex(false); +std::atomic_bool g_indexes_ready_to_sync{false}; bool fPruneMode = false; uint64_t nPruneTarget = 0; @@ -959,5 +960,6 @@ void ThreadImport(ChainstateManager& chainman, std::vector vImportFile } } // End scope of CImportingNow chainman.ActiveChainstate().LoadMempool(mempool_path); + g_indexes_ready_to_sync = true; } } // namespace node diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 66eb07f4b4e8..71424f616053 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -49,6 +49,7 @@ static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = CMessageHeader::MESSAG extern std::atomic_bool fImporting; extern std::atomic_bool fReindex; +extern std::atomic_bool g_indexes_ready_to_sync; /** Pruning-related variables and constants */ /** True if we're running in -prune mode. */ extern bool fPruneMode; diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 197898061d75..99ac447a39fc 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -241,6 +241,7 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve noui_connect(); noui_connected = true; } + node::g_indexes_ready_to_sync = true; bls::bls_legacy_scheme.store(true); } diff --git a/test/functional/feature_coinstatsindex.py b/test/functional/feature_coinstatsindex.py index b27f1b0b4759..e53def92e4ef 100755 --- a/test/functional/feature_coinstatsindex.py +++ b/test/functional/feature_coinstatsindex.py @@ -54,6 +54,7 @@ def run_test(self): self._test_use_index_option() self._test_reorg_index() self._test_index_rejects_hash_serialized() + self._test_init_index_after_reorg() def block_subsidy(self, prev_bits): # Subsidy calculations valid till block 4500 @@ -75,6 +76,9 @@ def block_sanity_check(self, block_info, prev_height): block_info['new_outputs_ex_coinbase'] + block_info['coinbase'] + block_info['unspendable'] ) + def sync_index_node(self): + self.wait_until(lambda: self.nodes[1].getindexinfo()['coinstatsindex']['synced'] is True) + def _test_coin_stats_index(self): node = self.nodes[0] index_node = self.nodes[1] @@ -237,18 +241,16 @@ def _test_coin_stats_index(self): self.log.info("Test that the index works with -reindex") self.restart_node(1, extra_args=["-coinstatsindex", "-reindex"]) + self.sync_index_node() res11 = index_node.gettxoutsetinfo('muhash') assert_equal(res11, res10) - self.log.info("Test that -reindex-chainstate is disallowed with coinstatsindex") + self.log.info("Test that the index works with -reindex-chainstate") - self.stop_node(1) - self.nodes[1].assert_start_raises_init_error( - expected_msg='Error: -reindex-chainstate option is not compatible with -coinstatsindex. ' - 'Please temporarily disable coinstatsindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.', - extra_args=['-coinstatsindex', '-reindex-chainstate'], - ) - self.restart_node(1, extra_args=["-coinstatsindex"]) + self.restart_node(1, extra_args=["-coinstatsindex", "-reindex-chainstate"]) + self.sync_index_node() + res12 = index_node.gettxoutsetinfo('muhash') + assert_equal(res12, res10) self.log.info("Test obtaining info for a non-existent block hash") assert_raises_rpc_error(-5, "Block not found", index_node.gettxoutsetinfo, hash_type="none", hash_or_height="ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", use_index=True) @@ -270,6 +272,7 @@ def _test_reorg_index(self): index_node = self.nodes[1] reorg_blocks = self.generatetoaddress(index_node, 2, getnewdestination()[2]) reorg_block = reorg_blocks[1] + self.sync_index_node() res_invalid = index_node.gettxoutsetinfo('muhash') index_node.invalidateblock(reorg_blocks[0]) assert_equal(index_node.gettxoutsetinfo('muhash')['height'], 110) @@ -309,6 +312,26 @@ def _test_index_rejects_hash_serialized(self): for use_index in {True, False, None}: assert_raises_rpc_error(-8, msg, self.nodes[1].gettxoutsetinfo, hash_type='hash_serialized_2', hash_or_height=111, use_index=use_index) + def _test_init_index_after_reorg(self): + self.log.info("Test a reorg while the index is deactivated") + index_node = self.nodes[1] + block = self.nodes[0].getbestblockhash() + self.generate(index_node, 2, sync_fun=self.no_op) + self.sync_index_node() + + # Restart without index + self.restart_node(1, extra_args=[]) + self.connect_nodes(0, 1) + index_node.invalidateblock(block) + self.generatetoaddress(index_node, 5, getnewdestination()[2]) + res = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=None, use_index=False) + + # Restart with index that still has its best block on the old chain + self.restart_node(1, extra_args=self.extra_args[1]) + self.sync_index_node() + res1 = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=None, use_index=True) + assert_equal(res["muhash"], res1["muhash"]) + if __name__ == '__main__': CoinStatsIndexTest().main() diff --git a/test/functional/p2p_blockfilters.py b/test/functional/p2p_blockfilters.py index 9df7f26ecca7..5aa8e6bba172 100755 --- a/test/functional/p2p_blockfilters.py +++ b/test/functional/p2p_blockfilters.py @@ -293,13 +293,6 @@ def run_test(self): msg = "Error: Unknown -blockfilterindex value abc." self.nodes[0].assert_start_raises_init_error(expected_msg=msg) - self.log.info("Test -blockfilterindex with -reindex-chainstate raises an error") - self.nodes[0].assert_start_raises_init_error( - expected_msg='Error: -reindex-chainstate option is not compatible with -blockfilterindex. ' - 'Please temporarily disable blockfilterindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.', - extra_args=['-blockfilterindex', '-reindex-chainstate'], - ) - self.test_special_transactions_in_filters() def create_simple_assetlock(self, base_tx, base_tx_value, amount_locked_1, amount_locked_2=0):