Bitcoin Core 29.99.0
P2P Digital Currency
blockstorage.cpp
Go to the documentation of this file.
1// Copyright (c) 2011-2022 The Bitcoin Core developers
2// Distributed under the MIT software license, see the accompanying
3// file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
5#include <node/blockstorage.h>
6
7#include <arith_uint256.h>
8#include <chain.h>
9#include <consensus/params.h>
11#include <dbwrapper.h>
12#include <flatfile.h>
13#include <hash.h>
15#include <kernel/chainparams.h>
18#include <logging.h>
19#include <pow.h>
20#include <primitives/block.h>
22#include <random.h>
23#include <serialize.h>
24#include <signet.h>
25#include <span.h>
26#include <streams.h>
27#include <sync.h>
28#include <tinyformat.h>
29#include <uint256.h>
30#include <undo.h>
31#include <util/batchpriority.h>
32#include <util/check.h>
33#include <util/fs.h>
34#include <util/obfuscation.h>
36#include <util/strencodings.h>
37#include <util/syserror.h>
38#include <util/translation.h>
39#include <validation.h>
40
41#include <cstddef>
42#include <map>
43#include <optional>
44#include <unordered_map>
45
46namespace kernel {
47static constexpr uint8_t DB_BLOCK_FILES{'f'};
48static constexpr uint8_t DB_BLOCK_INDEX{'b'};
49static constexpr uint8_t DB_FLAG{'F'};
50static constexpr uint8_t DB_REINDEX_FLAG{'R'};
51static constexpr uint8_t DB_LAST_BLOCK{'l'};
52// Keys used in previous version that might still be found in the DB:
53// BlockTreeDB::DB_TXINDEX_BLOCK{'T'};
54// BlockTreeDB::DB_TXINDEX{'t'}
55// BlockTreeDB::ReadFlag("txindex")
56
58{
59 return Read(std::make_pair(DB_BLOCK_FILES, nFile), info);
60}
61
62bool BlockTreeDB::WriteReindexing(bool fReindexing)
63{
64 if (fReindexing) {
65 return Write(DB_REINDEX_FLAG, uint8_t{'1'});
66 } else {
67 return Erase(DB_REINDEX_FLAG);
68 }
69}
70
71void BlockTreeDB::ReadReindexing(bool& fReindexing)
72{
73 fReindexing = Exists(DB_REINDEX_FLAG);
74}
75
77{
78 return Read(DB_LAST_BLOCK, nFile);
79}
80
81bool BlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*>>& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo)
82{
83 CDBBatch batch(*this);
84 for (const auto& [file, info] : fileInfo) {
85 batch.Write(std::make_pair(DB_BLOCK_FILES, file), *info);
86 }
87 batch.Write(DB_LAST_BLOCK, nLastFile);
88 for (const CBlockIndex* bi : blockinfo) {
89 batch.Write(std::make_pair(DB_BLOCK_INDEX, bi->GetBlockHash()), CDiskBlockIndex{bi});
90 }
91 return WriteBatch(batch, true);
92}
93
94bool BlockTreeDB::WriteFlag(const std::string& name, bool fValue)
95{
96 return Write(std::make_pair(DB_FLAG, name), fValue ? uint8_t{'1'} : uint8_t{'0'});
97}
98
99bool BlockTreeDB::ReadFlag(const std::string& name, bool& fValue)
100{
101 uint8_t ch;
102 if (!Read(std::make_pair(DB_FLAG, name), ch)) {
103 return false;
104 }
105 fValue = ch == uint8_t{'1'};
106 return true;
107}
108
109bool BlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex, const util::SignalInterrupt& interrupt)
110{
112 std::unique_ptr<CDBIterator> pcursor(NewIterator());
113 pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256()));
114
115 // Load m_block_index
116 while (pcursor->Valid()) {
117 if (interrupt) return false;
118 std::pair<uint8_t, uint256> key;
119 if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) {
120 CDiskBlockIndex diskindex;
121 if (pcursor->GetValue(diskindex)) {
122 // Construct block index object
123 CBlockIndex* pindexNew = insertBlockIndex(diskindex.ConstructBlockHash());
124 pindexNew->pprev = insertBlockIndex(diskindex.hashPrev);
125 pindexNew->nHeight = diskindex.nHeight;
126 pindexNew->nFile = diskindex.nFile;
127 pindexNew->nDataPos = diskindex.nDataPos;
128 pindexNew->nUndoPos = diskindex.nUndoPos;
129 pindexNew->nVersion = diskindex.nVersion;
130 pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot;
131 pindexNew->nTime = diskindex.nTime;
132 pindexNew->nBits = diskindex.nBits;
133 pindexNew->nNonce = diskindex.nNonce;
134 pindexNew->nStatus = diskindex.nStatus;
135 pindexNew->nTx = diskindex.nTx;
136
137 if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, consensusParams)) {
138 LogError("%s: CheckProofOfWork failed: %s\n", __func__, pindexNew->ToString());
139 return false;
140 }
141
142 pcursor->Next();
143 } else {
144 LogError("%s: failed to read value\n", __func__);
145 return false;
146 }
147 } else {
148 break;
149 }
150 }
151
152 return true;
153}
154} // namespace kernel
155
156namespace node {
157
159{
160 // First sort by most total work, ...
161 if (pa->nChainWork > pb->nChainWork) return false;
162 if (pa->nChainWork < pb->nChainWork) return true;
163
164 // ... then by earliest time received, ...
165 if (pa->nSequenceId < pb->nSequenceId) return false;
166 if (pa->nSequenceId > pb->nSequenceId) return true;
167
168 // Use pointer address as tie breaker (should only happen with blocks
169 // loaded from disk, as those all have id 0).
170 if (pa < pb) return false;
171 if (pa > pb) return true;
172
173 // Identical blocks.
174 return false;
175}
176
178{
179 return pa->nHeight < pb->nHeight;
180}
181
182std::vector<CBlockIndex*> BlockManager::GetAllBlockIndices()
183{
185 std::vector<CBlockIndex*> rv;
186 rv.reserve(m_block_index.size());
187 for (auto& [_, block_index] : m_block_index) {
188 rv.push_back(&block_index);
189 }
190 return rv;
191}
192
194{
196 BlockMap::iterator it = m_block_index.find(hash);
197 return it == m_block_index.end() ? nullptr : &it->second;
198}
199
201{
203 BlockMap::const_iterator it = m_block_index.find(hash);
204 return it == m_block_index.end() ? nullptr : &it->second;
205}
206
208{
210
211 auto [mi, inserted] = m_block_index.try_emplace(block.GetHash(), block);
212 if (!inserted) {
213 return &mi->second;
214 }
215 CBlockIndex* pindexNew = &(*mi).second;
216
217 // We assign the sequence id to blocks only when the full data is available,
218 // to avoid miners withholding blocks but broadcasting headers, to get a
219 // competitive advantage.
220 pindexNew->nSequenceId = 0;
221
222 pindexNew->phashBlock = &((*mi).first);
223 BlockMap::iterator miPrev = m_block_index.find(block.hashPrevBlock);
224 if (miPrev != m_block_index.end()) {
225 pindexNew->pprev = &(*miPrev).second;
226 pindexNew->nHeight = pindexNew->pprev->nHeight + 1;
227 pindexNew->BuildSkip();
228 }
229 pindexNew->nTimeMax = (pindexNew->pprev ? std::max(pindexNew->pprev->nTimeMax, pindexNew->nTime) : pindexNew->nTime);
230 pindexNew->nChainWork = (pindexNew->pprev ? pindexNew->pprev->nChainWork : 0) + GetBlockProof(*pindexNew);
232 if (best_header == nullptr || best_header->nChainWork < pindexNew->nChainWork) {
233 best_header = pindexNew;
234 }
235
236 m_dirty_blockindex.insert(pindexNew);
237
238 return pindexNew;
239}
240
241void BlockManager::PruneOneBlockFile(const int fileNumber)
242{
245
246 for (auto& entry : m_block_index) {
247 CBlockIndex* pindex = &entry.second;
248 if (pindex->nFile == fileNumber) {
249 pindex->nStatus &= ~BLOCK_HAVE_DATA;
250 pindex->nStatus &= ~BLOCK_HAVE_UNDO;
251 pindex->nFile = 0;
252 pindex->nDataPos = 0;
253 pindex->nUndoPos = 0;
254 m_dirty_blockindex.insert(pindex);
255
256 // Prune from m_blocks_unlinked -- any block we prune would have
257 // to be downloaded again in order to consider its chain, at which
258 // point it would be considered as a candidate for
259 // m_blocks_unlinked or setBlockIndexCandidates.
260 auto range = m_blocks_unlinked.equal_range(pindex->pprev);
261 while (range.first != range.second) {
262 std::multimap<CBlockIndex*, CBlockIndex*>::iterator _it = range.first;
263 range.first++;
264 if (_it->second == pindex) {
265 m_blocks_unlinked.erase(_it);
266 }
267 }
268 }
269 }
270
271 m_blockfile_info.at(fileNumber) = CBlockFileInfo{};
272 m_dirty_fileinfo.insert(fileNumber);
273}
274
276 std::set<int>& setFilesToPrune,
277 int nManualPruneHeight,
278 const Chainstate& chain,
279 ChainstateManager& chainman)
280{
281 assert(IsPruneMode() && nManualPruneHeight > 0);
282
284 if (chain.m_chain.Height() < 0) {
285 return;
286 }
287
288 const auto [min_block_to_prune, last_block_can_prune] = chainman.GetPruneRange(chain, nManualPruneHeight);
289
290 int count = 0;
291 for (int fileNumber = 0; fileNumber < this->MaxBlockfileNum(); fileNumber++) {
292 const auto& fileinfo = m_blockfile_info[fileNumber];
293 if (fileinfo.nSize == 0 || fileinfo.nHeightLast > (unsigned)last_block_can_prune || fileinfo.nHeightFirst < (unsigned)min_block_to_prune) {
294 continue;
295 }
296
297 PruneOneBlockFile(fileNumber);
298 setFilesToPrune.insert(fileNumber);
299 count++;
300 }
301 LogInfo("[%s] Prune (Manual): prune_height=%d removed %d blk/rev pairs",
302 chain.GetRole(), last_block_can_prune, count);
303}
304
306 std::set<int>& setFilesToPrune,
307 int last_prune,
308 const Chainstate& chain,
309 ChainstateManager& chainman)
310{
312 // Distribute our -prune budget over all chainstates.
313 const auto target = std::max(
315 const uint64_t target_sync_height = chainman.m_best_header->nHeight;
316
317 if (chain.m_chain.Height() < 0 || target == 0) {
318 return;
319 }
320 if (static_cast<uint64_t>(chain.m_chain.Height()) <= chainman.GetParams().PruneAfterHeight()) {
321 return;
322 }
323
324 const auto [min_block_to_prune, last_block_can_prune] = chainman.GetPruneRange(chain, last_prune);
325
326 uint64_t nCurrentUsage = CalculateCurrentUsage();
327 // We don't check to prune until after we've allocated new space for files
328 // So we should leave a buffer under our target to account for another allocation
329 // before the next pruning.
330 uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE;
331 uint64_t nBytesToPrune;
332 int count = 0;
333
334 if (nCurrentUsage + nBuffer >= target) {
335 // On a prune event, the chainstate DB is flushed.
336 // To avoid excessive prune events negating the benefit of high dbcache
337 // values, we should not prune too rapidly.
338 // So when pruning in IBD, increase the buffer to avoid a re-prune too soon.
339 const auto chain_tip_height = chain.m_chain.Height();
340 if (chainman.IsInitialBlockDownload() && target_sync_height > (uint64_t)chain_tip_height) {
341 // Since this is only relevant during IBD, we assume blocks are at least 1 MB on average
342 static constexpr uint64_t average_block_size = 1000000; /* 1 MB */
343 const uint64_t remaining_blocks = target_sync_height - chain_tip_height;
344 nBuffer += average_block_size * remaining_blocks;
345 }
346
347 for (int fileNumber = 0; fileNumber < this->MaxBlockfileNum(); fileNumber++) {
348 const auto& fileinfo = m_blockfile_info[fileNumber];
349 nBytesToPrune = fileinfo.nSize + fileinfo.nUndoSize;
350
351 if (fileinfo.nSize == 0) {
352 continue;
353 }
354
355 if (nCurrentUsage + nBuffer < target) { // are we below our target?
356 break;
357 }
358
359 // don't prune files that could have a block that's not within the allowable
360 // prune range for the chain being pruned.
361 if (fileinfo.nHeightLast > (unsigned)last_block_can_prune || fileinfo.nHeightFirst < (unsigned)min_block_to_prune) {
362 continue;
363 }
364
365 PruneOneBlockFile(fileNumber);
366 // Queue up the files for removal
367 setFilesToPrune.insert(fileNumber);
368 nCurrentUsage -= nBytesToPrune;
369 count++;
370 }
371 }
372
373 LogDebug(BCLog::PRUNE, "[%s] target=%dMiB actual=%dMiB diff=%dMiB min_height=%d max_prune_height=%d removed %d blk/rev pairs\n",
374 chain.GetRole(), target / 1024 / 1024, nCurrentUsage / 1024 / 1024,
375 (int64_t(target) - int64_t(nCurrentUsage)) / 1024 / 1024,
376 min_block_to_prune, last_block_can_prune, count);
377}
378
379void BlockManager::UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) {
381 m_prune_locks[name] = lock_info;
382}
383
385{
387
388 if (hash.IsNull()) {
389 return nullptr;
390 }
391
392 const auto [mi, inserted]{m_block_index.try_emplace(hash)};
393 CBlockIndex* pindex = &(*mi).second;
394 if (inserted) {
395 pindex->phashBlock = &((*mi).first);
396 }
397 return pindex;
398}
399
400bool BlockManager::LoadBlockIndex(const std::optional<uint256>& snapshot_blockhash)
401{
402 if (!m_block_tree_db->LoadBlockIndexGuts(
403 GetConsensus(), [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); }, m_interrupt)) {
404 return false;
405 }
406
407 if (snapshot_blockhash) {
408 const std::optional<AssumeutxoData> maybe_au_data = GetParams().AssumeutxoForBlockhash(*snapshot_blockhash);
409 if (!maybe_au_data) {
410 m_opts.notifications.fatalError(strprintf(_("Assumeutxo data not found for the given blockhash '%s'."), snapshot_blockhash->ToString()));
411 return false;
412 }
413 const AssumeutxoData& au_data = *Assert(maybe_au_data);
414 m_snapshot_height = au_data.height;
415 CBlockIndex* base{LookupBlockIndex(*snapshot_blockhash)};
416
417 // Since m_chain_tx_count (responsible for estimated progress) isn't persisted
418 // to disk, we must bootstrap the value for assumedvalid chainstates
419 // from the hardcoded assumeutxo chainparams.
420 base->m_chain_tx_count = au_data.m_chain_tx_count;
421 LogInfo("[snapshot] set m_chain_tx_count=%d for %s", au_data.m_chain_tx_count, snapshot_blockhash->ToString());
422 } else {
423 // If this isn't called with a snapshot blockhash, make sure the cached snapshot height
424 // is null. This is relevant during snapshot completion, when the blockman may be loaded
425 // with a height that then needs to be cleared after the snapshot is fully validated.
426 m_snapshot_height.reset();
427 }
428
429 Assert(m_snapshot_height.has_value() == snapshot_blockhash.has_value());
430
431 // Calculate nChainWork
432 std::vector<CBlockIndex*> vSortedByHeight{GetAllBlockIndices()};
433 std::sort(vSortedByHeight.begin(), vSortedByHeight.end(),
435
436 CBlockIndex* previous_index{nullptr};
437 for (CBlockIndex* pindex : vSortedByHeight) {
438 if (m_interrupt) return false;
439 if (previous_index && pindex->nHeight > previous_index->nHeight + 1) {
440 LogError("%s: block index is non-contiguous, index of height %d missing\n", __func__, previous_index->nHeight + 1);
441 return false;
442 }
443 previous_index = pindex;
444 pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex);
445 pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime);
446
447 // We can link the chain of blocks for which we've received transactions at some point, or
448 // blocks that are assumed-valid on the basis of snapshot load (see
449 // PopulateAndValidateSnapshot()).
450 // Pruned nodes may have deleted the block.
451 if (pindex->nTx > 0) {
452 if (pindex->pprev) {
453 if (m_snapshot_height && pindex->nHeight == *m_snapshot_height &&
454 pindex->GetBlockHash() == *snapshot_blockhash) {
455 // Should have been set above; don't disturb it with code below.
456 Assert(pindex->m_chain_tx_count > 0);
457 } else if (pindex->pprev->m_chain_tx_count > 0) {
458 pindex->m_chain_tx_count = pindex->pprev->m_chain_tx_count + pindex->nTx;
459 } else {
460 pindex->m_chain_tx_count = 0;
461 m_blocks_unlinked.insert(std::make_pair(pindex->pprev, pindex));
462 }
463 } else {
464 pindex->m_chain_tx_count = pindex->nTx;
465 }
466 }
467 if (!(pindex->nStatus & BLOCK_FAILED_MASK) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_MASK)) {
468 pindex->nStatus |= BLOCK_FAILED_CHILD;
469 m_dirty_blockindex.insert(pindex);
470 }
471 if (pindex->pprev) {
472 pindex->BuildSkip();
473 }
474 }
475
476 return true;
477}
478
479bool BlockManager::WriteBlockIndexDB()
480{
482 std::vector<std::pair<int, const CBlockFileInfo*>> vFiles;
483 vFiles.reserve(m_dirty_fileinfo.size());
484 for (std::set<int>::iterator it = m_dirty_fileinfo.begin(); it != m_dirty_fileinfo.end();) {
485 vFiles.emplace_back(*it, &m_blockfile_info[*it]);
486 m_dirty_fileinfo.erase(it++);
487 }
488 std::vector<const CBlockIndex*> vBlocks;
489 vBlocks.reserve(m_dirty_blockindex.size());
490 for (std::set<CBlockIndex*>::iterator it = m_dirty_blockindex.begin(); it != m_dirty_blockindex.end();) {
491 vBlocks.push_back(*it);
492 m_dirty_blockindex.erase(it++);
493 }
494 int max_blockfile = WITH_LOCK(cs_LastBlockFile, return this->MaxBlockfileNum());
495 if (!m_block_tree_db->WriteBatchSync(vFiles, max_blockfile, vBlocks)) {
496 return false;
497 }
498 return true;
499}
500
501bool BlockManager::LoadBlockIndexDB(const std::optional<uint256>& snapshot_blockhash)
502{
503 if (!LoadBlockIndex(snapshot_blockhash)) {
504 return false;
505 }
506 int max_blockfile_num{0};
507
508 // Load block file info
509 m_block_tree_db->ReadLastBlockFile(max_blockfile_num);
510 m_blockfile_info.resize(max_blockfile_num + 1);
511 LogInfo("Loading block index db: last block file = %i", max_blockfile_num);
512 for (int nFile = 0; nFile <= max_blockfile_num; nFile++) {
513 m_block_tree_db->ReadBlockFileInfo(nFile, m_blockfile_info[nFile]);
514 }
515 LogInfo("Loading block index db: last block file info: %s", m_blockfile_info[max_blockfile_num].ToString());
516 for (int nFile = max_blockfile_num + 1; true; nFile++) {
517 CBlockFileInfo info;
518 if (m_block_tree_db->ReadBlockFileInfo(nFile, info)) {
519 m_blockfile_info.push_back(info);
520 } else {
521 break;
522 }
523 }
524
525 // Check presence of blk files
526 LogInfo("Checking all blk files are present...");
527 std::set<int> setBlkDataFiles;
528 for (const auto& [_, block_index] : m_block_index) {
529 if (block_index.nStatus & BLOCK_HAVE_DATA) {
530 setBlkDataFiles.insert(block_index.nFile);
531 }
532 }
533 for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) {
534 FlatFilePos pos(*it, 0);
535 if (OpenBlockFile(pos, /*fReadOnly=*/true).IsNull()) {
536 return false;
537 }
538 }
539
540 {
541 // Initialize the blockfile cursors.
543 for (size_t i = 0; i < m_blockfile_info.size(); ++i) {
544 const auto last_height_in_file = m_blockfile_info[i].nHeightLast;
545 m_blockfile_cursors[BlockfileTypeForHeight(last_height_in_file)] = {static_cast<int>(i), 0};
546 }
547 }
548
549 // Check whether we have ever pruned block & undo files
550 m_block_tree_db->ReadFlag("prunedblockfiles", m_have_pruned);
551 if (m_have_pruned) {
552 LogInfo("Loading block index db: Block files have previously been pruned");
553 }
554
555 // Check whether we need to continue reindexing
556 bool fReindexing = false;
557 m_block_tree_db->ReadReindexing(fReindexing);
558 if (fReindexing) m_blockfiles_indexed = false;
559
560 return true;
561}
562
563void BlockManager::ScanAndUnlinkAlreadyPrunedFiles()
564{
566 int max_blockfile = WITH_LOCK(cs_LastBlockFile, return this->MaxBlockfileNum());
567 if (!m_have_pruned) {
568 return;
569 }
570
571 std::set<int> block_files_to_prune;
572 for (int file_number = 0; file_number < max_blockfile; file_number++) {
573 if (m_blockfile_info[file_number].nSize == 0) {
574 block_files_to_prune.insert(file_number);
575 }
576 }
577
578 UnlinkPrunedFiles(block_files_to_prune);
579}
580
581bool BlockManager::IsBlockPruned(const CBlockIndex& block) const
582{
584 return m_have_pruned && !(block.nStatus & BLOCK_HAVE_DATA) && (block.nTx > 0);
585}
586
587const CBlockIndex* BlockManager::GetFirstBlock(const CBlockIndex& upper_block, uint32_t status_mask, const CBlockIndex* lower_block) const
588{
590 const CBlockIndex* last_block = &upper_block;
591 assert((last_block->nStatus & status_mask) == status_mask); // 'upper_block' must satisfy the status mask
592 while (last_block->pprev && ((last_block->pprev->nStatus & status_mask) == status_mask)) {
593 if (lower_block) {
594 // Return if we reached the lower_block
595 if (last_block == lower_block) return lower_block;
596 // if range was surpassed, means that 'lower_block' is not part of the 'upper_block' chain
597 // and so far this is not allowed.
598 assert(last_block->nHeight >= lower_block->nHeight);
599 }
600 last_block = last_block->pprev;
601 }
602 assert(last_block != nullptr);
603 return last_block;
604}
605
606bool BlockManager::CheckBlockDataAvailability(const CBlockIndex& upper_block, const CBlockIndex& lower_block)
607{
608 if (!(upper_block.nStatus & BLOCK_HAVE_DATA)) return false;
609 return GetFirstBlock(upper_block, BLOCK_HAVE_DATA, &lower_block) == &lower_block;
610}
611
612// If we're using -prune with -reindex, then delete block files that will be ignored by the
613// reindex. Since reindexing works by starting at block file 0 and looping until a blockfile
614// is missing, do the same here to delete any later block files after a gap. Also delete all
615// rev files since they'll be rewritten by the reindex anyway. This ensures that m_blockfile_info
616// is in sync with what's actually on disk by the time we start downloading, so that pruning
617// works correctly.
619{
620 std::map<std::string, fs::path> mapBlockFiles;
621
622 // Glob all blk?????.dat and rev?????.dat files from the blocks directory.
623 // Remove the rev files immediately and insert the blk file paths into an
624 // ordered map keyed by block file index.
625 LogInfo("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune");
626 for (fs::directory_iterator it(m_opts.blocks_dir); it != fs::directory_iterator(); it++) {
627 const std::string path = fs::PathToString(it->path().filename());
628 if (fs::is_regular_file(*it) &&
629 path.length() == 12 &&
630 path.ends_with(".dat"))
631 {
632 if (path.starts_with("blk")) {
633 mapBlockFiles[path.substr(3, 5)] = it->path();
634 } else if (path.starts_with("rev")) {
635 remove(it->path());
636 }
637 }
638 }
639
640 // Remove all block files that aren't part of a contiguous set starting at
641 // zero by walking the ordered map (keys are block file indices) by
642 // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist)
643 // start removing block files.
644 int nContigCounter = 0;
645 for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) {
646 if (LocaleIndependentAtoi<int>(item.first) == nContigCounter) {
647 nContigCounter++;
648 continue;
649 }
650 remove(item.second);
651 }
652}
653
655{
657
658 return &m_blockfile_info.at(n);
659}
660
661bool BlockManager::ReadBlockUndo(CBlockUndo& blockundo, const CBlockIndex& index) const
662{
663 const FlatFilePos pos{WITH_LOCK(::cs_main, return index.GetUndoPos())};
664
665 // Open history file to read
666 AutoFile file{OpenUndoFile(pos, true)};
667 if (file.IsNull()) {
668 LogError("OpenUndoFile failed for %s while reading block undo", pos.ToString());
669 return false;
670 }
671 BufferedReader filein{std::move(file)};
672
673 try {
674 // Read block
675 HashVerifier verifier{filein}; // Use HashVerifier, as reserializing may lose data, c.f. commit d3424243
676
677 verifier << index.pprev->GetBlockHash();
678 verifier >> blockundo;
679
680 uint256 hashChecksum;
681 filein >> hashChecksum;
682
683 // Verify checksum
684 if (hashChecksum != verifier.GetHash()) {
685 LogError("Checksum mismatch at %s while reading block undo", pos.ToString());
686 return false;
687 }
688 } catch (const std::exception& e) {
689 LogError("Deserialize or I/O error - %s at %s while reading block undo", e.what(), pos.ToString());
690 return false;
691 }
692
693 return true;
694}
695
696bool BlockManager::FlushUndoFile(int block_file, bool finalize)
697{
698 FlatFilePos undo_pos_old(block_file, m_blockfile_info[block_file].nUndoSize);
699 if (!m_undo_file_seq.Flush(undo_pos_old, finalize)) {
700 m_opts.notifications.flushError(_("Flushing undo file to disk failed. This is likely the result of an I/O error."));
701 return false;
702 }
703 return true;
704}
705
706bool BlockManager::FlushBlockFile(int blockfile_num, bool fFinalize, bool finalize_undo)
707{
708 bool success = true;
710
711 if (m_blockfile_info.size() < 1) {
712 // Return if we haven't loaded any blockfiles yet. This happens during
713 // chainstate init, when we call ChainstateManager::MaybeRebalanceCaches() (which
714 // then calls FlushStateToDisk()), resulting in a call to this function before we
715 // have populated `m_blockfile_info` via LoadBlockIndexDB().
716 return true;
717 }
718 assert(static_cast<int>(m_blockfile_info.size()) > blockfile_num);
719
720 FlatFilePos block_pos_old(blockfile_num, m_blockfile_info[blockfile_num].nSize);
721 if (!m_block_file_seq.Flush(block_pos_old, fFinalize)) {
722 m_opts.notifications.flushError(_("Flushing block file to disk failed. This is likely the result of an I/O error."));
723 success = false;
724 }
725 // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks,
726 // e.g. during IBD or a sync after a node going offline
727 if (!fFinalize || finalize_undo) {
728 if (!FlushUndoFile(blockfile_num, finalize_undo)) {
729 success = false;
730 }
731 }
732 return success;
733}
734
736{
737 if (!m_snapshot_height) {
738 return BlockfileType::NORMAL;
739 }
740 return (height >= *m_snapshot_height) ? BlockfileType::ASSUMED : BlockfileType::NORMAL;
741}
742
744{
746 auto& cursor = m_blockfile_cursors[BlockfileTypeForHeight(tip_height)];
747 // If the cursor does not exist, it means an assumeutxo snapshot is loaded,
748 // but no blocks past the snapshot height have been written yet, so there
749 // is no data associated with the chainstate, and it is safe not to flush.
750 if (cursor) {
751 return FlushBlockFile(cursor->file_num, /*fFinalize=*/false, /*finalize_undo=*/false);
752 }
753 // No need to log warnings in this case.
754 return true;
755}
756
758{
760
761 uint64_t retval = 0;
762 for (const CBlockFileInfo& file : m_blockfile_info) {
763 retval += file.nSize + file.nUndoSize;
764 }
765 return retval;
766}
767
768void BlockManager::UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const
769{
770 std::error_code ec;
771 for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) {
772 FlatFilePos pos(*it, 0);
773 const bool removed_blockfile{fs::remove(m_block_file_seq.FileName(pos), ec)};
774 const bool removed_undofile{fs::remove(m_undo_file_seq.FileName(pos), ec)};
775 if (removed_blockfile || removed_undofile) {
776 LogDebug(BCLog::BLOCKSTORAGE, "Prune: %s deleted blk/rev (%05u)\n", __func__, *it);
777 }
778 }
779}
780
781AutoFile BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const
782{
783 return AutoFile{m_block_file_seq.Open(pos, fReadOnly), m_obfuscation};
784}
785
787AutoFile BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const
788{
789 return AutoFile{m_undo_file_seq.Open(pos, fReadOnly), m_obfuscation};
790}
791
793{
794 return m_block_file_seq.FileName(pos);
795}
796
797FlatFilePos BlockManager::FindNextBlockPos(unsigned int nAddSize, unsigned int nHeight, uint64_t nTime)
798{
800
801 const BlockfileType chain_type = BlockfileTypeForHeight(nHeight);
802
803 if (!m_blockfile_cursors[chain_type]) {
804 // If a snapshot is loaded during runtime, we may not have initialized this cursor yet.
805 assert(chain_type == BlockfileType::ASSUMED);
806 const auto new_cursor = BlockfileCursor{this->MaxBlockfileNum() + 1};
807 m_blockfile_cursors[chain_type] = new_cursor;
808 LogDebug(BCLog::BLOCKSTORAGE, "[%s] initializing blockfile cursor to %s\n", chain_type, new_cursor);
809 }
810 const int last_blockfile = m_blockfile_cursors[chain_type]->file_num;
811
812 int nFile = last_blockfile;
813 if (static_cast<int>(m_blockfile_info.size()) <= nFile) {
814 m_blockfile_info.resize(nFile + 1);
815 }
816
817 bool finalize_undo = false;
818 unsigned int max_blockfile_size{MAX_BLOCKFILE_SIZE};
819 // Use smaller blockfiles in test-only -fastprune mode - but avoid
820 // the possibility of having a block not fit into the block file.
821 if (m_opts.fast_prune) {
822 max_blockfile_size = 0x10000; // 64kiB
823 if (nAddSize >= max_blockfile_size) {
824 // dynamically adjust the blockfile size to be larger than the added size
825 max_blockfile_size = nAddSize + 1;
826 }
827 }
828 assert(nAddSize < max_blockfile_size);
829
830 while (m_blockfile_info[nFile].nSize + nAddSize >= max_blockfile_size) {
831 // when the undo file is keeping up with the block file, we want to flush it explicitly
832 // when it is lagging behind (more blocks arrive than are being connected), we let the
833 // undo block write case handle it
834 finalize_undo = (static_cast<int>(m_blockfile_info[nFile].nHeightLast) ==
835 Assert(m_blockfile_cursors[chain_type])->undo_height);
836
837 // Try the next unclaimed blockfile number
838 nFile = this->MaxBlockfileNum() + 1;
839 // Set to increment MaxBlockfileNum() for next iteration
840 m_blockfile_cursors[chain_type] = BlockfileCursor{nFile};
841
842 if (static_cast<int>(m_blockfile_info.size()) <= nFile) {
843 m_blockfile_info.resize(nFile + 1);
844 }
845 }
846 FlatFilePos pos;
847 pos.nFile = nFile;
848 pos.nPos = m_blockfile_info[nFile].nSize;
849
850 if (nFile != last_blockfile) {
851 LogDebug(BCLog::BLOCKSTORAGE, "Leaving block file %i: %s (onto %i) (height %i)\n",
852 last_blockfile, m_blockfile_info[last_blockfile].ToString(), nFile, nHeight);
853
854 // Do not propagate the return code. The flush concerns a previous block
855 // and undo file that has already been written to. If a flush fails
856 // here, and we crash, there is no expected additional block data
857 // inconsistency arising from the flush failure here. However, the undo
858 // data may be inconsistent after a crash if the flush is called during
859 // a reindex. A flush error might also leave some of the data files
860 // untrimmed.
861 if (!FlushBlockFile(last_blockfile, /*fFinalize=*/true, finalize_undo)) {
863 "Failed to flush previous block file %05i (finalize=1, finalize_undo=%i) before opening new block file %05i\n",
864 last_blockfile, finalize_undo, nFile);
865 }
866 // No undo data yet in the new file, so reset our undo-height tracking.
867 m_blockfile_cursors[chain_type] = BlockfileCursor{nFile};
868 }
869
870 m_blockfile_info[nFile].AddBlock(nHeight, nTime);
871 m_blockfile_info[nFile].nSize += nAddSize;
872
873 bool out_of_space;
874 size_t bytes_allocated = m_block_file_seq.Allocate(pos, nAddSize, out_of_space);
875 if (out_of_space) {
876 m_opts.notifications.fatalError(_("Disk space is too low!"));
877 return {};
878 }
879 if (bytes_allocated != 0 && IsPruneMode()) {
880 m_check_for_pruning = true;
881 }
882
883 m_dirty_fileinfo.insert(nFile);
884 return pos;
885}
886
887void BlockManager::UpdateBlockInfo(const CBlock& block, unsigned int nHeight, const FlatFilePos& pos)
888{
890
891 // Update the cursor so it points to the last file.
893 auto& cursor{m_blockfile_cursors[chain_type]};
894 if (!cursor || cursor->file_num < pos.nFile) {
895 m_blockfile_cursors[chain_type] = BlockfileCursor{pos.nFile};
896 }
897
898 // Update the file information with the current block.
899 const unsigned int added_size = ::GetSerializeSize(TX_WITH_WITNESS(block));
900 const int nFile = pos.nFile;
901 if (static_cast<int>(m_blockfile_info.size()) <= nFile) {
902 m_blockfile_info.resize(nFile + 1);
903 }
904 m_blockfile_info[nFile].AddBlock(nHeight, block.GetBlockTime());
905 m_blockfile_info[nFile].nSize = std::max(pos.nPos + added_size, m_blockfile_info[nFile].nSize);
906 m_dirty_fileinfo.insert(nFile);
907}
908
909bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize)
910{
911 pos.nFile = nFile;
912
914
915 pos.nPos = m_blockfile_info[nFile].nUndoSize;
916 m_blockfile_info[nFile].nUndoSize += nAddSize;
917 m_dirty_fileinfo.insert(nFile);
918
919 bool out_of_space;
920 size_t bytes_allocated = m_undo_file_seq.Allocate(pos, nAddSize, out_of_space);
921 if (out_of_space) {
922 return FatalError(m_opts.notifications, state, _("Disk space is too low!"));
923 }
924 if (bytes_allocated != 0 && IsPruneMode()) {
925 m_check_for_pruning = true;
926 }
927
928 return true;
929}
930
931bool BlockManager::WriteBlockUndo(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
932{
934 const BlockfileType type = BlockfileTypeForHeight(block.nHeight);
935 auto& cursor = *Assert(WITH_LOCK(cs_LastBlockFile, return m_blockfile_cursors[type]));
936
937 // Write undo information to disk
938 if (block.GetUndoPos().IsNull()) {
939 FlatFilePos pos;
940 const auto blockundo_size{static_cast<uint32_t>(GetSerializeSize(blockundo))};
941 if (!FindUndoPos(state, block.nFile, pos, blockundo_size + UNDO_DATA_DISK_OVERHEAD)) {
942 LogError("FindUndoPos failed for %s while writing block undo", pos.ToString());
943 return false;
944 }
945
946 // Open history file to append
947 AutoFile file{OpenUndoFile(pos)};
948 if (file.IsNull()) {
949 LogError("OpenUndoFile failed for %s while writing block undo", pos.ToString());
950 return FatalError(m_opts.notifications, state, _("Failed to write undo data."));
951 }
952 {
953 BufferedWriter fileout{file};
954
955 // Write index header
956 fileout << GetParams().MessageStart() << blockundo_size;
958 {
959 // Calculate checksum
960 HashWriter hasher{};
961 hasher << block.pprev->GetBlockHash() << blockundo;
962 // Write undo data & checksum
963 fileout << blockundo << hasher.GetHash();
964 }
965 // BufferedWriter will flush pending data to file when fileout goes out of scope.
966 }
967
968 // Make sure that the file is closed before we call `FlushUndoFile`.
969 if (file.fclose() != 0) {
970 LogError("Failed to close block undo file %s: %s", pos.ToString(), SysErrorString(errno));
971 return FatalError(m_opts.notifications, state, _("Failed to close block undo file."));
972 }
973
974 // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order)
975 // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height
976 // in the block file info as below; note that this does not catch the case where the undo writes are keeping up
977 // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in
978 // the FindNextBlockPos function
979 if (pos.nFile < cursor.file_num && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[pos.nFile].nHeightLast) {
980 // Do not propagate the return code, a failed flush here should not
981 // be an indication for a failed write. If it were propagated here,
982 // the caller would assume the undo data not to be written, when in
983 // fact it is. Note though, that a failed flush might leave the data
984 // file untrimmed.
985 if (!FlushUndoFile(pos.nFile, true)) {
986 LogPrintLevel(BCLog::BLOCKSTORAGE, BCLog::Level::Warning, "Failed to flush undo file %05i\n", pos.nFile);
987 }
988 } else if (pos.nFile == cursor.file_num && block.nHeight > cursor.undo_height) {
989 cursor.undo_height = block.nHeight;
990 }
991 // update nUndoPos in block index
992 block.nUndoPos = pos.nPos;
993 block.nStatus |= BLOCK_HAVE_UNDO;
994 m_dirty_blockindex.insert(&block);
995 }
996
997 return true;
998}
999
1000bool BlockManager::ReadBlock(CBlock& block, const FlatFilePos& pos, const std::optional<uint256>& expected_hash) const
1001{
1002 block.SetNull();
1003
1004 // Open history file to read
1005 std::vector<std::byte> block_data;
1006 if (!ReadRawBlock(block_data, pos)) {
1007 return false;
1008 }
1009
1010 try {
1011 // Read block
1012 SpanReader{block_data} >> TX_WITH_WITNESS(block);
1013 } catch (const std::exception& e) {
1014 LogError("Deserialize or I/O error - %s at %s while reading block", e.what(), pos.ToString());
1015 return false;
1016 }
1017
1018 const auto block_hash{block.GetHash()};
1019
1020 // Check the header
1021 if (!CheckProofOfWork(block_hash, block.nBits, GetConsensus())) {
1022 LogError("Errors in block header at %s while reading block", pos.ToString());
1023 return false;
1024 }
1025
1026 // Signet only: check block solution
1027 if (GetConsensus().signet_blocks && !CheckSignetBlockSolution(block, GetConsensus())) {
1028 LogError("Errors in block solution at %s while reading block", pos.ToString());
1029 return false;
1030 }
1031
1032 if (expected_hash && block_hash != *expected_hash) {
1033 LogError("GetHash() doesn't match index at %s while reading block (%s != %s)",
1034 pos.ToString(), block_hash.ToString(), expected_hash->ToString());
1035 return false;
1036 }
1037
1038 return true;
1039}
1040
1041bool BlockManager::ReadBlock(CBlock& block, const CBlockIndex& index) const
1042{
1043 const FlatFilePos block_pos{WITH_LOCK(cs_main, return index.GetBlockPos())};
1044 return ReadBlock(block, block_pos, index.GetBlockHash());
1045}
1046
1047bool BlockManager::ReadRawBlock(std::vector<std::byte>& block, const FlatFilePos& pos) const
1048{
1049 if (pos.nPos < STORAGE_HEADER_BYTES) {
1050 // If nPos is less than STORAGE_HEADER_BYTES, we can't read the header that precedes the block data
1051 // This would cause an unsigned integer underflow when trying to position the file cursor
1052 // This can happen after pruning or default constructed positions
1053 LogError("Failed for %s while reading raw block storage header", pos.ToString());
1054 return false;
1055 }
1056 AutoFile filein{OpenBlockFile({pos.nFile, pos.nPos - STORAGE_HEADER_BYTES}, /*fReadOnly=*/true)};
1057 if (filein.IsNull()) {
1058 LogError("OpenBlockFile failed for %s while reading raw block", pos.ToString());
1059 return false;
1060 }
1061
1062 try {
1063 MessageStartChars blk_start;
1064 unsigned int blk_size;
1065
1066 filein >> blk_start >> blk_size;
1067
1068 if (blk_start != GetParams().MessageStart()) {
1069 LogError("Block magic mismatch for %s: %s versus expected %s while reading raw block",
1070 pos.ToString(), HexStr(blk_start), HexStr(GetParams().MessageStart()));
1071 return false;
1072 }
1073
1074 if (blk_size > MAX_SIZE) {
1075 LogError("Block data is larger than maximum deserialization size for %s: %s versus %s while reading raw block",
1076 pos.ToString(), blk_size, MAX_SIZE);
1077 return false;
1078 }
1079
1080 block.resize(blk_size); // Zeroing of memory is intentional here
1081 filein.read(block);
1082 } catch (const std::exception& e) {
1083 LogError("Read from block file failed: %s for %s while reading raw block", e.what(), pos.ToString());
1084 return false;
1085 }
1086
1087 return true;
1088}
1089
1091{
1092 const unsigned int block_size{static_cast<unsigned int>(GetSerializeSize(TX_WITH_WITNESS(block)))};
1094 if (pos.IsNull()) {
1095 LogError("FindNextBlockPos failed for %s while writing block", pos.ToString());
1096 return FlatFilePos();
1097 }
1098 AutoFile file{OpenBlockFile(pos, /*fReadOnly=*/false)};
1099 if (file.IsNull()) {
1100 LogError("OpenBlockFile failed for %s while writing block", pos.ToString());
1101 m_opts.notifications.fatalError(_("Failed to write block."));
1102 return FlatFilePos();
1103 }
1104 {
1105 BufferedWriter fileout{file};
1106
1107 // Write index header
1108 fileout << GetParams().MessageStart() << block_size;
1110 // Write block
1111 fileout << TX_WITH_WITNESS(block);
1112 }
1113
1114 if (file.fclose() != 0) {
1115 LogError("Failed to close block file %s: %s", pos.ToString(), SysErrorString(errno));
1116 m_opts.notifications.fatalError(_("Failed to close file when writing block."));
1117 return FlatFilePos();
1118 }
1119
1120 return pos;
1121}
1122
1124{
1125 // Bytes are serialized without length indicator, so this is also the exact
1126 // size of the XOR-key file.
1127 std::array<std::byte, Obfuscation::KEY_SIZE> obfuscation{};
1128
1129 // Consider this to be the first run if the blocksdir contains only hidden
1130 // files (those which start with a .). Checking for a fully-empty dir would
1131 // be too aggressive as a .lock file may have already been written.
1132 bool first_run = true;
1133 for (const auto& entry : fs::directory_iterator(opts.blocks_dir)) {
1134 const std::string path = fs::PathToString(entry.path().filename());
1135 if (!entry.is_regular_file() || !path.starts_with('.')) {
1136 first_run = false;
1137 break;
1138 }
1139 }
1140
1141 if (opts.use_xor && first_run) {
1142 // Only use random fresh key when the boolean option is set and on the
1143 // very first start of the program.
1144 FastRandomContext{}.fillrand(obfuscation);
1145 }
1146
1147 const fs::path xor_key_path{opts.blocks_dir / "xor.dat"};
1148 if (fs::exists(xor_key_path)) {
1149 // A pre-existing xor key file has priority.
1150 AutoFile xor_key_file{fsbridge::fopen(xor_key_path, "rb")};
1151 xor_key_file >> obfuscation;
1152 } else {
1153 // Create initial or missing xor key file
1154 AutoFile xor_key_file{fsbridge::fopen(xor_key_path,
1155#ifdef __MINGW64__
1156 "wb" // Temporary workaround for https://github.com/bitcoin/bitcoin/issues/30210
1157#else
1158 "wbx"
1159#endif
1160 )};
1161 xor_key_file << obfuscation;
1162 if (xor_key_file.fclose() != 0) {
1163 throw std::runtime_error{strprintf("Error closing XOR key file %s: %s",
1164 fs::PathToString(xor_key_path),
1165 SysErrorString(errno))};
1166 }
1167 }
1168 // If the user disabled the key, it must be zero.
1169 if (!opts.use_xor && obfuscation != decltype(obfuscation){}) {
1170 throw std::runtime_error{
1171 strprintf("The blocksdir XOR-key can not be disabled when a random key was already stored! "
1172 "Stored key: '%s', stored path: '%s'.",
1173 HexStr(obfuscation), fs::PathToString(xor_key_path)),
1174 };
1175 }
1176 LogInfo("Using obfuscation key for blocksdir *.dat files (%s): '%s'\n", fs::PathToString(opts.blocks_dir), HexStr(obfuscation));
1177 return Obfuscation{obfuscation};
1178}
1179
1181 : m_prune_mode{opts.prune_target > 0},
1182 m_obfuscation{InitBlocksdirXorKey(opts)},
1183 m_opts{std::move(opts)},
1184 m_block_file_seq{FlatFileSeq{m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kB */ : BLOCKFILE_CHUNK_SIZE}},
1185 m_undo_file_seq{FlatFileSeq{m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE}},
1186 m_interrupt{interrupt}
1187{
1188 m_block_tree_db = std::make_unique<BlockTreeDB>(m_opts.block_tree_db_params);
1189
1191 m_block_tree_db->WriteReindexing(true);
1192 m_blockfiles_indexed = false;
1193 // If we're reindexing in prune mode, wipe away unusable block files and all undo data files
1194 if (m_prune_mode) {
1196 }
1197 }
1198}
1199
1201{
1202 std::atomic<bool>& m_importing;
1203
1204public:
1205 ImportingNow(std::atomic<bool>& importing) : m_importing{importing}
1206 {
1207 assert(m_importing == false);
1208 m_importing = true;
1209 }
1211 {
1212 assert(m_importing == true);
1213 m_importing = false;
1214 }
1215};
1216
1217void ImportBlocks(ChainstateManager& chainman, std::span<const fs::path> import_paths)
1218{
1219 ImportingNow imp{chainman.m_blockman.m_importing};
1220
1221 // -reindex
1222 if (!chainman.m_blockman.m_blockfiles_indexed) {
1223 int nFile = 0;
1224 // Map of disk positions for blocks with unknown parent (only used for reindex);
1225 // parent hash -> child disk position, multiple children can have the same parent.
1226 std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent;
1227 while (true) {
1228 FlatFilePos pos(nFile, 0);
1229 if (!fs::exists(chainman.m_blockman.GetBlockPosFilename(pos))) {
1230 break; // No block files left to reindex
1231 }
1232 AutoFile file{chainman.m_blockman.OpenBlockFile(pos, /*fReadOnly=*/true)};
1233 if (file.IsNull()) {
1234 break; // This error is logged in OpenBlockFile
1235 }
1236 LogInfo("Reindexing block file blk%05u.dat...", (unsigned int)nFile);
1237 chainman.LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent);
1238 if (chainman.m_interrupt) {
1239 LogInfo("Interrupt requested. Exit reindexing.");
1240 return;
1241 }
1242 nFile++;
1243 }
1244 WITH_LOCK(::cs_main, chainman.m_blockman.m_block_tree_db->WriteReindexing(false));
1245 chainman.m_blockman.m_blockfiles_indexed = true;
1246 LogInfo("Reindexing finished");
1247 // To avoid ending up in a situation without genesis block, re-try initializing (no-op if reindexing worked):
1248 chainman.ActiveChainstate().LoadGenesisBlock();
1249 }
1250
1251 // -loadblock=
1252 for (const fs::path& path : import_paths) {
1253 AutoFile file{fsbridge::fopen(path, "rb")};
1254 if (!file.IsNull()) {
1255 LogInfo("Importing blocks file %s...", fs::PathToString(path));
1256 chainman.LoadExternalBlockFile(file);
1257 if (chainman.m_interrupt) {
1258 LogInfo("Interrupt requested. Exit block importing.");
1259 return;
1260 }
1261 } else {
1262 LogPrintf("Warning: Could not open blocks file %s\n", fs::PathToString(path));
1263 }
1264 }
1265
1266 // scan for better chains in the block chain database, that are not yet connected in the active best chain
1267
1268 // We can't hold cs_main during ActivateBestChain even though we're accessing
1269 // the chainman unique_ptrs since ABC requires us not to be holding cs_main, so retrieve
1270 // the relevant pointers before the ABC call.
1271 for (Chainstate* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) {
1273 if (!chainstate->ActivateBestChain(state, nullptr)) {
1274 chainman.GetNotifications().fatalError(strprintf(_("Failed to connect best block (%s)."), state.ToString()));
1275 return;
1276 }
1277 }
1278 // End scope of ImportingNow
1279}
1280
1281std::ostream& operator<<(std::ostream& os, const BlockfileType& type) {
1282 switch(type) {
1283 case BlockfileType::NORMAL: os << "normal"; break;
1284 case BlockfileType::ASSUMED: os << "assumed"; break;
1285 default: os.setstate(std::ios_base::failbit);
1286 }
1287 return os;
1288}
1289
1290std::ostream& operator<<(std::ostream& os, const BlockfileCursor& cursor) {
1291 os << strprintf("BlockfileCursor(file_num=%d, undo_height=%d)", cursor.file_num, cursor.undo_height);
1292 return os;
1293}
1294} // namespace node
void CheckBlockDataAvailability(BlockManager &blockman, const CBlockIndex &blockindex, bool check_for_undo)
Definition: blockchain.cpp:646
arith_uint256 GetBlockProof(const CBlockIndex &block)
Definition: chain.cpp:126
@ BLOCK_VALID_TREE
All parent headers found, difficulty matches, timestamp >= median previous.
Definition: chain.h:97
@ BLOCK_HAVE_UNDO
undo data available in rev*.dat
Definition: chain.h:122
@ BLOCK_HAVE_DATA
full block available in blk*.dat
Definition: chain.h:121
@ BLOCK_FAILED_CHILD
descends from failed block
Definition: chain.h:126
@ BLOCK_FAILED_MASK
Definition: chain.h:127
#define Assert(val)
Identity function.
Definition: check.h:106
Non-refcounted RAII wrapper for FILE*.
Definition: streams.h:371
Wrapper that buffers reads from an underlying stream.
Definition: streams.h:625
Wrapper that buffers writes to an underlying stream.
Definition: streams.h:667
Nodes collect new transactions into a block, hash them into a hash tree, and scan through nonce value...
Definition: block.h:22
uint32_t nBits
Definition: block.h:29
int64_t GetBlockTime() const
Definition: block.h:61
uint256 hashPrevBlock
Definition: block.h:26
uint256 GetHash() const
Definition: block.cpp:11
Definition: block.h:69
void SetNull()
Definition: block.h:95
The block chain is a tree shaped structure starting with the genesis block at the root,...
Definition: chain.h:141
uint256 hashMerkleRoot
Definition: chain.h:188
std::string ToString() const
Definition: chain.cpp:15
CBlockIndex * pprev
pointer to the index of the predecessor of this block
Definition: chain.h:147
uint64_t m_chain_tx_count
(memory only) Number of transactions in the chain up to and including this block.
Definition: chain.h:176
void BuildSkip()
Build the skiplist pointer for this entry.
Definition: chain.cpp:120
arith_uint256 nChainWork
(memory only) Total amount of work (expected number of hashes) in the chain up to and including this ...
Definition: chain.h:165
uint32_t nTime
Definition: chain.h:189
unsigned int nTimeMax
(memory only) Maximum nTime in the chain up to and including this block.
Definition: chain.h:197
int32_t nSequenceId
(memory only) Sequential id assigned to distinguish order in which blocks are received.
Definition: chain.h:194
uint32_t nNonce
Definition: chain.h:191
uint256 GetBlockHash() const
Definition: chain.h:243
FlatFilePos GetUndoPos() const EXCLUSIVE_LOCKS_REQUIRED(
Definition: chain.h:219
uint32_t nBits
Definition: chain.h:190
bool RaiseValidity(enum BlockStatus nUpTo) EXCLUSIVE_LOCKS_REQUIRED(
Raise the validity level of this block index entry.
Definition: chain.h:307
unsigned int nTx
Number of transactions in this block.
Definition: chain.h:170
int32_t nVersion
block header
Definition: chain.h:187
int nHeight
height of the entry in the chain. The genesis block has height 0
Definition: chain.h:153
FlatFilePos GetBlockPos() const EXCLUSIVE_LOCKS_REQUIRED(
Definition: chain.h:208
const uint256 * phashBlock
pointer to the hash of the block, if any. Memory is owned by this CBlockIndex
Definition: chain.h:144
Undo information for a CBlock.
Definition: undo.h:63
int Height() const
Return the maximal height in the chain.
Definition: chain.h:462
const MessageStartChars & MessageStart() const
Definition: chainparams.h:82
std::optional< AssumeutxoData > AssumeutxoForBlockhash(const uint256 &blockhash) const
Definition: chainparams.h:114
uint64_t PruneAfterHeight() const
Definition: chainparams.h:93
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:72
void Write(const K &key, const V &value)
Definition: dbwrapper.h:96
bool WriteBatch(CDBBatch &batch, bool fSync=false)
Definition: dbwrapper.cpp:277
bool Read(const K &key, V &value) const
Definition: dbwrapper.h:213
CDBIterator * NewIterator()
Definition: dbwrapper.cpp:360
bool Erase(const K &key, bool fSync=false)
Definition: dbwrapper.h:258
bool Write(const K &key, const V &value, bool fSync=false)
Definition: dbwrapper.h:233
bool Exists(const K &key) const
Definition: dbwrapper.h:249
Used to marshal pointers into hashes for db storage.
Definition: chain.h:355
uint256 hashPrev
Definition: chain.h:365
uint256 ConstructBlockHash() const
Definition: chain.h:399
Chainstate stores and provides an API to update our local knowledge of the current best chain.
Definition: validation.h:531
CChain m_chain
The current chain of blockheaders we consult and build on.
Definition: validation.h:613
Provides an interface for creating and interacting with one or two chainstates: an IBD chainstate gen...
Definition: validation.h:898
SnapshotCompletionResult MaybeCompleteSnapshotValidation() EXCLUSIVE_LOCKS_REQUIRED(const CBlockIndex *GetSnapshotBaseBlock() const EXCLUSIVE_LOCKS_REQUIRED(Chainstate ActiveChainstate)() const
Once the background validation chainstate has reached the height which is the base of the UTXO snapsh...
Definition: validation.h:1122
kernel::Notifications & GetNotifications() const
Definition: validation.h:1011
bool IsInitialBlockDownload() const
Check whether we are doing an initial block download (synchronizing from disk or network)
const util::SignalInterrupt & m_interrupt
Definition: validation.h:1033
void LoadExternalBlockFile(AutoFile &file_in, FlatFilePos *dbp=nullptr, std::multimap< uint256, FlatFilePos > *blocks_with_unknown_parent=nullptr)
Import blocks from an external file.
const CChainParams & GetParams() const
Definition: validation.h:1006
Chainstate &InitializeChainstate(CTxMemPool *mempool) EXCLUSIVE_LOCKS_REQUIRED(std::vector< Chainstate * GetAll)()
Instantiate a new chainstate.
Definition: validation.h:1092
node::BlockManager m_blockman
A single BlockManager instance is shared across each constructed chainstate to avoid duplicating bloc...
Definition: validation.h:1037
Fast randomness source.
Definition: random.h:386
void fillrand(std::span< std::byte > output) noexcept
Fill a byte span with random bytes.
Definition: random.cpp:626
FlatFileSeq represents a sequence of numbered files storing raw data.
Definition: flatfile.h:46
FILE * Open(const FlatFilePos &pos, bool read_only=false) const
Open a handle to the file at the given position.
Definition: flatfile.cpp:33
fs::path FileName(const FlatFilePos &pos) const
Get the name of the file at the given position.
Definition: flatfile.cpp:28
bool Flush(const FlatFilePos &pos, bool finalize=false) const
Commit a file to disk, and optionally truncate off extra pre-allocated bytes if final.
Definition: flatfile.cpp:86
size_t Allocate(const FlatFilePos &pos, size_t add_size, bool &out_of_space) const
Allocate additional space in a file after the given starting position.
Definition: flatfile.cpp:57
Reads data from an underlying stream, while hashing the read data.
Definition: hash.h:151
A writer stream (for serialization) that computes a 256-bit hash.
Definition: hash.h:101
Minimal stream for reading from an existing byte array by std::span.
Definition: streams.h:83
std::string ToString() const
Definition: validation.h:111
constexpr bool IsNull() const
Definition: uint256.h:48
bool ReadLastBlockFile(int &nFile)
bool ReadFlag(const std::string &name, bool &fValue)
bool ReadBlockFileInfo(int nFile, CBlockFileInfo &info)
void ReadReindexing(bool &fReindexing)
bool WriteFlag(const std::string &name, bool fValue)
bool WriteBatchSync(const std::vector< std::pair< int, const CBlockFileInfo * > > &fileInfo, int nLastFile, const std::vector< const CBlockIndex * > &blockinfo)
bool WriteReindexing(bool fReindexing)
virtual void fatalError(const bilingual_str &message)
The fatal error notification is sent to notify the user when an error occurs in kernel code that can'...
virtual void flushError(const bilingual_str &message)
The flush error notification is sent to notify the user that an error occurred while flushing block d...
const kernel::BlockManagerOpts m_opts
Definition: blockstorage.h:256
std::set< int > m_dirty_fileinfo
Dirty block file entries.
Definition: blockstorage.h:244
const FlatFileSeq m_undo_file_seq
Definition: blockstorage.h:259
RecursiveMutex cs_LastBlockFile
Definition: blockstorage.h:204
const CChainParams & GetParams() const
Definition: blockstorage.h:144
bool FlushChainstateBlockFile(int tip_height)
void FindFilesToPrune(std::set< int > &setFilesToPrune, int last_prune, const Chainstate &chain, ChainstateManager &chainman)
Prune block and undo files (blk???.dat and rev???.dat) so that the disk space used is less than a use...
void UpdateBlockInfo(const CBlock &block, unsigned int nHeight, const FlatFilePos &pos)
Update blockfile info while processing a block during reindex.
const Obfuscation m_obfuscation
Definition: blockstorage.h:238
CBlockIndex * LookupBlockIndex(const uint256 &hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
bool ReadBlockUndo(CBlockUndo &blockundo, const CBlockIndex &index) const
void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
Mark one block file as pruned (modify associated database entries)
BlockfileType BlockfileTypeForHeight(int height)
std::atomic_bool m_blockfiles_indexed
Whether all blockfiles have been added to the block tree database.
Definition: blockstorage.h:275
std::vector< CBlockIndex * > GetAllBlockIndices() EXCLUSIVE_LOCKS_REQUIRED(std::multimap< CBlockIndex *, CBlockIndex * > m_blocks_unlinked
All pairs A->B, where A (or one of its ancestors) misses transactions, but B has transactions.
Definition: blockstorage.h:293
const Consensus::Params & GetConsensus() const
Definition: blockstorage.h:145
BlockManager(const util::SignalInterrupt &interrupt, Options opts)
std::set< CBlockIndex * > m_dirty_blockindex
Dirty block index entries.
Definition: blockstorage.h:241
fs::path GetBlockPosFilename(const FlatFilePos &pos) const
Translation to a filesystem path.
bool FlushBlockFile(int blockfile_num, bool fFinalize, bool finalize_undo)
Return false if block file or undo file flushing fails.
uint64_t GetPruneTarget() const
Attempt to stay below this number of bytes of block files.
Definition: blockstorage.h:352
int MaxBlockfileNum() const EXCLUSIVE_LOCKS_REQUIRED(cs_LastBlockFile)
Definition: blockstorage.h:222
void UnlinkPrunedFiles(const std::set< int > &setFilesToPrune) const
Actually unlink the specified files.
bool WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(bool LoadBlockIndexDB(const std::optional< uint256 > &snapshot_blockhash) EXCLUSIVE_LOCKS_REQUIRED(void ScanAndUnlinkAlreadyPrunedFiles() EXCLUSIVE_LOCKS_REQUIRED(CBlockIndex * AddToBlockIndex(const CBlockHeader &block, CBlockIndex *&best_header) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
Remove any pruned block & undo files that are still on disk.
Definition: blockstorage.h:314
FlatFilePos FindNextBlockPos(unsigned int nAddSize, unsigned int nHeight, uint64_t nTime)
Helper function performing various preparations before a block can be saved to disk: Returns the corr...
const bool m_prune_mode
Definition: blockstorage.h:236
bool CheckBlockDataAvailability(const CBlockIndex &upper_block LIFETIMEBOUND, const CBlockIndex &lower_block LIFETIMEBOUND) EXCLUSIVE_LOCKS_REQUIRED(const CBlockIndex *GetFirstBlock(const CBlockIndex &upper_block LIFETIMEBOUND, uint32_t status_mask, const CBlockIndex *lower_block=nullptr) const EXCLUSIVE_LOCKS_REQUIRED(boo m_have_pruned)
Check if all blocks in the [upper_block, lower_block] range have data available.
Definition: blockstorage.h:394
bool FlushUndoFile(int block_file, bool finalize=false)
Return false if undo file flushing fails.
uint64_t CalculateCurrentUsage()
Calculate the amount of disk space the block & undo files currently use.
bool ReadRawBlock(std::vector< std::byte > &block, const FlatFilePos &pos) const
const util::SignalInterrupt & m_interrupt
Definition: blockstorage.h:266
const FlatFileSeq m_block_file_seq
Definition: blockstorage.h:258
CBlockIndex * InsertBlockIndex(const uint256 &hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
Create a new block index entry for a given block hash.
bool ReadBlock(CBlock &block, const FlatFilePos &pos, const std::optional< uint256 > &expected_hash) const
Functions for disk access for blocks.
bool m_check_for_pruning
Global flag to indicate we should check to see if there are block/undo files that should be deleted.
Definition: blockstorage.h:234
bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize)
bool IsPruneMode() const
Whether running in -prune mode.
Definition: blockstorage.h:349
void CleanupBlockRevFiles() const
void FindFilesToPruneManual(std::set< int > &setFilesToPrune, int nManualPruneHeight, const Chainstate &chain, ChainstateManager &chainman)
std::atomic< bool > m_importing
Definition: blockstorage.h:267
bool WriteBlockUndo(const CBlockUndo &blockundo, BlockValidationState &state, CBlockIndex &block) EXCLUSIVE_LOCKS_REQUIRED(FlatFilePos WriteBlock(const CBlock &block, int nHeight)
Store block on disk and update block file statistics.
Definition: blockstorage.h:338
bool IsBlockPruned(const CBlockIndex &block) const EXCLUSIVE_LOCKS_REQUIRED(void UpdatePruneLock(const std::string &name, const PruneLockInfo &lock_info) EXCLUSIVE_LOCKS_REQUIRED(AutoFile OpenBlockFile(const FlatFilePos &pos, bool fReadOnly) const
Check whether the block associated with this index entry is pruned or not.
Definition: blockstorage.h:403
std::vector< CBlockFileInfo > m_blockfile_info
Definition: blockstorage.h:205
CBlockFileInfo * GetBlockFileInfo(size_t n)
Get block file info entry for one block file.
bool LoadBlockIndex(const std::optional< uint256 > &snapshot_blockhash) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
Load the blocktree off disk and into memory.
AutoFile OpenUndoFile(const FlatFilePos &pos, bool fReadOnly=false) const
Open an undo file (rev?????.dat)
std::optional< int > m_snapshot_height
The height of the base block of an assumeutxo snapshot, if one is in use.
Definition: blockstorage.h:291
ImportingNow(std::atomic< bool > &importing)
std::atomic< bool > & m_importing
256-bit opaque blob.
Definition: uint256.h:196
Helper class that manages an interrupt flag, and allows a thread or signal to interrupt another threa...
RecursiveMutex cs_main
Mutex to guard access to validation specific variables, such as reading or changing the chainstate.
Definition: cs_main.cpp:8
static bool exists(const path &p)
Definition: fs.h:89
static std::string PathToString(const path &path)
Convert path object to a byte string.
Definition: fs.h:151
std::string HexStr(const std::span< const uint8_t > s)
Convert a span of bytes to a lower-case hexadecimal string.
Definition: hex_base.cpp:29
#define LogPrintLevel(category, level,...)
Definition: logging.h:372
#define LogInfo(...)
Definition: logging.h:356
#define LogError(...)
Definition: logging.h:358
#define LogDebug(category,...)
Definition: logging.h:381
#define LogPrintf(...)
Definition: logging.h:361
unsigned int nHeight
std::array< uint8_t, 4 > MessageStartChars
@ BLOCKSTORAGE
Definition: logging.h:93
@ PRUNE
Definition: logging.h:80
FILE * fopen(const fs::path &p, const char *mode)
Definition: fs.cpp:26
static constexpr uint8_t DB_REINDEX_FLAG
static constexpr uint8_t DB_FLAG
static constexpr uint8_t DB_BLOCK_INDEX
static constexpr uint8_t DB_LAST_BLOCK
static constexpr uint8_t DB_BLOCK_FILES
Definition: messages.h:20
static const unsigned int UNDOFILE_CHUNK_SIZE
The pre-allocation chunk size for rev?????.dat files (since 0.8)
Definition: blockstorage.h:73
BlockfileType
Definition: blockstorage.h:102
@ ASSUMED
Definition: blockstorage.h:105
static auto InitBlocksdirXorKey(const BlockManager::Options &opts)
static const unsigned int BLOCKFILE_CHUNK_SIZE
The pre-allocation chunk size for blk?????.dat files (since 0.8)
Definition: blockstorage.h:71
static constexpr uint32_t STORAGE_HEADER_BYTES
Size of header written by WriteBlock before a serialized CBlock (8 bytes)
Definition: blockstorage.h:78
std::ostream & operator<<(std::ostream &os, const BlockfileType &type)
static constexpr uint32_t UNDO_DATA_DISK_OVERHEAD
Total overhead when writing undo data: header (8 bytes) plus checksum (32 bytes)
Definition: blockstorage.h:81
static const unsigned int MAX_BLOCKFILE_SIZE
The maximum size of a blk?????.dat file (since 0.8)
Definition: blockstorage.h:75
void ImportBlocks(ChainstateManager &chainman, std::span< const fs::path > import_paths)
std::string ToString(const T &t)
Locale-independent version of std::to_string.
Definition: string.h:245
bool CheckProofOfWork(uint256 hash, unsigned int nBits, const Consensus::Params &params)
Check whether a block hash satisfies the proof-of-work requirement specified by nBits.
Definition: pow.cpp:140
static constexpr TransactionSerParams TX_WITH_WITNESS
Definition: transaction.h:195
const char * name
Definition: rest.cpp:50
size_t GetSerializeSize(const T &t)
Definition: serialize.h:1094
static constexpr uint64_t MAX_SIZE
The maximum size of a serialized object in bytes or number of elements (for eg vectors) when the size...
Definition: serialize.h:32
bool CheckSignetBlockSolution(const CBlock &block, const Consensus::Params &consensusParams)
Extract signature and check whether a block has a valid solution.
Definition: signet.cpp:125
Holds configuration for use during UTXO snapshot load and validation.
Definition: chainparams.h:35
uint64_t m_chain_tx_count
Used to populate the m_chain_tx_count value, which is used during BlockManager::LoadBlockIndex().
Definition: chainparams.h:45
Parameters that influence chain consensus.
Definition: params.h:83
bool wipe_data
If true, remove all existing data.
Definition: dbwrapper.h:41
int nFile
Definition: flatfile.h:16
std::string ToString() const
Definition: flatfile.cpp:23
unsigned int nPos
Definition: flatfile.h:17
bool IsNull() const
Definition: flatfile.h:36
An options struct for BlockManager, more ergonomically referred to as BlockManager::Options due to th...
Notifications & notifications
bool operator()(const CBlockIndex *pa, const CBlockIndex *pb) const
bool operator()(const CBlockIndex *pa, const CBlockIndex *pb) const
#define LOCK2(cs1, cs2)
Definition: sync.h:260
#define LOCK(cs)
Definition: sync.h:259
#define WITH_LOCK(cs, code)
Run code while locking a mutex.
Definition: sync.h:290
std::string SysErrorString(int err)
Return system error string from errno value.
Definition: syserror.cpp:19
static int count
#define EXCLUSIVE_LOCKS_REQUIRED(...)
Definition: threadsafety.h:51
#define strprintf
Format arguments and return the string or write to given std::ostream (see tinyformat::format doc for...
Definition: tinyformat.h:1172
consteval auto _(util::TranslatedLiteral str)
Definition: translation.h:79
bool FatalError(Notifications &notifications, BlockValidationState &state, const bilingual_str &message)
AssertLockHeld(pool.cs)
assert(!tx.IsCoinBase())
static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES
Definition: validation.h:82